@lix-js/sdk 0.6.0-preview.1 → 0.6.0-preview.2

package/dist-engine-src/src/json_store/encoded.rs

@@ -0,0 +1,15 @@
+use crate::json_store::types::JsonRef;
+use std::borrow::Cow;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum JsonCodec {
+    Raw,
+    Zstd,
+}
+
+pub(super) struct EncodedJson<'a> {
+    pub(super) json_ref: JsonRef,
+    pub(super) codec: JsonCodec,
+    pub(super) uncompressed_len: usize,
+    pub(super) data: Cow<'a, [u8]>,
+}

package/dist-engine-src/src/json_store/mod.rs

@@ -0,0 +1,9 @@
+pub(crate) mod compression;
+pub(crate) mod context;
+mod encoded;
+pub(crate) mod store;
+pub(crate) mod types;
+
+#[allow(unused_imports)]
+pub(crate) use context::{JsonStoreContext, JsonStoreReader, JsonStoreWriter};
+pub(crate) use types::JsonRef;

package/dist-engine-src/src/json_store/store.rs

@@ -0,0 +1,236 @@
+use crate::json_store::compression::{compress_json_payload, decode_json_zstd_payload};
+use crate::json_store::encoded::{EncodedJson, JsonCodec};
+use crate::json_store::types::JsonRef;
+use crate::storage::{KvGetGroup, KvGetRequest, StorageReader};
+use crate::LixError;
+use std::borrow::Cow;
+
+pub(crate) const JSON_NAMESPACE: &str = "json_store.json";
+const STORED_JSON_MAGIC: &[u8] = b"lix-json:v1";
+const STORED_JSON_HEADER_LEN: usize = STORED_JSON_MAGIC.len() + 1 + 8;
+const ZSTD_MIN_JSON_BYTES: usize = 16 * 1024;
+const MIN_ZSTD_SAVINGS_BYTES: usize = 128;
+
+struct StoredJsonPayload<'a> {
+    codec: JsonCodec,
+    uncompressed_len: usize,
+    data: &'a [u8],
+}
+
+fn raw_json_ref_for_content(json: &str) -> JsonRef {
+    JsonRef::from_hash(blake3::hash(json.as_bytes()))
+}
+
+#[cfg(test)]
+fn encode_json(json: &str) -> Result<EncodedJson<'_>, LixError> {
+    encode_json_for_storage(json)
+}
+
+fn encode_json_for_storage(json: &str) -> Result<EncodedJson<'_>, LixError> {
+    let raw_ref = raw_json_ref_for_content(json);
+    encode_json_for_storage_with_ref(json, raw_ref)
+}
+
+fn encode_json_for_storage_with_ref(
+    json: &str,
+    raw_ref: JsonRef,
+) -> Result<EncodedJson<'_>, LixError> {
+    let raw_data = json.as_bytes();
+
+    if raw_data.len() >= ZSTD_MIN_JSON_BYTES {
+        let compressed = compress_json_payload(raw_data)?;
+        if raw_data.len().saturating_sub(compressed.len()) >= MIN_ZSTD_SAVINGS_BYTES {
+            return Ok(EncodedJson {
+                json_ref: raw_ref,
+                codec: JsonCodec::Zstd,
+                uncompressed_len: json.len(),
+                data: Cow::Owned(compressed),
+            });
+        }
+    }
+
+    Ok(EncodedJson {
+        json_ref: raw_ref,
+        codec: JsonCodec::Raw,
+        uncompressed_len: json.len(),
+        data: Cow::Borrowed(raw_data),
+    })
+}
+
+pub(crate) fn encode_json_bytes_for_storage(bytes: &[u8]) -> Result<(JsonRef, Vec<u8>), LixError> {
+    let json = std::str::from_utf8(bytes).map_err(|error| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("json bytes are invalid UTF-8: {error}"),
+        )
+    })?;
+    let json_ref = JsonRef::from_hash(blake3::hash(bytes));
+    encode_json_str_for_storage_with_ref(json, json_ref)
+}
+
+pub(crate) fn encode_json_str_for_storage_with_ref(
+    json: &str,
+    json_ref: JsonRef,
+) -> Result<(JsonRef, Vec<u8>), LixError> {
+    let encoded_json = encode_json_for_storage_with_ref(json, json_ref)?;
+    let json_ref = encoded_json.json_ref.clone();
+    Ok((json_ref, encode_stored_json_payload(&encoded_json)))
+}
+
+pub(crate) async fn load_json_bytes(
+    store: &mut impl StorageReader,
+    json_ref: &JsonRef,
+) -> Result<Option<Vec<u8>>, LixError> {
+    let result = store
+        .get_values(KvGetRequest {
+            groups: vec![KvGetGroup {
+                namespace: JSON_NAMESPACE.to_string(),
+                keys: vec![json_ref.as_hash_bytes().to_vec()],
+            }],
+        })
+        .await?
+        .groups
+        .into_iter()
+        .next()
+        .and_then(|group| group.single_value_owned());
+    let Some(bytes) = result else {
+        return Ok(None);
+    };
+    let stored_payload = decode_stored_json_payload(&bytes)?;
+    decode_json_payload(store, json_ref, stored_payload)
+        .await
+        .map(Some)
+}
+
+fn encode_stored_json_payload(encoded_json: &EncodedJson<'_>) -> Vec<u8> {
+    let mut out = Vec::with_capacity(STORED_JSON_HEADER_LEN + encoded_json.data.as_ref().len());
+    out.extend_from_slice(STORED_JSON_MAGIC);
+    out.push(json_codec_byte(encoded_json.codec));
+    out.extend_from_slice(&(encoded_json.uncompressed_len as u64).to_be_bytes());
+    out.extend_from_slice(encoded_json.data.as_ref());
+    out
+}
+
+fn decode_stored_json_payload(bytes: &[u8]) -> Result<StoredJsonPayload<'_>, LixError> {
+    if bytes.len() < STORED_JSON_HEADER_LEN {
+        return Err(LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            "stored JSON payload is truncated",
+        ));
+    }
+    if &bytes[..STORED_JSON_MAGIC.len()] != STORED_JSON_MAGIC {
+        return Err(LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            "stored JSON payload has invalid header",
+        ));
+    }
+    let codec = read_json_codec(bytes[STORED_JSON_MAGIC.len()])?;
+    let len_start = STORED_JSON_MAGIC.len() + 1;
+    let len_end = len_start + 8;
+    let uncompressed_len = u64::from_be_bytes(
+        bytes[len_start..len_end]
+            .try_into()
+            .expect("stored JSON length header is fixed size"),
+    ) as usize;
+    Ok(StoredJsonPayload {
+        codec,
+        uncompressed_len,
+        data: &bytes[len_end..],
+    })
+}
+
+fn json_codec_byte(codec: JsonCodec) -> u8 {
+    match codec {
+        JsonCodec::Raw => 0,
+        JsonCodec::Zstd => 1,
+    }
+}
+
+fn read_json_codec(byte: u8) -> Result<JsonCodec, LixError> {
+    match byte {
+        0 => Ok(JsonCodec::Raw),
+        1 => Ok(JsonCodec::Zstd),
+        _ => Err(LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("stored JSON payload has unknown codec byte {byte}"),
+        )),
+    }
+}
+
+async fn decode_json_payload(
+    _store: &mut impl StorageReader,
+    json_ref: &JsonRef,
+    stored_payload: StoredJsonPayload<'_>,
+) -> Result<Vec<u8>, LixError> {
+    let data = match stored_payload.codec {
+        JsonCodec::Raw => Ok(stored_payload.data.to_vec()),
+        JsonCodec::Zstd => decode_json_zstd_payload(
+            stored_payload.data,
+            stored_payload.uncompressed_len,
+            &json_ref.to_hex(),
+        ),
+    }?;
+    if data.len() != stored_payload.uncompressed_len {
+        return Err(LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!(
+                "json ref '{}' decoded to {} bytes, expected {}",
+                json_ref.to_hex(),
+                data.len(),
+                stored_payload.uncompressed_len
+            ),
+        ));
+    }
+    let actual_hash = blake3::hash(&data);
+    if actual_hash.as_bytes() != json_ref.as_hash_bytes() {
+        return Err(LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("json ref '{}' hash mismatch", json_ref.to_hex()),
+        ));
+    }
+    Ok(data)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use super::*;
+    use crate::backend::testing::UnitTestBackend;
+    use crate::storage::{StorageContext, StorageWriteSet};
+
+    #[tokio::test]
+    async fn json_roundtrips_raw_payload() {
+        let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
+        let json = "{\"value\":\"small\"}";
+        let encoded = encode_json(json).expect("json should encode");
+        assert_eq!(encoded.codec, JsonCodec::Raw);
+
+        let mut transaction = storage
+            .begin_write_transaction()
+            .await
+            .expect("transaction should open");
+        let mut writes = StorageWriteSet::new();
+        writes.put(
+            JSON_NAMESPACE,
+            encoded.json_ref.as_hash_bytes().to_vec(),
+            encode_stored_json_payload(&encoded),
+        );
+        writes
+            .apply(&mut transaction.as_mut())
+            .await
+            .expect("json should store");
+        transaction
+            .commit()
+            .await
+            .expect("transaction should commit");
+
+        let mut store = storage.clone();
+        assert_eq!(
+            load_json_bytes(&mut store, &encoded.json_ref)
+                .await
+                .expect("json should load"),
+            Some(json.as_bytes().to_vec())
+        );
+    }
+}

package/dist-engine-src/src/json_store/types.rs

@@ -0,0 +1,52 @@
+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub(crate) struct JsonRef {
+    hash: [u8; 32],
+}
+
+impl JsonRef {
+    pub(crate) fn from_hash(hash: blake3::Hash) -> Self {
+        Self {
+            hash: *hash.as_bytes(),
+        }
+    }
+
+    pub(crate) fn from_hash_bytes(hash: [u8; 32]) -> Self {
+        Self { hash }
+    }
+
+    pub(crate) fn as_hash_bytes(&self) -> &[u8] {
+        &self.hash
+    }
+
+    pub(crate) fn to_hex(&self) -> String {
+        self.hash.iter().map(|byte| format!("{byte:02x}")).collect()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct JsonProjectionPath(String);
+
+impl JsonProjectionPath {
+    pub(crate) fn new(pointer: impl Into<String>) -> Self {
+        Self(pointer.into())
+    }
+
+    pub(crate) fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct JsonProjection {
+    values: Vec<Option<serde_json::Value>>,
+}
+
+impl JsonProjection {
+    pub(crate) fn new(values: Vec<Option<serde_json::Value>>) -> Self {
+        Self { values }
+    }
+
+    pub(crate) fn values(&self) -> &[Option<serde_json::Value>] {
+        &self.values
+    }
+}

package/dist-engine-src/src/lib.rs

@@ -0,0 +1,61 @@
+mod backend;
+mod binary_cas;
+pub(crate) mod cel;
+pub(crate) mod changelog;
+pub(crate) mod commit_graph;
+mod common;
+pub mod engine;
+pub(crate) mod entity_identity;
+pub(crate) mod functions;
+pub(crate) mod init;
+#[allow(dead_code)]
+pub(crate) mod json_store;
+pub(crate) mod live_state;
+mod schema;
+pub(crate) mod schema_registry;
+pub mod session;
+pub(crate) mod sql2;
+#[allow(dead_code, unused_imports)]
+pub(crate) mod storage;
+#[cfg(feature = "storage-benches")]
+pub mod storage_bench;
+#[cfg(test)]
+pub(crate) mod test_support;
+pub(crate) mod tracked_state;
+pub mod transaction;
+pub(crate) mod untracked_state;
+pub(crate) mod version;
+pub mod wasm;
+
+pub use schema::{
+    lix_schema_definition, lix_schema_definition_json, validate_lix_schema,
+    validate_lix_schema_definition,
+};
+
+pub use backend::{
+    Backend, BackendKvEntryPage, BackendKvExistsBatch, BackendKvExistsGroup, BackendKvGetGroup,
+    BackendKvGetRequest, BackendKvKeyPage, BackendKvScanRange, BackendKvScanRequest,
+    BackendKvValueBatch, BackendKvValueGroup, BackendKvValuePage, BackendKvWriteBatch,
+    BackendKvWriteGroup, BackendKvWriteStats, BackendReadTransaction, BackendWriteTransaction,
+    BytePage, BytePageBuilder,
+};
+pub use common::LixError;
+pub(crate) use common::{
+    parse_row_metadata, serialize_row_metadata, validate_row_metadata, RowMetadata,
+};
+pub use common::{
+    CanonicalPluginKey, CanonicalSchemaKey, CanonicalSchemaVersion, EntityId, FileId, VersionId,
+};
+pub use common::{LixNotice, NullableKeyFilter, SqlQueryResult, Value, WriteReceipt};
+pub use common::{WireQueryResult, WireValue};
+pub use engine::Engine;
+pub use init::InitReceipt;
+pub use session::{
+    CreateVersionOptions, CreateVersionReceipt, MergeChangeStats, MergeConflict,
+    MergeConflictChangeKind, MergeConflictKind, MergeConflictSide, MergeVersionOptions,
+    MergeVersionOutcome, MergeVersionPreview, MergeVersionPreviewOptions, MergeVersionReceipt,
+    SessionContext, SwitchVersionOptions, SwitchVersionReceipt,
+};
+pub use session::{ExecuteResult, Row, RowRef, TryFromValue};
+
+pub(crate) const GLOBAL_VERSION_ID: &str = "global";