@lix-js/sdk 0.6.0-preview.4 → 0.6.0-preview.5

package/dist-engine-src/src/json_store/context.rs

@@ -4,11 +4,10 @@ use crate::json_store::types::{
     JsonProjectionLoadRequestRef, JsonRef, JsonValueBatch, JsonWritePlacementRef,
     NormalizedJsonRef,
 };
-use crate::storage::{KvGetGroup, StorageReader, StorageWriteSet};
+use crate::storage::{StorageKey, StorageRead, StorageValue, StorageWriteSet};
 use crate::LixError;
-use std::collections::{HashMap, HashSet};
-
-const PACK_LOCAL_MAX_JSON_BYTES: usize = 64 * 1024;
+use bytes::Bytes;
+use std::collections::HashSet;
 
 #[derive(Debug, Clone, Copy)]
 pub(crate) struct JsonStoreContext;
@@ -20,7 +19,7 @@ impl JsonStoreContext {
 
     pub(crate) fn reader<S>(&self, store: S) -> JsonStoreReader<S>
     where
-        S: StorageReader,
+        S: StorageRead,
     {
         JsonStoreReader { store }
     }
@@ -31,24 +30,13 @@ impl JsonStoreContext {
 
     pub(crate) async fn load_bytes_many(
         &self,
-        store: &mut impl StorageReader,
+        store: &(impl StorageRead + ?Sized),
         request: JsonLoadRequestRef<'_>,
     ) -> Result<JsonLoadBatch, LixError> {
         store::load_json_bytes_many_in_scope(store, request.refs, request.scope)
             .await
             .map(JsonLoadBatch::new)
     }
-
-    pub(crate) fn commit_pack_get_group(&self, commit_id: &str, pack_id: u32) -> KvGetGroup {
-        KvGetGroup {
-            namespace: store::JSON_PACK_NAMESPACE.to_string(),
-            keys: vec![store::pack_key(commit_id, pack_id)],
-        }
-    }
-
-    pub(crate) fn decode_pack_refs(&self, bytes: &[u8]) -> Result<Vec<JsonRef>, LixError> {
-        store::decode_json_pack_refs(bytes)
-    }
 }
 
 pub(crate) struct JsonStoreReader<S> {
@@ -68,13 +56,13 @@ where
 
 impl<S> JsonStoreReader<S>
 where
-    S: StorageReader,
+    S: StorageRead,
 {
     pub(crate) async fn load_bytes_many(
         &mut self,
         request: JsonLoadRequestRef<'_>,
     ) -> Result<JsonLoadBatch, LixError> {
-        store::load_json_bytes_many_in_scope(&mut self.store, request.refs, request.scope)
+        store::load_json_bytes_many_in_scope(&self.store, request.refs, request.scope)
             .await
             .map(JsonLoadBatch::new)
     }
@@ -136,12 +124,6 @@ where
 
 pub(crate) struct JsonStoreWriter;
 
-#[derive(Debug, Clone, Default)]
-pub(crate) struct JsonStageBatchReport {
-    pub(crate) refs: Vec<JsonRef>,
-    pub(crate) pack_indexes: HashMap<[u8; 32], usize>,
-}
-
 impl JsonStoreWriter {
     fn new() -> Self {
         Self
@@ -150,19 +132,10 @@ impl JsonStoreWriter {
     pub(crate) fn stage_batch<'a>(
         &mut self,
         writes: &mut StorageWriteSet,
-        placement: JsonWritePlacementRef<'a>,
+        placement: JsonWritePlacementRef,
         payloads: impl IntoIterator<Item = NormalizedJsonRef<'a>>,
     ) -> Result<Vec<JsonRef>, LixError> {
-        self.stage_batch_report(writes, placement, payloads)
-            .map(|report| report.refs)
-    }
-
-    pub(crate) fn stage_batch_report<'a>(
-        &mut self,
-        writes: &mut StorageWriteSet,
-        placement: JsonWritePlacementRef<'a>,
-        payloads: impl IntoIterator<Item = NormalizedJsonRef<'a>>,
-    ) -> Result<JsonStageBatchReport, LixError> {
+        let JsonWritePlacementRef::OutOfBand = placement;
         let mut unique_encoded = Vec::new();
         let mut order = Vec::new();
         let mut seen = HashSet::new();
@@ -184,240 +157,16 @@ impl JsonStoreWriter {
             }
         }
 
-        let pack_local = matches!(placement, JsonWritePlacementRef::CommitPack { .. });
-        let mut pack_indexes = HashMap::new();
-        if let JsonWritePlacementRef::CommitPack { commit_id, pack_id } = placement {
-            let pack_entries = unique_encoded
-                .iter()
-                .filter(|encoded| encoded.uncompressed_len <= PACK_LOCAL_MAX_JSON_BYTES)
-                .collect::<Vec<_>>();
-            for (index, encoded) in pack_entries.iter().enumerate() {
-                pack_indexes.insert(*encoded.json_ref.as_hash_array(), index);
-            }
-            if !pack_entries.is_empty() {
-                let encoded_pack = store::encode_json_pack(&pack_entries)?;
-                writes.put(
-                    store::JSON_PACK_NAMESPACE,
-                    store::pack_key(commit_id, pack_id),
-                    encoded_pack,
-                );
-            }
-        }
-
         for encoded in &unique_encoded {
-            if pack_local && encoded.uncompressed_len <= PACK_LOCAL_MAX_JSON_BYTES {
-                continue;
-            }
             writes.put(
-                store::JSON_NAMESPACE,
-                encoded.json_ref.as_hash_bytes().to_vec(),
-                store::encode_direct_json_payload(encoded),
+                store::JSON_SPACE,
+                StorageKey(Bytes::copy_from_slice(encoded.json_ref.as_hash_bytes())),
+                StorageValue {
+                    bytes: Bytes::from(store::encode_direct_json_payload(encoded)),
+                },
             );
         }
 
-        Ok(JsonStageBatchReport {
-            refs: order,
-            pack_indexes,
-        })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use super::*;
-    use crate::backend::testing::UnitTestBackend;
-    use crate::json_store::types::JsonReadScopeRef;
-    use crate::storage::StorageContext;
-
-    #[tokio::test]
-    async fn commit_local_batch_writes_pack_without_direct_rows() {
-        let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
-        let context = JsonStoreContext::new();
-        let first = "{\"value\":\"first\"}";
-        let second = "{\"value\":\"second\"}";
-
-        let mut transaction = storage
-            .begin_write_transaction()
-            .await
-            .expect("transaction should open");
-        let mut writes = StorageWriteSet::new();
-        context
-            .writer()
-            .stage_batch(
-                &mut writes,
-                JsonWritePlacementRef::CommitPack {
-                    commit_id: "commit-a",
-                    pack_id: 0,
-                },
-                [
-                    NormalizedJsonRef::new(first),
-                    NormalizedJsonRef::new(second),
-                ],
-            )
-            .expect("json pack should stage");
-        writes
-            .apply(&mut transaction.as_mut())
-            .await
-            .expect("json pack should apply");
-        transaction
-            .commit()
-            .await
-            .expect("transaction should commit");
-
-        let refs = [
-            JsonRef::for_content(first.as_bytes()),
-            JsonRef::for_content(second.as_bytes()),
-        ];
-        let unknown = context
-            .reader(storage.clone())
-            .load_bytes_many(JsonLoadRequestRef {
-                refs: &refs,
-                scope: JsonReadScopeRef::OutOfBand,
-            })
-            .await
-            .expect("unknown load should check direct rows");
-        assert_eq!(unknown.into_values(), vec![None, None]);
-
-        let pack_ids = [0];
-        let packed = context
-            .reader(storage.clone())
-            .load_bytes_many(JsonLoadRequestRef {
-                refs: &refs,
-                scope: JsonReadScopeRef::CommitPacks {
-                    commit_id: "commit-a",
-                    pack_ids: &pack_ids,
-                },
-            })
-            .await
-            .expect("packed load should hydrate");
-        assert_eq!(
-            packed.into_values(),
-            vec![
-                Some(first.as_bytes().to_vec()),
-                Some(second.as_bytes().to_vec())
-            ]
-        );
-    }
-
-    #[tokio::test]
-    async fn commit_local_batch_dedupes_pack_payloads_but_returns_request_order() {
-        let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
-        let context = JsonStoreContext::new();
-        let first = "{\"value\":\"first\"}";
-        let second = "{\"value\":\"second\"}";
-
-        let mut transaction = storage
-            .begin_write_transaction()
-            .await
-            .expect("transaction should open");
-        let mut writes = StorageWriteSet::new();
-        let staged_refs = context
-            .writer()
-            .stage_batch(
-                &mut writes,
-                JsonWritePlacementRef::CommitPack {
-                    commit_id: "commit-a",
-                    pack_id: 0,
-                },
-                [
-                    NormalizedJsonRef::new(first),
-                    NormalizedJsonRef::new(first),
-                    NormalizedJsonRef::new(second),
-                ],
-            )
-            .expect("json pack should stage");
-        writes
-            .apply(&mut transaction.as_mut())
-            .await
-            .expect("json pack should apply");
-        transaction
-            .commit()
-            .await
-            .expect("transaction should commit");
-
-        let first_ref = JsonRef::for_content(first.as_bytes());
-        let second_ref = JsonRef::for_content(second.as_bytes());
-        assert_eq!(staged_refs, vec![first_ref, first_ref, second_ref]);
-
-        let refs = [first_ref, second_ref];
-        let unknown = context
-            .reader(storage.clone())
-            .load_bytes_many(JsonLoadRequestRef {
-                refs: &refs,
-                scope: JsonReadScopeRef::OutOfBand,
-            })
-            .await
-            .expect("unknown load should check direct rows");
-        assert_eq!(unknown.into_values(), vec![None, None]);
-
-        let pack_ids = [0];
-        let packed = context
-            .reader(storage.clone())
-            .load_bytes_many(JsonLoadRequestRef {
-                refs: &refs,
-                scope: JsonReadScopeRef::CommitPacks {
-                    commit_id: "commit-a",
-                    pack_ids: &pack_ids,
-                },
-            })
-            .await
-            .expect("packed load should hydrate");
-        assert_eq!(
-            packed.into_values(),
-            vec![
-                Some(first.as_bytes().to_vec()),
-                Some(second.as_bytes().to_vec())
-            ]
-        );
-    }
-
-    #[tokio::test]
-    async fn commit_local_batch_accepts_trusted_prehashed_payload() {
-        let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
-        let context = JsonStoreContext::new();
-        let json = "{\"value\":\"prehashed\"}";
-        let json_ref = JsonRef::for_content(json.as_bytes());
-
-        let mut transaction = storage
-            .begin_write_transaction()
-            .await
-            .expect("transaction should open");
-        let mut writes = StorageWriteSet::new();
-        let refs = context
-            .writer()
-            .stage_batch(
-                &mut writes,
-                JsonWritePlacementRef::CommitPack {
-                    commit_id: "commit-a",
-                    pack_id: 0,
-                },
-                [NormalizedJsonRef::trusted_prehashed(json, json_ref)],
-            )
-            .expect("prehashed json should stage");
-        assert_eq!(refs, vec![json_ref]);
-        writes
-            .apply(&mut transaction.as_mut())
-            .await
-            .expect("json pack should apply");
-        transaction
-            .commit()
-            .await
-            .expect("transaction should commit");
-
-        let pack_ids = [0];
-        let packed = context
-            .reader(storage.clone())
-            .load_bytes_many(JsonLoadRequestRef {
-                refs: &refs,
-                scope: JsonReadScopeRef::CommitPacks {
-                    commit_id: "commit-a",
-                    pack_ids: &pack_ids,
-                },
-            })
-            .await
-            .expect("prehashed payload should hydrate");
-        assert_eq!(packed.into_values(), vec![Some(json.as_bytes().to_vec())]);
+        Ok(order)
     }
 }

package/dist-engine-src/src/json_store/mod.rs

@@ -4,9 +4,35 @@ mod encoded;
 pub(crate) mod store;
 pub(crate) mod types;
 
+use crate::storage::StorageWriteSet;
+use crate::LixError;
+
 #[allow(unused_imports)]
 pub(crate) use context::{JsonStoreContext, JsonStoreReader, JsonStoreWriter};
 pub(crate) use types::{
     JsonLoadRequestRef, JsonReadScopeRef, JsonRef, JsonWritePlacementRef, NormalizedJson,
     NormalizedJsonRef,
 };
+
+pub(crate) fn direct_json_payload_ref_from_key(key: &[u8]) -> Result<JsonRef, LixError> {
+    let hash: [u8; 32] = key.try_into().map_err(|_| {
+        LixError::unknown(format!(
+            "json_store direct payload key has {} bytes, expected 32",
+            key.len()
+        ))
+    })?;
+    Ok(JsonRef::from_hash_bytes(hash))
+}
+
+pub(crate) fn stage_direct_json_payload_delete(writes: &mut StorageWriteSet, json_ref: &JsonRef) {
+    writes.delete(store::JSON_SPACE, json_ref.as_hash_bytes().to_vec());
+}
+
+#[cfg(any(test, feature = "storage-benches"))]
+pub(crate) fn stage_direct_json_payload_put(
+    writes: &mut StorageWriteSet,
+    json_ref: &JsonRef,
+    bytes: Vec<u8>,
+) {
+    writes.put(store::JSON_SPACE, json_ref.as_hash_bytes().to_vec(), bytes);
+}