@lix-js/sdk 0.6.0-preview.3 → 0.6.0-preview.5

package/dist-engine-src/src/tracked_state/storage.rs

@@ -1,236 +1,97 @@
 use std::collections::HashMap;
 
-use crate::json_store::JsonStoreContext;
-use crate::storage::{KvGetGroup, KvGetRequest, StorageReader, StorageWriteSet};
+use crate::storage::{PointReadPlan, StorageRead, StorageSpace, StorageWriteSet};
+use crate::storage::{
+    StorageGetOptions, StorageKey, StorageProjectedValue, StorageSpaceId, StorageValue,
+};
 use crate::tracked_state::codec::PendingChunkWrite;
 use crate::tracked_state::types::{
-    TrackedStateDeltaEntry, TrackedStateDeltaRef, TrackedStateRootId, TRACKED_STATE_HASH_BYTES,
+    TrackedStateCommitRoot, TrackedStateCommitRootParent, TrackedStateRootId,
+    TRACKED_STATE_HASH_BYTES,
 };
 use crate::LixError;
+use bytes::Bytes;
+
+pub(crate) const TRACKED_STATE_TREE_CHUNK_NAMESPACE: &'static str = "tracked_state.tree_chunk";
+pub(crate) const TRACKED_STATE_COMMIT_ROOT_NAMESPACE: &'static str = "tracked_state.commit_root";
+pub(crate) const TRACKED_STATE_TREE_CHUNK_SPACE: StorageSpace = StorageSpace::new(
+    StorageSpaceId(0x0004_0001),
+    TRACKED_STATE_TREE_CHUNK_NAMESPACE,
+);
+pub(crate) const TRACKED_STATE_COMMIT_ROOT_SPACE: StorageSpace = StorageSpace::new(
+    StorageSpaceId(0x0004_0004),
+    TRACKED_STATE_COMMIT_ROOT_NAMESPACE,
+);
 
-pub(crate) const TRACKED_STATE_CHUNK_NAMESPACE: &'static str = "tracked_state.tree.chunk";
-pub(crate) const TRACKED_STATE_ROOT_NAMESPACE: &'static str = "tracked_state.tree.root";
-pub(crate) const TRACKED_STATE_BY_FILE_ROOT_NAMESPACE: &'static str =
-    "tracked_state.tree.root.by_file";
-pub(crate) const TRACKED_STATE_DELTA_PACK_NAMESPACE: &'static str = "tracked_state.delta_pack";
+const COMMIT_ROOT_MAGIC: &[u8; 5] = b"LXTR1";
 
 async fn get_one(
-    store: &mut (impl StorageReader + ?Sized),
-    namespace: &str,
+    store: &(impl StorageRead + ?Sized),
+    space: StorageSpace,
     key: Vec<u8>,
 ) -> Result<Option<Vec<u8>>, LixError> {
-    Ok(store
-        .get_values(KvGetRequest {
-            groups: vec![KvGetGroup {
-                namespace: namespace.to_string(),
-                keys: vec![key],
-            }],
-        })
-        .await?
-        .groups
+    let result = PointReadPlan::new(space, &[StorageKey(Bytes::from(key))])
+        .materialize(store, StorageGetOptions::default())?;
+    Ok(result
+        .value
         .into_iter()
         .next()
-        .and_then(|group| group.single_value_owned()))
+        .flatten()
+        .and_then(full_value))
 }
 
 pub(crate) async fn load_root(
-    store: &mut (impl StorageReader + ?Sized),
+    store: &(impl StorageRead + ?Sized),
     commit_id: &str,
 ) -> Result<Option<TrackedStateRootId>, LixError> {
-    let Some(bytes) = get_one(
-        store,
-        TRACKED_STATE_ROOT_NAMESPACE,
-        commit_id.as_bytes().to_vec(),
-    )
-    .await?
-    else {
-        return Ok(None);
-    };
-    TrackedStateRootId::from_slice(&bytes).map(Some)
-}
-
-pub(crate) fn stage_root(
-    writes: &mut StorageWriteSet,
-    commit_id: &str,
-    root_id: &TrackedStateRootId,
-) {
-    writes.put(
-        TRACKED_STATE_ROOT_NAMESPACE,
-        commit_id.as_bytes().to_vec(),
-        root_id.as_bytes().to_vec(),
-    );
+    Ok(load_commit_root(store, commit_id)
+        .await?
+        .map(|metadata| metadata.root_id))
 }
 
-pub(crate) async fn load_by_file_root(
-    store: &mut (impl StorageReader + ?Sized),
+pub(crate) async fn load_commit_root(
+    store: &(impl StorageRead + ?Sized),
     commit_id: &str,
-) -> Result<Option<TrackedStateRootId>, LixError> {
+) -> Result<Option<TrackedStateCommitRoot>, LixError> {
     let Some(bytes) = get_one(
         store,
-        TRACKED_STATE_BY_FILE_ROOT_NAMESPACE,
+        TRACKED_STATE_COMMIT_ROOT_SPACE,
         commit_id.as_bytes().to_vec(),
     )
     .await?
     else {
         return Ok(None);
     };
-    TrackedStateRootId::from_slice(&bytes).map(Some)
-}
-
-pub(crate) fn stage_by_file_root(
-    writes: &mut StorageWriteSet,
-    commit_id: &str,
-    root_id: &TrackedStateRootId,
-) {
-    writes.put(
-        TRACKED_STATE_BY_FILE_ROOT_NAMESPACE,
-        commit_id.as_bytes().to_vec(),
-        root_id.as_bytes().to_vec(),
-    );
-}
-
-pub(crate) async fn load_delta_pack(
-    store: &mut (impl StorageReader + ?Sized),
-    commit_id: &str,
-) -> Result<Option<Vec<TrackedStateDeltaEntry>>, LixError> {
-    let json_store = JsonStoreContext::new();
-    let result = store
-        .get_values(KvGetRequest {
-            groups: vec![
-                KvGetGroup {
-                    namespace: TRACKED_STATE_DELTA_PACK_NAMESPACE.to_string(),
-                    keys: vec![commit_id.as_bytes().to_vec()],
-                },
-                json_store.commit_pack_get_group(commit_id, 0),
-            ],
-        })
-        .await?;
-    let mut groups = result.groups.into_iter();
-    let delta_group = groups.next().ok_or_else(|| {
-        LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            "tracked-state delta pack load returned no delta result group",
-        )
-    })?;
-    let json_pack_group = groups.next().ok_or_else(|| {
-        LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            "tracked-state delta pack load returned no JSON pack result group",
-        )
-    })?;
-    let Some(bytes) = delta_group.single_value_owned() else {
-        return Ok(None);
-    };
-    let pack_refs = if crate::tracked_state::codec::delta_pack_uses_json_pack_indexes(&bytes)? {
-        json_pack_group
-            .single_value_owned()
-            .map(|bytes| json_store.decode_pack_refs(&bytes))
-            .transpose()?
-    } else {
-        None
-    };
-    let (stored_commit_id, entries) =
-        crate::tracked_state::codec::decode_delta_pack(&bytes, pack_refs.as_deref())?;
-    if stored_commit_id != commit_id {
+    let metadata = decode_commit_root(&bytes)?;
+    if metadata.commit_id != commit_id {
         return Err(LixError::new(
             LixError::CODE_INTERNAL_ERROR,
             format!(
-                "tracked-state delta pack identity mismatch: expected '{commit_id}', got '{stored_commit_id}'"
+                "tracked_state commit_root key for commit '{commit_id}' contains root metadata for commit '{}'",
+                metadata.commit_id
             ),
         ));
     }
-    Ok(Some(entries))
+    Ok(Some(metadata))
 }
 
-pub(crate) async fn delta_pack_exists(
-    store: &mut (impl StorageReader + ?Sized),
-    commit_id: &str,
-) -> Result<bool, LixError> {
-    let result = store
-        .exists_many(KvGetRequest {
-            groups: vec![KvGetGroup {
-                namespace: TRACKED_STATE_DELTA_PACK_NAMESPACE.to_string(),
-                keys: vec![commit_id.as_bytes().to_vec()],
-            }],
-        })
-        .await?;
-    let group = result.groups.into_iter().next().ok_or_else(|| {
-        LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            "tracked-state delta pack existence check returned no result group",
-        )
-    })?;
-    group.exists.into_iter().next().ok_or_else(|| {
-        LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            "tracked-state delta pack existence check returned no result",
-        )
-    })
-}
-
-pub(crate) fn stage_delta_pack_refs(
-    writes: &mut StorageWriteSet,
-    commit_id: &str,
-    deltas: &[TrackedStateDeltaRef<'_>],
-) -> Result<(), LixError> {
-    writes.put(
-        TRACKED_STATE_DELTA_PACK_NAMESPACE,
-        commit_id.as_bytes().to_vec(),
-        crate::tracked_state::codec::encode_delta_pack_refs(commit_id, deltas)?,
-    );
-    Ok(())
-}
-
-pub(crate) struct DeltaJsonPackIndexesRef<'a> {
-    pub(crate) commit_id: &'a str,
-    pub(crate) pack_id: u32,
-    pub(crate) indexes: &'a std::collections::HashMap<[u8; TRACKED_STATE_HASH_BYTES], usize>,
-}
-
-pub(crate) fn stage_delta_pack_refs_with_json_pack_indexes(
+pub(crate) fn stage_commit_root(
     writes: &mut StorageWriteSet,
-    commit_id: &str,
-    deltas: &[TrackedStateDeltaRef<'_>],
-    json_pack_indexes: DeltaJsonPackIndexesRef<'_>,
+    metadata: &TrackedStateCommitRoot,
 ) -> Result<(), LixError> {
-    if json_pack_indexes.commit_id != commit_id {
-        return Err(LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            format!(
-                "tracked-state delta JSON pack indexes for '{}' cannot encode delta pack '{}'",
-                json_pack_indexes.commit_id, commit_id
-            ),
-        ));
-    }
-    if json_pack_indexes.pack_id != 0 {
-        return Err(LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            format!(
-                "tracked-state delta JSON pack indexes only support pack 0, got pack {}",
-                json_pack_indexes.pack_id
-            ),
-        ));
-    }
-    if json_pack_indexes.indexes.is_empty() {
-        return stage_delta_pack_refs(writes, commit_id, deltas);
-    }
     writes.put(
-        TRACKED_STATE_DELTA_PACK_NAMESPACE,
-        commit_id.as_bytes().to_vec(),
-        crate::tracked_state::codec::encode_delta_pack_refs_with_json_pack_indexes(
-            commit_id,
-            deltas,
-            Some(json_pack_indexes.indexes),
-        )?,
+        TRACKED_STATE_COMMIT_ROOT_SPACE,
+        key(metadata.commit_id.as_bytes().to_vec()),
+        value(encode_commit_root(metadata)?),
     );
     Ok(())
 }
 
 pub(crate) async fn read_chunk(
-    store: &mut (impl StorageReader + ?Sized),
+    store: &(impl StorageRead + ?Sized),
     hash: &[u8; TRACKED_STATE_HASH_BYTES],
 ) -> Result<Option<Vec<u8>>, LixError> {
-    get_one(store, TRACKED_STATE_CHUNK_NAMESPACE, hash.to_vec()).await
+    get_one(store, TRACKED_STATE_TREE_CHUNK_SPACE, hash.to_vec()).await
 }
 
 pub(crate) fn verify_chunk_hash(
@@ -250,14 +111,13 @@ pub(crate) fn verify_chunk_hash(
 pub(crate) fn stage_chunks(writes: &mut StorageWriteSet, chunks: &[PendingChunkWrite]) {
     for chunk in chunks {
         writes.put(
-            TRACKED_STATE_CHUNK_NAMESPACE,
-            chunk.hash.to_vec(),
-            chunk.data.clone(),
+            TRACKED_STATE_TREE_CHUNK_SPACE,
+            key(chunk.hash.to_vec()),
+            value(chunk.data.clone()),
         );
     }
 }
 
-#[allow(dead_code)]
 #[derive(Debug, Default)]
 pub(crate) struct TrackedStateChunkOverlay {
     chunks: HashMap<[u8; TRACKED_STATE_HASH_BYTES], Vec<u8>>,
@@ -270,7 +130,7 @@ impl TrackedStateChunkOverlay {
 
     pub(crate) async fn read_chunk(
         &self,
-        store: &mut (impl StorageReader + ?Sized),
+        store: &(impl StorageRead + ?Sized),
         hash: &[u8; TRACKED_STATE_HASH_BYTES],
     ) -> Result<Option<Vec<u8>>, LixError> {
         if let Some(bytes) = self.chunks.get(hash) {
@@ -291,11 +151,203 @@ impl TrackedStateChunkOverlay {
     }
 }
 
+fn key(bytes: Vec<u8>) -> StorageKey {
+    StorageKey(Bytes::from(bytes))
+}
+
+fn value(bytes: Vec<u8>) -> StorageValue {
+    StorageValue {
+        bytes: Bytes::from(bytes),
+    }
+}
+
+fn full_value(value: StorageProjectedValue) -> Option<Vec<u8>> {
+    match value {
+        StorageProjectedValue::FullValue(bytes) => Some(bytes.to_vec()),
+        StorageProjectedValue::KeyOnly => None,
+    }
+}
+
+fn encode_commit_root(metadata: &TrackedStateCommitRoot) -> Result<Vec<u8>, LixError> {
+    let mut out = Vec::new();
+    out.extend_from_slice(COMMIT_ROOT_MAGIC);
+    write_string(&mut out, &metadata.commit_id)?;
+    out.extend_from_slice(metadata.root_id.as_bytes());
+    write_u32(&mut out, metadata.parent_roots.len(), "parent root count")?;
+    for parent in &metadata.parent_roots {
+        write_string(&mut out, &parent.commit_id)?;
+        out.extend_from_slice(parent.root_id.as_bytes());
+    }
+    out.extend_from_slice(&metadata.changed_key_count.to_le_bytes());
+    out.extend_from_slice(&metadata.row_count_estimate.to_le_bytes());
+    out.extend_from_slice(&metadata.tree_height.to_le_bytes());
+    out.extend_from_slice(&metadata.primary_chunk_count.to_le_bytes());
+    out.extend_from_slice(&metadata.primary_chunk_bytes.to_le_bytes());
+    Ok(out)
+}
+
+fn decode_commit_root(bytes: &[u8]) -> Result<TrackedStateCommitRoot, LixError> {
+    let mut cursor = CommitRootCursor { bytes, offset: 0 };
+    cursor.expect_magic()?;
+    let commit_id = cursor.read_string("commit_id")?;
+    let root_id = cursor.read_root_id("root_id")?;
+    let parent_count = cursor.read_u32("parent root count")? as usize;
+    let mut parent_roots = Vec::with_capacity(parent_count);
+    for _ in 0..parent_count {
+        parent_roots.push(TrackedStateCommitRootParent {
+            commit_id: cursor.read_string("parent commit_id")?,
+            root_id: cursor.read_root_id("parent root_id")?,
+        });
+    }
+    let changed_key_count = cursor.read_u64("changed_key_count")?;
+    let row_count_estimate = cursor.read_u64("row_count_estimate")?;
+    let tree_height = cursor.read_u32("tree_height")?;
+    let primary_chunk_count = cursor.read_u64("primary_chunk_count")?;
+    let primary_chunk_bytes = cursor.read_u64("primary_chunk_bytes")?;
+    cursor.expect_end()?;
+    Ok(TrackedStateCommitRoot {
+        commit_id,
+        root_id,
+        parent_roots,
+        changed_key_count,
+        row_count_estimate,
+        tree_height,
+        primary_chunk_count,
+        primary_chunk_bytes,
+    })
+}
+
+fn write_string(out: &mut Vec<u8>, value: &str) -> Result<(), LixError> {
+    write_u32(out, value.len(), "string length")?;
+    out.extend_from_slice(value.as_bytes());
+    Ok(())
+}
+
+fn write_u32(out: &mut Vec<u8>, value: usize, field: &str) -> Result<(), LixError> {
+    let value = u32::try_from(value).map_err(|_| {
+        LixError::new(
+            LixError::CODE_INTERNAL_ERROR,
+            format!("tracked_state commit root metadata {field} exceeds u32"),
+        )
+    })?;
+    out.extend_from_slice(&value.to_le_bytes());
+    Ok(())
+}
+
+struct CommitRootCursor<'a> {
+    bytes: &'a [u8],
+    offset: usize,
+}
+
+impl CommitRootCursor<'_> {
+    fn expect_magic(&mut self) -> Result<(), LixError> {
+        let magic = self.read_exact(COMMIT_ROOT_MAGIC.len(), "magic")?;
+        if magic != COMMIT_ROOT_MAGIC {
+            return Err(LixError::new(
+                LixError::CODE_INTERNAL_ERROR,
+                "failed to decode tracked_state commit_root: bad magic",
+            ));
+        }
+        Ok(())
+    }
+
+    fn expect_end(&self) -> Result<(), LixError> {
+        if self.offset != self.bytes.len() {
+            return Err(LixError::new(
+                LixError::CODE_INTERNAL_ERROR,
+                "failed to decode tracked_state commit_root: trailing bytes",
+            ));
+        }
+        Ok(())
+    }
+
+    fn read_string(&mut self, field: &str) -> Result<String, LixError> {
+        let len = self.read_u32(field)? as usize;
+        let bytes = self.read_exact(len, field)?;
+        String::from_utf8(bytes.to_vec()).map_err(|error| {
+            LixError::new(
+                LixError::CODE_INTERNAL_ERROR,
+                format!("failed to decode tracked_state commit_root {field}: {error}"),
+            )
+        })
+    }
+
+    fn read_root_id(&mut self, field: &str) -> Result<TrackedStateRootId, LixError> {
+        let bytes = self.read_exact(TRACKED_STATE_HASH_BYTES, field)?;
+        TrackedStateRootId::from_slice(bytes)
+    }
+
+    fn read_u32(&mut self, field: &str) -> Result<u32, LixError> {
+        let bytes = self.read_exact(std::mem::size_of::<u32>(), field)?;
+        Ok(u32::from_le_bytes(bytes.try_into().expect("fixed u32")))
+    }
+
+    fn read_u64(&mut self, field: &str) -> Result<u64, LixError> {
+        let bytes = self.read_exact(std::mem::size_of::<u64>(), field)?;
+        Ok(u64::from_le_bytes(bytes.try_into().expect("fixed u64")))
+    }
+
+    fn read_exact(&mut self, len: usize, field: &str) -> Result<&[u8], LixError> {
+        let end = self.offset.checked_add(len).ok_or_else(|| {
+            LixError::new(
+                LixError::CODE_INTERNAL_ERROR,
+                format!("failed to decode tracked_state commit_root {field}: overflow"),
+            )
+        })?;
+        if end > self.bytes.len() {
+            return Err(LixError::new(
+                LixError::CODE_INTERNAL_ERROR,
+                format!("failed to decode tracked_state commit_root {field}: truncated"),
+            ));
+        }
+        let out = &self.bytes[self.offset..end];
+        self.offset = end;
+        Ok(out)
+    }
+}
+
 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeMap;
     use std::fs;
     use std::path::{Path, PathBuf};
 
+    use crate::binary_cas::kv::{
+        BINARY_CAS_CHUNK_SPACE, BINARY_CAS_MANIFEST_CHUNK_SPACE, BINARY_CAS_MANIFEST_SPACE,
+    };
+    use crate::changelog::{CHANGE_SPACE, COMMIT_CHANGE_REF_CHUNK_SPACE, COMMIT_SPACE};
+    use crate::json_store::store::JSON_SPACE;
+    use crate::untracked_state::storage::UNTRACKED_STATE_ROW_SPACE;
+
+    use super::{TRACKED_STATE_COMMIT_ROOT_SPACE, TRACKED_STATE_TREE_CHUNK_SPACE};
+
+    #[test]
+    fn native_storage_space_ids_are_unique_across_owner_layouts() {
+        let spaces = [
+            UNTRACKED_STATE_ROW_SPACE,
+            JSON_SPACE,
+            TRACKED_STATE_TREE_CHUNK_SPACE,
+            TRACKED_STATE_COMMIT_ROOT_SPACE,
+            BINARY_CAS_MANIFEST_SPACE,
+            BINARY_CAS_MANIFEST_CHUNK_SPACE,
+            BINARY_CAS_CHUNK_SPACE,
+            COMMIT_SPACE,
+            CHANGE_SPACE,
+            COMMIT_CHANGE_REF_CHUNK_SPACE,
+        ];
+        let mut seen = BTreeMap::new();
+        for space in spaces {
+            assert_eq!(
+                seen.insert(space.id, space.name),
+                None,
+                "storage space id {:?} is reused by {} and {}",
+                space.id,
+                seen.get(&space.id).copied().unwrap_or(space.name),
+                space.name
+            );
+        }
+    }
+
     #[test]
     fn production_tracked_state_sources_do_not_call_storage_batch_writer() {
         let tracked_state_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/tracked_state");