@lix-js/sdk 0.6.0-preview.3 → 0.6.0-preview.5

package/dist-engine-src/src/changelog/mod.rs

@@ -0,0 +1,29 @@
+#[cfg(feature = "storage-benches")]
+mod bench_support;
+#[cfg(feature = "storage-benches")]
+pub mod bench {
+    pub use super::bench_support::*;
+}
+mod codec;
+mod context;
+mod store;
+#[cfg(test)]
+mod test_support;
+mod types;
+
+pub(crate) use codec::{
+    decode_change_record, decode_commit_change_ref_chunk, decode_commit_record,
+    encode_change_record, encode_commit_change_ref_chunk, encode_commit_record, view_change_record,
+    view_commit_change_ref_chunk, view_commit_record,
+};
+pub(crate) use context::{ChangelogContext, ChangelogStoreReader, ChangelogStoreWriter};
+pub(crate) use store::{ChangelogReader, ChangelogWriter};
+pub(crate) use store::{CHANGE_SPACE, COMMIT_CHANGE_REF_CHUNK_SPACE, COMMIT_SPACE};
+pub(crate) use types::{
+    ChangeId, ChangeIdRef, ChangeLoadBatch, ChangeLoadRequest, ChangeRecord, ChangeRecordView,
+    ChangeScanBatch, ChangeScanRequest, ChangelogAppend, CommitChangeRef, CommitChangeRefChunk,
+    CommitChangeRefChunkView, CommitChangeRefSet, CommitChangeRefView, CommitId, CommitIdRef,
+    CommitLoadBatch, CommitLoadEntry, CommitLoadRequest, CommitProjection, CommitRecord,
+    CommitRecordView, CommitScanBatch, CommitScanRequest, EntityPkRef, GcLiveSet, GcPlan,
+    GcRepairSet, GcRoot, GcSweepSet, RebuildIndexStats,
+};

package/dist-engine-src/src/changelog/store.rs

@@ -0,0 +1,163 @@
+use super::types::{
+    ChangeLoadBatch, ChangeLoadRequest, ChangeScanBatch, ChangeScanRequest, ChangelogAppend,
+    CommitLoadBatch, CommitLoadRequest, CommitScanBatch, CommitScanRequest, GcPlan, GcRoot,
+};
+use crate::common::LixError;
+use crate::storage::{StorageSpace, StorageSpaceId};
+use async_trait::async_trait;
+
+pub(crate) const COMMIT_NAMESPACE: &str = "changelog.commit";
+pub(crate) const CHANGE_NAMESPACE: &str = "changelog.change";
+pub(crate) const COMMIT_CHANGE_REF_CHUNK_NAMESPACE: &str = "changelog.commit_change_ref_chunk";
+
+pub(crate) const COMMIT_SPACE: StorageSpace =
+    StorageSpace::new(StorageSpaceId(0x0006_0001), COMMIT_NAMESPACE);
+pub(crate) const CHANGE_SPACE: StorageSpace =
+    StorageSpace::new(StorageSpaceId(0x0006_0002), CHANGE_NAMESPACE);
+pub(crate) const COMMIT_CHANGE_REF_CHUNK_SPACE: StorageSpace = StorageSpace::new(
+    StorageSpaceId(0x0006_0003),
+    COMMIT_CHANGE_REF_CHUNK_NAMESPACE,
+);
+
+pub(crate) fn commit_key(commit_id: &str) -> Vec<u8> {
+    identity_key(commit_id)
+}
+
+pub(crate) fn change_key(change_id: &str) -> Vec<u8> {
+    identity_key(change_id)
+}
+
+pub(crate) fn commit_change_ref_chunk_prefix(commit_id: &str) -> Vec<u8> {
+    let mut key = Vec::new();
+    push_ordered_str(&mut key, commit_id);
+    key
+}
+
+pub(crate) fn commit_change_ref_chunk_key(commit_id: &str, chunk_no: u32) -> Vec<u8> {
+    let mut key = commit_change_ref_chunk_prefix(commit_id);
+    key.extend_from_slice(&chunk_no.to_be_bytes());
+    key
+}
+
+pub(crate) fn commit_change_ref_chunk_ids_from_key(key: &[u8]) -> Result<(String, u32), LixError> {
+    let (commit_id, consumed_commit) = read_ordered_str(key)?;
+    let rest = &key[consumed_commit..];
+    if rest.len() != 4 {
+        return Err(LixError::new(
+            LixError::CODE_INTERNAL_ERROR,
+            "changelog commit_change_ref_chunk key has invalid chunk number",
+        ));
+    }
+    let chunk_no = u32::from_be_bytes([rest[0], rest[1], rest[2], rest[3]]);
+    Ok((commit_id, chunk_no))
+}
+
+fn identity_key(id: &str) -> Vec<u8> {
+    id.as_bytes().to_vec()
+}
+
+fn push_ordered_str(out: &mut Vec<u8>, value: &str) {
+    for byte in value.as_bytes() {
+        out.push(*byte);
+        if *byte == 0 {
+            out.push(0xff);
+        }
+    }
+    out.push(0);
+}
+
+fn read_ordered_str(bytes: &[u8]) -> Result<(String, usize), LixError> {
+    let mut out = Vec::new();
+    let mut index = 0;
+    while index < bytes.len() {
+        match bytes[index] {
+            0 => {
+                if bytes.get(index + 1) == Some(&0xff) {
+                    out.push(0);
+                    index += 2;
+                    continue;
+                }
+                let value = String::from_utf8(out).map_err(|error| {
+                    LixError::new(
+                        LixError::CODE_INTERNAL_ERROR,
+                        format!("changelog ordered key contains invalid UTF-8: {error}"),
+                    )
+                })?;
+                return Ok((value, index + 1));
+            }
+            byte => {
+                out.push(byte);
+                index += 1;
+            }
+        }
+    }
+    Err(LixError::new(
+        LixError::CODE_INTERNAL_ERROR,
+        "changelog ordered key component is missing terminator",
+    ))
+}
+
+#[async_trait]
+pub(crate) trait ChangelogReader {
+    async fn plan_gc(&mut self, roots: &[GcRoot]) -> Result<GcPlan, LixError>;
+
+    async fn load_commits(
+        &mut self,
+        request: CommitLoadRequest<'_>,
+    ) -> Result<CommitLoadBatch, LixError>;
+
+    async fn scan_commits(
+        &mut self,
+        request: CommitScanRequest<'_>,
+    ) -> Result<CommitScanBatch, LixError>;
+
+    async fn load_changes(
+        &mut self,
+        request: ChangeLoadRequest<'_>,
+    ) -> Result<ChangeLoadBatch, LixError>;
+
+    async fn scan_changes(
+        &mut self,
+        request: ChangeScanRequest<'_>,
+    ) -> Result<ChangeScanBatch, LixError>;
+}
+
+#[async_trait]
+pub(crate) trait ChangelogWriter {
+    async fn stage_append(&mut self, append: ChangelogAppend) -> Result<(), LixError>;
+
+    async fn collect_garbage(&mut self, roots: &[GcRoot]) -> Result<GcPlan, LixError>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn namespaces_are_stable() {
+        assert_eq!(COMMIT_NAMESPACE, "changelog.commit");
+        assert_eq!(CHANGE_NAMESPACE, "changelog.change");
+        assert_eq!(
+            COMMIT_CHANGE_REF_CHUNK_NAMESPACE,
+            "changelog.commit_change_ref_chunk"
+        );
+    }
+
+    #[test]
+    fn identity_keys_use_utf8_bytes() {
+        assert_eq!(commit_key("commit-1"), b"commit-1".to_vec());
+        assert_eq!(change_key("change-1"), b"change-1".to_vec());
+    }
+
+    #[test]
+    fn commit_change_ref_chunk_keys_are_prefixed_by_commit_id() {
+        let prefix = commit_change_ref_chunk_prefix("commit-1");
+        let key = commit_change_ref_chunk_key("commit-1", 42);
+        assert!(key.starts_with(&prefix));
+        assert!(!commit_change_ref_chunk_key("commit-10", 0).starts_with(&prefix));
+        assert_eq!(
+            commit_change_ref_chunk_ids_from_key(&key).unwrap(),
+            ("commit-1".to_string(), 42)
+        );
+    }
+}

package/dist-engine-src/src/changelog/test_support.rs

@@ -0,0 +1,54 @@
+use crate::changelog::{
+    ChangeRecord, ChangelogAppend, CommitChangeRef, CommitChangeRefSet, CommitRecord,
+};
+use crate::entity_pk::EntityPk;
+use crate::storage::{InMemoryStorageBackend, StorageContext};
+
+use super::ChangelogContext;
+
+pub(crate) fn changelog_test_context() -> (ChangelogContext, StorageContext) {
+    (
+        ChangelogContext::new(),
+        StorageContext::new(InMemoryStorageBackend::new()),
+    )
+}
+
+pub(crate) fn test_append() -> ChangelogAppend {
+    ChangelogAppend {
+        commits: vec![test_commit_record()],
+        changes: vec![test_change_record()],
+        commit_change_refs: vec![CommitChangeRefSet {
+            commit_id: "commit-1".to_string(),
+            entries: vec![CommitChangeRef {
+                schema_key: "message".to_string(),
+                file_id: Some("file-1".to_string()),
+                entity_pk: EntityPk::single("entity-1"),
+                change_id: "change-1".to_string(),
+            }],
+        }],
+    }
+}
+
+pub(crate) fn test_commit_record() -> CommitRecord {
+    CommitRecord {
+        format_version: 1,
+        commit_id: "commit-1".to_string(),
+        parent_commit_ids: Vec::new(),
+        change_id: "commit-row-change-1".to_string(),
+        author_account_ids: vec!["account-1".to_string()],
+        created_at: "2026-05-12T00:00:00Z".to_string(),
+    }
+}
+
+pub(crate) fn test_change_record() -> ChangeRecord {
+    ChangeRecord {
+        format_version: 1,
+        change_id: "change-1".to_string(),
+        schema_key: "message".to_string(),
+        entity_pk: EntityPk::single("entity-1"),
+        file_id: Some("file-1".to_string()),
+        snapshot_ref: None,
+        metadata_ref: None,
+        created_at: "2026-05-12T00:00:00Z".to_string(),
+    }
+}

package/dist-engine-src/src/changelog/types.rs

@@ -0,0 +1,213 @@
+use crate::entity_pk::EntityPk;
+use crate::json_store::JsonRef;
+
+pub(crate) type CommitId = String;
+pub(crate) type ChangeId = String;
+
+pub(crate) type CommitIdRef<'a> = &'a str;
+pub(crate) type ChangeIdRef<'a> = &'a str;
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub(crate) struct ChangelogAppend {
+    pub(crate) commits: Vec<CommitRecord>,
+    pub(crate) changes: Vec<ChangeRecord>,
+    pub(crate) commit_change_refs: Vec<CommitChangeRefSet>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitRecord {
+    pub(crate) format_version: u32,
+    pub(crate) commit_id: CommitId,
+    pub(crate) parent_commit_ids: Vec<CommitId>,
+    pub(crate) change_id: ChangeId,
+    pub(crate) author_account_ids: Vec<String>,
+    pub(crate) created_at: String,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitRecordView<'a> {
+    pub(crate) format_version: u32,
+    pub(crate) commit_id: CommitIdRef<'a>,
+    pub(crate) parent_commit_ids: Vec<CommitIdRef<'a>>,
+    pub(crate) change_id: ChangeIdRef<'a>,
+    pub(crate) author_account_ids: Vec<&'a str>,
+    pub(crate) created_at: &'a str,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitChangeRefSet {
+    pub(crate) commit_id: CommitId,
+    pub(crate) entries: Vec<CommitChangeRef>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitChangeRefChunk {
+    pub(crate) format_version: u32,
+    pub(crate) commit_id: CommitId,
+    pub(crate) entries: Vec<CommitChangeRef>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitChangeRefChunkView<'a> {
+    pub(crate) format_version: u32,
+    pub(crate) commit_id: CommitIdRef<'a>,
+    pub(crate) entries: Vec<CommitChangeRefView<'a>>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitChangeRef {
+    pub(crate) schema_key: String,
+    pub(crate) file_id: Option<String>,
+    pub(crate) entity_pk: EntityPk,
+    pub(crate) change_id: ChangeId,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitChangeRefView<'a> {
+    pub(crate) schema_key: &'a str,
+    pub(crate) file_id: Option<&'a str>,
+    pub(crate) entity_pk: EntityPkRef<'a>,
+    pub(crate) change_id: ChangeIdRef<'a>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct EntityPkRef<'a> {
+    pub(crate) parts: Vec<&'a str>,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum CommitProjection {
+    Record,
+    ChangeRefs,
+    Full,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct CommitLoadRequest<'a> {
+    pub(crate) commit_ids: &'a [CommitId],
+    pub(crate) projection: CommitProjection,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitLoadBatch {
+    pub(crate) entries: Vec<Option<CommitLoadEntry>>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct CommitScanRequest<'a> {
+    pub(crate) start_after: Option<&'a str>,
+    pub(crate) limit: Option<usize>,
+    pub(crate) projection: CommitProjection,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct CommitScanBatch {
+    pub(crate) entries: Vec<CommitLoadEntry>,
+    pub(crate) next_start_after: Option<CommitId>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) enum CommitLoadEntry {
+    Record(CommitRecord),
+    ChangeRefs(Vec<CommitChangeRefChunk>),
+    Full {
+        record: CommitRecord,
+        change_ref_chunks: Vec<CommitChangeRefChunk>,
+    },
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct ChangeRecord {
+    pub(crate) format_version: u32,
+    pub(crate) change_id: ChangeId,
+    pub(crate) schema_key: String,
+    pub(crate) entity_pk: EntityPk,
+    pub(crate) file_id: Option<String>,
+    pub(crate) snapshot_ref: Option<JsonRef>,
+    pub(crate) metadata_ref: Option<JsonRef>,
+    pub(crate) created_at: String,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct ChangeRecordView<'a> {
+    pub(crate) format_version: u32,
+    pub(crate) change_id: ChangeIdRef<'a>,
+    pub(crate) schema_key: &'a str,
+    pub(crate) entity_pk: EntityPkRef<'a>,
+    pub(crate) file_id: Option<&'a str>,
+    pub(crate) snapshot_ref: Option<JsonRef>,
+    pub(crate) metadata_ref: Option<JsonRef>,
+    pub(crate) created_at: &'a str,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ChangeLoadRequest<'a> {
+    pub(crate) change_ids: &'a [ChangeId],
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct ChangeLoadBatch {
+    pub(crate) entries: Vec<Option<ChangeRecord>>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ChangeScanRequest<'a> {
+    pub(crate) start_after: Option<&'a str>,
+    pub(crate) limit: Option<usize>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct ChangeScanBatch {
+    pub(crate) entries: Vec<ChangeRecord>,
+    pub(crate) next_start_after: Option<ChangeId>,
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub(crate) struct RebuildIndexStats {
+    pub(crate) expected: usize,
+    pub(crate) put: usize,
+    pub(crate) deleted: usize,
+    pub(crate) unchanged: usize,
+}
+
+impl RebuildIndexStats {
+    pub(crate) fn combine(self, other: Self) -> Self {
+        Self {
+            expected: self.expected + other.expected,
+            put: self.put + other.put,
+            deleted: self.deleted + other.deleted,
+            unchanged: self.unchanged + other.unchanged,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) enum GcRoot {
+    BranchHead(CommitId),
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub(crate) struct GcLiveSet {
+    pub(crate) commits: Vec<CommitId>,
+    pub(crate) changes: Vec<ChangeId>,
+    pub(crate) payloads: Vec<JsonRef>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub(crate) struct GcSweepSet {
+    pub(crate) commits: Vec<CommitId>,
+    pub(crate) changes: Vec<ChangeId>,
+    pub(crate) commit_change_ref_chunks: Vec<(CommitId, u32)>,
+    pub(crate) json_payloads: Vec<JsonRef>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub(crate) struct GcRepairSet {}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub(crate) struct GcPlan {
+    pub(crate) roots: Vec<GcRoot>,
+    pub(crate) live: GcLiveSet,
+    pub(crate) sweep: GcSweepSet,
+    pub(crate) repair: GcRepairSet,
+}