@lix-js/sdk 0.6.0-preview.1 → 0.6.0-preview.2

package/dist-engine-src/src/changelog/codec.rs

@@ -0,0 +1,321 @@
+use crate::changelog::CanonicalChange;
+use crate::entity_identity::EntityIdentity;
+use crate::json_store::JsonRef;
+use crate::LixError;
+
+const CHANGELOG_FILE_IDENTIFIER: &str = "LXCH";
+
+pub(crate) fn encode_change(change: &CanonicalChange) -> Result<Vec<u8>, LixError> {
+    let entity_id = change.entity_id.as_string().map_err(|error| {
+        LixError::unknown(format!(
+            "failed to encode changelog entity identity: {error}"
+        ))
+    })?;
+
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(256);
+    let id = builder.create_string(&change.id);
+    let entity_id = builder.create_string(&entity_id);
+    let schema_key = builder.create_string(&change.schema_key);
+    let schema_version = builder.create_string(&change.schema_version);
+    let file_id = change
+        .file_id
+        .as_ref()
+        .map(|value| builder.create_string(value));
+    let snapshot_ref = change
+        .snapshot_ref
+        .as_ref()
+        .map(|value| builder.create_vector(value.as_hash_bytes()));
+    let metadata_ref = change
+        .metadata_ref
+        .as_ref()
+        .map(|value| builder.create_vector(value.as_hash_bytes()));
+    let created_at = builder.create_string(&change.created_at);
+
+    let root = flatbuffer::create_canonical_change(
+        &mut builder,
+        &flatbuffer::CanonicalChangeArgs {
+            id,
+            entity_id,
+            schema_key,
+            schema_version,
+            file_id,
+            snapshot_ref,
+            metadata_ref,
+            created_at,
+        },
+    );
+    builder.finish(root, Some(CHANGELOG_FILE_IDENTIFIER));
+    Ok(builder.finished_data().to_vec())
+}
+
+pub(crate) fn decode_change(bytes: &[u8]) -> Result<CanonicalChange, LixError> {
+    if bytes.len() < flatbuffers::SIZE_UOFFSET + flatbuffers::FILE_IDENTIFIER_LENGTH
+        || !flatbuffers::buffer_has_identifier(bytes, CHANGELOG_FILE_IDENTIFIER, false)
+    {
+        return Err(LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            "failed to decode changelog change: invalid FlatBuffers file identifier",
+        ));
+    }
+
+    let change = flatbuffer::root_as_canonical_change(bytes).map_err(|error| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("failed to decode changelog change: {error}"),
+        )
+    })?;
+
+    let entity_id = required_str(change.entity_id(), "entity_id")?;
+    let entity_id = EntityIdentity::from_string(entity_id).map_err(|error| {
+        LixError::unknown(format!(
+            "failed to decode changelog entity identity: {error}"
+        ))
+    })?;
+
+    Ok(CanonicalChange {
+        id: required_str(change.id(), "id")?.to_string(),
+        entity_id,
+        schema_key: required_str(change.schema_key(), "schema_key")?.to_string(),
+        schema_version: required_str(change.schema_version(), "schema_version")?.to_string(),
+        file_id: change.file_id().map(ToString::to_string),
+        snapshot_ref: optional_json_ref(change.snapshot_ref(), "snapshot_ref")?,
+        metadata_ref: optional_json_ref(change.metadata_ref(), "metadata_ref")?,
+        created_at: required_str(change.created_at(), "created_at")?.to_string(),
+    })
+}
+
+fn required_str<'a>(value: Option<&'a str>, field: &str) -> Result<&'a str, LixError> {
+    value.ok_or_else(|| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("failed to decode changelog change: missing required field `{field}`"),
+        )
+    })
+}
+
+fn optional_json_ref(
+    value: Option<flatbuffers::Vector<'_, u8>>,
+    field: &str,
+) -> Result<Option<JsonRef>, LixError> {
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    let bytes = value.bytes();
+    let hash = <[u8; 32]>::try_from(bytes).map_err(|_| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("failed to decode changelog change: field `{field}` must be exactly 32 bytes"),
+        )
+    })?;
+    Ok(Some(JsonRef::from_hash_bytes(hash)))
+}
+
+mod flatbuffer {
+    #[derive(Copy, Clone, PartialEq)]
+    pub(super) struct CanonicalChange<'a> {
+        table: flatbuffers::Table<'a>,
+    }
+
+    impl<'a> flatbuffers::Follow<'a> for CanonicalChange<'a> {
+        type Inner = CanonicalChange<'a>;
+
+        #[inline]
+        unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+            Self {
+                table: unsafe { flatbuffers::Table::new(buf, loc) },
+            }
+        }
+    }
+
+    impl<'a> CanonicalChange<'a> {
+        const VT_ID: flatbuffers::VOffsetT = 4;
+        const VT_ENTITY_ID: flatbuffers::VOffsetT = 6;
+        const VT_SCHEMA_KEY: flatbuffers::VOffsetT = 8;
+        const VT_SCHEMA_VERSION: flatbuffers::VOffsetT = 10;
+        const VT_FILE_ID: flatbuffers::VOffsetT = 12;
+        const VT_SNAPSHOT_REF: flatbuffers::VOffsetT = 14;
+        const VT_METADATA_REF: flatbuffers::VOffsetT = 16;
+        const VT_CREATED_AT: flatbuffers::VOffsetT = 18;
+
+        #[inline]
+        pub(super) fn id(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_ID, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn entity_id(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_ENTITY_ID, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn schema_key(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_SCHEMA_KEY, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn schema_version(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_SCHEMA_VERSION, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn file_id(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_FILE_ID, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn snapshot_ref(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(
+                        Self::VT_SNAPSHOT_REF,
+                        None,
+                    )
+            }
+        }
+
+        #[inline]
+        pub(super) fn metadata_ref(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(
+                        Self::VT_METADATA_REF,
+                        None,
+                    )
+            }
+        }
+
+        #[inline]
+        pub(super) fn created_at(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_CREATED_AT, None)
+            }
+        }
+    }
+
+    impl flatbuffers::Verifiable for CanonicalChange<'_> {
+        #[inline]
+        fn run_verifier(
+            verifier: &mut flatbuffers::Verifier,
+            position: usize,
+        ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+            verifier
+                .visit_table(position)?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>("id", Self::VT_ID, true)?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "entity_id",
+                    Self::VT_ENTITY_ID,
+                    true,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "schema_key",
+                    Self::VT_SCHEMA_KEY,
+                    true,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "schema_version",
+                    Self::VT_SCHEMA_VERSION,
+                    true,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "file_id",
+                    Self::VT_FILE_ID,
+                    false,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>(
+                    "snapshot_ref",
+                    Self::VT_SNAPSHOT_REF,
+                    false,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>(
+                    "metadata_ref",
+                    Self::VT_METADATA_REF,
+                    false,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "created_at",
+                    Self::VT_CREATED_AT,
+                    true,
+                )?
+                .finish();
+            Ok(())
+        }
+    }
+
+    pub(super) struct CanonicalChangeArgs<'a> {
+        pub(super) id: flatbuffers::WIPOffset<&'a str>,
+        pub(super) entity_id: flatbuffers::WIPOffset<&'a str>,
+        pub(super) schema_key: flatbuffers::WIPOffset<&'a str>,
+        pub(super) schema_version: flatbuffers::WIPOffset<&'a str>,
+        pub(super) file_id: Option<flatbuffers::WIPOffset<&'a str>>,
+        pub(super) snapshot_ref: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+        pub(super) metadata_ref: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+        pub(super) created_at: flatbuffers::WIPOffset<&'a str>,
+    }
+
+    pub(super) fn create_canonical_change<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
+        builder: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
+        args: &'args CanonicalChangeArgs<'args>,
+    ) -> flatbuffers::WIPOffset<CanonicalChange<'bldr>> {
+        let start = builder.start_table();
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            CanonicalChange::VT_CREATED_AT,
+            args.created_at,
+        );
+        if let Some(metadata_ref) = args.metadata_ref {
+            builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+                CanonicalChange::VT_METADATA_REF,
+                metadata_ref,
+            );
+        }
+        if let Some(snapshot_ref) = args.snapshot_ref {
+            builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+                CanonicalChange::VT_SNAPSHOT_REF,
+                snapshot_ref,
+            );
+        }
+        if let Some(file_id) = args.file_id {
+            builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+                CanonicalChange::VT_FILE_ID,
+                file_id,
+            );
+        }
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            CanonicalChange::VT_SCHEMA_VERSION,
+            args.schema_version,
+        );
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            CanonicalChange::VT_SCHEMA_KEY,
+            args.schema_key,
+        );
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            CanonicalChange::VT_ENTITY_ID,
+            args.entity_id,
+        );
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(CanonicalChange::VT_ID, args.id);
+        let offset = builder.end_table(start);
+        flatbuffers::WIPOffset::new(offset.value())
+    }
+
+    #[inline]
+    pub(super) fn root_as_canonical_change(
+        bytes: &[u8],
+    ) -> Result<CanonicalChange<'_>, flatbuffers::InvalidFlatbuffer> {
+        flatbuffers::root::<CanonicalChange>(bytes)
+    }
+}

package/dist-engine-src/src/changelog/context.rs

@@ -0,0 +1,92 @@
+use crate::changelog::{CanonicalChange, ChangelogReader, ChangelogScanRequest};
+use crate::storage::{StorageReader, StorageWriteSet};
+use crate::LixError;
+use tokio::sync::Mutex;
+
+/// Durable append-only ledger for Lix changes.
+///
+/// This layer only records already-generated change facts. Transaction commit
+/// code is responsible for producing user changes plus normal `lix_commit`
+/// rows before appending them here.
+#[derive(Clone, Copy)]
+pub(crate) struct ChangelogContext;
+
+impl ChangelogContext {
+    pub(crate) fn new() -> Self {
+        Self
+    }
+
+    /// Creates a changelog reader over a caller-provided KV store.
+    ///
+    /// The caller decides which KV store supplies visibility for the read.
+    pub(crate) fn reader<S>(&self, store: S) -> ChangelogStoreReader<S>
+    where
+        S: StorageReader,
+    {
+        ChangelogStoreReader {
+            store: Mutex::new(store),
+        }
+    }
+
+    /// Creates a changelog writer over a transaction-local storage write set.
+    pub(crate) fn writer<'a>(&self, writes: &'a mut StorageWriteSet) -> ChangelogWriter<'a> {
+        ChangelogWriter { writes }
+    }
+}
+
+/// KV-backed changelog reader created by `ChangelogContext`.
+pub(crate) struct ChangelogStoreReader<S> {
+    store: Mutex<S>,
+}
+
+impl<S> ChangelogStoreReader<S>
+where
+    S: StorageReader,
+{
+    #[allow(dead_code)]
+    pub(crate) async fn load_change(
+        &self,
+        change_id: &str,
+    ) -> Result<Option<CanonicalChange>, LixError> {
+        let mut store = self.store.lock().await;
+        crate::changelog::storage::load_change(&mut *store, change_id).await
+    }
+
+    #[allow(dead_code)]
+    pub(crate) async fn scan_changes(
+        &self,
+        request: &ChangelogScanRequest,
+    ) -> Result<Vec<CanonicalChange>, LixError> {
+        let mut store = self.store.lock().await;
+        crate::changelog::storage::scan_changes(&mut *store, request).await
+    }
+}
+
+#[async_trait::async_trait]
+impl<S> ChangelogReader for ChangelogStoreReader<S>
+where
+    S: StorageReader,
+{
+    async fn load_change(&self, change_id: &str) -> Result<Option<CanonicalChange>, LixError> {
+        ChangelogStoreReader::load_change(self, change_id).await
+    }
+
+    async fn scan_changes(
+        &self,
+        request: &ChangelogScanRequest,
+    ) -> Result<Vec<CanonicalChange>, LixError> {
+        ChangelogStoreReader::scan_changes(self, request).await
+    }
+}
+
+/// Changelog writer over a transaction-local storage write set.
+pub(crate) struct ChangelogWriter<'a> {
+    writes: &'a mut StorageWriteSet,
+}
+
+impl ChangelogWriter<'_> {
+    #[allow(dead_code)]
+    pub(crate) fn stage_changes(&mut self, changes: &[CanonicalChange]) -> Result<(), LixError> {
+        crate::changelog::storage::stage_changes(self.writes, changes)
+    }
+}

package/dist-engine-src/src/changelog/materialization.rs

@@ -0,0 +1,121 @@
+use crate::changelog::{CanonicalChange, MaterializedCanonicalChange};
+use crate::json_store::{JsonRef, JsonStoreReader, JsonStoreWriter};
+use crate::storage::{StorageReader, StorageWriteSet};
+use crate::{serialize_row_metadata, validate_row_metadata, LixError, RowMetadata};
+
+pub(crate) fn canonicalize_materialized_change(
+    writes: &mut StorageWriteSet,
+    json_writer: &mut JsonStoreWriter,
+    change: &MaterializedCanonicalChange,
+) -> Result<CanonicalChange, LixError> {
+    let snapshot_ref =
+        stage_optional_json(writes, json_writer, change.snapshot_content.as_deref())?;
+    let metadata_ref = stage_optional_metadata(writes, json_writer, change.metadata.as_ref())?;
+    Ok(CanonicalChange {
+        id: change.id.clone(),
+        entity_id: change.entity_id.clone(),
+        schema_key: change.schema_key.clone(),
+        schema_version: change.schema_version.clone(),
+        file_id: change.file_id.clone(),
+        snapshot_ref,
+        metadata_ref,
+        created_at: change.created_at.clone(),
+    })
+}
+
+pub(crate) async fn materialize_change<S>(
+    json_reader: &mut JsonStoreReader<S>,
+    change: CanonicalChange,
+) -> Result<MaterializedCanonicalChange, LixError>
+where
+    S: StorageReader,
+{
+    let snapshot_content =
+        load_optional_json(json_reader, change.snapshot_ref.as_ref(), "snapshot_ref").await?;
+    let metadata = load_optional_metadata(json_reader, change.metadata_ref.as_ref()).await?;
+    Ok(MaterializedCanonicalChange {
+        id: change.id,
+        entity_id: change.entity_id,
+        schema_key: change.schema_key,
+        schema_version: change.schema_version,
+        file_id: change.file_id,
+        snapshot_content,
+        metadata,
+        created_at: change.created_at,
+    })
+}
+
+fn stage_optional_json(
+    writes: &mut StorageWriteSet,
+    json_writer: &mut JsonStoreWriter,
+    value: Option<&str>,
+) -> Result<Option<JsonRef>, LixError> {
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    json_writer.stage_bytes(writes, value.as_bytes()).map(Some)
+}
+
+fn stage_optional_metadata(
+    writes: &mut StorageWriteSet,
+    json_writer: &mut JsonStoreWriter,
+    value: Option<&RowMetadata>,
+) -> Result<Option<JsonRef>, LixError> {
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    let serialized = serialize_row_metadata(value);
+    json_writer
+        .stage_bytes(writes, serialized.as_bytes())
+        .map(Some)
+}
+
+async fn load_optional_metadata<S>(
+    json_reader: &mut JsonStoreReader<S>,
+    json_ref: Option<&JsonRef>,
+) -> Result<Option<RowMetadata>, LixError>
+where
+    S: StorageReader,
+{
+    let Some(json) = load_optional_json(json_reader, json_ref, "metadata_ref").await? else {
+        return Ok(None);
+    };
+    let metadata = serde_json::from_str::<RowMetadata>(&json).map_err(|error| {
+        LixError::new(
+            "LIX_ERROR_INVALID_JSON",
+            format!("metadata_ref is invalid JSON: {error}"),
+        )
+    })?;
+    validate_row_metadata(metadata, "metadata_ref").map(Some)
+}
+
+async fn load_optional_json<S>(
+    json_reader: &mut JsonStoreReader<S>,
+    json_ref: Option<&JsonRef>,
+    field: &str,
+) -> Result<Option<String>, LixError>
+where
+    S: StorageReader,
+{
+    let Some(json_ref) = json_ref else {
+        return Ok(None);
+    };
+    let bytes = json_reader.load_bytes(json_ref).await?.ok_or_else(|| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!(
+                "changelog {field} '{}' is missing from json_store",
+                json_ref.to_hex()
+            ),
+        )
+    })?;
+    String::from_utf8(bytes).map(Some).map_err(|error| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!(
+                "changelog {field} '{}' is not valid UTF-8 JSON bytes: {error}",
+                json_ref.to_hex()
+            ),
+        )
+    })
+}

package/dist-engine-src/src/changelog/mod.rs

@@ -0,0 +1,13 @@
+pub(crate) mod codec;
+mod context;
+mod materialization;
+mod reader;
+mod storage;
+mod types;
+
+#[allow(unused_imports)]
+pub(crate) use context::{ChangelogContext, ChangelogStoreReader, ChangelogWriter};
+pub(crate) use materialization::{canonicalize_materialized_change, materialize_change};
+pub(crate) use reader::ChangelogReader;
+#[allow(unused_imports)]
+pub(crate) use types::{CanonicalChange, ChangelogScanRequest, MaterializedCanonicalChange};

package/dist-engine-src/src/changelog/reader.rs

@@ -0,0 +1,20 @@
+use async_trait::async_trait;
+
+use crate::changelog::{CanonicalChange, ChangelogScanRequest};
+use crate::LixError;
+
+/// Read side for immutable changelog facts.
+///
+/// SQL providers and commit-graph readers depend on this role instead of
+/// knowing which KV store backs the changelog for the current execution.
+#[async_trait]
+pub(crate) trait ChangelogReader: Send + Sync {
+    #[allow(dead_code)]
+    async fn load_change(&self, change_id: &str) -> Result<Option<CanonicalChange>, LixError>;
+
+    #[allow(dead_code)]
+    async fn scan_changes(
+        &self,
+        request: &ChangelogScanRequest,
+    ) -> Result<Vec<CanonicalChange>, LixError>;
+}