@lix-js/sdk 0.6.0-preview.0 → 0.6.0-preview.2

package/dist-engine-src/src/untracked_state/codec.rs

@@ -0,0 +1,363 @@
+use crate::entity_identity::EntityIdentity;
+use crate::json_store::JsonRef;
+use crate::untracked_state::UntrackedStateRow;
+use crate::LixError;
+
+const UNTRACKED_STATE_FILE_IDENTIFIER: &str = "LXUS";
+
+pub(crate) fn encode_row(row: &UntrackedStateRow) -> Result<Vec<u8>, LixError> {
+    let entity_id = row.entity_id.as_string().map_err(|error| {
+        LixError::unknown(format!(
+            "failed to encode untracked-state entity identity: {error}"
+        ))
+    })?;
+
+    let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(256);
+    let entity_id = builder.create_string(&entity_id);
+    let schema_key = builder.create_string(&row.schema_key);
+    let file_id = row
+        .file_id
+        .as_ref()
+        .map(|value| builder.create_string(value));
+    let snapshot_ref = row
+        .snapshot_ref
+        .as_ref()
+        .map(|value| builder.create_vector(value.as_hash_bytes()));
+    let metadata_ref = row
+        .metadata_ref
+        .as_ref()
+        .map(|value| builder.create_vector(value.as_hash_bytes()));
+    let schema_version = builder.create_string(&row.schema_version);
+    let created_at = builder.create_string(&row.created_at);
+    let updated_at = builder.create_string(&row.updated_at);
+    let version_id = builder.create_string(&row.version_id);
+
+    let root = flatbuffer::create_untracked_state_row(
+        &mut builder,
+        &flatbuffer::UntrackedStateRowArgs {
+            entity_id,
+            schema_key,
+            file_id,
+            snapshot_ref,
+            metadata_ref,
+            schema_version,
+            created_at,
+            updated_at,
+            global: row.global,
+            version_id,
+        },
+    );
+    builder.finish(root, Some(UNTRACKED_STATE_FILE_IDENTIFIER));
+    Ok(builder.finished_data().to_vec())
+}
+
+pub(crate) fn decode_row(bytes: &[u8]) -> Result<UntrackedStateRow, LixError> {
+    if bytes.len() < flatbuffers::SIZE_UOFFSET + flatbuffers::FILE_IDENTIFIER_LENGTH
+        || !flatbuffers::buffer_has_identifier(bytes, UNTRACKED_STATE_FILE_IDENTIFIER, false)
+    {
+        return Err(LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            "failed to decode untracked-state row: invalid FlatBuffers file identifier",
+        ));
+    }
+
+    let row = flatbuffer::root_as_untracked_state_row(bytes).map_err(|error| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("failed to decode untracked-state row: {error}"),
+        )
+    })?;
+
+    let entity_id = required_str(row.entity_id(), "entity_id")?;
+    let entity_id = EntityIdentity::from_string(entity_id).map_err(|error| {
+        LixError::unknown(format!(
+            "failed to decode untracked-state entity identity: {error}"
+        ))
+    })?;
+
+    Ok(UntrackedStateRow {
+        entity_id,
+        schema_key: required_str(row.schema_key(), "schema_key")?.to_string(),
+        file_id: row.file_id().map(ToString::to_string),
+        snapshot_ref: optional_json_ref(row.snapshot_ref(), "snapshot_ref")?,
+        metadata_ref: optional_json_ref(row.metadata_ref(), "metadata_ref")?,
+        schema_version: required_str(row.schema_version(), "schema_version")?.to_string(),
+        created_at: required_str(row.created_at(), "created_at")?.to_string(),
+        updated_at: required_str(row.updated_at(), "updated_at")?.to_string(),
+        global: row.global(),
+        version_id: required_str(row.version_id(), "version_id")?.to_string(),
+    })
+}
+
+fn required_str<'a>(value: Option<&'a str>, field: &str) -> Result<&'a str, LixError> {
+    value.ok_or_else(|| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!("failed to decode untracked-state row: missing required field `{field}`"),
+        )
+    })
+}
+
+fn optional_json_ref(
+    value: Option<flatbuffers::Vector<'_, u8>>,
+    field: &str,
+) -> Result<Option<JsonRef>, LixError> {
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    let bytes = value.bytes();
+    let hash = <[u8; 32]>::try_from(bytes).map_err(|_| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!(
+                "failed to decode untracked-state row: field `{field}` must be exactly 32 bytes"
+            ),
+        )
+    })?;
+    Ok(Some(JsonRef::from_hash_bytes(hash)))
+}
+
+mod flatbuffer {
+    #[derive(Copy, Clone, PartialEq)]
+    pub(super) struct UntrackedStateRow<'a> {
+        table: flatbuffers::Table<'a>,
+    }
+
+    impl<'a> flatbuffers::Follow<'a> for UntrackedStateRow<'a> {
+        type Inner = UntrackedStateRow<'a>;
+
+        #[inline]
+        unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+            Self {
+                table: unsafe { flatbuffers::Table::new(buf, loc) },
+            }
+        }
+    }
+
+    impl<'a> UntrackedStateRow<'a> {
+        const VT_ENTITY_ID: flatbuffers::VOffsetT = 4;
+        const VT_SCHEMA_KEY: flatbuffers::VOffsetT = 6;
+        const VT_FILE_ID: flatbuffers::VOffsetT = 8;
+        const VT_SNAPSHOT_REF: flatbuffers::VOffsetT = 10;
+        const VT_METADATA_REF: flatbuffers::VOffsetT = 12;
+        const VT_SCHEMA_VERSION: flatbuffers::VOffsetT = 14;
+        const VT_CREATED_AT: flatbuffers::VOffsetT = 16;
+        const VT_UPDATED_AT: flatbuffers::VOffsetT = 18;
+        const VT_GLOBAL: flatbuffers::VOffsetT = 20;
+        const VT_VERSION_ID: flatbuffers::VOffsetT = 22;
+
+        #[inline]
+        pub(super) fn entity_id(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_ENTITY_ID, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn schema_key(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_SCHEMA_KEY, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn file_id(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_FILE_ID, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn snapshot_ref(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(
+                        Self::VT_SNAPSHOT_REF,
+                        None,
+                    )
+            }
+        }
+
+        #[inline]
+        pub(super) fn metadata_ref(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(
+                        Self::VT_METADATA_REF,
+                        None,
+                    )
+            }
+        }
+
+        #[inline]
+        pub(super) fn schema_version(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_SCHEMA_VERSION, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn created_at(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_CREATED_AT, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn updated_at(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_UPDATED_AT, None)
+            }
+        }
+
+        #[inline]
+        pub(super) fn global(&self) -> bool {
+            unsafe { self.table.get::<bool>(Self::VT_GLOBAL, Some(false)) }.unwrap_or(false)
+        }
+
+        #[inline]
+        pub(super) fn version_id(&self) -> Option<&'a str> {
+            unsafe {
+                self.table
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(Self::VT_VERSION_ID, None)
+            }
+        }
+    }
+
+    impl flatbuffers::Verifiable for UntrackedStateRow<'_> {
+        #[inline]
+        fn run_verifier(
+            verifier: &mut flatbuffers::Verifier,
+            position: usize,
+        ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+            verifier
+                .visit_table(position)?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "entity_id",
+                    Self::VT_ENTITY_ID,
+                    true,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "schema_key",
+                    Self::VT_SCHEMA_KEY,
+                    true,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "file_id",
+                    Self::VT_FILE_ID,
+                    false,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>(
+                    "snapshot_ref",
+                    Self::VT_SNAPSHOT_REF,
+                    false,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>(
+                    "metadata_ref",
+                    Self::VT_METADATA_REF,
+                    false,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "schema_version",
+                    Self::VT_SCHEMA_VERSION,
+                    true,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "created_at",
+                    Self::VT_CREATED_AT,
+                    true,
+                )?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "updated_at",
+                    Self::VT_UPDATED_AT,
+                    true,
+                )?
+                .visit_field::<bool>("global", Self::VT_GLOBAL, false)?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "version_id",
+                    Self::VT_VERSION_ID,
+                    true,
+                )?
+                .finish();
+            Ok(())
+        }
+    }
+
+    pub(super) struct UntrackedStateRowArgs<'a> {
+        pub(super) entity_id: flatbuffers::WIPOffset<&'a str>,
+        pub(super) schema_key: flatbuffers::WIPOffset<&'a str>,
+        pub(super) file_id: Option<flatbuffers::WIPOffset<&'a str>>,
+        pub(super) snapshot_ref: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+        pub(super) metadata_ref: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+        pub(super) schema_version: flatbuffers::WIPOffset<&'a str>,
+        pub(super) created_at: flatbuffers::WIPOffset<&'a str>,
+        pub(super) updated_at: flatbuffers::WIPOffset<&'a str>,
+        pub(super) global: bool,
+        pub(super) version_id: flatbuffers::WIPOffset<&'a str>,
+    }
+
+    pub(super) fn create_untracked_state_row<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
+        builder: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
+        args: &'args UntrackedStateRowArgs<'args>,
+    ) -> flatbuffers::WIPOffset<UntrackedStateRow<'bldr>> {
+        let start = builder.start_table();
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            UntrackedStateRow::VT_VERSION_ID,
+            args.version_id,
+        );
+        builder.push_slot::<bool>(UntrackedStateRow::VT_GLOBAL, args.global, false);
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            UntrackedStateRow::VT_UPDATED_AT,
+            args.updated_at,
+        );
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            UntrackedStateRow::VT_CREATED_AT,
+            args.created_at,
+        );
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            UntrackedStateRow::VT_SCHEMA_VERSION,
+            args.schema_version,
+        );
+        if let Some(metadata_ref) = args.metadata_ref {
+            builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+                UntrackedStateRow::VT_METADATA_REF,
+                metadata_ref,
+            );
+        }
+        if let Some(snapshot_ref) = args.snapshot_ref {
+            builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+                UntrackedStateRow::VT_SNAPSHOT_REF,
+                snapshot_ref,
+            );
+        }
+        if let Some(file_id) = args.file_id {
+            builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+                UntrackedStateRow::VT_FILE_ID,
+                file_id,
+            );
+        }
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            UntrackedStateRow::VT_SCHEMA_KEY,
+            args.schema_key,
+        );
+        builder.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            UntrackedStateRow::VT_ENTITY_ID,
+            args.entity_id,
+        );
+        let offset = builder.end_table(start);
+        flatbuffers::WIPOffset::new(offset.value())
+    }
+
+    #[inline]
+    pub(super) fn root_as_untracked_state_row(
+        bytes: &[u8],
+    ) -> Result<UntrackedStateRow<'_>, flatbuffers::InvalidFlatbuffer> {
+        flatbuffers::root::<UntrackedStateRow>(bytes)
+    }
+}

package/dist-engine-src/src/untracked_state/context.rs

@@ -0,0 +1,82 @@
+use crate::storage::{StorageReader, StorageWriteSet};
+use crate::untracked_state::{
+    MaterializedUntrackedStateRow, UntrackedStateIdentity, UntrackedStateRow,
+    UntrackedStateRowRequest, UntrackedStateScanRequest,
+};
+use crate::LixError;
+
+/// Durable local overlay excluded from changelog and commit membership.
+///
+/// Untracked state is not change-controlled, but it is still durable local
+/// state. It is read alongside tracked live state and can override tracked rows
+/// with the same identity.
+#[derive(Clone, Copy)]
+pub(crate) struct UntrackedStateContext;
+
+impl UntrackedStateContext {
+    pub(crate) fn new() -> Self {
+        Self
+    }
+
+    /// Creates a reader over a caller-provided KV store.
+    ///
+    /// The caller decides which KV store supplies visibility for the read.
+    pub(crate) fn reader<S>(&self, store: S) -> UntrackedStateStoreReader<S>
+    where
+        S: StorageReader,
+    {
+        UntrackedStateStoreReader { store }
+    }
+
+    /// Creates a writer over a transaction-local storage write set.
+    ///
+    /// The context never opens its own transaction; the caller applies the
+    /// write set to choose the durable commit or rollback boundary.
+    pub(crate) fn writer<'a>(&self, writes: &'a mut StorageWriteSet) -> UntrackedStateWriter<'a> {
+        UntrackedStateWriter { writes }
+    }
+}
+
+/// Store-backed untracked-state reader created by `UntrackedStateContext`.
+pub(crate) struct UntrackedStateStoreReader<S> {
+    store: S,
+}
+
+impl<S> UntrackedStateStoreReader<S>
+where
+    S: StorageReader,
+{
+    pub(crate) async fn scan_rows(
+        &mut self,
+        request: &UntrackedStateScanRequest,
+    ) -> Result<Vec<MaterializedUntrackedStateRow>, LixError> {
+        crate::untracked_state::storage::scan_rows(&mut self.store, request).await
+    }
+
+    pub(crate) async fn load_row(
+        &mut self,
+        request: &UntrackedStateRowRequest,
+    ) -> Result<Option<MaterializedUntrackedStateRow>, LixError> {
+        crate::untracked_state::storage::load_row(&mut self.store, request).await
+    }
+}
+
+/// Untracked-state writer over a transaction-local storage write set.
+pub(crate) struct UntrackedStateWriter<'a> {
+    writes: &'a mut StorageWriteSet,
+}
+
+impl UntrackedStateWriter<'_> {
+    /// Stages the latest untracked rows for their identities.
+    ///
+    /// A row with `snapshot_ref = None` is treated as removal because
+    /// untracked state keeps only the current local value, not tombstones.
+    pub(crate) fn stage_rows(&mut self, rows: &[UntrackedStateRow]) -> Result<(), LixError> {
+        crate::untracked_state::storage::stage_rows(self.writes, rows)
+    }
+
+    /// Removes untracked rows by exact identity.
+    pub(crate) fn stage_delete_rows(&mut self, identities: &[UntrackedStateIdentity]) {
+        crate::untracked_state::storage::stage_delete_rows(self.writes, identities)
+    }
+}

package/dist-engine-src/src/untracked_state/materialization.rs

@@ -0,0 +1,157 @@
+use crate::json_store::{JsonRef, JsonStoreReader, JsonStoreWriter};
+use crate::storage::{StorageReader, StorageWriteSet};
+use crate::untracked_state::{MaterializedUntrackedStateRow, UntrackedStateRow};
+use crate::{serialize_row_metadata, validate_row_metadata, LixError, RowMetadata};
+
+pub(crate) fn canonicalize_materialized_row(
+    writes: &mut StorageWriteSet,
+    json_writer: &mut JsonStoreWriter,
+    row: &MaterializedUntrackedStateRow,
+) -> Result<UntrackedStateRow, LixError> {
+    let snapshot_ref = stage_optional_json(writes, json_writer, row.snapshot_content.as_deref())?;
+    let metadata_ref = stage_optional_metadata(writes, json_writer, row.metadata.as_ref())?;
+    Ok(UntrackedStateRow {
+        entity_id: row.entity_id.clone(),
+        schema_key: row.schema_key.clone(),
+        file_id: row.file_id.clone(),
+        snapshot_ref,
+        metadata_ref,
+        schema_version: row.schema_version.clone(),
+        created_at: row.created_at.clone(),
+        updated_at: row.updated_at.clone(),
+        global: row.global,
+        version_id: row.version_id.clone(),
+    })
+}
+
+pub(crate) async fn materialize_row<S>(
+    json_reader: &mut JsonStoreReader<S>,
+    row: UntrackedStateRow,
+    projection: &UntrackedMaterializationProjection,
+) -> Result<MaterializedUntrackedStateRow, LixError>
+where
+    S: StorageReader,
+{
+    let snapshot_content = if projection.snapshot_content {
+        load_optional_json(json_reader, row.snapshot_ref.as_ref(), "snapshot_ref").await?
+    } else {
+        None
+    };
+    let metadata = if projection.metadata {
+        load_optional_metadata(json_reader, row.metadata_ref.as_ref()).await?
+    } else {
+        None
+    };
+    Ok(MaterializedUntrackedStateRow {
+        entity_id: row.entity_id,
+        schema_key: row.schema_key,
+        file_id: row.file_id,
+        snapshot_content,
+        metadata,
+        schema_version: row.schema_version,
+        created_at: row.created_at,
+        updated_at: row.updated_at,
+        global: row.global,
+        version_id: row.version_id,
+    })
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct UntrackedMaterializationProjection {
+    pub(crate) snapshot_content: bool,
+    pub(crate) metadata: bool,
+}
+
+impl UntrackedMaterializationProjection {
+    pub(crate) fn full() -> Self {
+        Self {
+            snapshot_content: true,
+            metadata: true,
+        }
+    }
+
+    pub(crate) fn from_columns(columns: &[String]) -> Self {
+        if columns.is_empty() {
+            return Self::full();
+        }
+        Self {
+            snapshot_content: columns.iter().any(|column| column == "snapshot_content"),
+            metadata: columns.iter().any(|column| column == "metadata"),
+        }
+    }
+}
+
+fn stage_optional_json(
+    writes: &mut StorageWriteSet,
+    json_writer: &mut JsonStoreWriter,
+    value: Option<&str>,
+) -> Result<Option<JsonRef>, LixError> {
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    json_writer.stage_bytes(writes, value.as_bytes()).map(Some)
+}
+
+fn stage_optional_metadata(
+    writes: &mut StorageWriteSet,
+    json_writer: &mut JsonStoreWriter,
+    value: Option<&RowMetadata>,
+) -> Result<Option<JsonRef>, LixError> {
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    let serialized = serialize_row_metadata(value);
+    json_writer
+        .stage_bytes(writes, serialized.as_bytes())
+        .map(Some)
+}
+
+async fn load_optional_metadata<S>(
+    json_reader: &mut JsonStoreReader<S>,
+    json_ref: Option<&JsonRef>,
+) -> Result<Option<RowMetadata>, LixError>
+where
+    S: StorageReader,
+{
+    let Some(json) = load_optional_json(json_reader, json_ref, "metadata_ref").await? else {
+        return Ok(None);
+    };
+    let metadata = serde_json::from_str::<RowMetadata>(&json).map_err(|error| {
+        LixError::new(
+            "LIX_ERROR_INVALID_JSON",
+            format!("untracked_state metadata_ref is invalid JSON: {error}"),
+        )
+    })?;
+    validate_row_metadata(metadata, "untracked_state metadata_ref").map(Some)
+}
+
+async fn load_optional_json<S>(
+    json_reader: &mut JsonStoreReader<S>,
+    json_ref: Option<&JsonRef>,
+    field: &str,
+) -> Result<Option<String>, LixError>
+where
+    S: StorageReader,
+{
+    let Some(json_ref) = json_ref else {
+        return Ok(None);
+    };
+    let bytes = json_reader.load_bytes(json_ref).await?.ok_or_else(|| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!(
+                "untracked_state {field} '{}' is missing from json_store",
+                json_ref.to_hex()
+            ),
+        )
+    })?;
+    String::from_utf8(bytes).map(Some).map_err(|error| {
+        LixError::new(
+            "LIX_ERROR_UNKNOWN",
+            format!(
+                "untracked_state {field} '{}' is not valid UTF-8 JSON bytes: {error}",
+                json_ref.to_hex()
+            ),
+        )
+    })
+}

package/dist-engine-src/src/untracked_state/mod.rs

@@ -0,0 +1,17 @@
+mod codec;
+mod context;
+mod materialization;
+pub(crate) mod storage;
+mod types;
+
+#[allow(unused_imports)]
+pub(crate) use context::{UntrackedStateContext, UntrackedStateStoreReader, UntrackedStateWriter};
+pub(crate) use materialization::{
+    canonicalize_materialized_row, materialize_row, UntrackedMaterializationProjection,
+};
+#[allow(unused_imports)]
+pub(crate) use types::{
+    MaterializedUntrackedStateRow, UntrackedStateFilter, UntrackedStateIdentity,
+    UntrackedStateProjection, UntrackedStateRow, UntrackedStateRowRequest,
+    UntrackedStateScanRequest,
+};