@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.4

package/dist-engine-src/src/sql2/lix_state_provider.rs

@@ -23,9 +23,10 @@ use datafusion::physical_plan::{
 use datafusion::prelude::SessionContext;
 use datafusion::scalar::ScalarValue;
 use futures_util::{stream, TryStreamExt};
+use serde_json::Value as JsonValue;
 
 use crate::entity_identity::EntityIdentity;
-use crate::live_state::LiveStateRow;
+use crate::live_state::MaterializedLiveStateRow;
 use crate::live_state::{
     LiveStateFilter, LiveStateProjection, LiveStateReader, LiveStateScanRequest,
 };
@@ -33,16 +34,17 @@ use crate::sql2::dml::{InsertExec, InsertSink};
 use crate::sql2::read_only::reject_read_only_stage_rows;
 use crate::sql2::version_scope::{resolve_provider_version_ids, VersionBinding};
 use crate::sql2::write_normalization::{InsertCell, SqlCell, UpdateAssignmentValues};
-use crate::transaction::types::StageRow;
+use crate::transaction::types::{TransactionJson, TransactionWriteRow};
 use crate::version::VersionRefReader;
 use crate::GLOBAL_VERSION_ID;
-use crate::{parse_row_metadata, serialize_row_metadata, LixError, NullableKeyFilter, RowMetadata};
+use crate::{parse_row_metadata_value, serialize_row_metadata, LixError, NullableKeyFilter};
 
 use crate::sql2::{
     SqlWriteContext, WriteAccess, WriteContextLiveStateReader, WriteContextVersionRefReader,
 };
-use crate::transaction::types::{StageWrite, StageWriteMode};
+use crate::transaction::types::{TransactionWrite, TransactionWriteMode};
 
+use super::predicate_typecheck::validate_json_predicate_filters;
 use super::result_metadata::json_field;
 
 pub(crate) async fn register_lix_state_providers(
@@ -265,6 +267,7 @@ impl TableProvider for LixStateProvider {
         let write_ctx = self.write_access.require_write("DELETE FROM lix_state")?;
 
         let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
+        validate_json_predicate_filters(self.schema.as_ref(), &filters)?;
         let physical_filters = filters
             .iter()
             .map(|expr| create_physical_expr(expr, &df_schema, state.execution_props()))
@@ -299,6 +302,7 @@ impl TableProvider for LixStateProvider {
         validate_lix_state_update_assignments(&self.schema, &assignments)?;
 
         let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
+        validate_json_predicate_filters(self.schema.as_ref(), &filters)?;
         let physical_assignments = assignments
             .iter()
             .map(|(column_name, expr)| {
@@ -378,8 +382,8 @@ impl InsertSink for LixStateInsertSink {
             .map_err(|_| DataFusionError::Execution("INSERT row count overflow".into()))?;
 
         self.write_ctx
-            .stage_write(StageWrite::Rows {
-                mode: StageWriteMode::Insert,
+            .stage_write(TransactionWrite::Rows {
+                mode: TransactionWriteMode::Insert,
                 rows,
             })
             .await
@@ -511,8 +515,8 @@ impl ExecutionPlan for LixStateDeleteExec {
 
             if count > 0 {
                 write_ctx
-                    .stage_write(StageWrite::Rows {
-                        mode: StageWriteMode::Replace,
+                    .stage_write(TransactionWrite::Rows {
+                        mode: TransactionWriteMode::Replace,
                         rows: write_rows,
                     })
                     .await
@@ -665,8 +669,8 @@ impl ExecutionPlan for LixStateUpdateExec {
 
             if count > 0 {
                 write_ctx
-                    .stage_write(StageWrite::Rows {
-                        mode: StageWriteMode::Replace,
+                    .stage_write(TransactionWrite::Rows {
+                        mode: TransactionWriteMode::Replace,
                         rows: write_rows,
                     })
                     .await
@@ -747,7 +751,7 @@ fn evaluate_lix_state_filters(
 fn lix_state_stageable_write_rows_from_batch(
     batch: &RecordBatch,
     version_binding: &str,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     let mut rows = lix_state_write_rows_from_batch(batch, version_binding)?;
     for row in &mut rows {
         row.created_at = None;
@@ -762,7 +766,7 @@ fn lix_state_update_write_rows_from_batch(
     batch: &RecordBatch,
     assignments: &[(String, Arc<dyn PhysicalExpr>)],
     version_binding: &str,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     let assignment_values = UpdateAssignmentValues::evaluate(batch, assignments)?;
     (0..batch.num_rows())
         .map(|row_index| {
@@ -776,9 +780,9 @@ fn lix_state_update_write_rows_from_batch(
                     }
                 });
 
-            Ok(StageRow {
+            Ok(TransactionWriteRow {
                 entity_id: Some(
-                    EntityIdentity::from_string(&required_string_value(
+                    EntityIdentity::from_json_array_text(&required_string_value(
                         batch,
                         row_index,
                         "entity_id",
@@ -791,7 +795,7 @@ fn lix_state_update_write_rows_from_batch(
                 ),
                 schema_key: required_string_value(batch, row_index, "schema_key")?,
                 file_id: optional_string_value(batch, row_index, "file_id")?,
-                snapshot_content: update_optional_string_value(
+                snapshot: update_optional_json_value(
                     batch,
                     &assignment_values,
                     row_index,
@@ -805,7 +809,6 @@ fn lix_state_update_write_rows_from_batch(
                     "lix_state",
                 )?,
                 origin: None,
-                schema_version: required_string_value(batch, row_index, "schema_version")?,
                 created_at: None,
                 updated_at: None,
                 global,
@@ -821,10 +824,10 @@ fn lix_state_update_write_rows_from_batch(
 fn lix_state_deletable_write_rows_from_batch(
     batch: &RecordBatch,
     version_binding: &str,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     let mut rows = lix_state_stageable_write_rows_from_batch(batch, version_binding)?;
     for row in &mut rows {
-        row.snapshot_content = None;
+        row.snapshot = None;
     }
     Ok(rows)
 }
@@ -854,14 +857,28 @@ fn update_optional_metadata_value(
     row_index: usize,
     column_name: &str,
     context: &str,
-) -> Result<Option<RowMetadata>> {
+) -> Result<Option<TransactionJson>> {
     update_optional_string_value(batch, assignment_values, row_index, column_name)?
         .map(|value| {
-            parse_row_metadata(&value, context).map_err(super::error::lix_error_to_datafusion_error)
+            let metadata = parse_row_metadata_value(&value, context)
+                .map_err(super::error::lix_error_to_datafusion_error)?;
+            TransactionJson::from_value(metadata, &format!("{context} metadata"))
+                .map_err(super::error::lix_error_to_datafusion_error)
         })
         .transpose()
 }
 
+fn update_optional_json_value(
+    batch: &RecordBatch,
+    assignment_values: &UpdateAssignmentValues,
+    row_index: usize,
+    column_name: &str,
+) -> Result<Option<TransactionJson>> {
+    update_optional_string_value(batch, assignment_values, row_index, column_name)?
+        .map(|value| parse_snapshot_json(&value, column_name))
+        .transpose()
+}
+
 fn dml_count_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![Field::new(
         "count",
@@ -881,7 +898,7 @@ fn dml_count_batch(schema: SchemaRef, count: u64) -> Result<RecordBatch> {
 fn lix_state_write_rows_from_batch(
     batch: &RecordBatch,
     version_binding: &str,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     (0..batch.num_rows())
         .map(|row_index| {
             let global = optional_bool_value(batch, row_index, "global")?.unwrap_or(false);
@@ -894,9 +911,9 @@ fn lix_state_write_rows_from_batch(
                     }
                 });
 
-            Ok(StageRow {
+            Ok(TransactionWriteRow {
                 entity_id: Some(
-                    EntityIdentity::from_string(&required_string_value(
+                    EntityIdentity::from_json_array_text(&required_string_value(
                         batch,
                         row_index,
                         "entity_id",
@@ -909,10 +926,9 @@ fn lix_state_write_rows_from_batch(
                 ),
                 schema_key: required_string_value(batch, row_index, "schema_key")?,
                 file_id: optional_string_value(batch, row_index, "file_id")?,
-                snapshot_content: optional_string_value(batch, row_index, "snapshot_content")?,
+                snapshot: optional_json_value(batch, row_index, "snapshot_content")?,
                 metadata: optional_metadata_value(batch, row_index, "metadata", "lix_state")?,
                 origin: None,
-                schema_version: required_string_value(batch, row_index, "schema_version")?,
                 created_at: optional_string_value(batch, row_index, "created_at")?,
                 updated_at: optional_string_value(batch, row_index, "updated_at")?,
                 global,
@@ -962,14 +978,37 @@ fn optional_metadata_value(
     row_index: usize,
     column_name: &str,
     context: &str,
-) -> Result<Option<RowMetadata>> {
+) -> Result<Option<TransactionJson>> {
     optional_string_value(batch, row_index, column_name)?
         .map(|value| {
-            parse_row_metadata(&value, context).map_err(super::error::lix_error_to_datafusion_error)
+            let metadata = parse_row_metadata_value(&value, context)
+                .map_err(super::error::lix_error_to_datafusion_error)?;
+            TransactionJson::from_value(metadata, &format!("{context} metadata"))
+                .map_err(super::error::lix_error_to_datafusion_error)
         })
         .transpose()
 }
 
+fn optional_json_value(
+    batch: &RecordBatch,
+    row_index: usize,
+    column_name: &str,
+) -> Result<Option<TransactionJson>> {
+    optional_string_value(batch, row_index, column_name)?
+        .map(|value| parse_snapshot_json(&value, column_name))
+        .transpose()
+}
+
+fn parse_snapshot_json(value: &str, column_name: &str) -> Result<TransactionJson> {
+    let parsed = serde_json::from_str::<JsonValue>(value).map_err(|error| {
+        DataFusionError::Execution(format!(
+            "lix_state expected valid JSON in column '{column_name}': {error}"
+        ))
+    })?;
+    TransactionJson::from_value(parsed, &format!("lix_state {column_name}"))
+        .map_err(super::error::lix_error_to_datafusion_error)
+}
+
 fn optional_bool_value(
     batch: &RecordBatch,
     row_index: usize,
@@ -1122,12 +1161,11 @@ impl ExecutionPlan for LixStateScanExec {
 
 fn lix_state_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![
-        Field::new("entity_id", DataType::Utf8, false),
+        json_field("entity_id", false),
         Field::new("schema_key", DataType::Utf8, false),
         Field::new("file_id", DataType::Utf8, true),
         json_field("snapshot_content", true),
         json_field("metadata", true),
-        Field::new("schema_version", DataType::Utf8, true),
         Field::new("created_at", DataType::Utf8, true),
         Field::new("updated_at", DataType::Utf8, true),
         Field::new("global", DataType::Boolean, true),
@@ -1139,12 +1177,11 @@ fn lix_state_schema() -> SchemaRef {
 
 fn lix_state_by_version_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![
-        Field::new("entity_id", DataType::Utf8, false),
+        json_field("entity_id", false),
         Field::new("schema_key", DataType::Utf8, false),
         Field::new("file_id", DataType::Utf8, true),
         json_field("snapshot_content", true),
         json_field("metadata", true),
-        Field::new("schema_version", DataType::Utf8, true),
         Field::new("created_at", DataType::Utf8, true),
         Field::new("updated_at", DataType::Utf8, true),
         Field::new("global", DataType::Boolean, true),
@@ -1238,7 +1275,7 @@ fn lix_state_scan_request(
             .map(|values| {
                 values
                     .iter()
-                    .map(|value| EntityIdentity::single(value))
+                    .filter_map(|value| EntityIdentity::from_json_array_text(value).ok())
                     .collect()
             })
             .unwrap_or_default(),
@@ -1360,7 +1397,7 @@ fn parse_lix_state_in_list_filter(in_list: &InList) -> Option<LixStateFilterPred
     match column.name.as_str() {
         "schema_key" => Some(LixStateFilterPredicate::SchemaKeys(values)),
         "version_id" => Some(LixStateFilterPredicate::VersionIds(values)),
-        "entity_id" => Some(LixStateFilterPredicate::EntityIds(values)),
+        "entity_id" => canonical_entity_id_values(values).map(LixStateFilterPredicate::EntityIds),
         _ => None,
     }
 }
@@ -1390,12 +1427,27 @@ fn parse_lix_state_column_literal_filter(
         "version_id" => string_expr_literal(literal_expr)
             .map(|value| LixStateFilterPredicate::VersionIds(BTreeSet::from([value]))),
         "entity_id" => string_expr_literal(literal_expr)
+            .and_then(|value| canonical_entity_id_value(&value))
             .map(|value| LixStateFilterPredicate::EntityIds(BTreeSet::from([value]))),
         "file_id" => nullable_key_literal(literal_expr).map(LixStateFilterPredicate::FileId),
         _ => None,
     }
 }
 
+fn canonical_entity_id_values(values: BTreeSet<String>) -> Option<BTreeSet<String>> {
+    values
+        .into_iter()
+        .map(|value| canonical_entity_id_value(&value))
+        .collect()
+}
+
+fn canonical_entity_id_value(value: &str) -> Option<String> {
+    EntityIdentity::from_json_array_text(value)
+        .ok()?
+        .as_json_array_text()
+        .ok()
+}
+
 fn nullable_key_literal(expr: &Expr) -> Option<NullableKeyFilter<String>> {
     if is_null_literal(expr) {
         return Some(NullableKeyFilter::Null);
@@ -1421,7 +1473,7 @@ fn is_null_literal(expr: &Expr) -> bool {
 
 fn lix_state_record_batch(
     schema: SchemaRef,
-    rows: &[LiveStateRow],
+    rows: &[MaterializedLiveStateRow],
 ) -> Result<RecordBatch, LixError> {
     if schema.fields().is_empty() {
         let options = RecordBatchOptions::new().with_row_count(Some(rows.len()));
@@ -1440,7 +1492,7 @@ fn lix_state_record_batch(
             Ok(match field.name().as_str() {
                 "entity_id" => Arc::new(StringArray::from(
                     rows.iter()
-                        .map(|row| row.entity_id.as_string().map(Some))
+                        .map(|row| row.entity_id.as_json_array_text().map(Some))
                         .collect::<std::result::Result<Vec<_>, LixError>>()?,
                 )) as ArrayRef,
                 "schema_key" => string_array(rows.iter().map(|row| Some(row.schema_key.as_str()))),
@@ -1453,9 +1505,6 @@ fn lix_state_record_batch(
                         .map(|row| row.metadata.as_ref().map(serialize_row_metadata))
                         .collect::<Vec<_>>(),
                 )),
-                "schema_version" => {
-                    string_array(rows.iter().map(|row| Some(row.schema_version.as_str())))
-                }
                 "created_at" => string_array(rows.iter().map(|row| Some(row.created_at.as_str()))),
                 "updated_at" => string_array(rows.iter().map(|row| Some(row.updated_at.as_str()))),
                 "global" => Arc::new(BooleanArray::from(
@@ -1525,11 +1574,16 @@ mod tests {
     };
     use crate::sql2::dml::{InsertExec, InsertSink};
     use crate::sql2::{SqlWriteContext, SqlWriteExecutionContext};
-    use crate::transaction::types::{StageRow, StageWrite, StageWriteMode, StageWriteOutcome};
+    use crate::transaction::types::{
+        TransactionJson, TransactionWrite, TransactionWriteMode, TransactionWriteOutcome,
+        TransactionWriteRow,
+    };
     use crate::version::{VersionHead, VersionRefReader};
     use crate::{
         entity_identity::EntityIdentity,
-        live_state::{LiveStateReader, LiveStateRow, LiveStateRowRequest, LiveStateScanRequest},
+        live_state::{
+            LiveStateReader, LiveStateRowRequest, LiveStateScanRequest, MaterializedLiveStateRow,
+        },
     };
     use crate::{LixError, NullableKeyFilter};
     use async_trait::async_trait;
@@ -1560,19 +1614,19 @@ mod tests {
     struct EmptyVersionRefReader;
     #[allow(dead_code)]
     struct RowsLiveStateReader {
-        rows: Vec<LiveStateRow>,
+        rows: Vec<MaterializedLiveStateRow>,
     }
     struct DummyBlobReader;
 
     #[derive(Default)]
     struct DummyWriteContext {
-        rows: Vec<LiveStateRow>,
+        rows: Vec<MaterializedLiveStateRow>,
     }
 
     #[derive(Default)]
     struct CapturingWriteContext {
-        rows: Vec<LiveStateRow>,
-        writes: Vec<StageWrite>,
+        rows: Vec<MaterializedLiveStateRow>,
+        writes: Vec<TransactionWrite>,
     }
 
     struct SingleBatchExec {
@@ -1663,14 +1717,14 @@ mod tests {
         async fn scan_rows(
             &self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(vec![])
         }
 
         async fn load_row(
             &self,
             _request: &LiveStateRowRequest,
-        ) -> Result<Option<LiveStateRow>, LixError> {
+        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
             Ok(None)
         }
     }
@@ -1695,14 +1749,14 @@ mod tests {
         async fn scan_rows(
             &self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(self.rows.clone())
         }
 
         async fn load_row(
             &self,
             _request: &LiveStateRowRequest,
-        ) -> Result<Option<LiveStateRow>, LixError> {
+        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
             Ok(None)
         }
     }
@@ -1719,7 +1773,10 @@ mod tests {
             &self,
             hashes: &[crate::binary_cas::BlobHash],
         ) -> Result<crate::binary_cas::BlobBytesBatch, LixError> {
-            Ok(crate::binary_cas::BlobBytesBatch::missing(hashes.len()))
+            Ok(crate::binary_cas::BlobBytesBatch::new(vec![
+                None;
+                hashes.len()
+            ]))
         }
     }
 
@@ -1747,7 +1804,7 @@ mod tests {
         async fn scan_live_state(
             &mut self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(self.rows.clone())
         }
 
@@ -1761,8 +1818,11 @@ mod tests {
             Ok(Some(format!("commit-{version_id}")))
         }
 
-        async fn stage_write(&mut self, _write: StageWrite) -> Result<StageWriteOutcome, LixError> {
-            Ok(StageWriteOutcome { count: 0 })
+        async fn stage_write(
+            &mut self,
+            _write: TransactionWrite,
+        ) -> Result<TransactionWriteOutcome, LixError> {
+            Ok(TransactionWriteOutcome { count: 0 })
         }
     }
 
@@ -1790,7 +1850,7 @@ mod tests {
         async fn scan_live_state(
             &mut self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(self.rows.clone())
         }
 
@@ -1804,9 +1864,12 @@ mod tests {
             Ok(Some(format!("commit-{version_id}")))
         }
 
-        async fn stage_write(&mut self, write: StageWrite) -> Result<StageWriteOutcome, LixError> {
+        async fn stage_write(
+            &mut self,
+            write: TransactionWrite,
+        ) -> Result<TransactionWriteOutcome, LixError> {
             self.writes.push(write);
-            Ok(StageWriteOutcome { count: 0 })
+            Ok(TransactionWriteOutcome { count: 0 })
         }
     }
 
@@ -1818,6 +1881,18 @@ mod tests {
         Expr::Literal(ScalarValue::Utf8(Some(value.to_string())), None)
     }
 
+    fn json_lit(value: &str) -> Expr {
+        Expr::Literal(
+            ScalarValue::Utf8(Some(value.to_string())),
+            Some(datafusion::common::metadata::FieldMetadata::new(
+                std::collections::BTreeMap::from([(
+                    crate::sql2::result_metadata::LIX_VALUE_TYPE_METADATA_KEY.to_string(),
+                    crate::sql2::result_metadata::LIX_VALUE_TYPE_JSON.to_string(),
+                )]),
+            )),
+        )
+    }
+
     fn string_column(values: Vec<Option<&str>>) -> ArrayRef {
         Arc::new(StringArray::from(values)) as ArrayRef
     }
@@ -1826,12 +1901,11 @@ mod tests {
         RecordBatch::try_new(
             lix_state_schema(),
             vec![
-                string_column(vec![Some("entity-1")]),
+                string_column(vec![Some("[\"entity-1\"]")]),
                 string_column(vec![Some("lix_key_value")]),
                 string_column(vec![None]),
                 string_column(vec![Some("{\"key\":\"hello\",\"value\":\"world\"}")]),
                 string_column(vec![Some("{\"source\":\"test\"}")]),
-                string_column(vec![Some("1")]),
                 string_column(vec![Some("2026-04-23T00:00:00Z")]),
                 string_column(vec![Some("2026-04-23T01:00:00Z")]),
                 Arc::new(BooleanArray::from(vec![global])) as ArrayRef,
@@ -1847,12 +1921,11 @@ mod tests {
         RecordBatch::try_new(
             lix_state_schema(),
             vec![
-                string_column(vec![Some("entity-1")]),
+                string_column(vec![Some("[\"entity-1\"]")]),
                 string_column(vec![Some("lix_key_value")]),
                 string_column(vec![None]),
                 string_column(vec![Some("{\"key\":\"hello\",\"value\":\"world\"}")]),
                 string_column(vec![None]),
-                string_column(vec![Some("1")]),
                 string_column(vec![None]),
                 string_column(vec![None]),
                 Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
@@ -1864,16 +1937,14 @@ mod tests {
         .expect("valid stageable lix_state batch")
     }
 
-    fn live_row(entity_id: &str, metadata: Option<&str>) -> LiveStateRow {
-        LiveStateRow {
-            entity_id: EntityIdentity::from_string(entity_id).expect("entity id should decode"),
+    fn live_row(entity_id: &str, metadata: Option<&str>) -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
+            entity_id: EntityIdentity::single(entity_id),
             schema_key: "lix_key_value".to_string(),
             file_id: None,
             snapshot_content: Some("{\"key\":\"hello\",\"value\":\"world\"}".to_string()),
-            metadata: metadata.map(|value| {
-                serde_json::from_str(value).expect("test metadata should be valid JSON")
-            }),
-            schema_version: "1".to_string(),
+            metadata: metadata.map(str::to_string),
+            deleted: false,
             version_id: "version-a".to_string(),
             change_id: Some(format!("change-{entity_id}")),
             commit_id: Some(format!("commit-{entity_id}")),
@@ -1935,7 +2006,7 @@ mod tests {
         ]);
 
         let request =
-            lix_state_scan_request(&schema, None, Some(&vec![0, 1, 12]), &route, Some(10));
+            lix_state_scan_request(&schema, None, Some(&vec![0, 1, 11]), &route, Some(10));
 
         assert_eq!(request.filter.schema_keys, vec!["profile".to_string()]);
         assert_eq!(request.filter.version_ids, vec!["v1".to_string()]);
@@ -1957,7 +2028,7 @@ mod tests {
             Box::new(Expr::BinaryExpr(BinaryExpr::new(
                 Box::new(col("entity_id")),
                 Operator::Eq,
-                Box::new(str_lit("entity-a")),
+                Box::new(str_lit("[\"entity-a\"]")),
             ))),
             Operator::And,
             Box::new(Expr::InList(InList::new(
@@ -1969,7 +2040,7 @@ mod tests {
 
         assert_eq!(
             route.entity_ids,
-            Some(BTreeSet::from(["entity-a".to_string()]))
+            Some(BTreeSet::from(["[\"entity-a\"]".to_string()]))
         );
         assert_eq!(
             route.version_ids,
@@ -2186,14 +2257,17 @@ mod tests {
 
         assert_eq!(
             rows,
-            vec![StageRow {
+            vec![TransactionWriteRow {
                 entity_id: Some(crate::entity_identity::EntityIdentity::single("entity-1")),
                 schema_key: "lix_key_value".to_string(),
                 file_id: None,
-                snapshot_content: Some("{\"key\":\"hello\",\"value\":\"world\"}".to_string()),
-                metadata: Some(json!({"source": "test"})),
+                snapshot: Some(TransactionJson::from_value_for_test(
+                    json!({"key":"hello","value":"world"})
+                )),
+                metadata: Some(TransactionJson::from_value_for_test(
+                    json!({"source": "test"})
+                )),
                 origin: None,
-                schema_version: "1".to_string(),
                 created_at: Some("2026-04-23T00:00:00Z".to_string()),
                 updated_at: Some("2026-04-23T01:00:00Z".to_string()),
                 global: false,
@@ -2228,16 +2302,19 @@ mod tests {
         assert_eq!(count, 1);
         assert_eq!(
             write_context.writes.as_slice(),
-            &[StageWrite::Rows {
-                mode: StageWriteMode::Insert,
-                rows: vec![StageRow {
+            &[TransactionWrite::Rows {
+                mode: TransactionWriteMode::Insert,
+                rows: vec![TransactionWriteRow {
                     entity_id: Some(crate::entity_identity::EntityIdentity::single("entity-1")),
                     schema_key: "lix_key_value".to_string(),
                     file_id: None,
-                    snapshot_content: Some("{\"key\":\"hello\",\"value\":\"world\"}".to_string()),
-                    metadata: Some(json!({"source": "test"})),
+                    snapshot: Some(TransactionJson::from_value_for_test(
+                        json!({"key":"hello","value":"world"})
+                    )),
+                    metadata: Some(TransactionJson::from_value_for_test(
+                        json!({"source": "test"})
+                    )),
                     origin: None,
-                    schema_version: "1".to_string(),
                     created_at: Some("2026-04-23T00:00:00Z".to_string()),
                     updated_at: Some("2026-04-23T01:00:00Z".to_string()),
                     global: false,
@@ -2312,7 +2389,7 @@ mod tests {
                 vec![Expr::BinaryExpr(BinaryExpr::new(
                     Box::new(col("metadata")),
                     Operator::Eq,
-                    Box::new(str_lit("{\"source\":\"match\"}")),
+                    Box::new(json_lit("{\"source\":\"match\"}")),
                 ))],
             )
             .await
@@ -2333,16 +2410,19 @@ mod tests {
 
         assert_eq!(
             write_context.writes.as_slice(),
-            &[StageWrite::Rows {
-                mode: StageWriteMode::Replace,
-                rows: vec![StageRow {
+            &[TransactionWrite::Rows {
+                mode: TransactionWriteMode::Replace,
+                rows: vec![TransactionWriteRow {
                     entity_id: Some(crate::entity_identity::EntityIdentity::single("entity-1")),
                     schema_key: "lix_key_value".to_string(),
                     file_id: None,
-                    snapshot_content: Some("{\"key\":\"hello\",\"value\":\"updated\"}".to_string()),
-                    metadata: Some(json!({"schema_key": "lix_key_value"})),
+                    snapshot: Some(TransactionJson::from_value_for_test(
+                        json!({"key":"hello","value":"updated"})
+                    )),
+                    metadata: Some(TransactionJson::from_value_for_test(
+                        json!({"schema_key": "lix_key_value"})
+                    )),
                     origin: None,
-                    schema_version: "1".to_string(),
                     created_at: None,
                     updated_at: None,
                     global: false,
@@ -2388,17 +2468,18 @@ mod tests {
 
         assert_eq!(
             write_context.writes.as_slice(),
-            &[StageWrite::Rows {
-                mode: StageWriteMode::Replace,
+            &[TransactionWrite::Rows {
+                mode: TransactionWriteMode::Replace,
                 rows: vec![
-                    StageRow {
+                    TransactionWriteRow {
                         entity_id: Some(crate::entity_identity::EntityIdentity::single("entity-1")),
                         schema_key: "lix_key_value".to_string(),
                         file_id: None,
-                        snapshot_content: None,
-                        metadata: Some(json!({"source": "one"})),
+                        snapshot: None,
+                        metadata: Some(TransactionJson::from_value_for_test(
+                            json!({"source": "one"})
+                        )),
                         origin: None,
-                        schema_version: "1".to_string(),
                         created_at: None,
                         updated_at: None,
                         global: false,
@@ -2407,14 +2488,15 @@ mod tests {
                         untracked: false,
                         version_id: "version-a".to_string(),
                     },
-                    StageRow {
+                    TransactionWriteRow {
                         entity_id: Some(crate::entity_identity::EntityIdentity::single("entity-2")),
                         schema_key: "lix_key_value".to_string(),
                         file_id: None,
-                        snapshot_content: None,
-                        metadata: Some(json!({"source": "two"})),
+                        snapshot: None,
+                        metadata: Some(TransactionJson::from_value_for_test(
+                            json!({"source": "two"})
+                        )),
                         origin: None,
-                        schema_version: "1".to_string(),
                         created_at: None,
                         updated_at: None,
                         global: false,