@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.4

package/dist-engine-src/src/sql2/file_provider.rs

@@ -14,7 +14,8 @@ use datafusion::common::{not_impl_err, DFSchema, DataFusionError, Result, Scalar
 use datafusion::datasource::TableType;
 use datafusion::execution::TaskContext;
 use datafusion::logical_expr::dml::InsertOp;
-use datafusion::logical_expr::{Expr, TableProviderFilterPushDown};
+use datafusion::logical_expr::expr::InList;
+use datafusion::logical_expr::{BinaryExpr, Expr, Operator, TableProviderFilterPushDown};
 use datafusion::physical_expr::{create_physical_expr, EquivalenceProperties, PhysicalExpr};
 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType, PlanProperties};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
@@ -26,8 +27,9 @@ use futures_util::{stream, TryStreamExt};
 use serde::Deserialize;
 
 use crate::binary_cas::{BlobDataReader, BlobHash};
+use crate::entity_identity::EntityIdentity;
 use crate::functions::FunctionProviderHandle;
-use crate::live_state::LiveStateRow;
+use crate::live_state::MaterializedLiveStateRow;
 use crate::live_state::{
     LiveStateFilter, LiveStateProjection, LiveStateReader, LiveStateScanRequest,
 };
@@ -35,6 +37,7 @@ use crate::sql2::dml::{InsertExec, InsertSink};
 use crate::sql2::filesystem_predicates::{
     canonicalize_filesystem_path_filters, FilesystemPathKind,
 };
+use crate::sql2::predicate_typecheck::validate_json_predicate_filters;
 use crate::sql2::version_scope::{
     explicit_version_ids_from_dml_filters, resolve_provider_version_ids,
     resolve_write_version_scope, VersionBinding,
@@ -45,9 +48,9 @@ use crate::sql2::write_normalization::{
     scalar_is_binary_or_null, InsertCell, InsertColumnIntents, SqlCell, UpdateAssignmentValues,
     UpdateCell,
 };
-use crate::transaction::types::StageRow;
+use crate::transaction::types::{TransactionJson, TransactionWriteRow};
 use crate::version::VersionRefReader;
-use crate::{parse_row_metadata, serialize_row_metadata, LixError, RowMetadata};
+use crate::{parse_row_metadata_value, serialize_row_metadata, LixError};
 
 const FILE_DESCRIPTOR_SCHEMA_KEY: &str = "lix_file_descriptor";
 const BLOB_REF_SCHEMA_KEY: &str = "lix_binary_blob_ref";
@@ -65,8 +68,8 @@ use crate::sql2::{
     SqlWriteContext, WriteAccess, WriteContextLiveStateReader, WriteContextVersionRefReader,
 };
 use crate::transaction::types::{
-    LogicalPrimaryKey, StageFileData, StageRowOrigin, StageWrite, StageWriteMode,
-    StageWriteOperation,
+    LogicalPrimaryKey, TransactionFileData, TransactionWrite, TransactionWriteMode,
+    TransactionWriteOperation, TransactionWriteOrigin,
 };
 
 pub(crate) async fn register_lix_file_providers(
@@ -226,9 +229,19 @@ impl TableProvider for LixFileProvider {
         &self,
         filters: &[&Expr],
     ) -> Result<Vec<TableProviderFilterPushDown>> {
+        let analyzer = LixFileIdFilterAnalyzer;
         Ok(filters
             .iter()
-            .map(|_| TableProviderFilterPushDown::Exact)
+            .map(|filter| {
+                if ExactStringColumnFilterAnalyzer::new("lixcol_version_id").supports(filter)
+                    || analyzer.supports(filter)
+                    || contains_column(filter, "path")
+                {
+                    TableProviderFilterPushDown::Exact
+                } else {
+                    TableProviderFilterPushDown::Unsupported
+                }
+            })
             .collect())
     }
 
@@ -241,8 +254,11 @@ impl TableProvider for LixFileProvider {
     ) -> Result<Arc<dyn ExecutionPlan>> {
         let projected_schema = projected_schema(&self.schema, projection)?;
         let scan_limit = if filters.is_empty() { limit } else { None };
-        let mut request =
-            lix_file_scan_request(self.version_binding.active_version_id(), scan_limit);
+        let mut request = lix_file_scan_request(
+            self.version_binding.active_version_id(),
+            Some(projected_schema.as_ref()),
+            scan_limit,
+        );
         if self.write_access.is_write() && matches!(self.version_binding, VersionBinding::Explicit)
         {
             request.filter.version_ids = explicit_version_ids_from_dml_filters(filters);
@@ -261,7 +277,9 @@ impl TableProvider for LixFileProvider {
         .await
         .map_err(lix_error_to_datafusion_error)?;
         let filters = canonicalize_filesystem_path_filters(filters, FilesystemPathKind::File)?;
+        let target_file_ids = file_id_constraint_from_filters(&filters)?;
         let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
+        validate_json_predicate_filters(self.schema.as_ref(), &filters)?;
         let physical_filters = filters
             .iter()
             .map(|expr| create_physical_expr(expr, &df_schema, _state.execution_props()))
@@ -273,6 +291,7 @@ impl TableProvider for LixFileProvider {
             projected_schema,
             projection.cloned(),
             request,
+            target_file_ids,
             physical_filters,
             limit,
         )))
@@ -314,11 +333,14 @@ impl TableProvider for LixFileProvider {
 
         let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
         let filters = canonicalize_filesystem_path_filters(&filters, FilesystemPathKind::File)?;
+        validate_json_predicate_filters(self.schema.as_ref(), &filters)?;
         let physical_filters = filters
             .iter()
             .map(|expr| create_physical_expr(expr, &df_schema, state.execution_props()))
             .collect::<Result<Vec<_>>>()?;
-        let mut request = lix_file_scan_request(self.version_binding.active_version_id(), None);
+        let target_file_ids = file_id_constraint_from_filters(&filters)?;
+        let mut request =
+            lix_file_scan_request(self.version_binding.active_version_id(), None, None);
         if matches!(self.version_binding, VersionBinding::Explicit) {
             request.filter.version_ids = explicit_version_ids_from_dml_filters(&filters);
             if request.filter.version_ids.is_empty() {
@@ -335,6 +357,7 @@ impl TableProvider for LixFileProvider {
             Arc::clone(&self.schema),
             self.version_binding.clone(),
             request,
+            target_file_ids,
             physical_filters,
         )))
     }
@@ -360,11 +383,13 @@ impl TableProvider for LixFileProvider {
             })
             .collect::<Result<Vec<_>>>()?;
         let filters = canonicalize_filesystem_path_filters(&filters, FilesystemPathKind::File)?;
+        let target_file_ids = file_id_constraint_from_filters(&filters)?;
+        validate_json_predicate_filters(self.schema.as_ref(), &filters)?;
         let physical_filters = filters
             .iter()
             .map(|expr| create_physical_expr(expr, &df_schema, state.execution_props()))
             .collect::<Result<Vec<_>>>()?;
-        let request = lix_file_scan_request(self.version_binding.active_version_id(), None);
+        let request = lix_file_scan_request(self.version_binding.active_version_id(), None, None);
 
         Ok(Arc::new(LixFileUpdateExec::new(
             Arc::clone(&self.blob_reader),
@@ -373,6 +398,7 @@ impl TableProvider for LixFileProvider {
             self.version_binding.clone(),
             self.functions.clone(),
             request,
+            target_file_ids,
             physical_assignments,
             physical_filters,
         )))
@@ -473,13 +499,13 @@ impl InsertSink for LixFileInsertSink {
 
         if !staged.state_rows.is_empty() || !staged.file_data_writes.is_empty() {
             let intent = if staged.file_data_writes.is_empty() {
-                StageWrite::Rows {
-                    mode: StageWriteMode::Insert,
+                TransactionWrite::Rows {
+                    mode: TransactionWriteMode::Insert,
                     rows: staged.state_rows,
                 }
             } else {
-                StageWrite::RowsWithFileData {
-                    mode: StageWriteMode::Insert,
+                TransactionWrite::RowsWithFileData {
+                    mode: TransactionWriteMode::Insert,
                     rows: staged.state_rows,
                     file_data: staged.file_data_writes,
                     count: staged.count,
@@ -509,6 +535,7 @@ struct LixFileDeleteExec {
     table_schema: SchemaRef,
     version_binding: VersionBinding,
     request: LiveStateScanRequest,
+    target_file_ids: FileIdConstraint,
     filters: Vec<Arc<dyn PhysicalExpr>>,
     result_schema: SchemaRef,
     properties: Arc<PlanProperties>,
@@ -527,6 +554,7 @@ impl LixFileDeleteExec {
         table_schema: SchemaRef,
         version_binding: VersionBinding,
         request: LiveStateScanRequest,
+        target_file_ids: FileIdConstraint,
         filters: Vec<Arc<dyn PhysicalExpr>>,
     ) -> Self {
         let result_schema = dml_count_schema();
@@ -542,6 +570,7 @@ impl LixFileDeleteExec {
             table_schema,
             version_binding,
             request,
+            target_file_ids,
             filters,
             result_schema,
             properties: Arc::new(properties),
@@ -605,15 +634,19 @@ impl ExecutionPlan for LixFileDeleteExec {
         let table_schema = Arc::clone(&self.table_schema);
         let version_binding = self.version_binding.clone();
         let request = self.request.clone();
+        let target_file_ids = self.target_file_ids.clone();
         let filters = self.filters.clone();
         let result_schema = Arc::clone(&self.result_schema);
         let stream_schema = Arc::clone(&result_schema);
 
         let stream = stream::once(async move {
-            let rows = write_ctx
-                .scan_live_state(&request)
-                .await
-                .map_err(lix_error_to_datafusion_error)?;
+            let rows = scan_lix_file_live_rows(
+                Arc::new(WriteContextLiveStateReader::new(write_ctx.clone())),
+                &request,
+                &target_file_ids,
+            )
+            .await
+            .map_err(lix_error_to_datafusion_error)?;
             let blob_ref_file_ids =
                 blob_ref_file_ids_from_live_rows(&rows).map_err(lix_error_to_datafusion_error)?;
             let source_batch = lix_file_record_batch(&table_schema, &blob_reader, rows)
@@ -629,8 +662,8 @@ impl ExecutionPlan for LixFileDeleteExec {
 
             if count > 0 {
                 write_ctx
-                    .stage_write(StageWrite::Rows {
-                        mode: StageWriteMode::Replace,
+                    .stage_write(TransactionWrite::Rows {
+                        mode: TransactionWriteMode::Replace,
                         rows: staged.state_rows,
                     })
                     .await
@@ -658,6 +691,7 @@ struct LixFileUpdateExec {
     version_binding: VersionBinding,
     functions: FunctionProviderHandle,
     request: LiveStateScanRequest,
+    target_file_ids: FileIdConstraint,
     assignments: Vec<(String, Arc<dyn PhysicalExpr>)>,
     filters: Vec<Arc<dyn PhysicalExpr>>,
     result_schema: SchemaRef,
@@ -678,6 +712,7 @@ impl LixFileUpdateExec {
         version_binding: VersionBinding,
         functions: FunctionProviderHandle,
         request: LiveStateScanRequest,
+        target_file_ids: FileIdConstraint,
         assignments: Vec<(String, Arc<dyn PhysicalExpr>)>,
         filters: Vec<Arc<dyn PhysicalExpr>>,
     ) -> Self {
@@ -695,6 +730,7 @@ impl LixFileUpdateExec {
             version_binding,
             functions,
             request,
+            target_file_ids,
             assignments,
             filters,
             result_schema,
@@ -765,16 +801,20 @@ impl ExecutionPlan for LixFileUpdateExec {
         let version_binding = self.version_binding.clone();
         let functions = self.functions.clone();
         let request = self.request.clone();
+        let target_file_ids = self.target_file_ids.clone();
         let assignments = self.assignments.clone();
         let filters = self.filters.clone();
         let result_schema = Arc::clone(&self.result_schema);
         let stream_schema = Arc::clone(&result_schema);
 
         let stream = stream::once(async move {
-            let rows = write_ctx
-                .scan_live_state(&request)
-                .await
-                .map_err(lix_error_to_datafusion_error)?;
+            let rows = scan_lix_file_live_rows(
+                Arc::new(WriteContextLiveStateReader::new(write_ctx.clone())),
+                &request,
+                &target_file_ids,
+            )
+            .await
+            .map_err(lix_error_to_datafusion_error)?;
             let source_batch = lix_file_record_batch(&table_schema, &blob_reader, rows)
                 .await
                 .map_err(lix_error_to_datafusion_error)?;
@@ -804,13 +844,13 @@ impl ExecutionPlan for LixFileUpdateExec {
 
             if count > 0 {
                 let intent = if staged.file_data_writes.is_empty() {
-                    StageWrite::Rows {
-                        mode: StageWriteMode::Replace,
+                    TransactionWrite::Rows {
+                        mode: TransactionWriteMode::Replace,
                         rows: staged.state_rows,
                     }
                 } else {
-                    StageWrite::RowsWithFileData {
-                        mode: StageWriteMode::Replace,
+                    TransactionWrite::RowsWithFileData {
+                        mode: TransactionWriteMode::Replace,
                         rows: staged.state_rows,
                         file_data: staged.file_data_writes,
                         count,
@@ -842,6 +882,7 @@ struct LixFileScanExec {
     output_schema: SchemaRef,
     projection: Option<Vec<usize>>,
     request: LiveStateScanRequest,
+    target_file_ids: FileIdConstraint,
     filters: Vec<Arc<dyn PhysicalExpr>>,
     limit: Option<usize>,
     properties: Arc<PlanProperties>,
@@ -861,6 +902,7 @@ impl LixFileScanExec {
         output_schema: SchemaRef,
         projection: Option<Vec<usize>>,
         request: LiveStateScanRequest,
+        target_file_ids: FileIdConstraint,
         filters: Vec<Arc<dyn PhysicalExpr>>,
         limit: Option<usize>,
     ) -> Self {
@@ -877,6 +919,7 @@ impl LixFileScanExec {
             output_schema,
             projection,
             request,
+            target_file_ids,
             filters,
             limit,
             properties: Arc::new(properties),
@@ -938,15 +981,18 @@ impl ExecutionPlan for LixFileScanExec {
         let live_state = Arc::clone(&self.live_state);
         let blob_reader = Arc::clone(&self.blob_reader);
         let request = self.request.clone();
+        let target_file_ids = self.target_file_ids.clone();
         let filters = self.filters.clone();
         let limit = self.limit;
         let output_schema = Arc::clone(&self.output_schema);
         let batch_schema = Arc::clone(&self.batch_schema);
         let projection = self.projection.clone();
         let fut = async move {
-            let rows = live_state.scan_rows(&request).await.map_err(|error| {
-                DataFusionError::Execution(format!("sql2 lix_file scan failed: {error}"))
-            })?;
+            let rows = scan_lix_file_live_rows(live_state, &request, &target_file_ids)
+                .await
+                .map_err(|error| {
+                    DataFusionError::Execution(format!("sql2 lix_file scan failed: {error}"))
+                })?;
             let batch = lix_file_record_batch(&batch_schema, &blob_reader, rows)
                 .await
                 .map_err(|error| {
@@ -976,7 +1022,7 @@ struct FileDescriptorRecord {
     directory_id: Option<String>,
     name: String,
     hidden: bool,
-    live: LiveStateRow,
+    live: MaterializedLiveStateRow,
 }
 
 #[derive(Debug, Clone)]
@@ -1015,8 +1061,8 @@ struct DirectoryDescriptorSnapshot {
 
 #[derive(Debug, Default)]
 struct LixFileStagedBatch {
-    state_rows: Vec<StageRow>,
-    file_data_writes: Vec<StageFileData>,
+    state_rows: Vec<TransactionWriteRow>,
+    file_data_writes: Vec<TransactionFileData>,
     count: u64,
 }
 
@@ -1043,7 +1089,7 @@ impl LixFileStagedBatch {
 fn lix_file_write_rows_from_batch(
     batch: &RecordBatch,
     version_binding: Option<&str>,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     Ok(lix_file_insert_stage_from_batch(batch, version_binding)?.state_rows)
 }
 
@@ -1066,7 +1112,7 @@ fn lix_file_delete_stage_from_batch(
 }
 
 fn blob_ref_file_ids_from_live_rows(
-    rows: &[LiveStateRow],
+    rows: &[MaterializedLiveStateRow],
 ) -> std::result::Result<BTreeSet<String>, LixError> {
     let mut file_ids = BTreeSet::new();
     for row in rows {
@@ -1462,7 +1508,7 @@ fn stage_lix_file_data_write(
     file_id: String,
     data: Vec<u8>,
     context: FilesystemRowContext,
-    origin: Option<StageRowOrigin>,
+    origin: Option<TransactionWriteOrigin>,
 ) -> Result<()> {
     let mut row = blob_ref_row(BlobRefRowInput {
         file_id: file_id.clone(),
@@ -1476,7 +1522,7 @@ fn stage_lix_file_data_write(
     .map_err(lix_error_to_datafusion_error)?;
     row.origin = origin;
     staged.state_rows.push(row);
-    staged.file_data_writes.push(StageFileData {
+    staged.file_data_writes.push(TransactionFileData {
         file_id,
         version_id: context.version_id,
         untracked: context.untracked,
@@ -1485,7 +1531,11 @@ fn stage_lix_file_data_write(
     Ok(())
 }
 
-fn attach_lix_file_insert_origin(rows: &mut [StageRow], surface_name: &str, file_id: &str) {
+fn attach_lix_file_insert_origin(
+    rows: &mut [TransactionWriteRow],
+    surface_name: &str,
+    file_id: &str,
+) {
     let origin = lix_file_insert_origin(surface_name, file_id);
     for row in rows {
         if row.schema_key == FILE_DESCRIPTOR_SCHEMA_KEY || row.schema_key == BLOB_REF_SCHEMA_KEY {
@@ -1494,10 +1544,10 @@ fn attach_lix_file_insert_origin(rows: &mut [StageRow], surface_name: &str, file
     }
 }
 
-fn lix_file_insert_origin(surface_name: &str, file_id: &str) -> StageRowOrigin {
-    StageRowOrigin {
+fn lix_file_insert_origin(surface_name: &str, file_id: &str) -> TransactionWriteOrigin {
+    TransactionWriteOrigin {
         surface: surface_name.to_string(),
-        operation: StageWriteOperation::Insert,
+        operation: TransactionWriteOperation::Insert,
         primary_key: Some(LogicalPrimaryKey {
             columns: vec!["id".to_string()],
             values: vec![file_id.to_string()],
@@ -1599,7 +1649,7 @@ async fn file_path_resolvers_from_live_state(
 async fn lix_file_record_batch(
     schema: &SchemaRef,
     blob_reader: &Arc<dyn BlobDataReader>,
-    rows: Vec<LiveStateRow>,
+    rows: Vec<MaterializedLiveStateRow>,
 ) -> Result<RecordBatch, LixError> {
     let projected_columns = schema
         .fields()
@@ -1688,7 +1738,6 @@ async fn lix_file_record_batch(
     let mut entity_ids = Vec::new();
     let mut schema_keys = Vec::new();
     let mut file_ids = Vec::new();
-    let mut schema_versions = Vec::new();
     let mut globals = Vec::new();
     let mut change_ids = Vec::new();
     let mut created_ats = Vec::new();
@@ -1734,10 +1783,9 @@ async fn lix_file_record_batch(
         names.push(Some(file.name));
         hiddens.push(Some(file.hidden));
         data_values.push(data);
-        entity_ids.push(Some(file.live.entity_id.as_string()?));
+        entity_ids.push(Some(file.live.entity_id.as_json_array_text()?));
         schema_keys.push(Some(file.live.schema_key));
         file_ids.push(file.live.file_id);
-        schema_versions.push(file.live.schema_version);
         globals.push(Some(file.live.global));
         change_ids.push(file.live.change_id);
         created_ats.push(file.live.created_at);
@@ -1765,7 +1813,6 @@ async fn lix_file_record_batch(
             "lixcol_entity_id" => Arc::new(StringArray::from(entity_ids.clone())),
             "lixcol_schema_key" => Arc::new(StringArray::from(schema_keys.clone())),
             "lixcol_file_id" => Arc::new(StringArray::from(file_ids.clone())),
-            "lixcol_schema_version" => Arc::new(StringArray::from(schema_versions.clone())),
             "lixcol_global" => Arc::new(BooleanArray::from(globals.clone())),
             "lixcol_change_id" => Arc::new(StringArray::from(change_ids.clone())),
             "lixcol_created_at" => Arc::new(StringArray::from(created_ats.clone())),
@@ -1896,6 +1943,7 @@ fn projected_schema(base_schema: &SchemaRef, projection: Option<&Vec<usize>>) ->
 
 fn lix_file_scan_request(
     version_binding: Option<&str>,
+    projected_schema: Option<&Schema>,
     limit: Option<usize>,
 ) -> LiveStateScanRequest {
     LiveStateScanRequest {
@@ -1910,11 +1958,261 @@ fn lix_file_scan_request(
                 .unwrap_or_default(),
             ..LiveStateFilter::default()
         },
-        projection: LiveStateProjection::default(),
+        projection: lix_file_live_state_projection(projected_schema),
         limit,
     }
 }
 
+fn lix_file_live_state_projection(projected_schema: Option<&Schema>) -> LiveStateProjection {
+    let Some(schema) = projected_schema else {
+        return LiveStateProjection::default();
+    };
+    let mut columns = Vec::new();
+    let needs_snapshot = schema.fields().iter().any(|field| {
+        matches!(
+            field.name().as_str(),
+            "path" | "directory_id" | "name" | "hidden" | "data"
+        )
+    });
+    if needs_snapshot {
+        columns.push("snapshot_content".to_string());
+    }
+    if schema
+        .fields()
+        .iter()
+        .any(|field| field.name() == "lixcol_metadata")
+    {
+        columns.push("metadata".to_string());
+    }
+    LiveStateProjection { columns }
+}
+
+async fn scan_lix_file_live_rows(
+    live_state: Arc<dyn LiveStateReader>,
+    request: &LiveStateScanRequest,
+    target_file_ids: &FileIdConstraint,
+) -> std::result::Result<Vec<MaterializedLiveStateRow>, LixError> {
+    let target_file_ids = match target_file_ids {
+        FileIdConstraint::All => return live_state.scan_rows(request).await,
+        FileIdConstraint::None => return Ok(Vec::new()),
+        FileIdConstraint::Ids(target_file_ids) => target_file_ids,
+    };
+
+    let mut file_request = request.clone();
+    file_request.filter.schema_keys = vec![
+        FILE_DESCRIPTOR_SCHEMA_KEY.to_string(),
+        BLOB_REF_SCHEMA_KEY.to_string(),
+    ];
+    file_request.filter.entity_ids = target_file_ids
+        .iter()
+        .map(|file_id| EntityIdentity::single(file_id.clone()))
+        .collect();
+
+    let mut rows = live_state.scan_rows(&file_request).await?;
+
+    let mut directory_request = request.clone();
+    directory_request.filter.schema_keys = vec![DIRECTORY_DESCRIPTOR_SCHEMA_KEY.to_string()];
+    directory_request.filter.entity_ids.clear();
+    directory_request.limit = None;
+    rows.extend(live_state.scan_rows(&directory_request).await?);
+
+    Ok(rows)
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum FileIdConstraint {
+    All,
+    None,
+    Ids(BTreeSet<String>),
+}
+
+impl FileIdConstraint {
+    fn from_ids(ids: Vec<String>) -> Self {
+        let ids = ids.into_iter().collect::<BTreeSet<_>>();
+        if ids.is_empty() {
+            Self::None
+        } else {
+            Self::Ids(ids)
+        }
+    }
+
+    fn intersect(self, other: Self) -> Self {
+        match (self, other) {
+            (Self::None, _) | (_, Self::None) => Self::None,
+            (Self::All, constraint) | (constraint, Self::All) => constraint,
+            (Self::Ids(left), Self::Ids(right)) => {
+                let ids = left.intersection(&right).cloned().collect::<BTreeSet<_>>();
+                if ids.is_empty() {
+                    Self::None
+                } else {
+                    Self::Ids(ids)
+                }
+            }
+        }
+    }
+
+    fn union(self, other: Self) -> Self {
+        match (self, other) {
+            (Self::All, _) | (_, Self::All) => Self::All,
+            (Self::None, constraint) | (constraint, Self::None) => constraint,
+            (Self::Ids(mut left), Self::Ids(right)) => {
+                left.extend(right);
+                Self::Ids(left)
+            }
+        }
+    }
+}
+
+fn file_id_constraint_from_filters(filters: &[Expr]) -> Result<FileIdConstraint> {
+    let analyzer = LixFileIdFilterAnalyzer;
+    let mut constraint = FileIdConstraint::All;
+    for filter in filters {
+        if let Some(filter_constraint) = analyzer.analyze(filter)? {
+            constraint = constraint.intersect(filter_constraint);
+        }
+    }
+    Ok(constraint)
+}
+
+struct LixFileIdFilterAnalyzer;
+
+impl LixFileIdFilterAnalyzer {
+    fn supports(&self, expr: &Expr) -> bool {
+        self.analyze(expr)
+            .is_ok_and(|constraint| constraint.is_some())
+    }
+
+    fn analyze(&self, expr: &Expr) -> Result<Option<FileIdConstraint>> {
+        ExactStringColumnFilterAnalyzer::new("id").analyze(expr)
+    }
+}
+
+struct ExactStringColumnFilterAnalyzer {
+    column_name: &'static str,
+}
+
+impl ExactStringColumnFilterAnalyzer {
+    fn new(column_name: &'static str) -> Self {
+        Self { column_name }
+    }
+
+    fn supports(&self, expr: &Expr) -> bool {
+        self.analyze(expr)
+            .is_ok_and(|constraint| constraint.is_some())
+    }
+
+    fn analyze(&self, expr: &Expr) -> Result<Option<FileIdConstraint>> {
+        match expr {
+            Expr::BinaryExpr(binary_expr) if binary_expr.op == Operator::And => {
+                let Some(left) = self.analyze(&binary_expr.left)? else {
+                    return Ok(None);
+                };
+                let Some(right) = self.analyze(&binary_expr.right)? else {
+                    return Ok(None);
+                };
+                Ok(Some(left.intersect(right)))
+            }
+            Expr::BinaryExpr(binary_expr) if binary_expr.op == Operator::Or => {
+                let Some(left) = self.analyze(&binary_expr.left)? else {
+                    return Ok(None);
+                };
+                let Some(right) = self.analyze(&binary_expr.right)? else {
+                    return Ok(None);
+                };
+                Ok(Some(left.union(right)))
+            }
+            Expr::BinaryExpr(binary_expr) => Ok(self
+                .value_from_binary_filter(binary_expr)
+                .map(|value| FileIdConstraint::Ids(BTreeSet::from([value])))),
+            Expr::InList(in_list) => Ok(self
+                .values_from_in_list_filter(in_list)
+                .map(FileIdConstraint::from_ids)),
+            _ => Ok(None),
+        }
+    }
+
+    fn value_from_binary_filter(&self, binary_expr: &BinaryExpr) -> Option<String> {
+        if binary_expr.op != Operator::Eq {
+            return None;
+        }
+        self.value_from_column_literal_filter(&binary_expr.left, &binary_expr.right)
+            .or_else(|| {
+                self.value_from_column_literal_filter(&binary_expr.right, &binary_expr.left)
+            })
+    }
+
+    fn values_from_in_list_filter(&self, in_list: &InList) -> Option<Vec<String>> {
+        if in_list.negated {
+            return None;
+        }
+        let Expr::Column(column) = in_list.expr.as_ref() else {
+            return None;
+        };
+        if column.name != self.column_name {
+            return None;
+        }
+        let values = in_list
+            .list
+            .iter()
+            .map(string_expr_literal)
+            .collect::<Option<Vec<_>>>()?;
+        Some(values)
+    }
+
+    fn value_from_column_literal_filter(
+        &self,
+        column_expr: &Expr,
+        literal_expr: &Expr,
+    ) -> Option<String> {
+        let Expr::Column(column) = column_expr else {
+            return None;
+        };
+        if column.name != self.column_name {
+            return None;
+        }
+        string_expr_literal(literal_expr)
+    }
+}
+
+fn string_expr_literal(expr: &Expr) -> Option<String> {
+    let Expr::Literal(literal, _) = expr else {
+        return None;
+    };
+    match literal {
+        ScalarValue::Utf8(Some(value))
+        | ScalarValue::Utf8View(Some(value))
+        | ScalarValue::LargeUtf8(Some(value)) => Some(value.clone()),
+        _ => None,
+    }
+}
+
+fn contains_column(expr: &Expr, column_name: &str) -> bool {
+    match expr {
+        Expr::Column(column) => column.name == column_name,
+        Expr::BinaryExpr(binary_expr) => {
+            contains_column(&binary_expr.left, column_name)
+                || contains_column(&binary_expr.right, column_name)
+        }
+        Expr::InList(in_list) => {
+            contains_column(&in_list.expr, column_name)
+                || in_list
+                    .list
+                    .iter()
+                    .any(|expr| contains_column(expr, column_name))
+        }
+        Expr::Between(between) => {
+            contains_column(&between.expr, column_name)
+                || contains_column(&between.low, column_name)
+                || contains_column(&between.high, column_name)
+        }
+        Expr::Not(expr) | Expr::IsNull(expr) | Expr::IsNotNull(expr) => {
+            contains_column(expr, column_name)
+        }
+        Expr::Negative(expr) => contains_column(expr, column_name),
+        _ => false,
+    }
+}
+
 fn validate_lix_file_update_assignments(
     schema: &SchemaRef,
     assignments: &[(String, Expr)],
@@ -2098,10 +2396,13 @@ fn update_optional_metadata_value(
     row_index: usize,
     column_name: &str,
     context: &str,
-) -> Result<Option<RowMetadata>> {
+) -> Result<Option<TransactionJson>> {
     update_optional_string_value(batch, assignment_values, row_index, column_name)?
         .map(|value| {
-            parse_row_metadata(&value, context).map_err(super::error::lix_error_to_datafusion_error)
+            let metadata = parse_row_metadata_value(&value, context)
+                .map_err(super::error::lix_error_to_datafusion_error)?;
+            TransactionJson::from_value(metadata, &format!("{context} metadata"))
+                .map_err(super::error::lix_error_to_datafusion_error)
         })
         .transpose()
 }
@@ -2174,10 +2475,13 @@ fn optional_metadata_value(
     row_index: usize,
     column_name: &str,
     context: &str,
-) -> Result<Option<RowMetadata>> {
+) -> Result<Option<TransactionJson>> {
     optional_string_value(batch, row_index, column_name)?
         .map(|value| {
-            parse_row_metadata(&value, context).map_err(super::error::lix_error_to_datafusion_error)
+            let metadata = parse_row_metadata_value(&value, context)
+                .map_err(super::error::lix_error_to_datafusion_error)?;
+            TransactionJson::from_value(metadata, &format!("{context} metadata"))
+                .map_err(super::error::lix_error_to_datafusion_error)
         })
         .transpose()
 }
@@ -2257,10 +2561,9 @@ fn lix_file_schema() -> SchemaRef {
         Field::new("name", DataType::Utf8, false),
         Field::new("hidden", DataType::Boolean, true),
         Field::new("data", DataType::Binary, true),
-        Field::new("lixcol_entity_id", DataType::Utf8, false),
+        json_field("lixcol_entity_id", false),
         Field::new("lixcol_schema_key", DataType::Utf8, false),
         Field::new("lixcol_file_id", DataType::Utf8, true),
-        Field::new("lixcol_schema_version", DataType::Utf8, false),
         Field::new("lixcol_global", DataType::Boolean, true),
         Field::new("lixcol_change_id", DataType::Utf8, true),
         Field::new("lixcol_created_at", DataType::Utf8, true),
@@ -2298,19 +2601,24 @@ mod tests {
     use datafusion::arrow::array::{ArrayRef, BinaryArray, BooleanArray, StringArray};
     use datafusion::arrow::datatypes::{DataType, Field, Schema};
     use datafusion::arrow::record_batch::RecordBatch;
+    use datafusion::common::{Column, ScalarValue};
     use datafusion::execution::TaskContext;
+    use datafusion::logical_expr::expr::InList;
     use datafusion::logical_expr::lit;
-    use serde_json::{json, Value as JsonValue};
+    use datafusion::logical_expr::{BinaryExpr, Expr, Operator};
+    use serde_json::Value as JsonValue;
 
     use crate::binary_cas::BlobDataReader;
     use crate::functions::{
         FunctionProvider, FunctionProviderHandle, SharedFunctionProvider, SystemFunctionProvider,
     };
-    use crate::live_state::LiveStateRow;
+    use crate::live_state::MaterializedLiveStateRow;
     use crate::live_state::{LiveStateReader, LiveStateRowRequest, LiveStateScanRequest};
     use crate::sql2::dml::InsertSink;
     use crate::sql2::{SqlWriteContext, SqlWriteExecutionContext};
-    use crate::transaction::types::{StageWrite, StageWriteMode, StageWriteOutcome};
+    use crate::transaction::types::{
+        TransactionJson, TransactionWrite, TransactionWriteMode, TransactionWriteOutcome,
+    };
     use crate::LixError;
 
     use super::{
@@ -2331,6 +2639,120 @@ mod tests {
         )
     }
 
+    fn string_literal(value: &str) -> Expr {
+        Expr::Literal(ScalarValue::Utf8(Some(value.to_string())), None)
+    }
+
+    fn column(name: &str) -> Expr {
+        Expr::Column(Column::from_name(name))
+    }
+
+    fn eq_filter(column_name: &str, value: &str) -> Expr {
+        Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(column(column_name)),
+            Operator::Eq,
+            Box::new(string_literal(value)),
+        ))
+    }
+
+    #[test]
+    fn file_id_filters_support_string_id_predicates() {
+        let analyzer = super::LixFileIdFilterAnalyzer;
+        let constraint = analyzer
+            .analyze(&Expr::InList(InList::new(
+                Box::new(column("id")),
+                vec![string_literal("file-b"), string_literal("file-a")],
+                false,
+            )))
+            .unwrap()
+            .unwrap();
+
+        assert_eq!(
+            constraint,
+            super::FileIdConstraint::Ids(BTreeSet::from([
+                "file-a".to_string(),
+                "file-b".to_string()
+            ]))
+        );
+        assert!(analyzer.supports(&eq_filter("id", "file-a")));
+        assert!(analyzer.supports(&Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(string_literal("file-a")),
+            Operator::Eq,
+            Box::new(column("id")),
+        ))));
+    }
+
+    #[test]
+    fn file_id_filters_intersect_and_union_boolean_predicates() {
+        let analyzer = super::LixFileIdFilterAnalyzer;
+        let left = Expr::InList(InList::new(
+            Box::new(column("id")),
+            vec![string_literal("file-a"), string_literal("file-b")],
+            false,
+        ));
+        let right = Expr::InList(InList::new(
+            Box::new(column("id")),
+            vec![string_literal("file-b"), string_literal("file-c")],
+            false,
+        ));
+
+        let and_constraint = analyzer
+            .analyze(&Expr::BinaryExpr(BinaryExpr::new(
+                Box::new(left.clone()),
+                Operator::And,
+                Box::new(right.clone()),
+            )))
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            and_constraint,
+            super::FileIdConstraint::Ids(BTreeSet::from(["file-b".to_string()]))
+        );
+
+        let or_constraint = analyzer
+            .analyze(&Expr::BinaryExpr(BinaryExpr::new(
+                Box::new(left),
+                Operator::Or,
+                Box::new(right),
+            )))
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            or_constraint,
+            super::FileIdConstraint::Ids(BTreeSet::from([
+                "file-a".to_string(),
+                "file-b".to_string(),
+                "file-c".to_string()
+            ]))
+        );
+    }
+
+    #[test]
+    fn file_id_filters_detect_contradictions() {
+        let filters = vec![Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(eq_filter("id", "file-a")),
+            Operator::And,
+            Box::new(eq_filter("id", "file-b")),
+        ))];
+
+        assert_eq!(
+            super::file_id_constraint_from_filters(&filters).unwrap(),
+            super::FileIdConstraint::None
+        );
+    }
+
+    #[test]
+    fn file_id_filters_ignore_non_id_and_negated_predicates() {
+        let analyzer = super::LixFileIdFilterAnalyzer;
+
+        assert!(!analyzer.supports(&eq_filter("name", "readme.md")));
+        assert!(!analyzer.supports(&Expr::InList(InList::new(
+            Box::new(column("id")),
+            vec![string_literal("file-a")],
+            true,
+        ))));
+    }
+
     fn lix_file_update_stage_from_batch_for_test(
         batch: &RecordBatch,
         version_binding: Option<&str>,
@@ -2361,8 +2783,8 @@ mod tests {
 
     #[derive(Default)]
     struct CapturingWriteContext {
-        rows: Vec<LiveStateRow>,
-        writes: Vec<StageWrite>,
+        rows: Vec<MaterializedLiveStateRow>,
+        writes: Vec<TransactionWrite>,
     }
 
     #[async_trait]
@@ -2371,7 +2793,10 @@ mod tests {
             &self,
             hashes: &[crate::binary_cas::BlobHash],
         ) -> Result<crate::binary_cas::BlobBytesBatch, LixError> {
-            Ok(crate::binary_cas::BlobBytesBatch::missing(hashes.len()))
+            Ok(crate::binary_cas::BlobBytesBatch::new(vec![
+                None;
+                hashes.len()
+            ]))
         }
     }
 
@@ -2399,7 +2824,7 @@ mod tests {
         async fn scan_live_state(
             &mut self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(self.rows.clone())
         }
 
@@ -2413,15 +2838,18 @@ mod tests {
             Ok(Some(format!("commit-{version_id}")))
         }
 
-        async fn stage_write(&mut self, write: StageWrite) -> Result<StageWriteOutcome, LixError> {
+        async fn stage_write(
+            &mut self,
+            write: TransactionWrite,
+        ) -> Result<TransactionWriteOutcome, LixError> {
             self.writes.push(write);
-            Ok(StageWriteOutcome { count: 0 })
+            Ok(TransactionWriteOutcome { count: 0 })
         }
     }
 
     #[derive(Default)]
     struct RowsLiveStateReader {
-        rows: Vec<LiveStateRow>,
+        rows: Vec<MaterializedLiveStateRow>,
     }
 
     #[async_trait]
@@ -2429,14 +2857,14 @@ mod tests {
         async fn scan_rows(
             &self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(self.rows.clone())
         }
 
         async fn load_row(
             &self,
             _request: &LiveStateRowRequest,
-        ) -> Result<Option<LiveStateRow>, LixError> {
+        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
             Ok(None)
         }
     }
@@ -2445,15 +2873,14 @@ mod tests {
         entity_id: &str,
         version_id: &str,
         snapshot_content: &str,
-    ) -> LiveStateRow {
-        LiveStateRow {
-            entity_id: crate::entity_identity::EntityIdentity::from_string(entity_id)
-                .expect("entity id should decode"),
+    ) -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
+            entity_id: crate::entity_identity::EntityIdentity::single(entity_id),
             schema_key: super::DIRECTORY_DESCRIPTOR_SCHEMA_KEY.to_string(),
             file_id: None,
             snapshot_content: Some(snapshot_content.to_string()),
             metadata: None,
-            schema_version: "1".to_string(),
+            deleted: false,
             version_id: version_id.to_string(),
             change_id: Some(format!("change-{entity_id}")),
             commit_id: Some(format!("commit-{entity_id}")),
@@ -2464,15 +2891,18 @@ mod tests {
         }
     }
 
-    fn live_file_row(entity_id: &str, version_id: &str, snapshot_content: &str) -> LiveStateRow {
-        LiveStateRow {
-            entity_id: crate::entity_identity::EntityIdentity::from_string(entity_id)
-                .expect("entity id should decode"),
+    fn live_file_row(
+        entity_id: &str,
+        version_id: &str,
+        snapshot_content: &str,
+    ) -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
+            entity_id: crate::entity_identity::EntityIdentity::single(entity_id),
             schema_key: super::FILE_DESCRIPTOR_SCHEMA_KEY.to_string(),
             file_id: None,
             snapshot_content: Some(snapshot_content.to_string()),
             metadata: None,
-            schema_version: "1".to_string(),
+            deleted: false,
             version_id: version_id.to_string(),
             change_id: Some(format!("change-{entity_id}")),
             commit_id: Some(format!("commit-{entity_id}")),
@@ -2653,11 +3083,13 @@ mod tests {
         );
         assert_eq!(rows[0].schema_key, "lix_file_descriptor");
         assert_eq!(rows[0].version_id, "version-b");
-        assert_eq!(rows[0].schema_version.as_str(), "1");
-        assert_eq!(rows[0].metadata.as_ref(), Some(&json!({"source": "file"})));
-        let snapshot: JsonValue =
-            serde_json::from_str(rows[0].snapshot_content.as_deref().unwrap())
-                .expect("descriptor snapshot JSON");
+        assert_eq!(
+            rows[0].metadata.as_ref(),
+            Some(&TransactionJson::from_value_for_test(
+                serde_json::json!({"source": "file"})
+            ))
+        );
+        let snapshot = rows[0].snapshot.as_ref().expect("descriptor snapshot JSON");
         assert_eq!(snapshot["id"], "file-readme");
         assert_eq!(snapshot["directory_id"], "dir-docs");
         assert_eq!(snapshot["name"], "readme.md");
@@ -2743,9 +3175,7 @@ mod tests {
             .iter()
             .find(|row| row.schema_key == "lix_file_descriptor")
             .expect("file descriptor row should be staged");
-        let snapshot: JsonValue =
-            serde_json::from_str(descriptor.snapshot_content.as_deref().unwrap())
-                .expect("descriptor snapshot JSON");
+        let snapshot: JsonValue = descriptor.snapshot.as_ref().unwrap().value().clone();
         assert_eq!(snapshot["id"], "file-readme");
         assert_eq!(snapshot["directory_id"], "dir-docs");
         assert_eq!(snapshot["name"], "renamed.md");
@@ -2825,9 +3255,12 @@ mod tests {
             .iter()
             .all(|row| row.schema_key != "lix_directory_descriptor"));
 
-        let snapshot: JsonValue =
-            serde_json::from_str(staged.state_rows[0].snapshot_content.as_deref().unwrap())
-                .expect("descriptor snapshot JSON");
+        let snapshot: JsonValue = staged.state_rows[0]
+            .snapshot
+            .as_ref()
+            .unwrap()
+            .value()
+            .clone();
         assert_eq!(snapshot["directory_id"], "dir-docs");
         assert_eq!(snapshot["name"], "renamed.md");
     }
@@ -2882,9 +3315,7 @@ mod tests {
             .iter()
             .find(|row| row.schema_key == "lix_file_descriptor")
             .expect("file descriptor should be staged");
-        let snapshot: JsonValue =
-            serde_json::from_str(descriptor.snapshot_content.as_deref().unwrap())
-                .expect("descriptor snapshot JSON");
+        let snapshot: JsonValue = descriptor.snapshot.as_ref().unwrap().value().clone();
         assert_eq!(snapshot["directory_id"], "dir-generated-docs");
     }
 
@@ -3003,7 +3434,7 @@ mod tests {
             ))
         );
         assert_eq!(descriptor.file_id, None);
-        assert_eq!(descriptor.snapshot_content, None);
+        assert_eq!(descriptor.snapshot, None);
 
         let blob_ref = staged
             .state_rows
@@ -3017,7 +3448,7 @@ mod tests {
             ))
         );
         assert_eq!(blob_ref.file_id.as_deref(), Some("file-readme"));
-        assert_eq!(blob_ref.snapshot_content, None);
+        assert_eq!(blob_ref.snapshot, None);
     }
 
     #[test]
@@ -3035,7 +3466,7 @@ mod tests {
                 "file-readme"
             ))
         );
-        assert_eq!(staged.state_rows[0].snapshot_content, None);
+        assert_eq!(staged.state_rows[0].snapshot, None);
     }
 
     #[test]
@@ -3069,9 +3500,7 @@ mod tests {
             .iter()
             .find(|row| row.schema_key == "lix_file_descriptor")
             .expect("file descriptor row should be staged");
-        let snapshot: JsonValue =
-            serde_json::from_str(descriptor.snapshot_content.as_deref().unwrap())
-                .expect("descriptor snapshot JSON");
+        let snapshot: JsonValue = descriptor.snapshot.as_ref().unwrap().value().clone();
         assert_eq!(snapshot["id"], "file-readme");
         assert_eq!(snapshot["directory_id"], "dir-guides");
         assert_eq!(snapshot["name"], "readme.md");
@@ -3105,9 +3534,7 @@ mod tests {
             .iter()
             .find(|row| row.schema_key == "lix_file_descriptor")
             .expect("file descriptor row should be staged");
-        let snapshot: JsonValue =
-            serde_json::from_str(descriptor.snapshot_content.as_deref().unwrap())
-                .expect("descriptor snapshot JSON");
+        let snapshot: JsonValue = descriptor.snapshot.as_ref().unwrap().value().clone();
         assert_eq!(snapshot["directory_id"], "dir-generated-guides");
     }
 
@@ -3133,8 +3560,8 @@ mod tests {
         let writes = &write_context.writes;
         assert_eq!(writes.len(), 1);
         match &writes[0] {
-            StageWrite::Rows { mode, rows } => {
-                assert_eq!(*mode, StageWriteMode::Insert);
+            TransactionWrite::Rows { mode, rows } => {
+                assert_eq!(*mode, TransactionWriteMode::Insert);
                 assert_eq!(rows.len(), 1);
                 assert_eq!(
                     rows[0].entity_id.as_ref(),
@@ -3170,14 +3597,14 @@ mod tests {
         let writes = &write_context.writes;
         assert_eq!(writes.len(), 1);
         match &writes[0] {
-            StageWrite::RowsWithFileData {
+            TransactionWrite::RowsWithFileData {
                 mode,
                 rows,
                 file_data,
                 count,
                 ..
             } => {
-                assert_eq!(*mode, StageWriteMode::Insert);
+                assert_eq!(*mode, TransactionWriteMode::Insert);
                 assert_eq!(*count, 1);
                 assert_eq!(rows.len(), 2);
                 assert!(rows
@@ -3230,7 +3657,7 @@ mod tests {
         let writes = &write_context.writes;
         assert_eq!(writes.len(), 1);
         match &writes[0] {
-            StageWrite::RowsWithFileData {
+            TransactionWrite::RowsWithFileData {
                 rows,
                 file_data,
                 count,
@@ -3243,9 +3670,7 @@ mod tests {
                     .iter()
                     .find(|row| row.schema_key == "lix_file_descriptor")
                     .expect("file descriptor row should be staged");
-                let snapshot: JsonValue =
-                    serde_json::from_str(descriptor.snapshot_content.as_deref().unwrap())
-                        .expect("descriptor snapshot JSON");
+                let snapshot: JsonValue = descriptor.snapshot.as_ref().unwrap().value().clone();
                 assert_eq!(snapshot["directory_id"], "dir-guides");
             }
             other => panic!("expected insert with file data staged write, got {other:?}"),