@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.4

package/dist-engine-src/src/sql2/entity_provider.rs

@@ -1,5 +1,5 @@
 use std::any::Any;
-use std::collections::BTreeSet;
+use std::collections::{BTreeMap, BTreeSet};
 use std::sync::Arc;
 
 use async_trait::async_trait;
@@ -14,7 +14,8 @@ use datafusion::common::{not_impl_err, DFSchema, DataFusionError, Result, Scalar
 use datafusion::datasource::TableType;
 use datafusion::execution::TaskContext;
 use datafusion::logical_expr::dml::InsertOp;
-use datafusion::logical_expr::{Expr, TableProviderFilterPushDown};
+use datafusion::logical_expr::expr::InList;
+use datafusion::logical_expr::{BinaryExpr, Expr, Operator, TableProviderFilterPushDown};
 use datafusion::physical_expr::{create_physical_expr, EquivalenceProperties, PhysicalExpr};
 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType, PlanProperties};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
@@ -27,11 +28,12 @@ use serde_json::Value as JsonValue;
 
 use crate::commit_graph::CommitGraphReader;
 use crate::entity_identity::EntityIdentity;
-use crate::live_state::LiveStateRow;
+use crate::live_state::MaterializedLiveStateRow;
 use crate::live_state::{
     LiveStateFilter, LiveStateProjection, LiveStateReader, LiveStateScanRequest,
 };
 use crate::sql2::dml::{InsertExec, InsertSink};
+use crate::sql2::predicate_typecheck::validate_json_predicate_filters;
 use crate::sql2::read_only::reject_read_only_entity_surface;
 use crate::sql2::version_scope::{
     explicit_version_ids_from_dml_filters, resolve_provider_version_ids,
@@ -40,23 +42,22 @@ use crate::sql2::version_scope::{
 use crate::sql2::write_normalization::{
     InsertCell, InsertColumnIntents, SqlCell, UpdateAssignmentValues, UpdateCell,
 };
-use crate::transaction::types::StageRow;
+use crate::transaction::types::{TransactionJson, TransactionWriteRow};
 use crate::version::VersionRefReader;
-use crate::{parse_row_metadata, serialize_row_metadata, LixError, RowMetadata};
+use crate::{parse_row_metadata_value, serialize_row_metadata, LixError};
 
 use super::entity_history_provider::EntityHistoryProvider;
 use super::history_route::{
     HISTORY_COL_CHANGE_ID, HISTORY_COL_COMMIT_CREATED_AT, HISTORY_COL_DEPTH, HISTORY_COL_ENTITY_ID,
     HISTORY_COL_FILE_ID, HISTORY_COL_METADATA, HISTORY_COL_OBSERVED_COMMIT_ID,
-    HISTORY_COL_SCHEMA_KEY, HISTORY_COL_SCHEMA_VERSION, HISTORY_COL_SNAPSHOT_CONTENT,
-    HISTORY_COL_START_COMMIT_ID,
+    HISTORY_COL_SCHEMA_KEY, HISTORY_COL_SNAPSHOT_CONTENT, HISTORY_COL_START_COMMIT_ID,
 };
 use super::result_metadata::{json_field, mark_json_field};
 use crate::sql2::{
-    SqlChangelogQuerySource, SqlWriteContext, WriteAccess, WriteContextLiveStateReader,
+    SqlCommitStoreQuerySource, SqlWriteContext, WriteAccess, WriteContextLiveStateReader,
     WriteContextVersionRefReader,
 };
-use crate::transaction::types::{StageWrite, StageWriteMode};
+use crate::transaction::types::{TransactionWrite, TransactionWriteMode};
 
 pub(crate) async fn register_entity_providers(
     ctx: &SessionContext,
@@ -64,7 +65,7 @@ pub(crate) async fn register_entity_providers(
     live_state: Arc<dyn LiveStateReader>,
     version_ref: Arc<dyn VersionRefReader>,
     commit_graph: Arc<tokio::sync::Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
     schema_definitions: &[JsonValue],
 ) -> Result<(), LixError> {
     for schema in schema_definitions {
@@ -99,16 +100,18 @@ pub(crate) async fn register_entity_providers(
         )
         .map_err(datafusion_error_to_lix_error)?;
 
-        let history_name = format!("{}_history", spec.schema_key);
-        ctx.register_table(
-            &history_name,
-            Arc::new(EntityHistoryProvider::new(
-                Arc::clone(&spec),
-                Arc::clone(&commit_graph),
-                query_source.clone(),
-            )),
-        )
-        .map_err(datafusion_error_to_lix_error)?;
+        if schema_exposed_as_entity_history_surface(&spec.schema_key) {
+            let history_name = format!("{}_history", spec.schema_key);
+            ctx.register_table(
+                &history_name,
+                Arc::new(EntityHistoryProvider::new(
+                    Arc::clone(&spec),
+                    Arc::clone(&commit_graph),
+                    query_source.clone(),
+                )),
+            )
+            .map_err(datafusion_error_to_lix_error)?;
+        }
     }
 
     Ok(())
@@ -177,7 +180,6 @@ pub(super) struct EntitySurfaceColumn {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub(super) struct EntitySurfaceSpec {
     pub(super) schema_key: String,
-    schema_version: Option<String>,
     pub(super) primary_key_paths: Vec<Vec<String>>,
     pub(super) columns: Vec<EntitySurfaceColumn>,
 }
@@ -300,13 +302,14 @@ impl TableProvider for EntityProvider {
         &self,
         filters: &[&Expr],
     ) -> Result<Vec<TableProviderFilterPushDown>> {
+        let analyzer = EntityPrimaryKeyFilterAnalyzer::new(&self.spec);
         Ok(filters
             .iter()
             .map(|filter| {
-                if explicit_version_ids_from_dml_filters(&[(*filter).clone()]).is_empty() {
-                    TableProviderFilterPushDown::Unsupported
+                if ExactVersionIdFilterAnalyzer.supports(filter) || analyzer.supports(filter) {
+                    TableProviderFilterPushDown::Exact
                 } else {
-                    TableProviderFilterPushDown::Inexact
+                    TableProviderFilterPushDown::Unsupported
                 }
             })
             .collect())
@@ -323,6 +326,7 @@ impl TableProvider for EntityProvider {
         let mut request = entity_live_state_scan_request(
             &self.spec.schema_key,
             self.version_binding.active_version_id(),
+            Some(projected_schema.as_ref()),
             limit,
         );
         if self.write_access.is_write() && matches!(self.version_binding, VersionBinding::Explicit)
@@ -342,6 +346,8 @@ impl TableProvider for EntityProvider {
         )
         .await
         .map_err(lix_error_to_datafusion_error)?;
+        apply_exact_version_id_filter(&mut request, exact_version_ids_from_filters(filters)?);
+        apply_exact_entity_id_filters(&mut request, &self.spec, filters)?;
 
         Ok(Arc::new(EntityScanExec::new(
             Arc::clone(&self.spec),
@@ -406,6 +412,7 @@ impl TableProvider for EntityProvider {
         };
 
         let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
+        validate_json_predicate_filters(self.schema.as_ref(), &filters)?;
         let physical_filters = filters
             .iter()
             .map(|expr| create_physical_expr(expr, &df_schema, state.execution_props()))
@@ -414,16 +421,19 @@ impl TableProvider for EntityProvider {
             &self.spec.schema_key,
             version_binding.active_version_id(),
             None,
+            None,
         );
         if matches!(version_binding, VersionBinding::Explicit) {
-            request.filter.version_ids = explicit_version_ids_from_dml_filters(&filters);
-            if request.filter.version_ids.is_empty() {
+            let exact_version_ids = exact_version_ids_from_filters(&filters)?;
+            if exact_version_ids.is_none() {
                 return Err(DataFusionError::Plan(format!(
                     "DELETE FROM {}_by_version requires an explicit lixcol_version_id predicate",
                     self.spec.schema_key
                 )));
             }
+            apply_exact_version_id_filter(&mut request, exact_version_ids);
         }
+        apply_exact_entity_id_filters(&mut request, &self.spec, &filters)?;
 
         Ok(Arc::new(EntityDeleteExec::new(
             Arc::clone(&self.spec),
@@ -458,6 +468,7 @@ impl TableProvider for EntityProvider {
         };
 
         let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
+        validate_json_predicate_filters(self.schema.as_ref(), &filters)?;
         let physical_assignments = assignments
             .iter()
             .map(|(column_name, expr)| {
@@ -471,11 +482,13 @@ impl TableProvider for EntityProvider {
             .iter()
             .map(|expr| create_physical_expr(expr, &df_schema, state.execution_props()))
             .collect::<Result<Vec<_>>>()?;
-        let request = entity_live_state_scan_request(
+        let mut request = entity_live_state_scan_request(
             &self.spec.schema_key,
             version_binding.active_version_id(),
             None,
+            None,
         );
+        apply_exact_entity_id_filters(&mut request, &self.spec, &filters)?;
 
         Ok(Arc::new(EntityUpdateExec::new(
             Arc::clone(&self.spec),
@@ -489,6 +502,413 @@ impl TableProvider for EntityProvider {
     }
 }
 
+fn entity_ids_from_primary_key_filters(
+    spec: &EntitySurfaceSpec,
+    filters: &[Expr],
+) -> Result<Option<Vec<EntityIdentity>>> {
+    let analyzer = EntityPrimaryKeyFilterAnalyzer::new(spec);
+    let mut entity_ids: Option<BTreeSet<EntityIdentity>> = None;
+    for filter in filters {
+        let Some(filter_ids) = analyzer.analyze(filter)? else {
+            continue;
+        };
+        entity_ids = Some(match entity_ids {
+            Some(existing_ids) => existing_ids.intersection(&filter_ids).cloned().collect(),
+            None => filter_ids,
+        });
+    }
+
+    Ok(entity_ids.map(|ids| ids.into_iter().collect()))
+}
+
+fn apply_exact_entity_id_filters(
+    request: &mut LiveStateScanRequest,
+    spec: &EntitySurfaceSpec,
+    filters: &[Expr],
+) -> Result<()> {
+    if let Some(entity_ids) = entity_ids_from_primary_key_filters(spec, filters)? {
+        if entity_ids.is_empty() {
+            request.limit = Some(0);
+        }
+        request.filter.entity_ids = entity_ids;
+    }
+    Ok(())
+}
+
+fn exact_version_ids_from_filters(filters: &[Expr]) -> Result<Option<Vec<String>>> {
+    let analyzer = ExactVersionIdFilterAnalyzer;
+    let mut version_ids: Option<BTreeSet<String>> = None;
+    for filter in filters {
+        let Some(filter_ids) = analyzer.analyze(filter)? else {
+            continue;
+        };
+        version_ids = Some(match version_ids {
+            Some(existing_ids) => existing_ids.intersection(&filter_ids).cloned().collect(),
+            None => filter_ids,
+        });
+    }
+    Ok(version_ids.map(|ids| ids.into_iter().collect()))
+}
+
+fn apply_exact_version_id_filter(
+    request: &mut LiveStateScanRequest,
+    version_ids: Option<Vec<String>>,
+) {
+    if let Some(version_ids) = version_ids {
+        if version_ids.is_empty() {
+            request.limit = Some(0);
+        }
+        request.filter.version_ids = version_ids;
+    }
+}
+
+struct EntityPrimaryKeyFilterAnalyzer<'a> {
+    primary_key_columns: Vec<&'a str>,
+}
+
+struct ExactVersionIdFilterAnalyzer;
+
+impl ExactVersionIdFilterAnalyzer {
+    fn supports(&self, expr: &Expr) -> bool {
+        self.analyze(expr)
+            .is_ok_and(|constraint| constraint.is_some())
+    }
+
+    fn analyze(&self, expr: &Expr) -> Result<Option<BTreeSet<String>>> {
+        match expr {
+            Expr::BinaryExpr(binary_expr) if binary_expr.op == Operator::And => {
+                let Some(left) = self.analyze(&binary_expr.left)? else {
+                    return Ok(None);
+                };
+                let Some(right) = self.analyze(&binary_expr.right)? else {
+                    return Ok(None);
+                };
+                Ok(Some(left.intersection(&right).cloned().collect()))
+            }
+            Expr::BinaryExpr(binary_expr) if binary_expr.op == Operator::Or => {
+                let Some(mut left) = self.analyze(&binary_expr.left)? else {
+                    return Ok(None);
+                };
+                let Some(right) = self.analyze(&binary_expr.right)? else {
+                    return Ok(None);
+                };
+                left.extend(right);
+                Ok(Some(left))
+            }
+            Expr::BinaryExpr(binary_expr) => {
+                Ok(version_id_from_binary_filter(binary_expr).map(|value| BTreeSet::from([value])))
+            }
+            Expr::InList(in_list) => {
+                Ok(version_ids_from_in_list_filter(in_list)
+                    .map(|values| values.into_iter().collect()))
+            }
+            _ => Ok(None),
+        }
+    }
+}
+
+fn version_id_from_binary_filter(binary_expr: &BinaryExpr) -> Option<String> {
+    if binary_expr.op != Operator::Eq {
+        return None;
+    }
+
+    version_id_from_column_literal_filter(&binary_expr.left, &binary_expr.right)
+        .or_else(|| version_id_from_column_literal_filter(&binary_expr.right, &binary_expr.left))
+}
+
+fn version_ids_from_in_list_filter(in_list: &InList) -> Option<Vec<String>> {
+    if in_list.negated {
+        return None;
+    }
+    let Expr::Column(column) = in_list.expr.as_ref() else {
+        return None;
+    };
+    if column.name != "lixcol_version_id" {
+        return None;
+    }
+
+    let values = in_list
+        .list
+        .iter()
+        .map(string_expr_literal)
+        .collect::<Option<Vec<_>>>()?;
+    if values.is_empty() {
+        return None;
+    }
+    Some(values)
+}
+
+fn version_id_from_column_literal_filter(
+    column_expr: &Expr,
+    literal_expr: &Expr,
+) -> Option<String> {
+    let Expr::Column(column) = column_expr else {
+        return None;
+    };
+    if column.name != "lixcol_version_id" {
+        return None;
+    }
+    string_expr_literal(literal_expr)
+}
+
+impl<'a> EntityPrimaryKeyFilterAnalyzer<'a> {
+    fn new(spec: &'a EntitySurfaceSpec) -> Self {
+        Self {
+            primary_key_columns: string_primary_key_columns(spec),
+        }
+    }
+
+    fn supports(&self, expr: &Expr) -> bool {
+        self.analyze(expr)
+            .is_ok_and(|constraint| constraint.is_some())
+    }
+
+    fn analyze(&self, expr: &Expr) -> Result<Option<BTreeSet<EntityIdentity>>> {
+        if self.primary_key_columns.is_empty() {
+            return Ok(None);
+        };
+        let Some(constraint) = self.analyze_constraint(expr)? else {
+            return Ok(None);
+        };
+        Ok(constraint.into_entity_ids(&self.primary_key_columns))
+    }
+
+    fn analyze_constraint(&self, expr: &Expr) -> Result<Option<EntityIdentityConstraint>> {
+        match expr {
+            Expr::BinaryExpr(binary_expr) if binary_expr.op == Operator::And => {
+                let Some(left) = self.analyze_constraint(&binary_expr.left)? else {
+                    return Ok(None);
+                };
+                let Some(right) = self.analyze_constraint(&binary_expr.right)? else {
+                    return Ok(None);
+                };
+                Ok(Some(left.intersect(right, &self.primary_key_columns)))
+            }
+            Expr::BinaryExpr(binary_expr) if binary_expr.op == Operator::Or => {
+                let Some(left) = self.analyze_constraint(&binary_expr.left)? else {
+                    return Ok(None);
+                };
+                let Some(right) = self.analyze_constraint(&binary_expr.right)? else {
+                    return Ok(None);
+                };
+                let Some(left_ids) = left.into_entity_ids(&self.primary_key_columns) else {
+                    return Ok(None);
+                };
+                let Some(mut right_ids) = right.into_entity_ids(&self.primary_key_columns) else {
+                    return Ok(None);
+                };
+                right_ids.extend(left_ids);
+                Ok(Some(EntityIdentityConstraint::Full(right_ids)))
+            }
+            Expr::BinaryExpr(binary_expr) => Ok(entity_identity_constraint_from_binary_filter(
+                binary_expr,
+                &self.primary_key_columns,
+            )),
+            Expr::InList(in_list) => Ok(entity_identity_constraint_from_in_list_filter(
+                in_list,
+                &self.primary_key_columns,
+            )),
+            _ => Ok(None),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum EntityIdentityConstraint {
+    Full(BTreeSet<EntityIdentity>),
+    Parts(BTreeMap<String, BTreeSet<String>>),
+}
+
+impl EntityIdentityConstraint {
+    fn intersect(self, other: Self, primary_key_columns: &[&str]) -> Self {
+        match (self, other) {
+            (Self::Full(left), Self::Full(right)) => {
+                Self::Full(left.intersection(&right).cloned().collect())
+            }
+            (Self::Full(ids), Self::Parts(parts)) | (Self::Parts(parts), Self::Full(ids)) => {
+                Self::Full(
+                    ids.into_iter()
+                        .filter(|identity| {
+                            identity_matches_parts(identity, primary_key_columns, &parts)
+                        })
+                        .collect(),
+                )
+            }
+            (Self::Parts(mut left), Self::Parts(right)) => {
+                for (column, right_values) in right {
+                    left.entry(column)
+                        .and_modify(|left_values| {
+                            *left_values =
+                                left_values.intersection(&right_values).cloned().collect();
+                        })
+                        .or_insert(right_values);
+                }
+                Self::Parts(left)
+            }
+        }
+    }
+
+    fn into_entity_ids(self, primary_key_columns: &[&str]) -> Option<BTreeSet<EntityIdentity>> {
+        match self {
+            Self::Full(ids) => Some(ids),
+            Self::Parts(parts) => entity_ids_from_primary_key_parts(primary_key_columns, parts),
+        }
+    }
+}
+
+fn string_primary_key_columns(spec: &EntitySurfaceSpec) -> Vec<&str> {
+    spec.primary_key_paths
+        .iter()
+        .map(|path| {
+            let [column_name] = path.as_slice() else {
+                return None;
+            };
+            let column = spec.visible_column(column_name)?;
+            (column.column_type == EntityColumnType::String).then_some(column.name.as_str())
+        })
+        .collect::<Option<Vec<_>>>()
+        .unwrap_or_default()
+}
+
+fn entity_identity_constraint_from_binary_filter(
+    binary_expr: &BinaryExpr,
+    primary_key_columns: &[&str],
+) -> Option<EntityIdentityConstraint> {
+    if binary_expr.op != Operator::Eq {
+        return None;
+    }
+    entity_identity_constraint_from_column_literal_filter(
+        &binary_expr.left,
+        &binary_expr.right,
+        primary_key_columns,
+    )
+    .or_else(|| {
+        entity_identity_constraint_from_column_literal_filter(
+            &binary_expr.right,
+            &binary_expr.left,
+            primary_key_columns,
+        )
+    })
+}
+
+fn entity_identity_constraint_from_in_list_filter(
+    in_list: &InList,
+    primary_key_columns: &[&str],
+) -> Option<EntityIdentityConstraint> {
+    if in_list.negated {
+        return None;
+    }
+    let Expr::Column(column) = in_list.expr.as_ref() else {
+        return None;
+    };
+    let values = in_list
+        .list
+        .iter()
+        .map(string_expr_literal)
+        .collect::<Option<Vec<_>>>()?;
+    if values.is_empty() {
+        return None;
+    }
+    match column.name.as_str() {
+        "lixcol_entity_id" => values
+            .into_iter()
+            .map(|value| EntityIdentity::from_json_array_text(&value).ok())
+            .collect::<Option<BTreeSet<_>>>()
+            .map(EntityIdentityConstraint::Full),
+        column_name if primary_key_columns.contains(&column_name) => {
+            Some(EntityIdentityConstraint::Parts(BTreeMap::from([(
+                column_name.to_string(),
+                values.into_iter().collect(),
+            )])))
+        }
+        _ => None,
+    }
+}
+
+fn entity_identity_constraint_from_column_literal_filter(
+    column_expr: &Expr,
+    literal_expr: &Expr,
+    primary_key_columns: &[&str],
+) -> Option<EntityIdentityConstraint> {
+    let Expr::Column(column) = column_expr else {
+        return None;
+    };
+    let value = string_expr_literal(literal_expr)?;
+    match column.name.as_str() {
+        "lixcol_entity_id" => EntityIdentity::from_json_array_text(&value)
+            .ok()
+            .map(|identity| EntityIdentityConstraint::Full(BTreeSet::from([identity]))),
+        column_name if primary_key_columns.contains(&column_name) => {
+            Some(EntityIdentityConstraint::Parts(BTreeMap::from([(
+                column_name.to_string(),
+                BTreeSet::from([value]),
+            )])))
+        }
+        _ => None,
+    }
+}
+
+fn entity_ids_from_primary_key_parts(
+    primary_key_columns: &[&str],
+    parts: BTreeMap<String, BTreeSet<String>>,
+) -> Option<BTreeSet<EntityIdentity>> {
+    if primary_key_columns
+        .iter()
+        .any(|column| !parts.contains_key(*column))
+    {
+        return None;
+    }
+
+    let mut identities = BTreeSet::from([Vec::<String>::new()]);
+    for column in primary_key_columns {
+        let values = parts.get(*column)?;
+        identities = identities
+            .into_iter()
+            .flat_map(|prefix| {
+                values.iter().map(move |value| {
+                    let mut parts = prefix.clone();
+                    parts.push(value.clone());
+                    parts
+                })
+            })
+            .collect();
+    }
+    Some(
+        identities
+            .into_iter()
+            .map(|parts| EntityIdentity { parts })
+            .collect(),
+    )
+}
+
+fn identity_matches_parts(
+    identity: &EntityIdentity,
+    primary_key_columns: &[&str],
+    parts: &BTreeMap<String, BTreeSet<String>>,
+) -> bool {
+    let identity_parts = identity.parts.as_slice();
+    primary_key_columns
+        .iter()
+        .zip(identity_parts.iter())
+        .all(|(column, value)| {
+            parts
+                .get(*column)
+                .is_none_or(|values| values.contains(value))
+        })
+}
+
+fn string_expr_literal(expr: &Expr) -> Option<String> {
+    let Expr::Literal(literal, _) = expr else {
+        return None;
+    };
+    match literal {
+        ScalarValue::Utf8(Some(value))
+        | ScalarValue::Utf8View(Some(value))
+        | ScalarValue::LargeUtf8(Some(value)) => Some(value.clone()),
+        _ => None,
+    }
+}
+
 struct EntityInsertSink {
     spec: Arc<EntitySurfaceSpec>,
     insert_column_intents: InsertColumnIntents,
@@ -552,8 +972,8 @@ impl InsertSink for EntityInsertSink {
             .map_err(|_| DataFusionError::Execution("entity INSERT row count overflow".into()))?;
 
         self.write_ctx
-            .stage_write(StageWrite::Rows {
-                mode: StageWriteMode::Insert,
+            .stage_write(TransactionWrite::Rows {
+                mode: TransactionWriteMode::Insert,
                 rows,
             })
             .await
@@ -616,12 +1036,7 @@ impl DisplayAs for EntityDeleteExec {
     fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
-                write!(
-                    f,
-                    "EntityDeleteExec(schema_key={}, filters={})",
-                    self.spec.schema_key,
-                    self.filters.len()
-                )
+                write!(f, "EntityDeleteExec(schema_key={})", self.spec.schema_key)
             }
             DisplayFormatType::TreeRender => write!(f, "EntityDeleteExec"),
         }
@@ -694,7 +1109,7 @@ impl ExecutionPlan for EntityDeleteExec {
                 version_binding.active_version_id(),
             )?;
             for row in &mut write_rows {
-                row.snapshot_content = None;
+                row.snapshot = None;
             }
             let count = u64::try_from(write_rows.len()).map_err(|_| {
                 DataFusionError::Execution("entity DELETE row count overflow".to_string())
@@ -702,8 +1117,8 @@ impl ExecutionPlan for EntityDeleteExec {
 
             if count > 0 {
                 write_ctx
-                    .stage_write(StageWrite::Rows {
-                        mode: StageWriteMode::Replace,
+                    .stage_write(TransactionWrite::Rows {
+                        mode: TransactionWriteMode::Replace,
                         rows: write_rows,
                     })
                     .await
@@ -781,10 +1196,9 @@ impl DisplayAs for EntityUpdateExec {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
                 write!(
                     f,
-                    "EntityUpdateExec(schema_key={}, assignments={}, filters={})",
+                    "EntityUpdateExec(schema_key={}, assignments={})",
                     self.spec.schema_key,
-                    self.assignments.len(),
-                    self.filters.len()
+                    self.assignments.len()
                 )
             }
             DisplayFormatType::TreeRender => write!(f, "EntityUpdateExec"),
@@ -865,8 +1279,8 @@ impl ExecutionPlan for EntityUpdateExec {
 
             if count > 0 {
                 write_ctx
-                    .stage_write(StageWrite::Rows {
-                        mode: StageWriteMode::Replace,
+                    .stage_write(TransactionWrite::Rows {
+                        mode: TransactionWriteMode::Replace,
                         rows: write_rows,
                     })
                     .await
@@ -953,7 +1367,7 @@ fn entity_update_write_rows_from_batch(
     batch: &RecordBatch,
     assignments: &[(String, Arc<dyn PhysicalExpr>)],
     version_binding: Option<&str>,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     let assignment_values = UpdateAssignmentValues::evaluate(batch, assignments)?;
     (0..batch.num_rows())
         .map(|row_index| {
@@ -965,19 +1379,10 @@ fn entity_update_write_rows_from_batch(
                 &spec.schema_key,
             )?;
 
-            let schema_version = optional_string_value(batch, row_index, "lixcol_schema_version")?
-                .or_else(|| spec.schema_version.clone())
-                .ok_or_else(|| {
-                    DataFusionError::Execution(format!(
-                        "UPDATE entity surface '{}' requires lixcol_schema_version",
-                        spec.schema_key
-                    ))
-                })?;
-
-            Ok(StageRow {
+            Ok(TransactionWriteRow {
                 entity_id: optional_string_value(batch, row_index, "lixcol_entity_id")?
                     .map(|entity_id| {
-                        EntityIdentity::from_string(&entity_id).map_err(|error| {
+                        EntityIdentity::from_json_array_text(&entity_id).map_err(|error| {
                             DataFusionError::Execution(format!(
                                 "UPDATE entity surface '{}' has invalid lixcol_entity_id: {error}",
                                 spec.schema_key
@@ -987,12 +1392,18 @@ fn entity_update_write_rows_from_batch(
                     .transpose()?,
                 schema_key: spec.schema_key.clone(),
                 file_id: optional_string_value(batch, row_index, "lixcol_file_id")?,
-                snapshot_content: Some(entity_update_snapshot_content_from_batch(
-                    spec,
-                    batch,
-                    &assignment_values,
-                    row_index,
-                )?),
+                snapshot: Some(
+                    TransactionJson::from_value(
+                        entity_update_snapshot_content_from_batch(
+                            spec,
+                            batch,
+                            &assignment_values,
+                            row_index,
+                        )?,
+                        &format!("{} update snapshot_content", spec.schema_key),
+                    )
+                    .map_err(super::error::lix_error_to_datafusion_error)?,
+                ),
                 metadata: entity_update_optional_metadata_value(
                     batch,
                     &assignment_values,
@@ -1001,7 +1412,6 @@ fn entity_update_write_rows_from_batch(
                     &spec.schema_key,
                 )?,
                 origin: None,
-                schema_version,
                 created_at: None,
                 updated_at: None,
                 global: scope.global,
@@ -1020,7 +1430,7 @@ fn entity_update_snapshot_content_from_batch(
     batch: &RecordBatch,
     assignment_values: &UpdateAssignmentValues,
     row_index: usize,
-) -> Result<String> {
+) -> Result<JsonValue> {
     let snapshot_content = optional_string_value(batch, row_index, "lixcol_snapshot_content")?
         .ok_or_else(|| {
             DataFusionError::Execution(format!(
@@ -1055,12 +1465,7 @@ fn entity_update_snapshot_content_from_batch(
         };
         object.insert(column.name.clone(), value);
     }
-    serde_json::to_string(&JsonValue::Object(object)).map_err(|error| {
-        DataFusionError::Execution(format!(
-            "failed to serialize entity surface '{}' snapshot_content: {error}",
-            spec.schema_key
-        ))
-    })
+    Ok(JsonValue::Object(object))
 }
 
 fn entity_update_optional_string_value(
@@ -1088,10 +1493,13 @@ fn entity_update_optional_metadata_value(
     row_index: usize,
     column_name: &str,
     context: &str,
-) -> Result<Option<RowMetadata>> {
+) -> Result<Option<TransactionJson>> {
     entity_update_optional_string_value(batch, assignment_values, row_index, column_name)?
         .map(|value| {
-            parse_row_metadata(&value, context).map_err(super::error::lix_error_to_datafusion_error)
+            let metadata = parse_row_metadata_value(&value, context)
+                .map_err(super::error::lix_error_to_datafusion_error)?;
+            TransactionJson::from_value(metadata, &format!("{context} metadata"))
+                .map_err(super::error::lix_error_to_datafusion_error)
         })
         .transpose()
 }
@@ -1132,7 +1540,7 @@ fn entity_lix_state_write_rows_from_batch(
     batch: &RecordBatch,
     insert_column_intents: &InsertColumnIntents,
     version_binding: Option<&str>,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     entity_lix_state_write_rows_from_batch_with_options(
         spec,
         batch,
@@ -1146,7 +1554,7 @@ fn entity_existing_lix_state_write_rows_from_batch(
     spec: &EntitySurfaceSpec,
     batch: &RecordBatch,
     version_binding: Option<&str>,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     entity_lix_state_write_rows_from_batch_with_options(
         spec,
         batch,
@@ -1162,7 +1570,7 @@ fn entity_lix_state_write_rows_from_batch_with_options(
     insert_column_intents: &InsertColumnIntents,
     version_binding: Option<&str>,
     reject_read_only_fields: bool,
-) -> Result<Vec<StageRow>> {
+) -> Result<Vec<TransactionWriteRow>> {
     (0..batch.num_rows())
         .map(|row_index| {
             let scope = resolve_write_version_scope(
@@ -1194,14 +1602,6 @@ fn entity_lix_state_write_rows_from_batch_with_options(
                 reject_present_entity_insert_field(batch, row_index, "lixcol_commit_id")?;
             }
 
-            let schema_version = optional_string_value(batch, row_index, "lixcol_schema_version")?
-                .or_else(|| spec.schema_version.clone())
-                .ok_or_else(|| {
-                    DataFusionError::Execution(format!(
-                        "INSERT into entity surface '{}' requires lixcol_schema_version",
-                        spec.schema_key
-                    ))
-                })?;
             let snapshot_content =
                 entity_snapshot_content_from_batch(spec, batch, insert_column_intents, row_index)?;
             let explicit_entity_id = optional_string_value(batch, row_index, "lixcol_entity_id")?;
@@ -1212,7 +1612,7 @@ fn entity_lix_state_write_rows_from_batch_with_options(
                         spec.schema_key
                     ))
                 })?;
-                Some(EntityIdentity::from_string(&entity_id).map_err(|error| {
+                Some(EntityIdentity::from_json_array_text(&entity_id).map_err(|error| {
                     DataFusionError::Execution(format!(
                         "INSERT into entity surface '{}' has invalid lixcol_entity_id: {error}",
                         spec.schema_key
@@ -1221,7 +1621,7 @@ fn entity_lix_state_write_rows_from_batch_with_options(
             } else {
                 explicit_entity_id
                     .map(|entity_id| {
-                        EntityIdentity::from_string(&entity_id).map_err(|error| {
+                        EntityIdentity::from_json_array_text(&entity_id).map_err(|error| {
                             DataFusionError::Execution(format!(
                                 "INSERT into entity surface '{}' has invalid lixcol_entity_id: {error}",
                                 spec.schema_key
@@ -1231,14 +1631,22 @@ fn entity_lix_state_write_rows_from_batch_with_options(
                     .transpose()?
             };
 
-            Ok(StageRow {
+            Ok(TransactionWriteRow {
                 entity_id,
                 schema_key: spec.schema_key.clone(),
                 file_id: optional_string_value(batch, row_index, "lixcol_file_id")?,
-                snapshot_content: Some(snapshot_content),
-                metadata: optional_metadata_value(batch, row_index, "lixcol_metadata", &spec.schema_key)?,
+                snapshot: Some(TransactionJson::from_value(
+                    snapshot_content,
+                    &format!("{} insert snapshot_content", spec.schema_key),
+                )
+                .map_err(super::error::lix_error_to_datafusion_error)?),
+                metadata: optional_metadata_value(
+                    batch,
+                    row_index,
+                    "lixcol_metadata",
+                    &spec.schema_key,
+                )?,
                 origin: None,
-                schema_version: schema_version,
                 created_at: None,
                 updated_at: None,
                 global: scope.global,
@@ -1257,7 +1665,7 @@ fn entity_snapshot_content_from_batch(
     batch: &RecordBatch,
     insert_column_intents: &InsertColumnIntents,
     row_index: usize,
-) -> Result<String> {
+) -> Result<JsonValue> {
     let mut object = serde_json::Map::new();
     for column in &spec.columns {
         let value = match insert_column_intents.cell(batch, row_index, &column.name)? {
@@ -1271,12 +1679,7 @@ fn entity_snapshot_content_from_batch(
         };
         object.insert(column.name.clone(), value);
     }
-    serde_json::to_string(&JsonValue::Object(object)).map_err(|error| {
-        DataFusionError::Execution(format!(
-            "failed to serialize entity surface '{}' snapshot_content: {error}",
-            spec.schema_key
-        ))
-    })
+    Ok(JsonValue::Object(object))
 }
 
 fn entity_json_value_from_scalar(
@@ -1404,10 +1807,13 @@ fn optional_metadata_value(
     row_index: usize,
     column_name: &str,
     context: &str,
-) -> Result<Option<RowMetadata>> {
+) -> Result<Option<TransactionJson>> {
     optional_string_value(batch, row_index, column_name)?
         .map(|value| {
-            parse_row_metadata(&value, context).map_err(super::error::lix_error_to_datafusion_error)
+            let metadata = parse_row_metadata_value(&value, context)
+                .map_err(super::error::lix_error_to_datafusion_error)?;
+            TransactionJson::from_value(metadata, &format!("{context} metadata"))
+                .map_err(super::error::lix_error_to_datafusion_error)
         })
         .transpose()
 }
@@ -1573,6 +1979,7 @@ impl ExecutionPlan for EntityScanExec {
 fn entity_live_state_scan_request(
     schema_key: &str,
     active_version_id: Option<&str>,
+    projected_schema: Option<&Schema>,
     limit: Option<usize>,
 ) -> LiveStateScanRequest {
     LiveStateScanRequest {
@@ -1583,15 +1990,40 @@ fn entity_live_state_scan_request(
                 .unwrap_or_default(),
             ..LiveStateFilter::default()
         },
-        projection: LiveStateProjection::default(),
+        projection: entity_live_state_projection(projected_schema),
         limit,
     }
 }
 
+fn entity_live_state_projection(projected_schema: Option<&Schema>) -> LiveStateProjection {
+    let Some(schema) = projected_schema else {
+        return LiveStateProjection::default();
+    };
+    let mut columns = projection_column_names(schema);
+    if schema
+        .fields()
+        .iter()
+        .any(|field| !field.name().starts_with("lixcol_"))
+        && !columns.iter().any(|column| column == "snapshot_content")
+    {
+        columns.push("snapshot_content".to_string());
+    }
+    LiveStateProjection { columns }
+}
+
+fn projection_column_names(schema: &Schema) -> Vec<String> {
+    schema
+        .fields()
+        .iter()
+        .filter_map(|field| field.name().strip_prefix("lixcol_"))
+        .map(str::to_string)
+        .collect()
+}
+
 fn entity_record_batch(
     spec: &EntitySurfaceSpec,
     schema: SchemaRef,
-    rows: &[LiveStateRow],
+    rows: &[MaterializedLiveStateRow],
 ) -> Result<RecordBatch> {
     if schema.fields().is_empty() {
         let options = RecordBatchOptions::new().with_row_count(Some(rows.len()));
@@ -1616,7 +2048,7 @@ fn entity_record_batch(
 fn entity_column_array(
     spec: &EntitySurfaceSpec,
     column_name: &str,
-    rows: &[LiveStateRow],
+    rows: &[MaterializedLiveStateRow],
     snapshots: &[Option<JsonValue>],
 ) -> Result<ArrayRef> {
     if let Some(property_name) = column_name.strip_prefix("lixcol_") {
@@ -1665,13 +2097,16 @@ fn entity_column_array(
     })
 }
 
-fn entity_system_column_array(column_name: &str, rows: &[LiveStateRow]) -> Result<ArrayRef> {
+fn entity_system_column_array(
+    column_name: &str,
+    rows: &[MaterializedLiveStateRow],
+) -> Result<ArrayRef> {
     Ok(match column_name {
         "entity_id" => Arc::new(StringArray::from(
             rows.iter()
                 .map(|row| {
                     row.entity_id
-                        .as_string()
+                        .as_json_array_text()
                         .map(Some)
                         .map_err(lix_error_to_datafusion_error)
                 })
@@ -1685,7 +2120,6 @@ fn entity_system_column_array(column_name: &str, rows: &[LiveStateRow]) -> Resul
                 .map(|row| row.metadata.as_ref().map(serialize_row_metadata))
                 .collect::<Vec<_>>(),
         )) as ArrayRef,
-        "schema_version" => string_array(rows.iter().map(|row| Some(row.schema_version.as_str()))),
         "created_at" => string_array(rows.iter().map(|row| Some(row.created_at.as_str()))),
         "updated_at" => string_array(rows.iter().map(|row| Some(row.updated_at.as_str()))),
         "global" => Arc::new(BooleanArray::from(
@@ -1801,12 +2235,11 @@ fn arrow_data_type_for_entity_column_type(column_type: EntityColumnType) -> Data
 pub(super) fn entity_system_fields(variant: EntityProviderVariant) -> Vec<Field> {
     if variant == EntityProviderVariant::History {
         return vec![
-            Field::new(HISTORY_COL_ENTITY_ID, DataType::Utf8, false),
+            json_field(HISTORY_COL_ENTITY_ID, false),
             Field::new(HISTORY_COL_SCHEMA_KEY, DataType::Utf8, false),
             Field::new(HISTORY_COL_FILE_ID, DataType::Utf8, true),
             json_field(HISTORY_COL_SNAPSHOT_CONTENT, true),
             json_field(HISTORY_COL_METADATA, true),
-            Field::new(HISTORY_COL_SCHEMA_VERSION, DataType::Utf8, false),
             Field::new(HISTORY_COL_CHANGE_ID, DataType::Utf8, false),
             Field::new(HISTORY_COL_OBSERVED_COMMIT_ID, DataType::Utf8, false),
             Field::new(HISTORY_COL_COMMIT_CREATED_AT, DataType::Utf8, false),
@@ -1816,12 +2249,11 @@ pub(super) fn entity_system_fields(variant: EntityProviderVariant) -> Vec<Field>
     }
 
     let mut fields = vec![
-        Field::new("lixcol_entity_id", DataType::Utf8, true),
+        json_field("lixcol_entity_id", true),
         Field::new("lixcol_schema_key", DataType::Utf8, false),
         Field::new("lixcol_file_id", DataType::Utf8, true),
         json_field("lixcol_snapshot_content", true),
         json_field("lixcol_metadata", true),
-        Field::new("lixcol_schema_version", DataType::Utf8, true),
         Field::new("lixcol_created_at", DataType::Utf8, true),
         Field::new("lixcol_updated_at", DataType::Utf8, true),
         Field::new("lixcol_global", DataType::Boolean, true),
@@ -1855,11 +2287,6 @@ fn derive_entity_surface_spec_from_schema(
             )
         })?;
 
-    let schema_version = schema
-        .get("x-lix-version")
-        .and_then(JsonValue::as_str)
-        .map(ToOwned::to_owned);
-
     let properties = schema
         .get("properties")
         .and_then(JsonValue::as_object)
@@ -1895,7 +2322,6 @@ fn derive_entity_surface_spec_from_schema(
 
     Ok(EntitySurfaceSpec {
         schema_key: schema_key.to_string(),
-        schema_version,
         primary_key_paths,
         columns,
     })
@@ -1927,7 +2353,7 @@ fn parse_primary_key_paths(schema: &JsonValue) -> std::result::Result<Vec<Vec<St
         .collect()
 }
 
-// TODO(engine2): share JSON Pointer parsing with schema/canonical validation once
+// TODO(engine): share JSON Pointer parsing with schema/canonical validation once
 // those helpers have a clean module boundary for SQL providers.
 fn parse_json_pointer(pointer: &str) -> std::result::Result<Vec<String>, LixError> {
     if pointer.is_empty() {
@@ -1968,14 +2394,11 @@ fn decode_json_pointer_segment(segment: &str) -> std::result::Result<String, Lix
 }
 
 fn schema_exposed_as_entity_surface(schema_key: &str) -> bool {
-    !matches!(
-        schema_key,
-        "lix_active_account"
-            | "lix_change"
-            | "lix_commit_edge"
-            | "lix_change_set"
-            | "lix_change_set_element"
-    )
+    !matches!(schema_key, "lix_active_account" | "lix_change")
+}
+
+fn schema_exposed_as_entity_history_surface(schema_key: &str) -> bool {
+    !matches!(schema_key, "lix_commit" | "lix_commit_edge")
 }
 
 fn entity_column_type_from_schema(schema: &JsonValue) -> Option<EntityColumnType> {
@@ -2039,7 +2462,10 @@ mod tests {
     use datafusion::arrow::array::{ArrayRef, BooleanArray, Float64Array, Int64Array, StringArray};
     use datafusion::arrow::datatypes::{DataType, Field, Schema};
     use datafusion::arrow::record_batch::RecordBatch;
+    use datafusion::common::{Column, ScalarValue};
     use datafusion::execution::TaskContext;
+    use datafusion::logical_expr::expr::InList;
+    use datafusion::logical_expr::{BinaryExpr, Expr, Operator};
     use serde_json::json;
 
     use super::{
@@ -2052,12 +2478,15 @@ mod tests {
         FunctionProvider, FunctionProviderHandle, SharedFunctionProvider, SystemFunctionProvider,
     };
     use crate::live_state::{
-        LiveStateReader, LiveStateRow, LiveStateRowRequest, LiveStateScanRequest,
+        LiveStateReader, LiveStateRowRequest, LiveStateScanRequest, MaterializedLiveStateRow,
     };
     use crate::sql2::dml::InsertSink;
     use crate::sql2::write_normalization::InsertColumnIntents;
     use crate::sql2::{SqlWriteContext, SqlWriteExecutionContext};
-    use crate::transaction::types::{StageRow, StageWrite, StageWriteMode, StageWriteOutcome};
+    use crate::transaction::types::{
+        TransactionJson, TransactionWrite, TransactionWriteMode, TransactionWriteOutcome,
+        TransactionWriteRow,
+    };
     use crate::version::{VersionHead, VersionRefReader};
     use crate::LixError;
 
@@ -2065,8 +2494,8 @@ mod tests {
     struct EmptyVersionRefReader;
     #[derive(Default)]
     struct CapturingWriteContext {
-        rows: Vec<LiveStateRow>,
-        writes: Vec<StageWrite>,
+        rows: Vec<MaterializedLiveStateRow>,
+        writes: Vec<TransactionWrite>,
     }
 
     #[async_trait]
@@ -2074,14 +2503,14 @@ mod tests {
         async fn scan_rows(
             &self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(vec![])
         }
 
         async fn load_row(
             &self,
             _request: &LiveStateRowRequest,
-        ) -> Result<Option<LiveStateRow>, LixError> {
+        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
             Ok(None)
         }
     }
@@ -2113,7 +2542,10 @@ mod tests {
             &self,
             hashes: &[crate::binary_cas::BlobHash],
         ) -> Result<crate::binary_cas::BlobBytesBatch, LixError> {
-            Ok(crate::binary_cas::BlobBytesBatch::missing(hashes.len()))
+            Ok(crate::binary_cas::BlobBytesBatch::new(vec![
+                None;
+                hashes.len()
+            ]))
         }
     }
 
@@ -2141,7 +2573,7 @@ mod tests {
         async fn scan_live_state(
             &mut self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(self.rows.clone())
         }
 
@@ -2155,14 +2587,17 @@ mod tests {
             Ok(Some(format!("commit-{version_id}")))
         }
 
-        async fn stage_write(&mut self, write: StageWrite) -> Result<StageWriteOutcome, LixError> {
+        async fn stage_write(
+            &mut self,
+            write: TransactionWrite,
+        ) -> Result<TransactionWriteOutcome, LixError> {
             self.writes.push(write);
-            Ok(StageWriteOutcome { count: 0 })
+            Ok(TransactionWriteOutcome { count: 0 })
         }
     }
 
-    fn live_row() -> LiveStateRow {
-        LiveStateRow {
+    fn live_row() -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
             entity_id: crate::entity_identity::EntityIdentity::single("entity-1"),
             schema_key: "project_message".to_string(),
             file_id: None,
@@ -2170,8 +2605,8 @@ mod tests {
                 "{\"body\":\"hello\",\"rating\":4.5,\"count\":7,\"enabled\":true,\"meta\":{\"x\":1}}"
                     .to_string(),
             ),
-            metadata: Some(json!({"source": "test"})),
-            schema_version: "1".to_string(),
+            metadata: Some(json!({"source": "test"}).to_string()),
+            deleted: false,
             version_id: "version-a".to_string(),
             change_id: Some("change-a".to_string()),
             commit_id: Some("commit-a".to_string()),
@@ -2186,7 +2621,6 @@ mod tests {
         Arc::new(
             derive_entity_surface_spec_from_schema(&json!({
                 "x-lix-key": "project_message",
-                "x-lix-version": "1",
                 "type": "object",
                 "properties": {
                     "body": { "type": "string" },
@@ -2204,7 +2638,6 @@ mod tests {
         Arc::new(
             derive_entity_surface_spec_from_schema(&json!({
                 "x-lix-key": "project_message",
-                "x-lix-version": "1",
                 "x-lix-primary-key": ["/id"],
                 "type": "object",
                 "properties": {
@@ -2221,6 +2654,22 @@ mod tests {
         Arc::new(StringArray::from(values)) as ArrayRef
     }
 
+    fn string_literal(value: &str) -> Expr {
+        Expr::Literal(ScalarValue::Utf8(Some(value.to_string())), None)
+    }
+
+    fn column(name: &str) -> Expr {
+        Expr::Column(Column::from_name(name))
+    }
+
+    fn eq_filter(column_name: &str, value: &str) -> Expr {
+        Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(column(column_name)),
+            Operator::Eq,
+            Box::new(string_literal(value)),
+        ))
+    }
+
     fn entity_insert_batch(include_version: bool, global: bool) -> RecordBatch {
         let mut fields = vec![
             Field::new("body", DataType::Utf8, true),
@@ -2239,7 +2688,7 @@ mod tests {
             Arc::new(BooleanArray::from(vec![true])) as ArrayRef,
             string_column(vec![Some("{\"x\":1}")]),
             Arc::new(Float64Array::from(vec![4.5])) as ArrayRef,
-            string_column(vec![Some("entity-1")]),
+            string_column(vec![Some("[\"entity-1\"]")]),
             string_column(vec![Some("{\"source\":\"entity\"}")]),
             Arc::new(BooleanArray::from(vec![global])) as ArrayRef,
             Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
@@ -2266,7 +2715,7 @@ mod tests {
         ];
         if include_entity_id {
             fields.push(Field::new("lixcol_entity_id", DataType::Utf8, false));
-            columns.push(string_column(vec![Some("message-1")]));
+            columns.push(string_column(vec![Some("[\"message-1\"]")]));
         }
 
         RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)
@@ -2283,7 +2732,6 @@ mod tests {
     fn derives_entity_surface_spec_from_schema_definition() {
         let spec = derive_entity_surface_spec_from_schema(&json!({
             "x-lix-key": "project_message",
-            "x-lix-version": "1",
             "type": "object",
             "properties": {
                 "body": { "type": "string" },
@@ -2295,7 +2743,6 @@ mod tests {
         .expect("schema should derive entity surface spec");
 
         assert_eq!(spec.schema_key, "project_message");
-        assert_eq!(spec.schema_version.as_deref(), Some("1"));
         assert_eq!(
             spec.visible_column_names().collect::<Vec<_>>(),
             vec!["body", "meta", "rating"]
@@ -2320,7 +2767,6 @@ mod tests {
     fn entity_surface_spec_rejects_properties_without_projection_type() {
         let error = derive_entity_surface_spec_from_schema(&json!({
             "x-lix-key": "project_message",
-            "x-lix-version": "1",
             "x-lix-primary-key": ["/id"],
             "type": "object",
             "properties": {
@@ -2463,7 +2909,7 @@ mod tests {
                 .downcast_ref::<datafusion::arrow::array::StringArray>()
                 .expect("entity id is string")
                 .value(0),
-            "entity-1"
+            "[\"entity-1\"]"
         );
         assert_eq!(
             batch
@@ -2498,6 +2944,80 @@ mod tests {
         assert!(provider.schema.field_with_name("lixcol_version_id").is_ok());
     }
 
+    #[test]
+    fn primary_key_filters_route_entity_ids_for_string_primary_key() {
+        let spec = entity_insert_spec_with_primary_key();
+        let filters = vec![
+            eq_filter("id", "entity-a"),
+            Expr::InList(InList::new(
+                Box::new(column("id")),
+                vec![string_literal("entity-b"), string_literal("entity-a")],
+                false,
+            )),
+        ];
+
+        let entity_ids = super::entity_ids_from_primary_key_filters(&spec, &filters)
+            .expect("primary-key filters should analyze")
+            .expect("primary-key filters should produce a constraint");
+
+        assert_eq!(
+            entity_ids,
+            vec![crate::entity_identity::EntityIdentity::single("entity-a")]
+        );
+    }
+
+    #[test]
+    fn primary_key_filter_analyzer_models_boolean_predicates() {
+        let spec = entity_insert_spec_with_primary_key();
+        let analyzer = super::EntityPrimaryKeyFilterAnalyzer::new(&spec);
+        let disjunction = Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(eq_filter("id", "entity-a")),
+            Operator::Or,
+            Box::new(eq_filter("id", "entity-b")),
+        ));
+        let contradiction = Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(eq_filter("id", "entity-a")),
+            Operator::And,
+            Box::new(eq_filter("id", "entity-b")),
+        ));
+
+        let disjunction_ids = analyzer
+            .analyze(&disjunction)
+            .expect("OR should analyze")
+            .expect("OR should produce an entity-id set");
+        let contradiction_ids = analyzer
+            .analyze(&contradiction)
+            .expect("AND should analyze")
+            .expect("AND should produce an entity-id set");
+
+        assert_eq!(
+            disjunction_ids.into_iter().collect::<Vec<_>>(),
+            vec![
+                crate::entity_identity::EntityIdentity::single("entity-a"),
+                crate::entity_identity::EntityIdentity::single("entity-b"),
+            ]
+        );
+        assert!(contradiction_ids.is_empty());
+    }
+
+    #[test]
+    fn primary_key_filters_ignore_non_key_and_negated_predicates() {
+        let spec = entity_insert_spec_with_primary_key();
+        let filters = vec![
+            eq_filter("body", "hello"),
+            Expr::InList(InList::new(
+                Box::new(column("id")),
+                vec![string_literal("entity-a")],
+                true,
+            )),
+        ];
+
+        assert!(super::entity_ids_from_primary_key_filters(&spec, &filters)
+            .expect("ignored filters should analyze")
+            .unwrap_or_default()
+            .is_empty());
+    }
+
     #[test]
     fn decodes_by_version_entity_insert_into_lix_state_write_row() {
         let spec = entity_insert_spec();
@@ -2515,22 +3035,17 @@ mod tests {
             Some(&crate::entity_identity::EntityIdentity::single("entity-1"))
         );
         assert_eq!(rows[0].schema_key, "project_message");
-        assert_eq!(rows[0].schema_version.as_str(), "1");
         assert_eq!(rows[0].version_id, "version-a");
         assert_eq!(
             rows[0].metadata.as_ref(),
-            Some(&json!({"source": "entity"}))
+            Some(&TransactionJson::from_value_for_test(
+                json!({"source": "entity"})
+            ))
         );
         assert!(!rows[0].global);
         assert_eq!(
-            serde_json::from_str::<serde_json::Value>(
-                rows[0]
-                    .snapshot_content
-                    .as_deref()
-                    .expect("snapshot_content")
-            )
-            .expect("snapshot_content JSON"),
-            json!({
+            rows[0].snapshot.as_ref().expect("snapshot_content"),
+            &json!({
                 "body": "hello",
                 "count": 7,
                 "enabled": true,
@@ -2554,14 +3069,8 @@ mod tests {
         assert_eq!(rows.len(), 1);
         assert_eq!(rows[0].entity_id, None);
         assert_eq!(
-            serde_json::from_str::<serde_json::Value>(
-                rows[0]
-                    .snapshot_content
-                    .as_deref()
-                    .expect("snapshot_content")
-            )
-            .expect("snapshot_content JSON"),
-            json!({
+            rows[0].snapshot.as_ref().expect("snapshot_content"),
+            &json!({
                 "body": "hello",
                 "id": "message-1"
             })
@@ -2675,17 +3184,19 @@ mod tests {
         assert_eq!(count, 1);
         assert_eq!(
             write_context.writes.as_slice(),
-            &[StageWrite::Rows { mode: StageWriteMode::Insert, rows: vec![StageRow {
+            &[TransactionWrite::Rows {
+                mode: TransactionWriteMode::Insert,
+                rows: vec![TransactionWriteRow {
                     entity_id: Some(crate::entity_identity::EntityIdentity::single("entity-1")),
                     schema_key: "project_message".to_string(),
                     file_id: None,
-                    snapshot_content: Some(
-                        "{\"body\":\"hello\",\"count\":7,\"enabled\":true,\"meta\":{\"x\":1},\"rating\":4.5}"
-                            .to_string()
-                    ),
-                    metadata: Some(json!({"source": "entity"})),
+                    snapshot: Some(TransactionJson::from_value_for_test(
+                        json!({"body":"hello","count":7,"enabled":true,"meta":{"x":1},"rating":4.5})
+                    )),
+                    metadata: Some(TransactionJson::from_value_for_test(
+                        json!({"source": "entity"})
+                    )),
                     origin: None,
-                    schema_version: "1".to_string(),
                     created_at: None,
                     updated_at: None,
                     global: false,