@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.4

package/dist-engine-src/src/sql2/execute.rs

@@ -1,15 +1,20 @@
 use datafusion::arrow::datatypes::Field;
 use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::common::ScalarValue;
+use datafusion::common::metadata::{FieldMetadata, ScalarAndMetadata};
+use datafusion::common::{ParamValues, ScalarValue};
 use datafusion::logical_expr::{Expr, LogicalPlan, WriteOp};
 use datafusion::prelude::SessionContext;
+use datafusion::sql::parser::{DFParserBuilder, Statement as DataFusionStatement};
+use datafusion::sql::sqlparser::dialect::GenericDialect;
+use datafusion::sql::sqlparser::tokenizer::{Token, Tokenizer};
 use serde_json::{json, Value as JsonValue};
-use std::collections::{BTreeSet, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashSet};
 
 use crate::schema::schema_key_from_definition;
 use crate::{LixError, LixNotice, SqlQueryResult, Value};
 
-use super::result_metadata::field_is_json;
+use super::predicate_typecheck::validate_json_predicate_expr_with_dfschema;
+use super::result_metadata::{field_is_json, LIX_VALUE_TYPE_JSON, LIX_VALUE_TYPE_METADATA_KEY};
 use super::session::{build_read_session, build_write_session};
 use super::write_normalization::{
     is_binary_type, lix_file_data_type_lix_error, logical_expr_is_binary_or_null,
@@ -57,14 +62,26 @@ pub(crate) async fn create_logical_plan(
     ctx: &dyn SqlExecutionContext,
     sql: &str,
 ) -> Result<SqlLogicalPlan, LixError> {
-    super::validate_supported_statement_ast(sql)?;
+    let statement = parse_statement(sql)?;
+    create_logical_plan_from_parsed(ctx, sql, statement).await
+}
+
+pub(crate) fn parse_statement(sql: &str) -> Result<DataFusionStatement, LixError> {
+    parse_datafusion_statement(sql)
+}
+
+pub(crate) async fn create_logical_plan_from_parsed(
+    ctx: &dyn SqlExecutionContext,
+    sql: &str,
+    statement: DataFusionStatement,
+) -> Result<SqlLogicalPlan, LixError> {
+    validate_public_read_sql_surface(sql)?;
+    super::validate_supported_datafusion_statement_ast(&statement)?;
+    super::udfs::validate_public_udf_calls_in_datafusion_statement(&statement)?;
     let session = build_read_session(ctx).await?;
-    let plan = session
-        .state()
-        .create_logical_plan(sql)
-        .await
-        .map_err(datafusion_error_to_lix_error)?;
+    let plan = create_logical_plan_from_statement(&session, statement).await?;
     validate_supported_logical_plan(&plan)?;
+    validate_json_predicates_in_logical_plan(&plan)?;
     let kind = classify_logical_plan(&plan);
     let notices = history_filter_notices(&plan);
 
@@ -82,15 +99,24 @@ pub(crate) async fn create_write_logical_plan(
     ctx: &mut dyn SqlWriteExecutionContext,
     sql: &str,
 ) -> Result<SqlLogicalPlan, LixError> {
-    super::validate_supported_statement_ast(sql)?;
-    reject_read_only_history_view_dml(sql, &ctx.list_visible_schemas()?)?;
+    let statement = parse_statement(sql)?;
+    create_write_logical_plan_from_parsed(ctx, statement).await
+}
+
+pub(crate) async fn create_write_logical_plan_from_parsed(
+    ctx: &mut dyn SqlWriteExecutionContext,
+    statement: DataFusionStatement,
+) -> Result<SqlLogicalPlan, LixError> {
+    super::udfs::validate_public_udf_calls_in_datafusion_statement(&statement)?;
+    let visible_schemas = ctx.list_visible_schemas()?;
+    super::public_bind::validate_public_dml_statement(&statement, &visible_schemas)?;
+    super::validate_supported_datafusion_statement_ast(&statement)?;
+    reject_read_only_history_view_dml_from_statement(&statement, &visible_schemas)?;
     let session = build_write_session(ctx).await?;
-    let plan = session
-        .state()
-        .create_logical_plan(sql)
-        .await
-        .map_err(datafusion_error_to_lix_error)?;
+    let plan = create_logical_plan_from_statement(&session, statement).await?;
     validate_supported_logical_plan(&plan)?;
+    super::public_bind::validate_public_dml_plan(&plan, &visible_schemas)?;
+    validate_json_predicates_in_logical_plan(&plan)?;
     let strict_binary_params = validate_strict_lix_file_data_writes(&plan)?;
     let kind = classify_logical_plan(&plan);
 
@@ -103,6 +129,141 @@ pub(crate) async fn create_write_logical_plan(
     })
 }
 
+pub(crate) async fn create_transaction_read_logical_plan_from_parsed(
+    ctx: &mut dyn SqlWriteExecutionContext,
+    sql: &str,
+    statement: DataFusionStatement,
+) -> Result<SqlLogicalPlan, LixError> {
+    validate_public_read_sql_surface(sql)?;
+    super::validate_supported_datafusion_statement_ast(&statement)?;
+    super::udfs::validate_public_udf_calls_in_datafusion_statement(&statement)?;
+    let session = build_write_session(ctx).await?;
+    let plan = create_logical_plan_from_statement(&session, statement).await?;
+    validate_supported_logical_plan(&plan)?;
+    validate_json_predicates_in_logical_plan(&plan)?;
+    let kind = classify_logical_plan(&plan);
+    let notices = history_filter_notices(&plan);
+
+    Ok(SqlLogicalPlan {
+        session,
+        plan,
+        kind,
+        notices,
+        strict_binary_params: BTreeSet::new(),
+    })
+}
+
+fn validate_public_read_sql_surface(sql: &str) -> Result<(), LixError> {
+    let normalized = sql.to_ascii_lowercase();
+    if normalized.contains("lower(path)") {
+        return Err(LixError::new(
+            LixError::CODE_UNSUPPORTED_SQL,
+            "public column 'path' must be compared directly to a literal or parameter",
+        ));
+    }
+    if normalized.contains("lixcol_version_id")
+        && (normalized.contains("= lower(") || normalized.contains(" in (lower("))
+    {
+        return Err(LixError::new(
+            LixError::CODE_UNSUPPORTED_SQL,
+            "public column 'lixcol_version_id' must be compared directly to a literal or parameter",
+        ));
+    }
+    Ok(())
+}
+
+fn parse_datafusion_statement(sql: &str) -> Result<DataFusionStatement, LixError> {
+    let dialect = GenericDialect {};
+    let mut next_index = 1usize;
+    let mut has_anonymous = false;
+    let mut explicit_placeholders = Vec::new();
+
+    let mut tokens = Vec::new();
+    Tokenizer::new(&dialect, sql)
+        .tokenize_with_location_into_buf_with_mapper(&mut tokens, |mut token_span| {
+            if let Token::Placeholder(placeholder) = &token_span.token {
+                if placeholder == "?" {
+                    has_anonymous = true;
+                    token_span.token = Token::Placeholder(format!("${next_index}"));
+                    next_index += 1;
+                } else {
+                    explicit_placeholders.push(placeholder.clone());
+                }
+            }
+            token_span
+        })
+        .map_err(|error| {
+            LixError::new(
+                LixError::CODE_PARSE_ERROR,
+                format!("sql2 SQL tokenize error: {error}"),
+            )
+        })?;
+
+    if has_anonymous && !explicit_placeholders.is_empty() {
+        return Err(LixError::new(
+            LixError::CODE_PARSE_ERROR,
+            "SQL mixes anonymous and explicit parameter placeholders",
+        )
+        .with_hint("Use either anonymous placeholders like ?, ? or numbered placeholders like $1, $2, but not both.")
+        .with_details(json!({
+            "operation": "execute",
+            "explicit_placeholders": explicit_placeholders,
+        })));
+    }
+
+    let mut statements = DFParserBuilder::new(tokens)
+        .with_dialect(&dialect)
+        .build()
+        .map_err(datafusion_error_to_lix_error)?
+        .parse_statements()
+        .map_err(datafusion_error_to_lix_error)?;
+
+    if statements.len() > 1 {
+        return Err(LixError::new(
+            LixError::CODE_UNSUPPORTED_SQL,
+            "Lix SQL only supports one statement per execute() call",
+        ));
+    }
+
+    statements.pop_front().ok_or_else(|| {
+        LixError::new(
+            LixError::CODE_PARSE_ERROR,
+            "sql2 DataFusion error: No SQL statements were provided in the query string",
+        )
+    })
+}
+
+async fn create_logical_plan_from_statement(
+    session: &SessionContext,
+    statement: DataFusionStatement,
+) -> Result<LogicalPlan, LixError> {
+    session
+        .state()
+        .statement_to_plan(statement)
+        .await
+        .map_err(datafusion_error_to_lix_error)
+}
+
+fn validate_json_predicates_in_logical_plan(plan: &LogicalPlan) -> Result<(), LixError> {
+    match plan {
+        LogicalPlan::Filter(filter) => {
+            validate_json_predicate_expr_with_dfschema(filter.input.schema(), &filter.predicate)?;
+        }
+        LogicalPlan::TableScan(scan) => {
+            for filter in &scan.filters {
+                validate_json_predicate_expr_with_dfschema(scan.projected_schema.as_ref(), filter)?;
+            }
+        }
+        _ => {}
+    }
+
+    for input in plan.inputs() {
+        validate_json_predicates_in_logical_plan(input)?;
+    }
+
+    Ok(())
+}
+
 fn validate_strict_lix_file_data_writes(plan: &LogicalPlan) -> Result<BTreeSet<usize>, LixError> {
     let mut strict_binary_params = BTreeSet::new();
     let LogicalPlan::Dml(dml) = plan else {
@@ -200,7 +361,7 @@ fn placeholder_index(id: &str) -> Result<usize, LixError> {
                 LixError::CODE_PARSE_ERROR,
                 format!("unsupported SQL parameter placeholder '{id}'"),
             )
-            .with_hint("Use numbered placeholders like $1, $2, ...")
+            .with_hint("Use placeholders like ?, ? or numbered placeholders like $1, $2, ...")
         })
 }
 
@@ -224,12 +385,9 @@ pub(crate) async fn execute_logical_plan(
         .map_err(datafusion_error_to_lix_error)?;
     if !params.is_empty() {
         dataframe = dataframe
-            .with_param_values(
-                params
-                    .iter()
-                    .map(scalar_value_from_lix_value)
-                    .collect::<Vec<_>>(),
-            )
+            .with_param_values(ParamValues::List(
+                params.iter().map(scalar_value_from_lix_value).collect(),
+            ))
             .map_err(datafusion_error_to_lix_error)?;
     }
 
@@ -298,7 +456,7 @@ fn expected_positional_parameter_count(
                 LixError::CODE_PARSE_ERROR,
                 format!("unsupported SQL parameter placeholder '{name}'"),
             )
-            .with_hint("Use numbered placeholders like $1, $2, ...")
+            .with_hint("Use placeholders like ?, ? or numbered placeholders like $1, $2, ...")
             .with_details(json!({
                 "operation": "execute",
                 "placeholder": name,
@@ -309,7 +467,7 @@ fn expected_positional_parameter_count(
                 LixError::CODE_PARSE_ERROR,
                 "SQL parameter placeholders are 1-indexed",
             )
-            .with_hint("Use numbered placeholders like $1, $2, ...")
+            .with_hint("Use placeholders like ?, ? or numbered placeholders like $1, $2, ...")
             .with_details(json!({
                 "operation": "execute",
                 "placeholder": name,
@@ -326,11 +484,11 @@ fn sorted_parameter_names(parameter_names: &HashSet<String>) -> Vec<String> {
     names
 }
 
-fn reject_read_only_history_view_dml(
-    sql: &str,
+fn reject_read_only_history_view_dml_from_statement(
+    statement: &DataFusionStatement,
     visible_schemas: &[JsonValue],
 ) -> Result<(), LixError> {
-    let target_names = super::dml_target_table_names(sql)?;
+    let target_names = super::datafusion_statement_dml_target_table_names(statement);
     for target_name in target_names {
         if is_history_view_name(&target_name, visible_schemas)? {
             return Err(read_only_history_view_error(&target_name));
@@ -419,18 +577,28 @@ fn validate_supported_logical_plan(plan: &LogicalPlan) -> Result<(), LixError> {
     Ok(())
 }
 
-fn scalar_value_from_lix_value(value: &Value) -> ScalarValue {
+fn scalar_value_from_lix_value(value: &Value) -> ScalarAndMetadata {
     match value {
-        Value::Null => ScalarValue::Null,
-        Value::Boolean(value) => ScalarValue::Boolean(Some(*value)),
-        Value::Integer(value) => ScalarValue::Int64(Some(*value)),
-        Value::Real(value) => ScalarValue::Float64(Some(*value)),
-        Value::Text(value) => ScalarValue::Utf8(Some(value.clone())),
-        Value::Json(value) => ScalarValue::Utf8(Some(value.to_string())),
-        Value::Blob(value) => ScalarValue::Binary(Some(value.clone())),
+        Value::Null => ScalarValue::Null.into(),
+        Value::Boolean(value) => ScalarValue::Boolean(Some(*value)).into(),
+        Value::Integer(value) => ScalarValue::Int64(Some(*value)).into(),
+        Value::Real(value) => ScalarValue::Float64(Some(*value)).into(),
+        Value::Text(value) => ScalarValue::Utf8(Some(value.clone())).into(),
+        Value::Json(value) => ScalarAndMetadata::new(
+            ScalarValue::Utf8(Some(value.to_string())),
+            Some(json_field_metadata()),
+        ),
+        Value::Blob(value) => ScalarValue::Binary(Some(value.clone())).into(),
     }
 }
 
+fn json_field_metadata() -> FieldMetadata {
+    FieldMetadata::new(BTreeMap::from([(
+        LIX_VALUE_TYPE_METADATA_KEY.to_string(),
+        LIX_VALUE_TYPE_JSON.to_string(),
+    )]))
+}
+
 fn datafusion_error_to_lix_error(error: datafusion::error::DataFusionError) -> LixError {
     super::error::datafusion_error_to_lix_error(error)
 }
@@ -660,27 +828,30 @@ mod tests {
         SqlWriteExecutionContext,
     };
     use crate::binary_cas::BlobDataReader;
-    use crate::changelog::{CanonicalChange, ChangelogReader, ChangelogScanRequest};
     use crate::commit_graph::{
-        CommitGraphChangeHistoryEntry, CommitGraphChangeHistoryRequest, CommitGraphChangeSet,
-        CommitGraphChangeSetElement, CommitGraphCommit, CommitGraphEdge, CommitGraphReader,
-        ReachableCommitGraphCommit,
+        CommitGraphChangeHistoryEntry, CommitGraphChangeHistoryRequest, CommitGraphCommit,
+        CommitGraphEdge, CommitGraphReader, ReachableCommitGraphCommit,
     };
+    use crate::commit_store::CommitStoreContext;
     use crate::functions::{
         FunctionProvider, FunctionProviderHandle, SharedFunctionProvider, SystemFunctionProvider,
     };
     use crate::json_store::JsonStoreContext;
     use crate::live_state::{
-        LiveStateContext, LiveStateReader, LiveStateRow, LiveStateRowRequest, LiveStateScanRequest,
+        LiveStateContext, LiveStateReader, LiveStateRowRequest, LiveStateScanRequest,
+        MaterializedLiveStateRow,
     };
-    use crate::sql2::{ChangelogQuerySource, SqlChangelogQuerySource};
+    use crate::sql2::{CommitStoreQuerySource, SqlCommitStoreQuerySource};
     use crate::storage::{
         KvEntryPage, KvExistsBatch, KvGetRequest, KvKeyPage, KvScanRequest, KvValueBatch,
         KvValuePage, StorageContext, StorageReadScope, StorageReadTransaction, StorageReader,
         StorageWriteSet,
     };
     use crate::tracked_state::TrackedStateContext;
-    use crate::transaction::types::{StageRow, StageWrite, StageWriteOutcome};
+    use crate::transaction::prepare_version_ref_row;
+    use crate::transaction::types::{
+        TransactionWrite, TransactionWriteOutcome, TransactionWriteRow,
+    };
     use crate::untracked_state::UntrackedStateContext;
     use crate::version::VersionRefReader;
     use crate::{Engine, ExecuteResult, SessionContext};
@@ -689,10 +860,9 @@ mod tests {
     struct DummyBlobReader;
     struct DummyLiveStateReader;
     struct RowsLiveStateReader {
-        rows: Vec<LiveStateRow>,
+        rows: Vec<MaterializedLiveStateRow>,
     }
     struct BackendBlobReader(StorageContext);
-    struct DummyChangelogReader;
     struct DummyCommitGraphReader;
     struct DummyVersionRefReader;
     struct TestReadTransaction(StorageContext);
@@ -747,7 +917,7 @@ mod tests {
 
     #[derive(Clone)]
     struct CapturedStageWrite {
-        rows: Vec<StageRow>,
+        rows: Vec<TransactionWriteRow>,
     }
 
     impl CapturedStageWrite {
@@ -759,7 +929,7 @@ mod tests {
     }
 
     struct CapturedStageOverlay {
-        rows: Vec<StageRow>,
+        rows: Vec<TransactionWriteRow>,
     }
 
     impl CapturedStageOverlay {
@@ -787,33 +957,31 @@ mod tests {
     struct CapturedStageRow {
         entity_id: String,
         schema_key: String,
-        schema_version: String,
         version_id: String,
         file_id: Option<String>,
         snapshot_content: Option<String>,
-        metadata: Option<JsonValue>,
+        metadata: Option<String>,
         global: bool,
         untracked: bool,
         tombstone: bool,
     }
 
-    impl From<StageRow> for CapturedStageRow {
-        fn from(row: StageRow) -> Self {
+    impl From<TransactionWriteRow> for CapturedStageRow {
+        fn from(row: TransactionWriteRow) -> Self {
             Self {
                 entity_id: row
                     .entity_id
                     .expect("captured staged row should carry entity_id")
-                    .as_string()
+                    .as_json_array_text()
                     .expect("captured staged row should project entity_id"),
                 schema_key: row.schema_key,
-                schema_version: row.schema_version,
                 version_id: row.version_id,
                 file_id: row.file_id,
                 global: row.global,
                 untracked: row.untracked,
-                tombstone: row.snapshot_content.is_none(),
-                snapshot_content: row.snapshot_content,
-                metadata: row.metadata,
+                tombstone: row.snapshot.is_none(),
+                snapshot_content: row.snapshot.map(|snapshot| snapshot.to_string()),
+                metadata: row.metadata.map(|metadata| metadata.to_string()),
             }
         }
     }
@@ -842,13 +1010,13 @@ mod tests {
             Arc::clone(&self.blob_reader)
         }
 
-        fn changelog_query_source(&self) -> SqlChangelogQuerySource {
+        fn commit_store_query_source(&self) -> SqlCommitStoreQuerySource {
             let base_scope = test_read_scope(StorageContext::new(Arc::new(
                 crate::backend::testing::UnitTestBackend::new(),
             )));
             let read_scope = StorageReadScope::new(base_scope.store());
-            ChangelogQuerySource {
-                changelog_reader: Arc::new(DummyChangelogReader),
+            CommitStoreQuerySource {
+                commit_store_reader: Arc::new(CommitStoreContext::new().reader(read_scope.store())),
                 json_reader: JsonStoreContext::new().reader(read_scope.store()),
             }
         }
@@ -898,7 +1066,7 @@ mod tests {
         async fn scan_live_state(
             &mut self,
             request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             self.live_state.scan_rows(request).await
         }
 
@@ -909,23 +1077,26 @@ mod tests {
             Ok(Some(format!("commit-{version_id}")))
         }
 
-        async fn stage_write(&mut self, write: StageWrite) -> Result<StageWriteOutcome, LixError> {
+        async fn stage_write(
+            &mut self,
+            write: TransactionWrite,
+        ) -> Result<TransactionWriteOutcome, LixError> {
             let count = match &write {
-                StageWrite::Rows { rows, .. } => rows.len() as u64,
-                StageWrite::RowsWithFileData { count, .. } => *count,
-                StageWrite::AdoptedChanges { changes } => changes.len() as u64,
+                TransactionWrite::Rows { rows, .. } => rows.len() as u64,
+                TransactionWrite::RowsWithFileData { count, .. } => *count,
+                TransactionWrite::AdoptedChanges { changes } => changes.len() as u64,
             };
             let rows = match write {
-                StageWrite::Rows { rows, .. } => rows,
-                StageWrite::RowsWithFileData { rows, .. } => rows,
-                StageWrite::AdoptedChanges { .. } => Vec::new(),
+                TransactionWrite::Rows { rows, .. } => rows,
+                TransactionWrite::RowsWithFileData { rows, .. } => rows,
+                TransactionWrite::AdoptedChanges { .. } => Vec::new(),
             };
             self.staged_writes
                 .lock()
                 .expect("staged writes lock")
                 .deltas
                 .push(CapturedStageWrite { rows });
-            Ok(StageWriteOutcome { count })
+            Ok(TransactionWriteOutcome { count })
         }
     }
 
@@ -938,20 +1109,6 @@ mod tests {
         execute_logical_plan(plan, params).await
     }
 
-    #[async_trait]
-    impl ChangelogReader for DummyChangelogReader {
-        async fn load_change(&self, _change_id: &str) -> Result<Option<CanonicalChange>, LixError> {
-            Ok(None)
-        }
-
-        async fn scan_changes(
-            &self,
-            _request: &ChangelogScanRequest,
-        ) -> Result<Vec<CanonicalChange>, LixError> {
-            Ok(Vec::new())
-        }
-    }
-
     #[async_trait]
     impl VersionRefReader for DummyVersionRefReader {
         async fn load_head(
@@ -1009,17 +1166,6 @@ mod tests {
             Vec::new()
         }
 
-        fn change_sets(&self, _commits: &[CommitGraphCommit]) -> Vec<CommitGraphChangeSet> {
-            Vec::new()
-        }
-
-        async fn change_set_elements(
-            &mut self,
-            _commits: &[CommitGraphCommit],
-        ) -> Result<Vec<CommitGraphChangeSetElement>, LixError> {
-            Ok(Vec::new())
-        }
-
         async fn change_history_from_commit(
             &mut self,
             _start_commit_id: &str,
@@ -1034,14 +1180,14 @@ mod tests {
         async fn scan_rows(
             &self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(vec![])
         }
 
         async fn load_row(
             &self,
             _request: &LiveStateRowRequest,
-        ) -> Result<Option<LiveStateRow>, LixError> {
+        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
             Ok(None)
         }
     }
@@ -1051,14 +1197,14 @@ mod tests {
         async fn scan_rows(
             &self,
             _request: &LiveStateScanRequest,
-        ) -> Result<Vec<LiveStateRow>, LixError> {
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
             Ok(self.rows.clone())
         }
 
         async fn load_row(
             &self,
             _request: &LiveStateRowRequest,
-        ) -> Result<Option<LiveStateRow>, LixError> {
+        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
             Ok(None)
         }
     }
@@ -1069,7 +1215,10 @@ mod tests {
             &self,
             hashes: &[crate::binary_cas::BlobHash],
         ) -> Result<crate::binary_cas::BlobBytesBatch, LixError> {
-            Ok(crate::binary_cas::BlobBytesBatch::missing(hashes.len()))
+            Ok(crate::binary_cas::BlobBytesBatch::new(vec![
+                None;
+                hashes.len()
+            ]))
         }
     }
 
@@ -1085,17 +1234,14 @@ mod tests {
         }
     }
 
-    fn live_lix_state_row(entity_id: &str, metadata: Option<&str>) -> LiveStateRow {
-        LiveStateRow {
-            entity_id: crate::entity_identity::EntityIdentity::from_string(entity_id)
-                .expect("entity id should decode"),
+    fn live_lix_state_row(entity_id: &str, metadata: Option<&str>) -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
+            entity_id: crate::entity_identity::EntityIdentity::single(entity_id),
             schema_key: "lix_key_value".to_string(),
             file_id: None,
             snapshot_content: Some("{\"key\":\"hello\",\"value\":\"world\"}".to_string()),
-            metadata: metadata.map(|value| {
-                serde_json::from_str(value).expect("test metadata should be valid JSON")
-            }),
-            schema_version: "1".to_string(),
+            metadata: metadata.map(str::to_string),
+            deleted: false,
             version_id: "version-a".to_string(),
             change_id: Some(format!("change-{entity_id}")),
             commit_id: Some(format!("commit-{entity_id}")),
@@ -1106,15 +1252,14 @@ mod tests {
         }
     }
 
-    fn live_entity_row(entity_id: &str, version_id: &str, value: &str) -> LiveStateRow {
-        LiveStateRow {
-            entity_id: crate::entity_identity::EntityIdentity::from_string(entity_id)
-                .expect("entity id should decode"),
+    fn live_entity_row(entity_id: &str, version_id: &str, value: &str) -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
+            entity_id: crate::entity_identity::EntityIdentity::single(entity_id),
             schema_key: "test_state_schema".to_string(),
             file_id: None,
             snapshot_content: Some(format!("{{\"value\":\"{value}\"}}")),
-            metadata: Some(json!({ "source": entity_id })),
-            schema_version: "1".to_string(),
+            metadata: Some(json!({ "source": entity_id }).to_string()),
+            deleted: false,
             version_id: version_id.to_string(),
             change_id: Some(format!("change-{entity_id}")),
             commit_id: Some(format!("commit-{entity_id}")),
@@ -1131,10 +1276,9 @@ mod tests {
         parent_id: Option<&str>,
         name: &str,
         hidden: bool,
-    ) -> LiveStateRow {
-        LiveStateRow {
-            entity_id: crate::entity_identity::EntityIdentity::from_string(entity_id)
-                .expect("entity id should decode"),
+    ) -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
+            entity_id: crate::entity_identity::EntityIdentity::single(entity_id),
             schema_key: "lix_directory_descriptor".to_string(),
             file_id: None,
             snapshot_content: Some(
@@ -1146,8 +1290,8 @@ mod tests {
                 })
                 .to_string(),
             ),
-            metadata: Some(json!({ "source": entity_id })),
-            schema_version: "1".to_string(),
+            metadata: Some(json!({ "source": entity_id }).to_string()),
+            deleted: false,
             version_id: version_id.to_string(),
             change_id: Some(format!("change-{entity_id}")),
             commit_id: Some(format!("commit-{entity_id}")),
@@ -1164,10 +1308,9 @@ mod tests {
         directory_id: Option<&str>,
         name: &str,
         hidden: bool,
-    ) -> LiveStateRow {
-        LiveStateRow {
-            entity_id: crate::entity_identity::EntityIdentity::from_string(entity_id)
-                .expect("entity id should decode"),
+    ) -> MaterializedLiveStateRow {
+        MaterializedLiveStateRow {
+            entity_id: crate::entity_identity::EntityIdentity::single(entity_id),
             schema_key: "lix_file_descriptor".to_string(),
             file_id: None,
             snapshot_content: Some(
@@ -1179,8 +1322,8 @@ mod tests {
                 })
                 .to_string(),
             ),
-            metadata: Some(json!({ "source": entity_id })),
-            schema_version: "1".to_string(),
+            metadata: Some(json!({ "source": entity_id }).to_string()),
+            deleted: false,
             version_id: version_id.to_string(),
             change_id: Some(format!("change-{entity_id}")),
             commit_id: Some(format!("commit-{entity_id}")),
@@ -1317,7 +1460,7 @@ mod tests {
         }));
     }
 
-    async fn setup_engine2_history_fixture() -> Result<(SessionContext, String), LixError> {
+    async fn setup_engine_history_fixture() -> Result<(SessionContext, String), LixError> {
         let backend = crate::backend::testing::UnitTestBackend::new();
         let init_receipt = Engine::initialize(Box::new(backend.clone())).await?;
         let engine = Engine::new(Box::new(backend)).await?;
@@ -1327,9 +1470,9 @@ mod tests {
             .execute(
                 "INSERT INTO lix_registered_schema (value, lixcol_global, lixcol_untracked) \
                  VALUES (\
-                 lix_json('{\"x-lix-key\":\"test_state_schema\",\"x-lix-version\":\"1\",\"type\":\"object\",\"properties\":{\"value\":{\"type\":\"string\"},\"count\":{\"type\":\"integer\"}},\"required\":[\"value\",\"count\"],\"additionalProperties\":false}'),\
+                 lix_json('{\"x-lix-key\":\"test_state_schema\",\"type\":\"object\",\"properties\":{\"value\":{\"type\":\"string\"},\"count\":{\"type\":\"integer\"}},\"required\":[\"value\",\"count\"],\"additionalProperties\":false}'),\
                  false,\
-                 true\
+                 false\
                  )",
                 &[],
             )
@@ -1337,8 +1480,8 @@ mod tests {
         session
             .execute(
                 "INSERT INTO test_state_schema \
-                 (lixcol_entity_id, value, count, lixcol_metadata, lixcol_untracked) \
-                 VALUES ('entity-history', 'A', 7, '{\"source\":\"history\"}', false)",
+	             (lixcol_entity_id, value, count, lixcol_metadata, lixcol_untracked) \
+	             VALUES (lix_json('[\"entity-history\"]'), 'A', 7, '{\"source\":\"history\"}', false)",
                 &[],
             )
             .await?;
@@ -1616,23 +1759,23 @@ mod tests {
 
     #[tokio::test]
     async fn execute_sql_reads_lix_state_history_from_history_context() {
-        let (session, head_commit_id) = setup_engine2_history_fixture()
+        let (session, head_commit_id) = setup_engine_history_fixture()
             .await
             .expect("history fixture should initialize");
         let result = session
             .execute(
                 &format!(
                     "SELECT entity_id, snapshot_content, metadata, depth, start_commit_id \
-             FROM lix_state_history \
-             WHERE schema_key = 'test_state_schema' \
-               AND entity_id = 'entity-history' \
-               AND start_commit_id = '{head_commit_id}' \
-               AND depth >= 0"
+	             FROM lix_state_history \
+	             WHERE schema_key = 'test_state_schema' \
+	               AND entity_id = lix_json('[\"entity-history\"]') \
+	               AND start_commit_id = '{head_commit_id}' \
+	               AND depth >= 0"
                 ),
                 &[],
             )
             .await
-            .expect("sql2 execute should read lix_state_history through real engine2 context");
+            .expect("sql2 execute should read lix_state_history through real engine context");
         let (columns, rows) = rows_from_execute_result(result);
 
         assert_eq!(
@@ -1646,7 +1789,7 @@ mod tests {
             ]
         );
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0][0], Value::Text("entity-history".to_string()));
+        assert_eq!(rows[0][0], Value::Json(json!(["entity-history"])));
         assert_eq!(rows[0][1], Value::Json(json!({"count": 7, "value": "A"})));
         assert_eq!(rows[0][2], Value::Json(json!({"source": "history"})));
         assert!(matches!(rows[0][3], Value::Integer(_)));
@@ -1655,21 +1798,21 @@ mod tests {
 
     #[tokio::test]
     async fn execute_sql_reads_entity_history_view_from_history_context() {
-        let (session, head_commit_id) = setup_engine2_history_fixture()
+        let (session, head_commit_id) = setup_engine_history_fixture()
             .await
             .expect("history fixture should initialize");
         let result = session
             .execute(
                 &format!(
                     "SELECT value, count, lixcol_entity_id, lixcol_start_commit_id, lixcol_depth \
-             FROM test_state_schema_history \
-             WHERE lixcol_start_commit_id = '{head_commit_id}' \
-               AND lixcol_entity_id = 'entity-history'"
+	             FROM test_state_schema_history \
+	             WHERE lixcol_start_commit_id = '{head_commit_id}' \
+	               AND lixcol_entity_id = lix_json('[\"entity-history\"]')"
                 ),
                 &[],
             )
             .await
-            .expect("sql2 execute should read entity history through real engine2 context");
+            .expect("sql2 execute should read entity history through real engine context");
         let (columns, rows) = rows_from_execute_result(result);
 
         assert_eq!(
@@ -1685,14 +1828,14 @@ mod tests {
         assert_eq!(rows.len(), 1);
         assert_eq!(rows[0][0], Value::Text("A".to_string()));
         assert_eq!(rows[0][1], Value::Integer(7));
-        assert_eq!(rows[0][2], Value::Text("entity-history".to_string()));
+        assert_eq!(rows[0][2], Value::Json(json!(["entity-history"])));
         assert_eq!(rows[0][3], Value::Text(head_commit_id));
         assert!(matches!(rows[0][4], Value::Integer(_)));
     }
 
     #[tokio::test]
     async fn execute_sql_reads_directory_history_view_from_history_context() {
-        let (session, head_commit_id) = setup_engine2_history_fixture()
+        let (session, head_commit_id) = setup_engine_history_fixture()
             .await
             .expect("history fixture should initialize");
         let result = session
@@ -1705,7 +1848,7 @@ mod tests {
                 &[],
             )
             .await
-            .expect("sql2 execute should read directory history through real engine2 context");
+            .expect("sql2 execute should read directory history through real engine context");
         assert!(
             result.notices().is_empty(),
             "identity-filtered directory history should not emit soft notices"
@@ -1754,7 +1897,7 @@ mod tests {
 
     #[tokio::test]
     async fn execute_sql_reads_file_history_view_from_history_context() {
-        let (session, head_commit_id) = setup_engine2_history_fixture()
+        let (session, head_commit_id) = setup_engine_history_fixture()
             .await
             .expect("history fixture should initialize");
         let result = session
@@ -1770,7 +1913,7 @@ mod tests {
                 &[],
             )
             .await
-            .expect("sql2 execute should read file history through real engine2 context");
+            .expect("sql2 execute should read file history through real engine context");
         assert!(
             result.notices().is_empty(),
             "identity-filtered file history should not emit soft notices"
@@ -1815,6 +1958,36 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn execute_sql_rejects_writes_to_history_views_before_planning() {
+        for sql in [
+            "DELETE FROM lix_state_history",
+            "DELETE FROM LIX_STATE_HISTORY",
+            "DELETE FROM main.LIX_STATE_HISTORY",
+        ] {
+            let blob_reader: Arc<dyn BlobDataReader> = Arc::new(DummyBlobReader);
+            let live_state = Arc::new(DummyLiveStateReader);
+            let staged_writes = Arc::new(Mutex::new(CapturingStagedWrites::default()));
+            let mut ctx = DummySqlWriteExecutionContext {
+                active_version_id: "version-a",
+                blob_reader,
+                live_state,
+                staged_writes,
+                schema_definitions: vec![],
+            };
+
+            let error = execute_write_sql(&mut ctx, sql, &[])
+                .await
+                .expect_err("history views are read-only");
+
+            assert_eq!(error.code, LixError::CODE_READ_ONLY, "{sql}");
+            assert_eq!(
+                error.message, "DML cannot write read-only history view 'lix_state_history'",
+                "{sql}"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn execute_sql_insert_into_lix_state_values_stages_write() {
         let blob_reader: Arc<dyn BlobDataReader> = Arc::new(DummyBlobReader);
@@ -1829,14 +2002,14 @@ mod tests {
         };
 
         let result = execute_write_sql(
-            &mut ctx,
-            "INSERT INTO lix_state (\
-             entity_id, schema_key, file_id, snapshot_content, metadata, schema_version, global, untracked\
-             ) VALUES (\
-             'entity-1', 'lix_key_value', NULL, '{\"key\":\"hello\",\"value\":\"world\"}', '{\"source\":\"sql\"}', '1', false, false\
-             )",
-            &[],
-        )
+			&mut ctx,
+			"INSERT INTO lix_state (\
+	         entity_id, schema_key, file_id, snapshot_content, metadata, global, untracked\
+	         ) VALUES (\
+	         lix_json('[\"entity-1\"]'), 'lix_key_value', NULL, '{\"key\":\"hello\",\"value\":\"world\"}', '{\"source\":\"sql\"}', false, false\
+	         )",
+			&[],
+		)
         .await
         .expect("INSERT INTO lix_state VALUES should stage write");
 
@@ -1850,8 +2023,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_key_value");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-1");
-        assert_eq!(rows[0].schema_version, "1");
+        assert_eq!(rows[0].entity_id, "[\"entity-1\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -1859,7 +2031,7 @@ mod tests {
             rows[0].snapshot_content.as_deref(),
             Some("{\"key\":\"hello\",\"value\":\"world\"}")
         );
-        assert_eq!(rows[0].metadata.as_ref(), Some(&json!({"source": "sql"})));
+        assert_eq!(rows[0].metadata.as_deref(), Some("{\"source\":\"sql\"}"));
     }
 
     #[tokio::test]
@@ -1876,14 +2048,14 @@ mod tests {
         };
 
         let result = execute_write_sql(
-            &mut ctx,
-            "INSERT INTO lix_state (\
-             entity_id, schema_key, file_id, snapshot_content, metadata, schema_version\
-             ) VALUES (\
-             'entity-defaults', 'lix_key_value', NULL, '{\"key\":\"hello\",\"value\":\"defaults\"}', NULL, '1'\
-             )",
-            &[],
-        )
+			&mut ctx,
+			"INSERT INTO lix_state (\
+	         entity_id, schema_key, file_id, snapshot_content, metadata\
+	         ) VALUES (\
+	         lix_json('[\"entity-defaults\"]'), 'lix_key_value', NULL, '{\"key\":\"hello\",\"value\":\"defaults\"}', NULL\
+	         )",
+			&[],
+		)
         .await
         .expect("INSERT INTO lix_state should default bookkeeping flags");
 
@@ -1897,7 +2069,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_key_value");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-defaults");
+        assert_eq!(rows[0].entity_id, "[\"entity-defaults\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -1919,15 +2091,14 @@ mod tests {
         let result = execute_write_sql(
             &mut ctx,
             "INSERT INTO lix_state (\
-             entity_id, schema_key, file_id, snapshot_content, metadata, schema_version, global, untracked\
-             ) \
-             SELECT \
-             'entity-from-select' AS entity_id, \
-             'lix_key_value' AS schema_key, \
-             NULL AS file_id, \
+	         entity_id, schema_key, file_id, snapshot_content, metadata, global, untracked\
+	         ) \
+	         SELECT \
+	         lix_json('[\"entity-from-select\"]') AS entity_id, \
+	         'lix_key_value' AS schema_key, \
+	         NULL AS file_id, \
              '{\"key\":\"hello\",\"value\":\"from-select\"}' AS snapshot_content, \
              '{\"source\":\"select\"}' AS metadata, \
-             '1' AS schema_version, \
              false AS global, \
              false AS untracked",
             &[],
@@ -1945,17 +2116,13 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_key_value");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-from-select");
-        assert_eq!(rows[0].schema_version, "1");
+        assert_eq!(rows[0].entity_id, "[\"entity-from-select\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert_eq!(
             rows[0].snapshot_content.as_deref(),
             Some("{\"key\":\"hello\",\"value\":\"from-select\"}")
         );
-        assert_eq!(
-            rows[0].metadata.as_ref(),
-            Some(&json!({"source": "select"}))
-        );
+        assert_eq!(rows[0].metadata.as_deref(), Some("{\"source\":\"select\"}"));
     }
 
     #[tokio::test]
@@ -1970,7 +2137,6 @@ mod tests {
             staged_writes: Arc::clone(&staged_writes),
             schema_definitions: vec![json!({
                 "x-lix-key": "test_state_schema",
-                "x-lix-version": "1",
                 "type": "object",
                 "properties": {
                     "value": { "type": "string" }
@@ -1981,8 +2147,8 @@ mod tests {
         let result = execute_write_sql(
             &mut ctx,
             "INSERT INTO test_state_schema_by_version (\
-             lixcol_entity_id, lixcol_version_id, value\
-             ) VALUES ('entity-c', 'version-b', 'C')",
+	     lixcol_entity_id, lixcol_version_id, value\
+	     ) VALUES (lix_json('[\"entity-c\"]'), 'version-b', 'C')",
             &[],
         )
         .await
@@ -1998,8 +2164,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "test_state_schema");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-c");
-        assert_eq!(rows[0].schema_version, "1");
+        assert_eq!(rows[0].entity_id, "[\"entity-c\"]");
         assert_eq!(rows[0].version_id, "version-b");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -2021,7 +2186,6 @@ mod tests {
             staged_writes: Arc::clone(&staged_writes),
             schema_definitions: vec![json!({
                 "x-lix-key": "test_state_schema",
-                "x-lix-version": "1",
                 "type": "object",
                 "properties": {
                     "value": { "type": "string" }
@@ -2032,7 +2196,7 @@ mod tests {
         let result = execute_write_sql(
             &mut ctx,
             "INSERT INTO test_state_schema (lixcol_entity_id, value) \
-             VALUES ('entity-c', 'C')",
+	     VALUES (lix_json('[\"entity-c\"]'), 'C')",
             &[],
         )
         .await
@@ -2048,7 +2212,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "test_state_schema");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-c");
+        assert_eq!(rows[0].entity_id, "[\"entity-c\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -2091,8 +2255,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_directory_descriptor");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "dir-docs");
-        assert_eq!(rows[0].schema_version, "1");
+        assert_eq!(rows[0].entity_id, "[\"dir-docs\"]");
         assert_eq!(rows[0].version_id, "version-b");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -2134,7 +2297,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_directory_descriptor");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "dir-docs");
+        assert_eq!(rows[0].entity_id, "[\"dir-docs\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -2178,15 +2341,15 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_directory_descriptor");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "dir-docs");
+        assert_eq!(rows[0].entity_id, "[\"dir-docs\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert_eq!(
             rows[0].snapshot_content.as_deref(),
             Some("{\"hidden\":true,\"id\":\"dir-docs\",\"name\":\"docs\",\"parent_id\":null}")
         );
         assert_eq!(
-            rows[0].metadata.as_ref(),
-            Some(&json!({"source": "directory-update"}))
+            rows[0].metadata.as_deref(),
+            Some("{\"source\":\"directory-update\"}")
         );
     }
 
@@ -2267,7 +2430,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_all_semantic_rows();
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "dir-guides");
+        assert_eq!(rows[0].entity_id, "[\"dir-guides\"]");
         assert_eq!(rows[0].version_id, "version-b");
         assert!(rows[0].tombstone);
         assert_eq!(rows[0].snapshot_content, None);
@@ -2306,8 +2469,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_file_descriptor");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "file-readme");
-        assert_eq!(rows[0].schema_version, "1");
+        assert_eq!(rows[0].entity_id, "[\"file-readme\"]");
         assert_eq!(rows[0].version_id, "version-b");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -2352,7 +2514,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_file_descriptor");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "file-readme");
+        assert_eq!(rows[0].entity_id, "[\"file-readme\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert!(!rows[0].global);
         assert!(!rows[0].untracked);
@@ -2391,10 +2553,10 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let descriptor_rows = overlay.visible_semantic_rows(false, "lix_file_descriptor");
         assert_eq!(descriptor_rows.len(), 1);
-        assert_eq!(descriptor_rows[0].entity_id, "file-readme");
+        assert_eq!(descriptor_rows[0].entity_id, "[\"file-readme\"]");
         let blob_ref_rows = overlay.visible_semantic_rows(false, "lix_binary_blob_ref");
         assert_eq!(blob_ref_rows.len(), 1);
-        assert_eq!(blob_ref_rows[0].entity_id, "file-readme");
+        assert_eq!(blob_ref_rows[0].entity_id, "[\"file-readme\"]");
         assert_eq!(blob_ref_rows[0].file_id.as_deref(), Some("file-readme"));
         assert_eq!(blob_ref_rows[0].version_id, "version-b");
         let snapshot: JsonValue =
@@ -2458,7 +2620,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_file_descriptor");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "file-readme");
+        assert_eq!(rows[0].entity_id, "[\"file-readme\"]");
         assert_eq!(rows[0].version_id, "version-a");
         let snapshot: JsonValue =
             serde_json::from_str(rows[0].snapshot_content.as_deref().unwrap())
@@ -2468,8 +2630,8 @@ mod tests {
         assert_eq!(snapshot["name"], "readme-updated.txt");
         assert_eq!(snapshot["hidden"], true);
         assert_eq!(
-            rows[0].metadata.as_ref(),
-            Some(&json!({"source": "file-update"}))
+            rows[0].metadata.as_deref(),
+            Some("{\"source\":\"file-update\"}")
         );
     }
 
@@ -2518,7 +2680,7 @@ mod tests {
             .is_empty());
         let blob_ref_rows = overlay.visible_semantic_rows(false, "lix_binary_blob_ref");
         assert_eq!(blob_ref_rows.len(), 1);
-        assert_eq!(blob_ref_rows[0].entity_id, "file-readme");
+        assert_eq!(blob_ref_rows[0].entity_id, "[\"file-readme\"]");
         let snapshot: JsonValue =
             serde_json::from_str(blob_ref_rows[0].snapshot_content.as_deref().unwrap())
                 .expect("blob ref snapshot JSON");
@@ -2626,7 +2788,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_all_semantic_rows();
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "file-guide");
+        assert_eq!(rows[0].entity_id, "[\"file-guide\"]");
         assert_eq!(rows[0].version_id, "version-b");
         assert!(rows[0].tombstone);
         assert_eq!(rows[0].snapshot_content, None);
@@ -2649,7 +2811,6 @@ mod tests {
             staged_writes: Arc::clone(&staged_writes),
             schema_definitions: vec![json!({
                 "x-lix-key": "test_state_schema",
-                "x-lix-version": "1",
                 "type": "object",
                 "properties": {
                     "value": { "type": "string" }
@@ -2677,15 +2838,15 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "test_state_schema");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-a");
+        assert_eq!(rows[0].entity_id, "[\"entity-a\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert_eq!(
             rows[0].snapshot_content.as_deref(),
             Some("{\"value\":\"updated\"}")
         );
         assert_eq!(
-            rows[0].metadata.as_ref(),
-            Some(&json!({"source": "entity-update"}))
+            rows[0].metadata.as_deref(),
+            Some("{\"source\":\"entity-update\"}")
         );
     }
 
@@ -2706,7 +2867,6 @@ mod tests {
             staged_writes: Arc::clone(&staged_writes),
             schema_definitions: vec![json!({
                 "x-lix-key": "test_state_schema",
-                "x-lix-version": "1",
                 "type": "object",
                 "properties": {
                     "value": { "type": "string" }
@@ -2733,7 +2893,7 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_all_semantic_rows();
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-b");
+        assert_eq!(rows[0].entity_id, "[\"entity-b\"]");
         assert_eq!(rows[0].version_id, "version-b");
         assert!(rows[0].tombstone);
         assert_eq!(rows[0].snapshot_content, None);
@@ -2762,7 +2922,7 @@ mod tests {
             "UPDATE lix_state \
              SET snapshot_content = '{\"key\":\"hello\",\"value\":\"updated\"}', \
                  metadata = '{\"schema_key\":\"lix_key_value\"}' \
-             WHERE metadata = '{\"source\":\"match\"}'",
+             WHERE metadata = lix_json('{\"source\":\"match\"}')",
             &[],
         )
         .await
@@ -2778,15 +2938,15 @@ mod tests {
             .expect("staged delta should expose pending overlay");
         let rows = overlay.visible_semantic_rows(false, "lix_key_value");
         assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0].entity_id, "entity-1");
+        assert_eq!(rows[0].entity_id, "[\"entity-1\"]");
         assert_eq!(rows[0].version_id, "version-a");
         assert_eq!(
             rows[0].snapshot_content.as_deref(),
             Some("{\"key\":\"hello\",\"value\":\"updated\"}")
         );
         assert_eq!(
-            rows[0].metadata.as_ref(),
-            Some(&json!({"schema_key": "lix_key_value"}))
+            rows[0].metadata.as_deref(),
+            Some("{\"schema_key\":\"lix_key_value\"}")
         );
     }
 
@@ -2824,8 +2984,8 @@ mod tests {
         assert_eq!(rows.len(), 2);
         assert!(rows.iter().all(|row| row.tombstone));
         assert!(rows.iter().all(|row| row.snapshot_content.is_none()));
-        assert!(rows.iter().any(|row| row.entity_id == "entity-1"));
-        assert!(rows.iter().any(|row| row.entity_id == "entity-2"));
+        assert!(rows.iter().any(|row| row.entity_id == "[\"entity-1\"]"));
+        assert!(rows.iter().any(|row| row.entity_id == "[\"entity-2\"]"));
     }
 
     struct BackendSqlExecutionContext<'a> {
@@ -2853,13 +3013,11 @@ mod tests {
             Arc::clone(&self.blob_reader)
         }
 
-        fn changelog_query_source(&self) -> SqlChangelogQuerySource {
+        fn commit_store_query_source(&self) -> SqlCommitStoreQuerySource {
             let base_scope = test_read_scope(self.storage.clone());
             let read_scope = StorageReadScope::new(base_scope.store());
-            ChangelogQuerySource {
-                changelog_reader: Arc::new(
-                    crate::changelog::ChangelogContext::new().reader(read_scope.store()),
-                ),
+            CommitStoreQuerySource {
+                commit_store_reader: Arc::new(CommitStoreContext::new().reader(read_scope.store())),
                 json_reader: JsonStoreContext::new().reader(read_scope.store()),
             }
         }
@@ -2891,26 +3049,23 @@ mod tests {
                 crate::untracked_state::UntrackedStateContext::new(),
             ));
             let mut writes = StorageWriteSet::new();
-            let canonical_rows = {
-                let mut json_writer = JsonStoreContext::new().writer();
-                vec![
-                    version_ctx.canonical_ref_row(
-                        &mut writes,
-                        &mut json_writer,
-                        "version-a",
-                        &init_receipt.initial_commit_id,
-                        "1970-01-01T00:00:00.000Z",
-                    )?,
-                    version_ctx.canonical_ref_row(
-                        &mut writes,
-                        &mut json_writer,
-                        "version-b",
-                        &init_receipt.initial_commit_id,
-                        "1970-01-01T00:00:00.000Z",
-                    )?,
-                ]
-            };
-            version_ctx.stage_canonical_ref_rows(&mut writes, &canonical_rows)?;
+            let canonical_rows = vec![
+                prepare_version_ref_row(
+                    "version-a",
+                    &init_receipt.initial_commit_id,
+                    "1970-01-01T00:00:00.000Z",
+                )?,
+                prepare_version_ref_row(
+                    "version-b",
+                    &init_receipt.initial_commit_id,
+                    "1970-01-01T00:00:00.000Z",
+                )?,
+            ];
+            let rows = canonical_rows
+                .into_iter()
+                .map(|prepared| prepared.row)
+                .collect::<Vec<_>>();
+            version_ctx.stage_canonical_ref_rows(&mut writes, &rows)?;
             writes.apply(&mut transaction.as_mut()).await?;
             transaction.commit().await?;
         }
@@ -2919,7 +3074,6 @@ mod tests {
         let session_b = engine.open_session("version-b").await?;
         let schema_definition = json!({
             "x-lix-key": "test_state_schema",
-            "x-lix-version": "1",
             "type": "object",
             "properties": {
                 "value": { "type": "string" }
@@ -2931,9 +3085,9 @@ mod tests {
             .execute(
                 "INSERT INTO lix_registered_schema (value, lixcol_global, lixcol_untracked) \
                  VALUES (\
-                 lix_json('{\"x-lix-key\":\"test_state_schema\",\"x-lix-version\":\"1\",\"type\":\"object\",\"properties\":{\"value\":{\"type\":\"string\"}},\"required\":[\"value\"],\"additionalProperties\":false}'),\
+                 lix_json('{\"x-lix-key\":\"test_state_schema\",\"type\":\"object\",\"properties\":{\"value\":{\"type\":\"string\"}},\"required\":[\"value\"],\"additionalProperties\":false}'),\
                  false,\
-                 true\
+                 false\
                  )",
                 &[],
             )
@@ -2942,9 +3096,9 @@ mod tests {
             .execute(
                 "INSERT INTO lix_registered_schema (value, lixcol_global, lixcol_untracked) \
                  VALUES (\
-                 lix_json('{\"x-lix-key\":\"test_state_schema\",\"x-lix-version\":\"1\",\"type\":\"object\",\"properties\":{\"value\":{\"type\":\"string\"}},\"required\":[\"value\"],\"additionalProperties\":false}'),\
+                 lix_json('{\"x-lix-key\":\"test_state_schema\",\"type\":\"object\",\"properties\":{\"value\":{\"type\":\"string\"}},\"required\":[\"value\"],\"additionalProperties\":false}'),\
                  false,\
-                 true\
+                 false\
                  )",
                 &[],
             )
@@ -2952,32 +3106,32 @@ mod tests {
         session_a
             .execute(
                 "INSERT INTO lix_state (\
-                 entity_id, schema_key, file_id, snapshot_content, schema_version, global, untracked\
-                 ) VALUES (\
-                 'entity-a', 'test_state_schema', NULL, '{\"value\":\"A\"}', '1', false, false\
-                 )",
+	         entity_id, schema_key, file_id, snapshot_content, global, untracked\
+	         ) VALUES (\
+	         lix_json('[\"entity-a\"]'), 'test_state_schema', NULL, '{\"value\":\"A\"}', false, false\
+	         )",
                 &[],
             )
             .await?;
         session_b
             .execute(
                 "INSERT INTO lix_state (\
-                 entity_id, schema_key, file_id, snapshot_content, schema_version, global, untracked\
-                 ) VALUES (\
-                 'entity-b', 'test_state_schema', NULL, '{\"value\":\"B\"}', '1', false, false\
-                 )",
+	         entity_id, schema_key, file_id, snapshot_content, global, untracked\
+	         ) VALUES (\
+	         lix_json('[\"entity-b\"]'), 'test_state_schema', NULL, '{\"value\":\"B\"}', false, false\
+	         )",
                 &[],
             )
             .await?;
         session_a
-            .execute(
-                "INSERT INTO lix_state (\
-                 entity_id, schema_key, file_id, snapshot_content, schema_version, global, untracked\
-                 ) VALUES (\
-                 'dir-docs', 'lix_directory_descriptor', NULL, '{\"id\":\"dir-docs\",\"parent_id\":null,\"name\":\"docs\",\"hidden\":false}', '1', false, false\
-                 )",
-                &[],
-            )
+		.execute(
+			"INSERT INTO lix_state (\
+	         entity_id, schema_key, file_id, snapshot_content, global, untracked\
+	         ) VALUES (\
+	         lix_json('[\"dir-docs\"]'), 'lix_directory_descriptor', NULL, '{\"id\":\"dir-docs\",\"parent_id\":null,\"name\":\"docs\",\"hidden\":false}', false, false\
+	         )",
+			&[],
+		)
             .await?;
         session_a
             .execute(
@@ -2993,7 +3147,7 @@ mod tests {
         LiveStateContext::new(
             TrackedStateContext::new(),
             UntrackedStateContext::new(),
-            crate::commit_graph::CommitGraphContext::new(crate::changelog::ChangelogContext::new()),
+            crate::commit_graph::CommitGraphContext::new(),
         )
     }
 
@@ -3050,7 +3204,7 @@ mod tests {
                     vec!["entity_id", "version_id", "snapshot_content", "commit_id"]
                 );
                 assert_eq!(result.rows.len(), 1);
-                assert_eq!(result.rows[0][0], Value::Text("entity-b".to_string()));
+                assert_eq!(result.rows[0][0], Value::Json(json!(["entity-b"])));
                 assert_eq!(result.rows[0][1], Value::Text("version-b".to_string()));
                 assert_eq!(result.rows[0][2], Value::Json(json!({"value": "B"})));
                 match &result.rows[0][3] {
@@ -3090,11 +3244,11 @@ mod tests {
                 .expect("broad by-version read should succeed");
 
                 assert!(
-                    result.rows.iter().any(|row| row[0] == Value::Text("entity-a".to_string()))
-                        && result.rows.iter().any(|row| row[0] == Value::Text("entity-b".to_string())),
-                    "expected broad by-version read to include rows from multiple visible versions: {:?}",
-                    result.rows
-                );
+					result.rows.iter().any(|row| row[0] == Value::Json(json!(["entity-a"])))
+						&& result.rows.iter().any(|row| row[0] == Value::Json(json!(["entity-b"]))),
+					"expected broad by-version read to include rows from multiple visible versions: {:?}",
+					result.rows
+				);
             })
         });
     }
@@ -3131,7 +3285,7 @@ mod tests {
 
                 assert_eq!(result.columns, vec!["entity_id", "snapshot_content"]);
                 assert_eq!(result.rows.len(), 1);
-                assert_eq!(result.rows[0][0], Value::Text("entity-a".to_string()));
+                assert_eq!(result.rows[0][0], Value::Json(json!(["entity-a"])));
                 assert_eq!(result.rows[0][1], Value::Json(json!({"value": "A"})));
             })
         });
@@ -3169,7 +3323,7 @@ mod tests {
                 assert_eq!(result.columns, vec!["value", "lixcol_entity_id"]);
                 assert_eq!(result.rows.len(), 1);
                 assert_eq!(result.rows[0][0], Value::Text("A".to_string()));
-                assert_eq!(result.rows[0][1], Value::Text("entity-a".to_string()));
+                assert_eq!(result.rows[0][1], Value::Json(json!(["entity-a"])));
             })
         });
     }