@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.3

package/dist-engine-src/src/sql2/history_provider.rs

@@ -21,19 +21,19 @@ use futures_util::{stream, TryStreamExt};
 use tokio::sync::Mutex;
 
 use crate::commit_graph::CommitGraphReader;
-use crate::{serialize_row_metadata, LixError, RowMetadata};
+use crate::{serialize_row_metadata, LixError};
 
 use super::history_route::{
     load_history_entries, parse_history_filter, HistoryColumnStyle, HistoryRoute,
     HistoryViewDescriptor,
 };
 use super::result_metadata::json_field;
-use super::SqlChangelogQuerySource;
+use super::SqlCommitStoreQuerySource;
 
 pub(crate) async fn register_history_providers(
     session: &SessionContext,
     commit_graph: Box<dyn CommitGraphReader>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
 ) -> Result<Arc<dyn TableProvider>, LixError> {
     let provider: Arc<dyn TableProvider> = Arc::new(LixStateHistoryProvider::new(
         Arc::new(Mutex::new(commit_graph)),
@@ -48,7 +48,7 @@ pub(crate) async fn register_history_providers(
 pub(crate) struct LixStateHistoryProvider {
     schema: SchemaRef,
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
 }
 
 impl std::fmt::Debug for LixStateHistoryProvider {
@@ -60,7 +60,7 @@ impl std::fmt::Debug for LixStateHistoryProvider {
 impl LixStateHistoryProvider {
     pub(crate) fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlChangelogQuerySource,
+        query_source: SqlCommitStoreQuerySource,
     ) -> Self {
         Self {
             schema: lix_state_history_schema(),
@@ -121,7 +121,7 @@ impl TableProvider for LixStateHistoryProvider {
 
 struct LixStateHistoryScanExec {
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
     schema: SchemaRef,
     projection: Option<Vec<usize>>,
     route: HistoryRoute,
@@ -141,7 +141,7 @@ impl std::fmt::Debug for LixStateHistoryScanExec {
 impl LixStateHistoryScanExec {
     fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlChangelogQuerySource,
+        query_source: SqlCommitStoreQuerySource,
         schema: SchemaRef,
         projection: Option<Vec<usize>>,
         route: HistoryRoute,
@@ -268,12 +268,11 @@ impl ExecutionPlan for LixStateHistoryScanExec {
 
 fn lix_state_history_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![
-        Field::new("entity_id", DataType::Utf8, false),
+        json_field("entity_id", false),
         Field::new("schema_key", DataType::Utf8, false),
         Field::new("file_id", DataType::Utf8, true),
         json_field("snapshot_content", true),
         json_field("metadata", true),
-        Field::new("schema_version", DataType::Utf8, false),
         Field::new("change_id", DataType::Utf8, false),
         Field::new("observed_commit_id", DataType::Utf8, false),
         Field::new("commit_created_at", DataType::Utf8, false),
@@ -303,8 +302,7 @@ struct StateHistorySqlRow {
     schema_key: String,
     file_id: Option<String>,
     snapshot_content: Option<String>,
-    metadata: Option<RowMetadata>,
-    schema_version: String,
+    metadata: Option<String>,
     change_id: String,
     observed_commit_id: String,
     commit_created_at: String,
@@ -332,9 +330,6 @@ fn state_history_record_batch(
                         .map(|row| row.metadata.as_ref().map(serialize_row_metadata))
                         .collect::<Vec<_>>(),
                 )),
-                "schema_version" => {
-                    string_array(rows.iter().map(|row| Some(row.schema_version.as_str())))
-                }
                 "change_id" => string_array(rows.iter().map(|row| Some(row.change_id.as_str()))),
                 "observed_commit_id" => {
                     string_array(rows.iter().map(|row| Some(row.observed_commit_id.as_str())))
@@ -365,7 +360,7 @@ fn string_array<'a>(values: impl Iterator<Item = Option<&'a str>>) -> ArrayRef {
 
 async fn load_state_history_rows(
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
     route: &HistoryRoute,
 ) -> Result<Vec<StateHistorySqlRow>, LixError> {
     let entries = load_history_entries(
@@ -383,12 +378,11 @@ async fn load_state_history_rows(
         .into_iter()
         .map(|entry| -> Result<StateHistorySqlRow, LixError> {
             Ok(StateHistorySqlRow {
-                entity_id: entry.change.entity_id.as_string()?,
+                entity_id: entry.change.entity_id.as_json_array_text()?,
                 schema_key: entry.change.schema_key,
                 file_id: entry.change.file_id,
                 snapshot_content: entry.change.snapshot_content,
                 metadata: entry.change.metadata,
-                schema_version: entry.change.schema_version,
                 change_id: entry.change.id,
                 observed_commit_id: entry.observed_commit_id,
                 commit_created_at: entry.commit_created_at,

package/dist-engine-src/src/sql2/history_route.rs

@@ -6,12 +6,12 @@ use datafusion::logical_expr::expr::InList;
 use datafusion::logical_expr::{Expr, Operator};
 use tokio::sync::Mutex;
 
-use crate::changelog::{materialize_change, MaterializedCanonicalChange};
 use crate::commit_graph::{CommitGraphChangeHistoryRequest, CommitGraphReader};
 use crate::entity_identity::EntityIdentity;
 use crate::LixError;
 
 use super::SqlJsonReader;
+use crate::commit_store::{materialize_change, MaterializedChange};
 
 /// Shared routing state for commit-shaped history SQL surfaces.
 ///
@@ -133,7 +133,7 @@ impl HistoryRoute {
 /// history surfaces.
 #[derive(Debug, Clone)]
 pub(crate) struct HistoryEntry {
-    pub(crate) change: MaterializedCanonicalChange,
+    pub(crate) change: MaterializedChange,
     pub(crate) observed_commit_id: String,
     pub(crate) commit_created_at: String,
     pub(crate) start_commit_id: String,
@@ -145,7 +145,6 @@ pub(crate) const HISTORY_COL_SCHEMA_KEY: &str = "lixcol_schema_key";
 pub(crate) const HISTORY_COL_FILE_ID: &str = "lixcol_file_id";
 pub(crate) const HISTORY_COL_SNAPSHOT_CONTENT: &str = "lixcol_snapshot_content";
 pub(crate) const HISTORY_COL_METADATA: &str = "lixcol_metadata";
-pub(crate) const HISTORY_COL_SCHEMA_VERSION: &str = "lixcol_schema_version";
 pub(crate) const HISTORY_COL_CHANGE_ID: &str = "lixcol_change_id";
 pub(crate) const HISTORY_COL_OBSERVED_COMMIT_ID: &str = "lixcol_observed_commit_id";
 pub(crate) const HISTORY_COL_COMMIT_CREATED_AT: &str = "lixcol_commit_created_at";
@@ -191,7 +190,7 @@ pub(crate) fn commit_graph_history_request(
         entity_ids: route
             .entity_ids
             .iter()
-            .map(|entity_id| EntityIdentity::single(entity_id))
+            .filter_map(|entity_id| EntityIdentity::from_json_array_text(entity_id).ok())
             .collect(),
         schema_keys,
         file_ids: route.file_ids.clone(),
@@ -253,7 +252,7 @@ pub(crate) async fn load_history_entries(
             .collect::<BTreeMap<_, _>>();
 
         for entry in entries {
-            let change = materialize_change(&mut json_reader, entry.change).await?;
+            let change = materialize_change(&mut json_reader, entry.located_change).await?;
             rows.push(HistoryEntry {
                 commit_created_at: commit_created_at_by_id
                     .get(&entry.observed_commit_id)
@@ -411,17 +410,18 @@ fn parse_history_binary_filter(
     let right = &*binary_expr.right;
     match (column_name, &binary_expr.op, right) {
         ("start_commit_id", Operator::Eq, Expr::Literal(ScalarValue::Utf8(Some(value)), _))
-        | ("entity_id", Operator::Eq, Expr::Literal(ScalarValue::Utf8(Some(value)), _))
         | ("schema_key", Operator::Eq, Expr::Literal(ScalarValue::Utf8(Some(value)), _))
         | ("file_id", Operator::Eq, Expr::Literal(ScalarValue::Utf8(Some(value)), _)) => {
             Some(match column_name {
                 "start_commit_id" => HistoryFilterTerm::StartCommitIds(vec![value.clone()]),
-                "entity_id" => HistoryFilterTerm::EntityIds(vec![value.clone()]),
                 "schema_key" => HistoryFilterTerm::SchemaKeys(vec![value.clone()]),
                 "file_id" => HistoryFilterTerm::FileIds(vec![value.clone()]),
                 _ => unreachable!(),
             })
         }
+        ("entity_id", Operator::Eq, Expr::Literal(ScalarValue::Utf8(Some(value)), _)) => {
+            canonical_entity_id_value(value).map(|value| HistoryFilterTerm::EntityIds(vec![value]))
+        }
         ("depth", Operator::Eq, depth_expr) => {
             scalar_i64_literal(depth_expr).map(HistoryFilterTerm::ExactDepth)
         }
@@ -464,7 +464,7 @@ fn parse_history_in_list_filter(
 
     match column_name {
         "start_commit_id" => Some(HistoryFilterTerm::StartCommitIds(values)),
-        "entity_id" => Some(HistoryFilterTerm::EntityIds(values)),
+        "entity_id" => canonical_entity_id_values(values).map(HistoryFilterTerm::EntityIds),
         "schema_key" => Some(HistoryFilterTerm::SchemaKeys(values)),
         "file_id" => Some(HistoryFilterTerm::FileIds(values)),
         _ => None,
@@ -518,6 +518,20 @@ fn apply_conjunctive_values_filter(bucket: &mut Vec<String>, incoming_values: Ve
     bucket.is_empty()
 }
 
+fn canonical_entity_id_values(values: Vec<String>) -> Option<Vec<String>> {
+    values
+        .into_iter()
+        .map(|value| canonical_entity_id_value(&value))
+        .collect()
+}
+
+fn canonical_entity_id_value(value: &str) -> Option<String> {
+    EntityIdentity::from_json_array_text(value)
+        .ok()?
+        .as_json_array_text()
+        .ok()
+}
+
 fn canonical_history_column_name(name: &str, column_style: HistoryColumnStyle) -> Option<&str> {
     match (column_style, name) {
         (HistoryColumnStyle::Bare, "start_commit_id")