@lix-js/sdk 0.6.0-preview.4 → 0.6.0-preview.5

package/dist-engine-src/src/sql2/{change_provider.rs → providers/change.rs}

@@ -18,38 +18,54 @@ use datafusion::physical_plan::{
 };
 use futures_util::stream;
 
-use crate::commit_store::ChangeScanRequest;
+use crate::changelog::{
+    ChangeRecord, ChangeScanRequest, ChangelogContext, ChangelogReader, CommitLoadEntry,
+    CommitProjection, CommitScanRequest,
+};
 use crate::serialize_row_metadata;
 use crate::LixError;
 
-use super::record_batch::record_batch_with_row_count;
-use super::result_metadata::json_field;
-use super::SqlCommitStoreQuerySource;
-use crate::commit_store::{materialize_change, MaterializedChange};
+use crate::sql2::change_materialization::{
+    materialize_changelog_change_record, materialize_commit_graph_change, MaterializedChange,
+};
+use crate::sql2::record_batch::record_batch_with_row_count;
+use crate::sql2::result_metadata::json_field;
+use crate::sql2::SqlChangelogQuerySource;
+use crate::storage::StorageRead;
 
-pub(crate) async fn register_lix_change_provider(
+pub(super) async fn register_lix_change_read_provider<S>(
     session: &datafusion::prelude::SessionContext,
-    query_source: SqlCommitStoreQuerySource,
-) -> Result<(), LixError> {
+    surface_name: &str,
+    query_source: SqlChangelogQuerySource<S>,
+) -> Result<(), LixError>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     session
-        .register_table("lix_change", Arc::new(LixChangeProvider::new(query_source)))
+        .register_table(surface_name, Arc::new(LixChangeProvider::new(query_source)))
         .map_err(datafusion_error_to_lix_error)?;
     Ok(())
 }
 
-struct LixChangeProvider {
+/// SQL provider for `lix_change`.
+///
+/// `lix_change` is the unscoped durable change surface: it scans direct
+/// `changelog.change` records and unions derived `lix_commit` changes from
+/// `changelog.commit`. It does not prove branch reachability. History
+/// providers are the reachability-aware SQL surfaces.
+struct LixChangeProvider<S> {
     schema: SchemaRef,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlChangelogQuerySource<S>,
 }
 
-impl std::fmt::Debug for LixChangeProvider {
+impl<S> std::fmt::Debug for LixChangeProvider<S> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LixChangeProvider").finish()
     }
 }
 
-impl LixChangeProvider {
-    fn new(query_source: SqlCommitStoreQuerySource) -> Self {
+impl<S> LixChangeProvider<S> {
+    fn new(query_source: SqlChangelogQuerySource<S>) -> Self {
         Self {
             schema: lix_change_schema(),
             query_source,
@@ -58,7 +74,10 @@ impl LixChangeProvider {
 }
 
 #[async_trait]
-impl TableProvider for LixChangeProvider {
+impl<S> TableProvider for LixChangeProvider<S>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -85,35 +104,36 @@ impl TableProvider for LixChangeProvider {
         &self,
         _state: &dyn Session,
         projection: Option<&Vec<usize>>,
-        _filters: &[Expr],
+        filters: &[Expr],
         limit: Option<usize>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
+        let pushed_limit = if filters.is_empty() { limit } else { None };
         Ok(Arc::new(LixChangeScanExec::new(
             self.query_source.clone(),
             projected_schema(&self.schema, projection),
             projection.cloned(),
-            limit,
+            pushed_limit,
         )))
     }
 }
 
-struct LixChangeScanExec {
-    query_source: SqlCommitStoreQuerySource,
+struct LixChangeScanExec<S> {
+    query_source: SqlChangelogQuerySource<S>,
     schema: SchemaRef,
     projection: Option<Vec<usize>>,
     limit: Option<usize>,
     properties: Arc<PlanProperties>,
 }
 
-impl std::fmt::Debug for LixChangeScanExec {
+impl<S> std::fmt::Debug for LixChangeScanExec<S> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LixChangeScanExec").finish()
     }
 }
 
-impl LixChangeScanExec {
+impl<S> LixChangeScanExec<S> {
     fn new(
-        query_source: SqlCommitStoreQuerySource,
+        query_source: SqlChangelogQuerySource<S>,
         schema: SchemaRef,
         projection: Option<Vec<usize>>,
         limit: Option<usize>,
@@ -134,7 +154,7 @@ impl LixChangeScanExec {
     }
 }
 
-impl DisplayAs for LixChangeScanExec {
+impl<S> DisplayAs for LixChangeScanExec<S> {
     fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
@@ -145,7 +165,10 @@ impl DisplayAs for LixChangeScanExec {
     }
 }
 
-impl ExecutionPlan for LixChangeScanExec {
+impl<S> ExecutionPlan for LixChangeScanExec<S>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     fn name(&self) -> &str {
         "LixChangeScanExec"
     }
@@ -191,18 +214,23 @@ impl ExecutionPlan for LixChangeScanExec {
         let schema = Arc::clone(&self.schema);
         let stream = stream::once(async move {
             let mut json_reader = query_source.json_reader;
-            let canonical_changes = query_source
-                .commit_store_reader
-                .scan_changes(&ChangeScanRequest { limit })
+            let canonical_changes = scan_changelog_changes(query_source.store, limit)
                 .await
                 .map_err(lix_error_to_datafusion_error)?;
             let mut changes = Vec::with_capacity(canonical_changes.len());
             for change in canonical_changes {
-                changes.push(
-                    materialize_change(&mut json_reader, change)
-                        .await
-                        .map_err(lix_error_to_datafusion_error)?,
-                );
+                match change {
+                    LixChangeRow::Direct(change) => changes.push(
+                        materialize_changelog_change_record(&mut json_reader, change)
+                            .await
+                            .map_err(lix_error_to_datafusion_error)?,
+                    ),
+                    LixChangeRow::DerivedCommit(change) => changes.push(
+                        materialize_commit_graph_change(&mut json_reader, change)
+                            .await
+                            .map_err(lix_error_to_datafusion_error)?,
+                    ),
+                }
             }
             change_record_batch(&projection, &changes)
         });
@@ -210,10 +238,96 @@ impl ExecutionPlan for LixChangeScanExec {
     }
 }
 
+async fn scan_changelog_changes<S>(
+    store: S,
+    limit: Option<usize>,
+) -> Result<Vec<LixChangeRow>, LixError>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
+    let mut reader = ChangelogContext::new().reader(store);
+    let mut changes = Vec::<LixChangeRow>::new();
+    let mut start_after = None::<String>;
+    loop {
+        let scan = reader
+            .scan_changes(ChangeScanRequest {
+                start_after: start_after.as_deref(),
+                limit: Some(1024),
+            })
+            .await?;
+        changes.extend(scan.entries.into_iter().map(LixChangeRow::Direct));
+        let Some(next) = scan.next_start_after else {
+            break;
+        };
+        start_after = Some(next);
+    }
+    let mut start_after = None::<String>;
+    loop {
+        let scan = reader
+            .scan_commits(CommitScanRequest {
+                start_after: start_after.as_deref(),
+                limit: Some(1024),
+                projection: CommitProjection::Record,
+            })
+            .await?;
+        for entry in scan.entries {
+            let CommitLoadEntry::Record(commit) = entry else {
+                continue;
+            };
+            changes.push(LixChangeRow::DerivedCommit(commit_record_canonical_change(
+                &commit,
+            )));
+        }
+        let Some(next) = scan.next_start_after else {
+            break;
+        };
+        start_after = Some(next);
+    }
+    changes.sort_by(|left, right| left.change_id().cmp(right.change_id()));
+    if let Some(limit) = limit {
+        changes.truncate(limit);
+    }
+    Ok(changes)
+}
+
+enum LixChangeRow {
+    Direct(ChangeRecord),
+    DerivedCommit(crate::commit_graph::CommitGraphChange),
+}
+
+impl LixChangeRow {
+    fn change_id(&self) -> &str {
+        match self {
+            Self::Direct(change) => &change.change_id,
+            Self::DerivedCommit(change) => &change.id,
+        }
+    }
+}
+
+fn commit_record_canonical_change(
+    commit: &crate::changelog::CommitRecord,
+) -> crate::commit_graph::CommitGraphChange {
+    let snapshot_content = serde_json::to_string(&serde_json::json!({
+        "id": commit.commit_id,
+    }))
+    .expect("lix_commit snapshot serialization should not fail");
+    crate::commit_graph::CommitGraphChange {
+        id: commit.change_id.clone(),
+        entity_pk: crate::entity_pk::EntityPk::single(&commit.commit_id),
+        schema_key: "lix_commit".to_string(),
+        file_id: None,
+        snapshot_ref: Some(crate::json_store::JsonRef::for_content(
+            snapshot_content.as_bytes(),
+        )),
+        metadata_ref: None,
+        created_at: commit.created_at.clone(),
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 enum ChangeColumn {
     Id,
-    EntityId,
+    EntityPk,
     SchemaKey,
     FileId,
     Metadata,
@@ -221,10 +335,10 @@ enum ChangeColumn {
     SnapshotContent,
 }
 
-fn lix_change_schema() -> SchemaRef {
+pub(super) fn lix_change_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![
         Field::new("id", DataType::Utf8, false),
-        json_field("entity_id", false),
+        json_field("entity_pk", false),
         Field::new("schema_key", DataType::Utf8, false),
         Field::new("file_id", DataType::Utf8, true),
         json_field("metadata", true),
@@ -236,7 +350,7 @@ fn lix_change_schema() -> SchemaRef {
 fn change_projection_for_scan(projection: Option<&Vec<usize>>) -> Vec<ChangeColumn> {
     let all_columns = vec![
         ChangeColumn::Id,
-        ChangeColumn::EntityId,
+        ChangeColumn::EntityPk,
         ChangeColumn::SchemaKey,
         ChangeColumn::FileId,
         ChangeColumn::Metadata,
@@ -266,14 +380,14 @@ fn change_record_batch(
         .iter()
         .map(|column| match column {
             ChangeColumn::Id => string_array(changes.iter().map(|row| Some(row.id.as_str()))),
-            ChangeColumn::EntityId => Arc::new(StringArray::from(
+            ChangeColumn::EntityPk => Arc::new(StringArray::from(
                 changes
                     .iter()
                     .map(|row| {
                         Some(
-                            row.entity_id
+                            row.entity_pk
                                 .as_json_array_text()
-                                .expect("canonical change entity identity should project"),
+                                .expect("canonical change entity primary key should project"),
                         )
                     })
                     .collect::<Vec<_>>(),
@@ -307,7 +421,7 @@ fn change_schema(projection: &[ChangeColumn]) -> SchemaRef {
             .iter()
             .map(|column| match column {
                 ChangeColumn::Id => Field::new("id", DataType::Utf8, false),
-                ChangeColumn::EntityId => json_field("entity_id", false),
+                ChangeColumn::EntityPk => json_field("entity_pk", false),
                 ChangeColumn::SchemaKey => Field::new("schema_key", DataType::Utf8, false),
                 ChangeColumn::FileId => Field::new("file_id", DataType::Utf8, true),
                 ChangeColumn::Metadata => json_field("metadata", true),
@@ -323,9 +437,9 @@ fn string_array<'a>(values: impl Iterator<Item = Option<&'a str>>) -> ArrayRef {
 }
 
 fn datafusion_error_to_lix_error(error: DataFusionError) -> LixError {
-    super::error::datafusion_error_to_lix_error(error)
+    crate::sql2::error::datafusion_error_to_lix_error(error)
 }
 
 fn lix_error_to_datafusion_error(error: LixError) -> DataFusionError {
-    super::error::lix_error_to_datafusion_error(error)
+    crate::sql2::error::lix_error_to_datafusion_error(error)
 }