@lix-js/sdk 0.6.0-preview.3 → 0.6.0-preview.5

package/dist-engine-src/src/sql2/{file_history_provider.rs → providers/file_history.rs}

@@ -26,31 +26,36 @@ use crate::commit_graph::CommitGraphReader;
 use crate::serialize_row_metadata;
 use crate::LixError;
 
-use super::history_projection::{tombstone_identity_column_value, HistoryIdentityProjection};
-use super::history_route::{
+use crate::sql2::change_materialization::MaterializedChange;
+use crate::sql2::history_projection::{tombstone_identity_column_value, HistoryIdentityProjection};
+use crate::sql2::history_route::{
     history_descriptor_event_matches, load_history_entries, parse_history_filter,
     HistoryColumnStyle, HistoryEntry, HistoryRoute, HistoryViewDescriptor, HISTORY_COL_CHANGE_ID,
-    HISTORY_COL_COMMIT_CREATED_AT, HISTORY_COL_DEPTH, HISTORY_COL_ENTITY_ID, HISTORY_COL_FILE_ID,
+    HISTORY_COL_COMMIT_CREATED_AT, HISTORY_COL_DEPTH, HISTORY_COL_ENTITY_PK, HISTORY_COL_FILE_ID,
     HISTORY_COL_METADATA, HISTORY_COL_OBSERVED_COMMIT_ID, HISTORY_COL_SCHEMA_KEY,
     HISTORY_COL_SNAPSHOT_CONTENT, HISTORY_COL_START_COMMIT_ID,
 };
-use super::result_metadata::json_field;
-use super::SqlCommitStoreQuerySource;
-use crate::commit_store::MaterializedChange;
+use crate::sql2::result_metadata::json_field;
+use crate::sql2::SqlHistoryQuerySource;
+use crate::storage::StorageRead;
 
 const FILE_DESCRIPTOR_SCHEMA_KEY: &str = "lix_file_descriptor";
 const DIRECTORY_DESCRIPTOR_SCHEMA_KEY: &str = "lix_directory_descriptor";
 const BLOB_REF_SCHEMA_KEY: &str = "lix_binary_blob_ref";
 
-pub(crate) async fn register_lix_file_history_provider(
+pub(super) async fn register_lix_file_history_surface<S>(
     session: &datafusion::prelude::SessionContext,
+    surface_name: &str,
     commit_graph: Box<dyn CommitGraphReader>,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlHistoryQuerySource<S>,
     blob_reader: Arc<dyn BlobDataReader>,
-) -> Result<(), LixError> {
+) -> Result<(), LixError>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     session
         .register_table(
-            "lix_file_history",
+            surface_name,
             Arc::new(LixFileHistoryProvider::new(
                 Arc::new(Mutex::new(commit_graph)),
                 query_source,
@@ -61,23 +66,23 @@ pub(crate) async fn register_lix_file_history_provider(
     Ok(())
 }
 
-struct LixFileHistoryProvider {
+struct LixFileHistoryProvider<S> {
     schema: SchemaRef,
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlHistoryQuerySource<S>,
     blob_reader: Arc<dyn BlobDataReader>,
 }
 
-impl std::fmt::Debug for LixFileHistoryProvider {
+impl<S> std::fmt::Debug for LixFileHistoryProvider<S> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LixFileHistoryProvider").finish()
     }
 }
 
-impl LixFileHistoryProvider {
+impl<S> LixFileHistoryProvider<S> {
     fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlCommitStoreQuerySource,
+        query_source: SqlHistoryQuerySource<S>,
         blob_reader: Arc<dyn BlobDataReader>,
     ) -> Self {
         Self {
@@ -90,7 +95,10 @@ impl LixFileHistoryProvider {
 }
 
 #[async_trait]
-impl TableProvider for LixFileHistoryProvider {
+impl<S> TableProvider for LixFileHistoryProvider<S>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -148,9 +156,9 @@ impl TableProvider for LixFileHistoryProvider {
     }
 }
 
-struct LixFileHistoryScanExec {
+struct LixFileHistoryScanExec<S> {
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlHistoryQuerySource<S>,
     blob_reader: Arc<dyn BlobDataReader>,
     schema: SchemaRef,
     needs_data: bool,
@@ -159,7 +167,7 @@ struct LixFileHistoryScanExec {
     properties: Arc<PlanProperties>,
 }
 
-impl std::fmt::Debug for LixFileHistoryScanExec {
+impl<S> std::fmt::Debug for LixFileHistoryScanExec<S> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LixFileHistoryScanExec")
             .field("route", &self.route)
@@ -168,10 +176,10 @@ impl std::fmt::Debug for LixFileHistoryScanExec {
     }
 }
 
-impl LixFileHistoryScanExec {
+impl<S> LixFileHistoryScanExec<S> {
     fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlCommitStoreQuerySource,
+        query_source: SqlHistoryQuerySource<S>,
         blob_reader: Arc<dyn BlobDataReader>,
         schema: SchemaRef,
         needs_data: bool,
@@ -197,7 +205,7 @@ impl LixFileHistoryScanExec {
     }
 }
 
-impl DisplayAs for LixFileHistoryScanExec {
+impl<S> DisplayAs for LixFileHistoryScanExec<S> {
     fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => write!(
@@ -210,7 +218,10 @@ impl DisplayAs for LixFileHistoryScanExec {
     }
 }
 
-impl ExecutionPlan for LixFileHistoryScanExec {
+impl<S> ExecutionPlan for LixFileHistoryScanExec<S>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     fn name(&self) -> &str {
         "LixFileHistoryScanExec"
     }
@@ -319,7 +330,7 @@ struct FileHistoryEvent {
 
 #[derive(Debug, Clone)]
 struct FileHistoryOutputRow {
-    entity_id: String,
+    entity_pk: String,
     id: String,
     path: Option<String>,
     directory_id: Option<String>,
@@ -351,13 +362,16 @@ struct BlobRefSnapshot {
     blob_hash: String,
 }
 
-async fn load_file_history_rows(
+async fn load_file_history_rows<S>(
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlHistoryQuerySource<S>,
     blob_reader: &Arc<dyn BlobDataReader>,
     route: &HistoryRoute,
     needs_data: bool,
-) -> Result<Vec<FileHistoryOutputRow>, LixError> {
+) -> Result<Vec<FileHistoryOutputRow>, LixError>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     let event_route = route.traversal_only();
     let event_entries = load_history_entries(
         HistoryViewDescriptor {
@@ -428,7 +442,7 @@ async fn load_file_history_rows(
         .unwrap_or_else(|| descriptor.id.clone());
 
         output.push(FileHistoryOutputRow {
-            entity_id: descriptor.id.clone(),
+            entity_pk: descriptor.id.clone(),
             id,
             path,
             directory_id: descriptor.directory_id.clone(),
@@ -440,18 +454,18 @@ async fn load_file_history_rows(
         });
     }
     output.retain(|row| {
-        let entity_id = entity_id_json_array(&row.entity_id).ok();
+        let entity_pk = entity_pk_json_array(&row.entity_pk).ok();
         route.matches_surface_row(
             FILE_DESCRIPTOR_SCHEMA_KEY,
-            entity_id.as_deref().unwrap_or(&row.entity_id),
-            Some(&row.entity_id),
+            entity_pk.as_deref().unwrap_or(&row.entity_pk),
+            Some(&row.entity_pk),
             row.event.depth,
         )
     });
 
     output.sort_by(|left, right| {
-        left.entity_id
-            .cmp(&right.entity_id)
+        left.entity_pk
+            .cmp(&right.entity_pk)
             .then(left.event.start_commit_id.cmp(&right.event.start_commit_id))
             .then(left.event.depth.cmp(&right.event.depth))
             .then(
@@ -575,7 +589,7 @@ fn parse_file_history_descriptors(
         .map(|entry| {
             let Some(snapshot_content) = entry.change.snapshot_content.as_deref() else {
                 return Ok(FileHistoryDescriptorRecord {
-                    id: entry.change.entity_id.as_single_string_owned()?,
+                    id: entry.change.entity_pk.as_single_string_owned()?,
                     directory_id: None,
                     name: None,
                     hidden: None,
@@ -640,9 +654,9 @@ fn parse_file_history_blobs(
                     file_id: entry.change.file_id.clone().unwrap_or_else(|| {
                         entry
                             .change
-                            .entity_id
+                            .entity_pk
                             .as_single_string_owned()
-                            .expect("canonical change entity identity should project")
+                            .expect("canonical change entity primary key should project")
                     }),
                     blob_hash: None,
                     entry: entry.clone(),
@@ -807,15 +821,15 @@ fn file_history_column_array(
                 .map(|row| row.data.as_deref())
                 .collect::<Vec<_>>(),
         )) as ArrayRef,
-        HISTORY_COL_ENTITY_ID => Arc::new(StringArray::from(
+        HISTORY_COL_ENTITY_PK => Arc::new(StringArray::from(
             rows.iter()
-                .map(|row| entity_id_json_array(&row.entity_id).map(Some))
+                .map(|row| entity_pk_json_array(&row.entity_pk).map(Some))
                 .collect::<std::result::Result<Vec<_>, _>>()?,
         )) as ArrayRef,
         HISTORY_COL_SCHEMA_KEY => {
             string_array(rows.iter().map(|_| Some(FILE_DESCRIPTOR_SCHEMA_KEY)))
         }
-        HISTORY_COL_FILE_ID => string_array(rows.iter().map(|row| Some(row.entity_id.as_str()))),
+        HISTORY_COL_FILE_ID => string_array(rows.iter().map(|row| Some(row.entity_pk.as_str()))),
         HISTORY_COL_CHANGE_ID => {
             string_array(rows.iter().map(|row| Some(row.event.change.id.as_str())))
         }
@@ -861,7 +875,7 @@ fn file_history_column_array(
     })
 }
 
-fn lix_file_history_schema() -> SchemaRef {
+pub(super) fn lix_file_history_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![
         Field::new("id", DataType::Utf8, false),
         Field::new("path", DataType::Utf8, true),
@@ -869,7 +883,7 @@ fn lix_file_history_schema() -> SchemaRef {
         Field::new("name", DataType::Utf8, true),
         Field::new("hidden", DataType::Boolean, true),
         Field::new("data", DataType::Binary, true),
-        json_field(HISTORY_COL_ENTITY_ID, false),
+        json_field(HISTORY_COL_ENTITY_PK, false),
         Field::new(HISTORY_COL_SCHEMA_KEY, DataType::Utf8, false),
         Field::new(HISTORY_COL_FILE_ID, DataType::Utf8, true),
         json_field(HISTORY_COL_SNAPSHOT_CONTENT, true),
@@ -894,17 +908,17 @@ fn string_array<'a>(values: impl Iterator<Item = Option<&'a str>>) -> ArrayRef {
 }
 
 fn datafusion_error_to_lix_error(error: DataFusionError) -> LixError {
-    super::error::datafusion_error_to_lix_error(error)
+    crate::sql2::error::datafusion_error_to_lix_error(error)
 }
 
-fn entity_id_json_array(entity_id: &str) -> Result<String, LixError> {
-    serde_json::to_string(&[entity_id]).map_err(|error| {
+fn entity_pk_json_array(entity_pk: &str) -> Result<String, LixError> {
+    serde_json::to_string(&[entity_pk]).map_err(|error| {
         LixError::unknown(format!(
-            "failed to encode history entity id as JSON: {error}"
+            "failed to encode history entity pk as JSON: {error}"
         ))
     })
 }
 
 fn lix_error_to_datafusion_error(error: LixError) -> DataFusionError {
-    super::error::lix_error_to_datafusion_error(error)
+    crate::sql2::error::lix_error_to_datafusion_error(error)
 }

package/dist-engine-src/src/sql2/{history_provider.rs → providers/history.rs}

@@ -23,44 +23,49 @@ use tokio::sync::Mutex;
 use crate::commit_graph::CommitGraphReader;
 use crate::{serialize_row_metadata, LixError};
 
-use super::history_route::{
+use crate::sql2::history_route::{
     load_history_entries, parse_history_filter, HistoryColumnStyle, HistoryRoute,
     HistoryViewDescriptor,
 };
-use super::result_metadata::json_field;
-use super::SqlCommitStoreQuerySource;
+use crate::sql2::result_metadata::json_field;
+use crate::sql2::SqlHistoryQuerySource;
+use crate::storage::StorageRead;
 
-pub(crate) async fn register_history_providers(
+pub(super) async fn register_history_provider<S>(
     session: &SessionContext,
+    surface_name: &str,
     commit_graph: Box<dyn CommitGraphReader>,
-    query_source: SqlCommitStoreQuerySource,
-) -> Result<Arc<dyn TableProvider>, LixError> {
+    query_source: SqlHistoryQuerySource<S>,
+) -> Result<Arc<dyn TableProvider>, LixError>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     let provider: Arc<dyn TableProvider> = Arc::new(LixStateHistoryProvider::new(
         Arc::new(Mutex::new(commit_graph)),
         query_source,
     ));
     session
-        .register_table("lix_state_history", Arc::clone(&provider))
+        .register_table(surface_name, Arc::clone(&provider))
         .map_err(datafusion_error_to_lix_error)?;
     Ok(provider)
 }
 
-pub(crate) struct LixStateHistoryProvider {
+pub(crate) struct LixStateHistoryProvider<S> {
     schema: SchemaRef,
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlHistoryQuerySource<S>,
 }
 
-impl std::fmt::Debug for LixStateHistoryProvider {
+impl<S> std::fmt::Debug for LixStateHistoryProvider<S> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LixStateHistoryProvider").finish()
     }
 }
 
-impl LixStateHistoryProvider {
+impl<S> LixStateHistoryProvider<S> {
     pub(crate) fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlCommitStoreQuerySource,
+        query_source: SqlHistoryQuerySource<S>,
     ) -> Self {
         Self {
             schema: lix_state_history_schema(),
@@ -71,7 +76,10 @@ impl LixStateHistoryProvider {
 }
 
 #[async_trait]
-impl TableProvider for LixStateHistoryProvider {
+impl<S> TableProvider for LixStateHistoryProvider<S>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -119,9 +127,9 @@ impl TableProvider for LixStateHistoryProvider {
     }
 }
 
-struct LixStateHistoryScanExec {
+struct LixStateHistoryScanExec<S> {
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlHistoryQuerySource<S>,
     schema: SchemaRef,
     projection: Option<Vec<usize>>,
     route: HistoryRoute,
@@ -129,7 +137,7 @@ struct LixStateHistoryScanExec {
     properties: Arc<PlanProperties>,
 }
 
-impl std::fmt::Debug for LixStateHistoryScanExec {
+impl<S> std::fmt::Debug for LixStateHistoryScanExec<S> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LixStateHistoryScanExec")
             .field("limit", &self.limit)
@@ -138,10 +146,10 @@ impl std::fmt::Debug for LixStateHistoryScanExec {
     }
 }
 
-impl LixStateHistoryScanExec {
+impl<S> LixStateHistoryScanExec<S> {
     fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlCommitStoreQuerySource,
+        query_source: SqlHistoryQuerySource<S>,
         schema: SchemaRef,
         projection: Option<Vec<usize>>,
         route: HistoryRoute,
@@ -165,7 +173,7 @@ impl LixStateHistoryScanExec {
     }
 }
 
-impl DisplayAs for LixStateHistoryScanExec {
+impl<S> DisplayAs for LixStateHistoryScanExec<S> {
     fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
@@ -180,7 +188,10 @@ impl DisplayAs for LixStateHistoryScanExec {
     }
 }
 
-impl ExecutionPlan for LixStateHistoryScanExec {
+impl<S> ExecutionPlan for LixStateHistoryScanExec<S>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     fn name(&self) -> &str {
         "LixStateHistoryScanExec"
     }
@@ -266,9 +277,9 @@ impl ExecutionPlan for LixStateHistoryScanExec {
     }
 }
 
-fn lix_state_history_schema() -> SchemaRef {
+pub(super) fn lix_state_history_schema() -> SchemaRef {
     Arc::new(Schema::new(vec![
-        json_field("entity_id", false),
+        json_field("entity_pk", false),
         Field::new("schema_key", DataType::Utf8, false),
         Field::new("file_id", DataType::Utf8, true),
         json_field("snapshot_content", true),
@@ -298,7 +309,7 @@ fn projected_schema(base_schema: &SchemaRef, projection: Option<&Vec<usize>>) ->
 
 #[derive(Debug, Clone)]
 struct StateHistorySqlRow {
-    entity_id: String,
+    entity_pk: String,
     schema_key: String,
     file_id: Option<String>,
     snapshot_content: Option<String>,
@@ -319,7 +330,7 @@ fn state_history_record_batch(
         .iter()
         .map(|field| {
             Ok(match field.name().as_str() {
-                "entity_id" => string_array(rows.iter().map(|row| Some(row.entity_id.as_str()))),
+                "entity_pk" => string_array(rows.iter().map(|row| Some(row.entity_pk.as_str()))),
                 "schema_key" => string_array(rows.iter().map(|row| Some(row.schema_key.as_str()))),
                 "file_id" => string_array(rows.iter().map(|row| row.file_id.as_deref())),
                 "snapshot_content" => {
@@ -358,11 +369,14 @@ fn string_array<'a>(values: impl Iterator<Item = Option<&'a str>>) -> ArrayRef {
     Arc::new(StringArray::from(values.collect::<Vec<_>>())) as ArrayRef
 }
 
-async fn load_state_history_rows(
+async fn load_state_history_rows<S>(
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlCommitStoreQuerySource,
+    query_source: SqlHistoryQuerySource<S>,
     route: &HistoryRoute,
-) -> Result<Vec<StateHistorySqlRow>, LixError> {
+) -> Result<Vec<StateHistorySqlRow>, LixError>
+where
+    S: StorageRead + Clone + Send + Sync + 'static,
+{
     let entries = load_history_entries(
         HistoryViewDescriptor {
             view_name: "lix_state_history",
@@ -378,7 +392,7 @@ async fn load_state_history_rows(
         .into_iter()
         .map(|entry| -> Result<StateHistorySqlRow, LixError> {
             Ok(StateHistorySqlRow {
-                entity_id: entry.change.entity_id.as_json_array_text()?,
+                entity_pk: entry.change.entity_pk.as_json_array_text()?,
                 schema_key: entry.change.schema_key,
                 file_id: entry.change.file_id,
                 snapshot_content: entry.change.snapshot_content,
@@ -393,8 +407,8 @@ async fn load_state_history_rows(
         .collect::<Result<Vec<_>, _>>()?;
 
     rows.sort_by(|left, right| {
-        left.entity_id
-            .cmp(&right.entity_id)
+        left.entity_pk
+            .cmp(&right.entity_pk)
             .then(left.file_id.cmp(&right.file_id))
             .then(left.schema_key.cmp(&right.schema_key))
             .then(left.depth.cmp(&right.depth))
@@ -404,9 +418,9 @@ async fn load_state_history_rows(
 }
 
 fn datafusion_error_to_lix_error(error: DataFusionError) -> LixError {
-    super::error::datafusion_error_to_lix_error(error)
+    crate::sql2::error::datafusion_error_to_lix_error(error)
 }
 
 fn lix_error_to_datafusion_error(error: LixError) -> DataFusionError {
-    super::error::lix_error_to_datafusion_error(error)
+    crate::sql2::error::lix_error_to_datafusion_error(error)
 }