@lix-js/sdk 0.6.0-preview.5 → 0.6.1

package/dist-engine-src/src/sql2/change_materialization.rs

@@ -1,122 +0,0 @@
-use crate::changelog::ChangeRecord;
-use crate::entity_pk::EntityPk;
-use crate::json_store::{JsonLoadRequestRef, JsonReadScopeRef, JsonRef, JsonStoreReader};
-use crate::storage::StorageRead;
-use crate::{parse_row_metadata, LixError};
-
-/// Read-boundary view of a changelog change with JSON refs resolved.
-///
-/// `lix_change` materializes direct durable `changelog.change` facts and
-/// derived `lix_commit` changes from `changelog.commit`. History surfaces
-/// materialize reachability-aware commit-graph changes, while traversal context
-/// stays outside this row shape.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) struct MaterializedChange {
-    pub(crate) id: String,
-    pub(crate) entity_pk: EntityPk,
-    pub(crate) schema_key: String,
-    pub(crate) file_id: Option<String>,
-    pub(crate) snapshot_content: Option<String>,
-    pub(crate) metadata: Option<String>,
-    pub(crate) created_at: String,
-}
-
-pub(crate) async fn materialize_located_history_change<S>(
-    json_reader: &mut JsonStoreReader<S>,
-    change: crate::commit_graph::CommitGraphChange,
-) -> Result<MaterializedChange, LixError>
-where
-    S: StorageRead,
-{
-    materialize_commit_graph_change(json_reader, change).await
-}
-
-pub(crate) async fn materialize_changelog_change_record<S>(
-    json_reader: &mut JsonStoreReader<S>,
-    change: ChangeRecord,
-) -> Result<MaterializedChange, LixError>
-where
-    S: StorageRead,
-{
-    materialize_commit_graph_change(
-        json_reader,
-        crate::commit_graph::CommitGraphChange {
-            id: change.change_id,
-            entity_pk: change.entity_pk,
-            schema_key: change.schema_key,
-            file_id: change.file_id,
-            snapshot_ref: change.snapshot_ref,
-            metadata_ref: change.metadata_ref,
-            created_at: change.created_at,
-        },
-    )
-    .await
-}
-
-pub(crate) async fn materialize_commit_graph_change<S>(
-    json_reader: &mut JsonStoreReader<S>,
-    change: crate::commit_graph::CommitGraphChange,
-) -> Result<MaterializedChange, LixError>
-where
-    S: StorageRead,
-{
-    let snapshot_content = load_optional_changelog_json_text(
-        json_reader,
-        change.snapshot_ref.as_ref(),
-        "snapshot_ref",
-    )
-    .await?;
-    let metadata = match load_optional_changelog_json_text(
-        json_reader,
-        change.metadata_ref.as_ref(),
-        "metadata_ref",
-    )
-    .await?
-    {
-        Some(value) => Some(parse_row_metadata(&value, "changelog change metadata_ref")?),
-        None => None,
-    };
-    Ok(MaterializedChange {
-        id: change.id,
-        entity_pk: change.entity_pk,
-        schema_key: change.schema_key,
-        file_id: change.file_id,
-        snapshot_content,
-        metadata,
-        created_at: change.created_at,
-    })
-}
-
-async fn load_optional_changelog_json_text<S>(
-    json_reader: &mut JsonStoreReader<S>,
-    json_ref: Option<&JsonRef>,
-    field: &str,
-) -> Result<Option<String>, LixError>
-where
-    S: StorageRead,
-{
-    let Some(json_ref) = json_ref else {
-        return Ok(None);
-    };
-    let batch = json_reader
-        .load_bytes_many(JsonLoadRequestRef {
-            refs: std::slice::from_ref(json_ref),
-            scope: JsonReadScopeRef::OutOfBand,
-        })
-        .await?;
-    let Some(bytes) = batch.into_values().into_iter().next().flatten() else {
-        return Err(LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            format!(
-                "changelog change {field} '{}' is missing",
-                json_ref.to_hex()
-            ),
-        ));
-    };
-    String::from_utf8(bytes).map(Some).map_err(|error| {
-        LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            format!("changelog change {field} is not UTF-8 JSON: {error}"),
-        )
-    })
-}

package/dist-engine-src/src/sql2/context.rs

@@ -1,317 +0,0 @@
-use std::ptr::NonNull;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use serde_json::Value as JsonValue;
-use tokio::sync::Mutex;
-
-use crate::binary_cas::{BlobBytesBatch, BlobDataReader, BlobHash};
-use crate::branch::{BranchHead, BranchRefReader};
-use crate::commit_graph::CommitGraphReader;
-use crate::functions::FunctionProviderHandle;
-use crate::json_store::JsonStoreReader;
-use crate::live_state::{
-    LiveStateFilter, LiveStateReader, LiveStateRowRequest, LiveStateScanRequest,
-    MaterializedLiveStateRow,
-};
-use crate::storage::StorageRead;
-use crate::transaction::types::{TransactionWrite, TransactionWriteOutcome};
-use crate::LixError;
-
-pub(crate) type SqlChangelogQuerySource<S> = ChangelogQuerySource<S>;
-pub(crate) type SqlHistoryQuerySource<S> = HistoryQuerySource<S>;
-pub(crate) type SqlJsonReader<S> = JsonStoreReader<S>;
-
-#[derive(Clone)]
-pub(crate) struct HistoryQuerySource<S> {
-    pub(crate) json_reader: JsonStoreReader<S>,
-}
-
-#[derive(Clone)]
-pub(crate) struct ChangelogQuerySource<S> {
-    pub(crate) store: S,
-    pub(crate) json_reader: JsonStoreReader<S>,
-}
-
-/// Read-only execution boundary for `sql2::execute_sql(...)`.
-///
-/// Session and transaction orchestration stay above `sql2`. They provide the
-/// execution-scoped committed read context for each call.
-///
-/// This trait is for read SQL session construction. Write SQL should use
-/// `SqlWriteExecutionContext` so transaction-scoped reads and staging stay in
-/// the transaction capability instead of flowing through committed read
-/// sources.
-#[allow(dead_code)]
-pub(crate) trait SqlExecutionContext {
-    type ReadStore: StorageRead + Clone + Send + Sync + 'static;
-
-    fn active_branch_id(&self) -> &str;
-    fn live_state(&self) -> Arc<dyn LiveStateReader>;
-    fn functions(&self) -> FunctionProviderHandle;
-    fn history_query_source(&self) -> SqlHistoryQuerySource<Self::ReadStore>;
-    fn changelog_query_source(&self) -> SqlChangelogQuerySource<Self::ReadStore>;
-    fn commit_graph(&self) -> Box<dyn CommitGraphReader>;
-    fn branch_ref(&self) -> Arc<dyn BranchRefReader>;
-    fn blob_reader(&self) -> Arc<dyn BlobDataReader>;
-    fn list_visible_schemas(&self) -> Result<Vec<JsonValue>, LixError>;
-}
-
-/// Write-capable SQL runtime boundary.
-///
-/// Providers that mutate engine state should target this shape instead of
-/// reaching through session/backend escape hatches. The request and write
-/// payloads stay in the existing engine forms so this boundary centralizes
-/// authority without adding another translation layer.
-#[async_trait]
-#[allow(dead_code)]
-pub(crate) trait SqlWriteExecutionContext {
-    fn active_branch_id(&self) -> &str;
-    fn functions(&self) -> FunctionProviderHandle;
-    fn list_visible_schemas(&self) -> Result<Vec<JsonValue>, LixError>;
-
-    async fn load_bytes_many(&mut self, hashes: &[BlobHash]) -> Result<BlobBytesBatch, LixError>;
-
-    async fn scan_live_state(
-        &mut self,
-        request: &LiveStateScanRequest,
-    ) -> Result<Vec<MaterializedLiveStateRow>, LixError>;
-
-    async fn load_branch_head(&mut self, branch_id: &str) -> Result<Option<String>, LixError>;
-
-    async fn stage_write(
-        &mut self,
-        write: TransactionWrite,
-    ) -> Result<TransactionWriteOutcome, LixError>;
-}
-
-#[derive(Clone)]
-pub(crate) struct SqlWriteContext {
-    ptr: Arc<SqlWriteContextPtr>,
-    gate: Arc<Mutex<()>>,
-}
-
-struct SqlWriteContextPtr(NonNull<dyn SqlWriteExecutionContext>);
-
-// DataFusion stores providers as owned Send + Sync trait objects. This context
-// is only constructed for one write execution and never outlives the borrowed
-// transaction context that owns it.
-unsafe impl Send for SqlWriteContextPtr {}
-unsafe impl Sync for SqlWriteContextPtr {}
-
-impl SqlWriteContext {
-    pub(crate) fn new(ctx: &mut dyn SqlWriteExecutionContext) -> Self {
-        let ptr = NonNull::from(ctx);
-        let ptr = unsafe {
-            std::mem::transmute::<
-                NonNull<dyn SqlWriteExecutionContext + '_>,
-                NonNull<dyn SqlWriteExecutionContext + 'static>,
-            >(ptr)
-        };
-        Self {
-            ptr: Arc::new(SqlWriteContextPtr(ptr)),
-            gate: Arc::new(Mutex::new(())),
-        }
-    }
-
-    pub(crate) fn functions(&self) -> FunctionProviderHandle {
-        unsafe { self.ptr.0.as_ref().functions() }
-    }
-
-    pub(crate) fn blob_reader(&self) -> Arc<dyn BlobDataReader> {
-        Arc::new(WriteContextBlobDataReader::new(self.clone()))
-    }
-
-    pub(crate) fn list_visible_schemas(&self) -> Result<Vec<JsonValue>, LixError> {
-        unsafe { self.ptr.0.as_ref().list_visible_schemas() }
-    }
-
-    pub(crate) fn active_branch_id(&self) -> String {
-        unsafe { self.ptr.0.as_ref().active_branch_id().to_string() }
-    }
-
-    pub(crate) async fn scan_live_state(
-        &self,
-        request: &LiveStateScanRequest,
-    ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
-        let _guard = self.gate.lock().await;
-        unsafe {
-            self.ptr
-                .0
-                .as_ptr()
-                .as_mut()
-                .unwrap()
-                .scan_live_state(request)
-                .await
-        }
-    }
-
-    pub(crate) async fn load_bytes_many(
-        &self,
-        hashes: &[BlobHash],
-    ) -> Result<BlobBytesBatch, LixError> {
-        let _guard = self.gate.lock().await;
-        unsafe {
-            self.ptr
-                .0
-                .as_ptr()
-                .as_mut()
-                .unwrap()
-                .load_bytes_many(hashes)
-                .await
-        }
-    }
-
-    pub(crate) async fn load_branch_head(
-        &self,
-        branch_id: &str,
-    ) -> Result<Option<String>, LixError> {
-        let _guard = self.gate.lock().await;
-        unsafe {
-            self.ptr
-                .0
-                .as_ptr()
-                .as_mut()
-                .unwrap()
-                .load_branch_head(branch_id)
-                .await
-        }
-    }
-
-    pub(crate) async fn stage_write(
-        &self,
-        write: TransactionWrite,
-    ) -> Result<TransactionWriteOutcome, LixError> {
-        let _guard = self.gate.lock().await;
-        unsafe {
-            self.ptr
-                .0
-                .as_ptr()
-                .as_mut()
-                .unwrap()
-                .stage_write(write)
-                .await
-        }
-    }
-}
-
-pub(crate) struct WriteContextBlobDataReader {
-    ctx: SqlWriteContext,
-}
-
-impl WriteContextBlobDataReader {
-    pub(crate) fn new(ctx: SqlWriteContext) -> Self {
-        Self { ctx }
-    }
-}
-
-#[async_trait]
-impl BlobDataReader for WriteContextBlobDataReader {
-    async fn load_bytes_many(&self, hashes: &[BlobHash]) -> Result<BlobBytesBatch, LixError> {
-        self.ctx.load_bytes_many(hashes).await
-    }
-}
-
-#[derive(Clone)]
-pub(crate) enum WriteAccess {
-    ReadOnly,
-    Write { ctx: SqlWriteContext },
-}
-
-impl WriteAccess {
-    pub(crate) fn read_only() -> Self {
-        Self::ReadOnly
-    }
-
-    pub(crate) fn write(ctx: SqlWriteContext) -> Self {
-        Self::Write { ctx }
-    }
-
-    pub(crate) fn require_write(
-        &self,
-        action: &str,
-    ) -> Result<SqlWriteContext, datafusion::error::DataFusionError> {
-        match self {
-            Self::Write { ctx } => Ok(ctx.clone()),
-            Self::ReadOnly => Err(datafusion::error::DataFusionError::Execution(format!(
-                "{action} requires a write transaction"
-            ))),
-        }
-    }
-
-    pub(crate) fn is_write(&self) -> bool {
-        matches!(self, Self::Write { .. })
-    }
-}
-
-pub(crate) struct WriteContextLiveStateReader {
-    ctx: SqlWriteContext,
-}
-
-impl WriteContextLiveStateReader {
-    pub(crate) fn new(ctx: SqlWriteContext) -> Self {
-        Self { ctx }
-    }
-}
-
-#[async_trait]
-impl LiveStateReader for WriteContextLiveStateReader {
-    async fn scan_rows(
-        &self,
-        request: &LiveStateScanRequest,
-    ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
-        self.ctx.scan_live_state(request).await
-    }
-
-    async fn load_row(
-        &self,
-        request: &LiveStateRowRequest,
-    ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
-        let mut rows = self
-            .ctx
-            .scan_live_state(&LiveStateScanRequest {
-                filter: LiveStateFilter {
-                    schema_keys: vec![request.schema_key.clone()],
-                    entity_pks: vec![request.entity_pk.clone()],
-                    branch_ids: vec![request.branch_id.clone()],
-                    file_ids: vec![request.file_id.clone()],
-                    ..LiveStateFilter::default()
-                },
-                projection: Default::default(),
-                limit: Some(1),
-            })
-            .await?;
-        Ok(rows.pop())
-    }
-}
-
-pub(crate) struct WriteContextBranchRefReader {
-    ctx: SqlWriteContext,
-}
-
-impl WriteContextBranchRefReader {
-    pub(crate) fn new(ctx: SqlWriteContext) -> Self {
-        Self { ctx }
-    }
-}
-
-#[async_trait]
-impl BranchRefReader for WriteContextBranchRefReader {
-    async fn load_head(&self, branch_id: &str) -> Result<Option<BranchHead>, LixError> {
-        Ok(self
-            .ctx
-            .load_branch_head(branch_id)
-            .await?
-            .map(|commit_id| BranchHead {
-                branch_id: branch_id.to_string(),
-                commit_id,
-            }))
-    }
-
-    async fn scan_heads(&self) -> Result<Vec<BranchHead>, LixError> {
-        Err(LixError::new(
-            "LIX_ERROR_UNKNOWN",
-            "scan_heads is not available through sql2 write context",
-        ))
-    }
-}

package/dist-engine-src/src/sql2/dml.rs

@@ -1,148 +0,0 @@
-use std::any::Any;
-use std::fmt::Debug;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use datafusion::arrow::array::{ArrayRef, UInt64Array};
-use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::common::{DataFusionError, Result};
-use datafusion::execution::TaskContext;
-use datafusion::physical_expr::EquivalenceProperties;
-use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType, PlanProperties};
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use datafusion::physical_plan::{
-    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream,
-};
-use futures_util::stream;
-
-use super::runtime;
-
-#[async_trait]
-pub(crate) trait InsertSink: Debug + DisplayAs + Send + Sync {
-    async fn write_batches(
-        &self,
-        batches: Vec<RecordBatch>,
-        context: &Arc<TaskContext>,
-    ) -> Result<u64>;
-}
-
-pub(crate) struct InsertExec {
-    input: Arc<dyn ExecutionPlan>,
-    sink: Arc<dyn InsertSink>,
-    result_schema: SchemaRef,
-    properties: Arc<PlanProperties>,
-}
-
-impl InsertExec {
-    pub(crate) fn new(input: Arc<dyn ExecutionPlan>, sink: Arc<dyn InsertSink>) -> Self {
-        let result_schema = dml_count_schema();
-        let properties = PlanProperties::new(
-            EquivalenceProperties::new(Arc::clone(&result_schema)),
-            Partitioning::UnknownPartitioning(1),
-            EmissionType::Final,
-            Boundedness::Bounded,
-        );
-        Self {
-            input,
-            sink,
-            result_schema,
-            properties: Arc::new(properties),
-        }
-    }
-}
-
-impl Debug for InsertExec {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("InsertExec").finish()
-    }
-}
-
-impl DisplayAs for InsertExec {
-    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match t {
-            DisplayFormatType::Default | DisplayFormatType::Verbose => {
-                write!(f, "InsertExec: sink=")?;
-                self.sink.fmt_as(t, f)
-            }
-            DisplayFormatType::TreeRender => write!(f, "InsertExec"),
-        }
-    }
-}
-
-impl ExecutionPlan for InsertExec {
-    fn name(&self) -> &str {
-        "InsertExec"
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn properties(&self) -> &Arc<PlanProperties> {
-        &self.properties
-    }
-
-    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-        vec![&self.input]
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        mut children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        if children.len() != 1 {
-            return Err(DataFusionError::Execution(format!(
-                "InsertExec expects one input child, got {}",
-                children.len()
-            )));
-        }
-        Ok(Arc::new(Self::new(
-            children.swap_remove(0),
-            Arc::clone(&self.sink),
-        )))
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        context: Arc<TaskContext>,
-    ) -> Result<SendableRecordBatchStream> {
-        if partition != 0 {
-            return Err(DataFusionError::Execution(format!(
-                "InsertExec only exposes one partition, got {partition}"
-            )));
-        }
-
-        let input = Arc::clone(&self.input);
-        let sink = Arc::clone(&self.sink);
-        let stream_schema = Arc::clone(&self.result_schema);
-        let result_schema = Arc::clone(&self.result_schema);
-        let stream = stream::once(async move {
-            let batches = runtime::collect_input_plan(input, Arc::clone(&context)).await?;
-            let count = sink.write_batches(batches, &context).await?;
-            dml_count_batch(stream_schema, count)
-        });
-
-        Ok(Box::pin(RecordBatchStreamAdapter::new(
-            result_schema,
-            stream,
-        )))
-    }
-}
-
-fn dml_count_schema() -> SchemaRef {
-    Arc::new(Schema::new(vec![Field::new(
-        "count",
-        DataType::UInt64,
-        false,
-    )]))
-}
-
-fn dml_count_batch(schema: SchemaRef, count: u64) -> Result<RecordBatch> {
-    RecordBatch::try_new(
-        schema,
-        vec![Arc::new(UInt64Array::from(vec![count])) as ArrayRef],
-    )
-    .map_err(DataFusionError::from)
-}