@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +4 -5
- package/dist/engine-wasm/wasm/lix_engine.js +1 -1
- package/dist/engine-wasm/wasm/lix_engine.wasm +0 -0
- package/dist/generated/builtin-schemas.d.ts +87 -162
- package/dist/generated/builtin-schemas.js +139 -236
- package/dist/open-lix.d.ts +1 -1
- package/dist-engine-src/src/binary_cas/types.rs +0 -6
- package/dist-engine-src/src/catalog/context.rs +412 -0
- package/dist-engine-src/src/catalog/mod.rs +10 -0
- package/dist-engine-src/src/catalog/schema.rs +4 -0
- package/dist-engine-src/src/catalog/snapshot.rs +1114 -0
- package/dist-engine-src/src/cel/mod.rs +1 -1
- package/dist-engine-src/src/cel/provider.rs +1 -1
- package/dist-engine-src/src/commit_graph/context.rs +328 -1015
- package/dist-engine-src/src/commit_graph/mod.rs +2 -3
- package/dist-engine-src/src/commit_graph/types.rs +7 -43
- package/dist-engine-src/src/commit_graph/walker.rs +57 -81
- package/dist-engine-src/src/commit_store/codec.rs +887 -0
- package/dist-engine-src/src/commit_store/context.rs +944 -0
- package/dist-engine-src/src/commit_store/materialization.rs +84 -0
- package/dist-engine-src/src/commit_store/mod.rs +16 -0
- package/dist-engine-src/src/commit_store/storage.rs +600 -0
- package/dist-engine-src/src/commit_store/types.rs +215 -0
- package/dist-engine-src/src/common/identity.rs +15 -5
- package/dist-engine-src/src/common/json_pointer.rs +67 -0
- package/dist-engine-src/src/common/metadata.rs +17 -12
- package/dist-engine-src/src/common/mod.rs +5 -5
- package/dist-engine-src/src/domain.rs +324 -0
- package/dist-engine-src/src/engine.rs +29 -43
- package/dist-engine-src/src/entity_identity.rs +238 -118
- package/dist-engine-src/src/functions/context.rs +17 -52
- package/dist-engine-src/src/functions/deterministic.rs +1 -1
- package/dist-engine-src/src/functions/mod.rs +1 -1
- package/dist-engine-src/src/functions/provider.rs +4 -4
- package/dist-engine-src/src/functions/state.rs +39 -66
- package/dist-engine-src/src/functions/types.rs +1 -1
- package/dist-engine-src/src/init.rs +204 -151
- package/dist-engine-src/src/json_store/context.rs +354 -60
- package/dist-engine-src/src/json_store/encoded.rs +6 -6
- package/dist-engine-src/src/json_store/mod.rs +4 -1
- package/dist-engine-src/src/json_store/store.rs +884 -11
- package/dist-engine-src/src/json_store/types.rs +166 -1
- package/dist-engine-src/src/lib.rs +10 -9
- package/dist-engine-src/src/live_state/context.rs +608 -830
- package/dist-engine-src/src/live_state/mod.rs +3 -3
- package/dist-engine-src/src/live_state/overlay.rs +7 -7
- package/dist-engine-src/src/live_state/reader.rs +5 -5
- package/dist-engine-src/src/live_state/types.rs +19 -36
- package/dist-engine-src/src/live_state/visibility.rs +19 -14
- package/dist-engine-src/src/plugin/archive.rs +3 -6
- package/dist-engine-src/src/plugin/install.rs +0 -18
- package/dist-engine-src/src/plugin/plugin_manifest.json +0 -1
- package/dist-engine-src/src/schema/annotations/defaults.rs +2 -7
- package/dist-engine-src/src/schema/builtin/lix_account.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_active_account.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_binary_blob_ref.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_change.json +11 -10
- package/dist-engine-src/src/schema/builtin/lix_change_author.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_commit.json +8 -46
- package/dist-engine-src/src/schema/builtin/lix_commit_edge.json +29 -22
- package/dist-engine-src/src/schema/builtin/lix_directory_descriptor.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_file_descriptor.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_key_value.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_label.json +10 -3
- package/dist-engine-src/src/schema/builtin/lix_label_assignment.json +74 -0
- package/dist-engine-src/src/schema/builtin/lix_registered_schema.json +2 -8
- package/dist-engine-src/src/schema/builtin/lix_version_descriptor.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_version_ref.json +0 -1
- package/dist-engine-src/src/schema/builtin/mod.rs +10 -59
- package/dist-engine-src/src/schema/compatibility.rs +787 -0
- package/dist-engine-src/src/schema/definition.json +47 -17
- package/dist-engine-src/src/schema/definition.rs +202 -96
- package/dist-engine-src/src/schema/key.rs +9 -77
- package/dist-engine-src/src/schema/mod.rs +4 -4
- package/dist-engine-src/src/schema/tests.rs +133 -92
- package/dist-engine-src/src/session/context.rs +40 -42
- package/dist-engine-src/src/session/create_version.rs +22 -14
- package/dist-engine-src/src/session/execute.rs +45 -14
- package/dist-engine-src/src/session/merge/apply.rs +4 -4
- package/dist-engine-src/src/session/merge/conflicts.rs +3 -2
- package/dist-engine-src/src/session/merge/stats.rs +1 -1
- package/dist-engine-src/src/session/merge/version.rs +35 -45
- package/dist-engine-src/src/session/mod.rs +4 -2
- package/dist-engine-src/src/session/optimization9_sql2_bench.rs +100 -0
- package/dist-engine-src/src/session/switch_version.rs +16 -28
- package/dist-engine-src/src/sql2/change_provider.rs +14 -20
- package/dist-engine-src/src/sql2/classify.rs +61 -26
- package/dist-engine-src/src/sql2/context.rs +22 -18
- package/dist-engine-src/src/sql2/directory_history_provider.rs +28 -20
- package/dist-engine-src/src/sql2/directory_provider.rs +131 -83
- package/dist-engine-src/src/sql2/entity_history_provider.rs +10 -14
- package/dist-engine-src/src/sql2/entity_provider.rs +680 -169
- package/dist-engine-src/src/sql2/error.rs +21 -1
- package/dist-engine-src/src/sql2/execute.rs +325 -264
- package/dist-engine-src/src/sql2/file_history_provider.rs +29 -21
- package/dist-engine-src/src/sql2/file_provider.rs +533 -108
- package/dist-engine-src/src/sql2/filesystem_planner.rs +58 -94
- package/dist-engine-src/src/sql2/filesystem_visibility.rs +37 -23
- package/dist-engine-src/src/sql2/history_projection.rs +3 -27
- package/dist-engine-src/src/sql2/history_provider.rs +11 -17
- package/dist-engine-src/src/sql2/history_route.rs +22 -8
- package/dist-engine-src/src/sql2/lix_state_provider.rs +178 -96
- package/dist-engine-src/src/sql2/mod.rs +6 -3
- package/dist-engine-src/src/sql2/predicate_typecheck.rs +246 -0
- package/dist-engine-src/src/sql2/public_bind/assignment.rs +46 -0
- package/dist-engine-src/src/sql2/public_bind/capability.rs +41 -0
- package/dist-engine-src/src/sql2/public_bind/dml.rs +166 -0
- package/dist-engine-src/src/sql2/public_bind/mod.rs +25 -0
- package/dist-engine-src/src/sql2/public_bind/table.rs +168 -0
- package/dist-engine-src/src/sql2/read_only.rs +10 -12
- package/dist-engine-src/src/sql2/session.rs +7 -10
- package/dist-engine-src/src/sql2/udfs/lix_timestamp.rs +76 -0
- package/dist-engine-src/src/sql2/udfs/mod.rs +8 -1
- package/dist-engine-src/src/sql2/udfs/public_call.rs +211 -0
- package/dist-engine-src/src/sql2/version_provider.rs +46 -31
- package/dist-engine-src/src/sql2/version_scope.rs +4 -4
- package/dist-engine-src/src/storage_bench.rs +1782 -325
- package/dist-engine-src/src/test_support.rs +183 -36
- package/dist-engine-src/src/tracked_state/by_file_index.rs +20 -24
- package/dist-engine-src/src/tracked_state/codec.rs +1519 -181
- package/dist-engine-src/src/tracked_state/context.rs +1155 -271
- package/dist-engine-src/src/tracked_state/diff.rs +249 -57
- package/dist-engine-src/src/tracked_state/materialization.rs +365 -103
- package/dist-engine-src/src/tracked_state/materializer.rs +488 -0
- package/dist-engine-src/src/tracked_state/merge.rs +37 -19
- package/dist-engine-src/src/tracked_state/mod.rs +8 -7
- package/dist-engine-src/src/tracked_state/storage.rs +138 -6
- package/dist-engine-src/src/tracked_state/tree.rs +695 -252
- package/dist-engine-src/src/tracked_state/types.rs +176 -6
- package/dist-engine-src/src/transaction/commit.rs +695 -435
- package/dist-engine-src/src/transaction/context.rs +551 -310
- package/dist-engine-src/src/transaction/live_state_overlay.rs +9 -8
- package/dist-engine-src/src/transaction/mod.rs +2 -0
- package/dist-engine-src/src/transaction/normalization.rs +311 -447
- package/dist-engine-src/src/transaction/prep.rs +37 -0
- package/dist-engine-src/src/transaction/schema_resolver.rs +93 -71
- package/dist-engine-src/src/transaction/staging.rs +701 -406
- package/dist-engine-src/src/transaction/types.rs +231 -122
- package/dist-engine-src/src/transaction/validation.rs +2717 -1698
- package/dist-engine-src/src/untracked_state/codec.rs +40 -96
- package/dist-engine-src/src/untracked_state/context.rs +21 -5
- package/dist-engine-src/src/untracked_state/materialization.rs +10 -104
- package/dist-engine-src/src/untracked_state/mod.rs +3 -5
- package/dist-engine-src/src/untracked_state/storage.rs +105 -57
- package/dist-engine-src/src/untracked_state/types.rs +63 -13
- package/dist-engine-src/src/version/context.rs +1 -13
- package/dist-engine-src/src/version/lifecycle.rs +221 -0
- package/dist-engine-src/src/version/mod.rs +3 -2
- package/dist-engine-src/src/version/refs.rs +12 -103
- package/dist-engine-src/src/version/stage_rows.rs +15 -19
- package/package.json +1 -1
- package/dist-engine-src/src/changelog/codec.rs +0 -321
- package/dist-engine-src/src/changelog/context.rs +0 -92
- package/dist-engine-src/src/changelog/materialization.rs +0 -121
- package/dist-engine-src/src/changelog/mod.rs +0 -13
- package/dist-engine-src/src/changelog/reader.rs +0 -20
- package/dist-engine-src/src/changelog/storage.rs +0 -220
- package/dist-engine-src/src/changelog/types.rs +0 -38
- package/dist-engine-src/src/schema/builtin/lix_change_set.json +0 -18
- package/dist-engine-src/src/schema/builtin/lix_change_set_element.json +0 -75
- package/dist-engine-src/src/schema/builtin/lix_entity_label.json +0 -63
- package/dist-engine-src/src/schema_registry.rs +0 -294
- package/dist-engine-src/src/sql2/commit_derived_provider.rs +0 -591
- package/dist-engine-src/src/tracked_state/rebuild.rs +0 -771
- package/dist-engine-src/src/tracked_state/tree_types.rs +0 -176
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
use crate::functions::FunctionContext;
|
|
2
|
+
use crate::session::context::{SessionContext, SessionSqlExecutionContext};
|
|
3
|
+
use crate::sql2::{self, SqlLogicalPlan};
|
|
4
|
+
use crate::storage::StorageReadScope;
|
|
5
|
+
use crate::transaction::open_transaction;
|
|
6
|
+
use crate::{LixError, SqlQueryResult, Value};
|
|
7
|
+
|
|
8
|
+
/// Opaque read plan used by the Optimization 9 SQL2 diagnostic benchmark.
|
|
9
|
+
///
|
|
10
|
+
/// This module is gated behind `storage-benches` and exists only to split SQL2
|
|
11
|
+
/// planning cost from SQL2 execution cost without widening the normal session
|
|
12
|
+
/// API.
|
|
13
|
+
pub struct PreparedReadPlan {
|
|
14
|
+
plan: SqlLogicalPlan,
|
|
15
|
+
read_scope:
|
|
16
|
+
StorageReadScope<Box<dyn crate::storage::StorageReadTransaction + Send + Sync + 'static>>,
|
|
17
|
+
runtime_functions: FunctionContext,
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
pub async fn plan_read_only(session: &SessionContext, sql: &str) -> Result<(), LixError> {
|
|
21
|
+
let prepared = prepare_read_plan(session, sql).await?;
|
|
22
|
+
drop(prepared.plan);
|
|
23
|
+
drop(prepared.runtime_functions);
|
|
24
|
+
prepared.read_scope.rollback().await
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
pub async fn plan_write_only(session: &SessionContext, sql: &str) -> Result<(), LixError> {
|
|
28
|
+
session.ensure_open()?;
|
|
29
|
+
let opened = open_transaction(
|
|
30
|
+
&session.mode,
|
|
31
|
+
session.storage.clone(),
|
|
32
|
+
std::sync::Arc::clone(&session.live_state),
|
|
33
|
+
std::sync::Arc::clone(&session.tracked_state),
|
|
34
|
+
std::sync::Arc::clone(&session.binary_cas),
|
|
35
|
+
std::sync::Arc::clone(&session.commit_store),
|
|
36
|
+
std::sync::Arc::clone(&session.version_ctx),
|
|
37
|
+
std::sync::Arc::clone(&session.catalog_context),
|
|
38
|
+
)
|
|
39
|
+
.await?;
|
|
40
|
+
let mut transaction = opened.transaction;
|
|
41
|
+
let runtime_functions = opened.runtime_functions;
|
|
42
|
+
let plan = sql2::create_write_logical_plan(&mut transaction, sql).await?;
|
|
43
|
+
drop(plan);
|
|
44
|
+
drop(runtime_functions);
|
|
45
|
+
transaction.rollback().await
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
pub async fn prepare_read_plan(
|
|
49
|
+
session: &SessionContext,
|
|
50
|
+
sql: &str,
|
|
51
|
+
) -> Result<PreparedReadPlan, LixError> {
|
|
52
|
+
session.ensure_open()?;
|
|
53
|
+
let read_scope = StorageReadScope::new(session.storage.begin_read_transaction().await?);
|
|
54
|
+
let mut read_store = read_scope.store();
|
|
55
|
+
let live_state: std::sync::Arc<dyn crate::live_state::LiveStateReader> =
|
|
56
|
+
std::sync::Arc::new(session.live_state.reader(read_store.clone()));
|
|
57
|
+
let runtime_functions = FunctionContext::prepare(live_state.as_ref()).await?;
|
|
58
|
+
let functions = runtime_functions.provider();
|
|
59
|
+
let active_version_id = session
|
|
60
|
+
.active_version_id_from_reader(&mut read_store)
|
|
61
|
+
.await?;
|
|
62
|
+
let visible_schemas = session
|
|
63
|
+
.catalog_context
|
|
64
|
+
.schema_jsons_for_sql_read_planning(live_state.as_ref(), &active_version_id)
|
|
65
|
+
.await?;
|
|
66
|
+
let ctx = SessionSqlExecutionContext {
|
|
67
|
+
active_version_id: &active_version_id,
|
|
68
|
+
read_store,
|
|
69
|
+
live_state: std::sync::Arc::clone(&session.live_state),
|
|
70
|
+
binary_cas: std::sync::Arc::clone(&session.binary_cas),
|
|
71
|
+
commit_store: std::sync::Arc::clone(&session.commit_store),
|
|
72
|
+
version_ctx: std::sync::Arc::clone(&session.version_ctx),
|
|
73
|
+
visible_schemas,
|
|
74
|
+
functions: functions.clone(),
|
|
75
|
+
};
|
|
76
|
+
let plan = sql2::create_logical_plan(&ctx, sql).await?;
|
|
77
|
+
drop(ctx);
|
|
78
|
+
drop(live_state);
|
|
79
|
+
|
|
80
|
+
Ok(PreparedReadPlan {
|
|
81
|
+
plan,
|
|
82
|
+
read_scope,
|
|
83
|
+
runtime_functions,
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
pub async fn execute_read_plan(
|
|
88
|
+
prepared: PreparedReadPlan,
|
|
89
|
+
params: &[Value],
|
|
90
|
+
) -> Result<SqlQueryResult, LixError> {
|
|
91
|
+
let PreparedReadPlan {
|
|
92
|
+
plan,
|
|
93
|
+
read_scope,
|
|
94
|
+
runtime_functions,
|
|
95
|
+
} = prepared;
|
|
96
|
+
let result = sql2::execute_logical_plan(plan, params).await;
|
|
97
|
+
read_scope.rollback().await?;
|
|
98
|
+
drop(runtime_functions);
|
|
99
|
+
result
|
|
100
|
+
}
|
|
@@ -2,15 +2,14 @@ use std::sync::Arc;
|
|
|
2
2
|
|
|
3
3
|
use serde_json::json;
|
|
4
4
|
|
|
5
|
-
use crate::transaction::types::
|
|
6
|
-
use crate::version::
|
|
5
|
+
use crate::transaction::types::{TransactionJson, TransactionWriteRow};
|
|
6
|
+
use crate::version::{VersionLifecycle, VersionOperation, VersionReferenceRole};
|
|
7
7
|
use crate::LixError;
|
|
8
8
|
use crate::GLOBAL_VERSION_ID;
|
|
9
9
|
|
|
10
10
|
use super::context::{SessionContext, SessionMode, WORKSPACE_VERSION_KEY};
|
|
11
11
|
|
|
12
12
|
const KEY_VALUE_SCHEMA_KEY: &str = "lix_key_value";
|
|
13
|
-
const KEY_VALUE_SCHEMA_VERSION: &str = "1";
|
|
14
13
|
|
|
15
14
|
/// Options for switching a session to another version.
|
|
16
15
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
@@ -40,17 +39,16 @@ impl SessionContext {
|
|
|
40
39
|
let next_mode = self
|
|
41
40
|
.with_write_transaction(|transaction| {
|
|
42
41
|
Box::pin(async move {
|
|
43
|
-
|
|
42
|
+
{
|
|
44
43
|
let reader = transaction.version_ref_reader();
|
|
45
|
-
|
|
44
|
+
VersionLifecycle::new(&reader)
|
|
45
|
+
.require_existing_commit_id(
|
|
46
|
+
&version_id,
|
|
47
|
+
VersionOperation::SwitchVersion,
|
|
48
|
+
VersionReferenceRole::Target,
|
|
49
|
+
)
|
|
50
|
+
.await?
|
|
46
51
|
};
|
|
47
|
-
if head.is_none() {
|
|
48
|
-
return Err(LixError::version_not_found(
|
|
49
|
-
version_id.clone(),
|
|
50
|
-
"switch_version",
|
|
51
|
-
"target",
|
|
52
|
-
));
|
|
53
|
-
}
|
|
54
52
|
|
|
55
53
|
match current_mode {
|
|
56
54
|
SessionMode::Pinned { .. } => Ok(SessionMode::Pinned {
|
|
@@ -73,9 +71,9 @@ impl SessionContext {
|
|
|
73
71
|
Arc::clone(&self.live_state),
|
|
74
72
|
Arc::clone(&self.tracked_state),
|
|
75
73
|
Arc::clone(&self.binary_cas),
|
|
76
|
-
Arc::clone(&self.
|
|
74
|
+
Arc::clone(&self.commit_store),
|
|
77
75
|
Arc::clone(&self.version_ctx),
|
|
78
|
-
Arc::clone(&self.
|
|
76
|
+
Arc::clone(&self.catalog_context),
|
|
79
77
|
self.closed_flag(),
|
|
80
78
|
);
|
|
81
79
|
Ok((
|
|
@@ -87,20 +85,19 @@ impl SessionContext {
|
|
|
87
85
|
}
|
|
88
86
|
}
|
|
89
87
|
|
|
90
|
-
fn workspace_version_stage_row(version_id: &str) -> Result<
|
|
91
|
-
Ok(
|
|
88
|
+
fn workspace_version_stage_row(version_id: &str) -> Result<TransactionWriteRow, LixError> {
|
|
89
|
+
Ok(TransactionWriteRow {
|
|
92
90
|
entity_id: Some(crate::entity_identity::EntityIdentity::single(
|
|
93
91
|
WORKSPACE_VERSION_KEY,
|
|
94
92
|
)),
|
|
95
93
|
schema_key: KEY_VALUE_SCHEMA_KEY.to_string(),
|
|
96
94
|
file_id: None,
|
|
97
|
-
|
|
95
|
+
snapshot: Some(TransactionJson::from_value_unchecked(json!({
|
|
98
96
|
"key": WORKSPACE_VERSION_KEY,
|
|
99
97
|
"value": version_id,
|
|
100
|
-
}))
|
|
98
|
+
}))),
|
|
101
99
|
metadata: None,
|
|
102
100
|
origin: None,
|
|
103
|
-
schema_version: KEY_VALUE_SCHEMA_VERSION.to_string(),
|
|
104
101
|
created_at: None,
|
|
105
102
|
updated_at: None,
|
|
106
103
|
global: true,
|
|
@@ -110,12 +107,3 @@ fn workspace_version_stage_row(version_id: &str) -> Result<StageRow, LixError> {
|
|
|
110
107
|
version_id: GLOBAL_VERSION_ID.to_string(),
|
|
111
108
|
})
|
|
112
109
|
}
|
|
113
|
-
|
|
114
|
-
fn encode_snapshot(value: serde_json::Value) -> Result<String, LixError> {
|
|
115
|
-
serde_json::to_string(&value).map_err(|error| {
|
|
116
|
-
LixError::new(
|
|
117
|
-
"LIX_ERROR_UNKNOWN",
|
|
118
|
-
format!("engine2 switch_version snapshot serialization failed: {error}"),
|
|
119
|
-
)
|
|
120
|
-
})
|
|
121
|
-
}
|
|
@@ -18,17 +18,18 @@ use datafusion::physical_plan::{
|
|
|
18
18
|
};
|
|
19
19
|
use futures_util::stream;
|
|
20
20
|
|
|
21
|
-
use crate::
|
|
21
|
+
use crate::commit_store::ChangeScanRequest;
|
|
22
22
|
use crate::serialize_row_metadata;
|
|
23
23
|
use crate::LixError;
|
|
24
24
|
|
|
25
25
|
use super::record_batch::record_batch_with_row_count;
|
|
26
26
|
use super::result_metadata::json_field;
|
|
27
|
-
use super::
|
|
27
|
+
use super::SqlCommitStoreQuerySource;
|
|
28
|
+
use crate::commit_store::{materialize_change, MaterializedChange};
|
|
28
29
|
|
|
29
30
|
pub(crate) async fn register_lix_change_provider(
|
|
30
31
|
session: &datafusion::prelude::SessionContext,
|
|
31
|
-
query_source:
|
|
32
|
+
query_source: SqlCommitStoreQuerySource,
|
|
32
33
|
) -> Result<(), LixError> {
|
|
33
34
|
session
|
|
34
35
|
.register_table("lix_change", Arc::new(LixChangeProvider::new(query_source)))
|
|
@@ -38,7 +39,7 @@ pub(crate) async fn register_lix_change_provider(
|
|
|
38
39
|
|
|
39
40
|
struct LixChangeProvider {
|
|
40
41
|
schema: SchemaRef,
|
|
41
|
-
query_source:
|
|
42
|
+
query_source: SqlCommitStoreQuerySource,
|
|
42
43
|
}
|
|
43
44
|
|
|
44
45
|
impl std::fmt::Debug for LixChangeProvider {
|
|
@@ -48,7 +49,7 @@ impl std::fmt::Debug for LixChangeProvider {
|
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
impl LixChangeProvider {
|
|
51
|
-
fn new(query_source:
|
|
52
|
+
fn new(query_source: SqlCommitStoreQuerySource) -> Self {
|
|
52
53
|
Self {
|
|
53
54
|
schema: lix_change_schema(),
|
|
54
55
|
query_source,
|
|
@@ -97,7 +98,7 @@ impl TableProvider for LixChangeProvider {
|
|
|
97
98
|
}
|
|
98
99
|
|
|
99
100
|
struct LixChangeScanExec {
|
|
100
|
-
query_source:
|
|
101
|
+
query_source: SqlCommitStoreQuerySource,
|
|
101
102
|
schema: SchemaRef,
|
|
102
103
|
projection: Option<Vec<usize>>,
|
|
103
104
|
limit: Option<usize>,
|
|
@@ -112,7 +113,7 @@ impl std::fmt::Debug for LixChangeScanExec {
|
|
|
112
113
|
|
|
113
114
|
impl LixChangeScanExec {
|
|
114
115
|
fn new(
|
|
115
|
-
query_source:
|
|
116
|
+
query_source: SqlCommitStoreQuerySource,
|
|
116
117
|
schema: SchemaRef,
|
|
117
118
|
projection: Option<Vec<usize>>,
|
|
118
119
|
limit: Option<usize>,
|
|
@@ -191,8 +192,8 @@ impl ExecutionPlan for LixChangeScanExec {
|
|
|
191
192
|
let stream = stream::once(async move {
|
|
192
193
|
let mut json_reader = query_source.json_reader;
|
|
193
194
|
let canonical_changes = query_source
|
|
194
|
-
.
|
|
195
|
-
.scan_changes(&
|
|
195
|
+
.commit_store_reader
|
|
196
|
+
.scan_changes(&ChangeScanRequest { limit })
|
|
196
197
|
.await
|
|
197
198
|
.map_err(lix_error_to_datafusion_error)?;
|
|
198
199
|
let mut changes = Vec::with_capacity(canonical_changes.len());
|
|
@@ -214,7 +215,6 @@ enum ChangeColumn {
|
|
|
214
215
|
Id,
|
|
215
216
|
EntityId,
|
|
216
217
|
SchemaKey,
|
|
217
|
-
SchemaVersion,
|
|
218
218
|
FileId,
|
|
219
219
|
Metadata,
|
|
220
220
|
CreatedAt,
|
|
@@ -224,9 +224,8 @@ enum ChangeColumn {
|
|
|
224
224
|
fn lix_change_schema() -> SchemaRef {
|
|
225
225
|
Arc::new(Schema::new(vec![
|
|
226
226
|
Field::new("id", DataType::Utf8, false),
|
|
227
|
-
|
|
227
|
+
json_field("entity_id", false),
|
|
228
228
|
Field::new("schema_key", DataType::Utf8, false),
|
|
229
|
-
Field::new("schema_version", DataType::Utf8, false),
|
|
230
229
|
Field::new("file_id", DataType::Utf8, true),
|
|
231
230
|
json_field("metadata", true),
|
|
232
231
|
Field::new("created_at", DataType::Utf8, false),
|
|
@@ -239,7 +238,6 @@ fn change_projection_for_scan(projection: Option<&Vec<usize>>) -> Vec<ChangeColu
|
|
|
239
238
|
ChangeColumn::Id,
|
|
240
239
|
ChangeColumn::EntityId,
|
|
241
240
|
ChangeColumn::SchemaKey,
|
|
242
|
-
ChangeColumn::SchemaVersion,
|
|
243
241
|
ChangeColumn::FileId,
|
|
244
242
|
ChangeColumn::Metadata,
|
|
245
243
|
ChangeColumn::CreatedAt,
|
|
@@ -262,7 +260,7 @@ fn projected_schema(schema: &SchemaRef, projection: Option<&Vec<usize>>) -> Sche
|
|
|
262
260
|
|
|
263
261
|
fn change_record_batch(
|
|
264
262
|
projection: &[ChangeColumn],
|
|
265
|
-
changes: &[
|
|
263
|
+
changes: &[MaterializedChange],
|
|
266
264
|
) -> Result<RecordBatch> {
|
|
267
265
|
let arrays = projection
|
|
268
266
|
.iter()
|
|
@@ -274,7 +272,7 @@ fn change_record_batch(
|
|
|
274
272
|
.map(|row| {
|
|
275
273
|
Some(
|
|
276
274
|
row.entity_id
|
|
277
|
-
.
|
|
275
|
+
.as_json_array_text()
|
|
278
276
|
.expect("canonical change entity identity should project"),
|
|
279
277
|
)
|
|
280
278
|
})
|
|
@@ -283,9 +281,6 @@ fn change_record_batch(
|
|
|
283
281
|
ChangeColumn::SchemaKey => {
|
|
284
282
|
string_array(changes.iter().map(|row| Some(row.schema_key.as_str())))
|
|
285
283
|
}
|
|
286
|
-
ChangeColumn::SchemaVersion => {
|
|
287
|
-
string_array(changes.iter().map(|row| Some(row.schema_version.as_str())))
|
|
288
|
-
}
|
|
289
284
|
ChangeColumn::FileId => string_array(changes.iter().map(|row| row.file_id.as_deref())),
|
|
290
285
|
ChangeColumn::Metadata => Arc::new(StringArray::from(
|
|
291
286
|
changes
|
|
@@ -312,9 +307,8 @@ fn change_schema(projection: &[ChangeColumn]) -> SchemaRef {
|
|
|
312
307
|
.iter()
|
|
313
308
|
.map(|column| match column {
|
|
314
309
|
ChangeColumn::Id => Field::new("id", DataType::Utf8, false),
|
|
315
|
-
ChangeColumn::EntityId =>
|
|
310
|
+
ChangeColumn::EntityId => json_field("entity_id", false),
|
|
316
311
|
ChangeColumn::SchemaKey => Field::new("schema_key", DataType::Utf8, false),
|
|
317
|
-
ChangeColumn::SchemaVersion => Field::new("schema_version", DataType::Utf8, false),
|
|
318
312
|
ChangeColumn::FileId => Field::new("file_id", DataType::Utf8, true),
|
|
319
313
|
ChangeColumn::Metadata => json_field("metadata", true),
|
|
320
314
|
ChangeColumn::CreatedAt => Field::new("created_at", DataType::Utf8, false),
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
use datafusion::sql::parser::Statement as DataFusionStatement;
|
|
1
2
|
use datafusion::sql::sqlparser::ast::{
|
|
2
|
-
FromTable, ObjectName, Query, SetExpr, Statement, TableFactor, TableObject,
|
|
3
|
+
FromTable, ObjectName, Query, SetExpr, Statement as SqlStatement, TableFactor, TableObject,
|
|
4
|
+
TableWithJoins,
|
|
3
5
|
};
|
|
4
6
|
use datafusion::sql::sqlparser::dialect::GenericDialect;
|
|
5
7
|
use datafusion::sql::sqlparser::parser::Parser;
|
|
@@ -31,17 +33,29 @@ pub(crate) fn validate_supported_statement_ast(sql: &str) -> Result<(), LixError
|
|
|
31
33
|
validate_supported_ast_statement(statement)
|
|
32
34
|
}
|
|
33
35
|
|
|
34
|
-
pub(crate) fn
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
pub(crate) fn validate_supported_datafusion_statement_ast(
|
|
37
|
+
statement: &DataFusionStatement,
|
|
38
|
+
) -> Result<(), LixError> {
|
|
39
|
+
match statement {
|
|
40
|
+
DataFusionStatement::Statement(statement) => validate_supported_ast_statement(statement),
|
|
41
|
+
DataFusionStatement::Explain(explain) => {
|
|
42
|
+
validate_supported_datafusion_statement_ast(explain.statement.as_ref())
|
|
43
|
+
}
|
|
44
|
+
_ => Err(unsupported_sql_error(format!(
|
|
45
|
+
"SQL statement is not supported by Lix SQL: {statement}"
|
|
46
|
+
))),
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
pub(crate) fn datafusion_statement_dml_target_table_names(
|
|
51
|
+
statement: &DataFusionStatement,
|
|
52
|
+
) -> Vec<String> {
|
|
39
53
|
let mut targets = Vec::new();
|
|
40
|
-
|
|
41
|
-
|
|
54
|
+
collect_datafusion_statement_dml_target_table_names(statement, &mut targets);
|
|
55
|
+
targets
|
|
42
56
|
}
|
|
43
57
|
|
|
44
|
-
fn parse_sql_statements(sql: &str) -> Result<Vec<
|
|
58
|
+
fn parse_sql_statements(sql: &str) -> Result<Vec<SqlStatement>, LixError> {
|
|
45
59
|
Parser::parse_sql(&GenericDialect {}, sql).map_err(|error| {
|
|
46
60
|
LixError::new(
|
|
47
61
|
LixError::CODE_PARSE_ERROR,
|
|
@@ -50,19 +64,37 @@ fn parse_sql_statements(sql: &str) -> Result<Vec<Statement>, LixError> {
|
|
|
50
64
|
})
|
|
51
65
|
}
|
|
52
66
|
|
|
53
|
-
fn
|
|
67
|
+
fn collect_datafusion_statement_dml_target_table_names(
|
|
68
|
+
statement: &DataFusionStatement,
|
|
69
|
+
targets: &mut Vec<String>,
|
|
70
|
+
) {
|
|
71
|
+
match statement {
|
|
72
|
+
DataFusionStatement::Statement(statement) => {
|
|
73
|
+
collect_dml_target_table_names(statement, targets);
|
|
74
|
+
}
|
|
75
|
+
DataFusionStatement::Explain(explain) => {
|
|
76
|
+
collect_datafusion_statement_dml_target_table_names(
|
|
77
|
+
explain.statement.as_ref(),
|
|
78
|
+
targets,
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
_ => {}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
fn collect_dml_target_table_names(statement: &SqlStatement, targets: &mut Vec<String>) {
|
|
54
86
|
match statement {
|
|
55
|
-
|
|
87
|
+
SqlStatement::Insert(insert) => {
|
|
56
88
|
if let TableObject::TableName(name) = &insert.table {
|
|
57
89
|
if let Some(table_name) = object_name_table_part(name) {
|
|
58
90
|
targets.push(table_name);
|
|
59
91
|
}
|
|
60
92
|
}
|
|
61
93
|
}
|
|
62
|
-
|
|
94
|
+
SqlStatement::Update(update) => {
|
|
63
95
|
collect_table_with_joins_target(&update.table, targets);
|
|
64
96
|
}
|
|
65
|
-
|
|
97
|
+
SqlStatement::Delete(delete) => {
|
|
66
98
|
let tables = match &delete.from {
|
|
67
99
|
FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables,
|
|
68
100
|
};
|
|
@@ -70,7 +102,7 @@ fn collect_dml_target_table_names(statement: &Statement, targets: &mut Vec<Strin
|
|
|
70
102
|
collect_table_with_joins_target(table, targets);
|
|
71
103
|
}
|
|
72
104
|
}
|
|
73
|
-
|
|
105
|
+
SqlStatement::Explain { statement, .. } => {
|
|
74
106
|
collect_dml_target_table_names(statement.as_ref(), targets);
|
|
75
107
|
}
|
|
76
108
|
_ => {}
|
|
@@ -86,28 +118,31 @@ fn collect_table_with_joins_target(table: &TableWithJoins, targets: &mut Vec<Str
|
|
|
86
118
|
}
|
|
87
119
|
|
|
88
120
|
fn object_name_table_part(name: &ObjectName) -> Option<String> {
|
|
89
|
-
name.0
|
|
90
|
-
.
|
|
91
|
-
|
|
92
|
-
|
|
121
|
+
name.0.last().and_then(|part| part.as_ident()).map(|ident| {
|
|
122
|
+
if ident.quote_style.is_some() {
|
|
123
|
+
ident.value.clone()
|
|
124
|
+
} else {
|
|
125
|
+
ident.value.to_ascii_lowercase()
|
|
126
|
+
}
|
|
127
|
+
})
|
|
93
128
|
}
|
|
94
129
|
|
|
95
|
-
fn classify_ast_statement(statement: &
|
|
130
|
+
fn classify_ast_statement(statement: &SqlStatement) -> SqlStatementKind {
|
|
96
131
|
match statement {
|
|
97
|
-
|
|
132
|
+
SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => {
|
|
98
133
|
SqlStatementKind::Write
|
|
99
134
|
}
|
|
100
|
-
|
|
101
|
-
|
|
135
|
+
SqlStatement::Query(_) => SqlStatementKind::Read,
|
|
136
|
+
SqlStatement::Explain { statement, .. } => classify_ast_statement(statement.as_ref()),
|
|
102
137
|
_ => SqlStatementKind::Other,
|
|
103
138
|
}
|
|
104
139
|
}
|
|
105
140
|
|
|
106
|
-
fn validate_supported_ast_statement(statement: &
|
|
141
|
+
fn validate_supported_ast_statement(statement: &SqlStatement) -> Result<(), LixError> {
|
|
107
142
|
match statement {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
143
|
+
SqlStatement::Query(query) => validate_supported_query(query),
|
|
144
|
+
SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => Ok(()),
|
|
145
|
+
SqlStatement::Explain { statement, .. } => validate_supported_ast_statement(statement),
|
|
111
146
|
_ => Err(unsupported_sql_error(format!(
|
|
112
147
|
"SQL statement is not supported by Lix SQL: {statement}"
|
|
113
148
|
))),
|
|
@@ -6,26 +6,27 @@ use serde_json::Value as JsonValue;
|
|
|
6
6
|
use tokio::sync::Mutex;
|
|
7
7
|
|
|
8
8
|
use crate::binary_cas::{BlobBytesBatch, BlobDataReader, BlobHash};
|
|
9
|
-
use crate::changelog::ChangelogReader;
|
|
10
9
|
use crate::commit_graph::CommitGraphReader;
|
|
10
|
+
use crate::commit_store::CommitStoreReader;
|
|
11
11
|
use crate::functions::FunctionProviderHandle;
|
|
12
12
|
use crate::json_store::JsonStoreReader;
|
|
13
13
|
use crate::live_state::{
|
|
14
|
-
LiveStateFilter, LiveStateReader,
|
|
14
|
+
LiveStateFilter, LiveStateReader, LiveStateRowRequest, LiveStateScanRequest,
|
|
15
|
+
MaterializedLiveStateRow,
|
|
15
16
|
};
|
|
16
17
|
use crate::storage::{ScopedStorageReader, StorageReadTransaction};
|
|
17
|
-
use crate::transaction::types::{
|
|
18
|
+
use crate::transaction::types::{TransactionWrite, TransactionWriteOutcome};
|
|
18
19
|
use crate::version::{VersionHead, VersionRefReader};
|
|
19
20
|
use crate::LixError;
|
|
20
21
|
|
|
21
22
|
pub(crate) type SqlReadStore =
|
|
22
23
|
ScopedStorageReader<Box<dyn StorageReadTransaction + Send + Sync + 'static>>;
|
|
23
|
-
pub(crate) type
|
|
24
|
+
pub(crate) type SqlCommitStoreQuerySource = CommitStoreQuerySource<SqlReadStore>;
|
|
24
25
|
pub(crate) type SqlJsonReader = JsonStoreReader<ScopedStorageReader<SqlReadStore>>;
|
|
25
26
|
|
|
26
27
|
#[derive(Clone)]
|
|
27
|
-
pub(crate) struct
|
|
28
|
-
pub(crate)
|
|
28
|
+
pub(crate) struct CommitStoreQuerySource<S> {
|
|
29
|
+
pub(crate) commit_store_reader: Arc<CommitStoreReader<ScopedStorageReader<S>>>,
|
|
29
30
|
pub(crate) json_reader: JsonStoreReader<ScopedStorageReader<S>>,
|
|
30
31
|
}
|
|
31
32
|
|
|
@@ -43,7 +44,7 @@ pub(crate) trait SqlExecutionContext {
|
|
|
43
44
|
fn active_version_id(&self) -> &str;
|
|
44
45
|
fn live_state(&self) -> Arc<dyn LiveStateReader>;
|
|
45
46
|
fn functions(&self) -> FunctionProviderHandle;
|
|
46
|
-
fn
|
|
47
|
+
fn commit_store_query_source(&self) -> SqlCommitStoreQuerySource;
|
|
47
48
|
fn commit_graph(&self) -> Box<dyn CommitGraphReader>;
|
|
48
49
|
fn version_ref(&self) -> Arc<dyn VersionRefReader>;
|
|
49
50
|
fn blob_reader(&self) -> Arc<dyn BlobDataReader>;
|
|
@@ -52,9 +53,9 @@ pub(crate) trait SqlExecutionContext {
|
|
|
52
53
|
|
|
53
54
|
/// Write-capable SQL runtime boundary.
|
|
54
55
|
///
|
|
55
|
-
/// Providers that mutate
|
|
56
|
+
/// Providers that mutate engine state should target this shape instead of
|
|
56
57
|
/// reaching through session/backend escape hatches. The request and write
|
|
57
|
-
/// payloads stay in the existing
|
|
58
|
+
/// payloads stay in the existing engine forms so this boundary centralizes
|
|
58
59
|
/// authority without adding another translation layer.
|
|
59
60
|
#[async_trait]
|
|
60
61
|
#[allow(dead_code)]
|
|
@@ -68,11 +69,14 @@ pub(crate) trait SqlWriteExecutionContext {
|
|
|
68
69
|
async fn scan_live_state(
|
|
69
70
|
&mut self,
|
|
70
71
|
request: &LiveStateScanRequest,
|
|
71
|
-
) -> Result<Vec<
|
|
72
|
+
) -> Result<Vec<MaterializedLiveStateRow>, LixError>;
|
|
72
73
|
|
|
73
74
|
async fn load_version_head(&mut self, version_id: &str) -> Result<Option<String>, LixError>;
|
|
74
75
|
|
|
75
|
-
async fn stage_write(
|
|
76
|
+
async fn stage_write(
|
|
77
|
+
&mut self,
|
|
78
|
+
write: TransactionWrite,
|
|
79
|
+
) -> Result<TransactionWriteOutcome, LixError>;
|
|
76
80
|
}
|
|
77
81
|
|
|
78
82
|
#[derive(Clone)]
|
|
@@ -123,7 +127,7 @@ impl SqlWriteContext {
|
|
|
123
127
|
pub(crate) async fn scan_live_state(
|
|
124
128
|
&self,
|
|
125
129
|
request: &LiveStateScanRequest,
|
|
126
|
-
) -> Result<Vec<
|
|
130
|
+
) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
|
|
127
131
|
let _guard = self.gate.lock().await;
|
|
128
132
|
unsafe {
|
|
129
133
|
self.ptr
|
|
@@ -170,8 +174,8 @@ impl SqlWriteContext {
|
|
|
170
174
|
|
|
171
175
|
pub(crate) async fn stage_write(
|
|
172
176
|
&self,
|
|
173
|
-
write:
|
|
174
|
-
) -> Result<
|
|
177
|
+
write: TransactionWrite,
|
|
178
|
+
) -> Result<TransactionWriteOutcome, LixError> {
|
|
175
179
|
let _guard = self.gate.lock().await;
|
|
176
180
|
unsafe {
|
|
177
181
|
self.ptr
|
|
@@ -219,12 +223,12 @@ impl WriteAccess {
|
|
|
219
223
|
|
|
220
224
|
pub(crate) fn require_write(
|
|
221
225
|
&self,
|
|
222
|
-
|
|
226
|
+
action: &str,
|
|
223
227
|
) -> Result<SqlWriteContext, datafusion::error::DataFusionError> {
|
|
224
228
|
match self {
|
|
225
229
|
Self::Write { ctx } => Ok(ctx.clone()),
|
|
226
230
|
Self::ReadOnly => Err(datafusion::error::DataFusionError::Execution(format!(
|
|
227
|
-
"{
|
|
231
|
+
"{action} requires a write transaction"
|
|
228
232
|
))),
|
|
229
233
|
}
|
|
230
234
|
}
|
|
@@ -249,14 +253,14 @@ impl LiveStateReader for WriteContextLiveStateReader {
|
|
|
249
253
|
async fn scan_rows(
|
|
250
254
|
&self,
|
|
251
255
|
request: &LiveStateScanRequest,
|
|
252
|
-
) -> Result<Vec<
|
|
256
|
+
) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
|
|
253
257
|
self.ctx.scan_live_state(request).await
|
|
254
258
|
}
|
|
255
259
|
|
|
256
260
|
async fn load_row(
|
|
257
261
|
&self,
|
|
258
262
|
request: &LiveStateRowRequest,
|
|
259
|
-
) -> Result<Option<
|
|
263
|
+
) -> Result<Option<MaterializedLiveStateRow>, LixError> {
|
|
260
264
|
let mut rows = self
|
|
261
265
|
.ctx
|
|
262
266
|
.scan_live_state(&LiveStateScanRequest {
|