@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/SKILL.md +4 -5
  2. package/dist/engine-wasm/wasm/lix_engine.js +1 -1
  3. package/dist/engine-wasm/wasm/lix_engine.wasm +0 -0
  4. package/dist/generated/builtin-schemas.d.ts +87 -162
  5. package/dist/generated/builtin-schemas.js +139 -236
  6. package/dist/open-lix.d.ts +1 -1
  7. package/dist-engine-src/src/binary_cas/types.rs +0 -6
  8. package/dist-engine-src/src/catalog/context.rs +412 -0
  9. package/dist-engine-src/src/catalog/mod.rs +10 -0
  10. package/dist-engine-src/src/catalog/schema.rs +4 -0
  11. package/dist-engine-src/src/catalog/snapshot.rs +1114 -0
  12. package/dist-engine-src/src/cel/mod.rs +1 -1
  13. package/dist-engine-src/src/cel/provider.rs +1 -1
  14. package/dist-engine-src/src/commit_graph/context.rs +328 -1015
  15. package/dist-engine-src/src/commit_graph/mod.rs +2 -3
  16. package/dist-engine-src/src/commit_graph/types.rs +7 -43
  17. package/dist-engine-src/src/commit_graph/walker.rs +57 -81
  18. package/dist-engine-src/src/commit_store/codec.rs +887 -0
  19. package/dist-engine-src/src/commit_store/context.rs +944 -0
  20. package/dist-engine-src/src/commit_store/materialization.rs +84 -0
  21. package/dist-engine-src/src/commit_store/mod.rs +16 -0
  22. package/dist-engine-src/src/commit_store/storage.rs +600 -0
  23. package/dist-engine-src/src/commit_store/types.rs +215 -0
  24. package/dist-engine-src/src/common/identity.rs +15 -5
  25. package/dist-engine-src/src/common/json_pointer.rs +67 -0
  26. package/dist-engine-src/src/common/metadata.rs +17 -12
  27. package/dist-engine-src/src/common/mod.rs +5 -5
  28. package/dist-engine-src/src/domain.rs +324 -0
  29. package/dist-engine-src/src/engine.rs +29 -43
  30. package/dist-engine-src/src/entity_identity.rs +238 -118
  31. package/dist-engine-src/src/functions/context.rs +17 -52
  32. package/dist-engine-src/src/functions/deterministic.rs +1 -1
  33. package/dist-engine-src/src/functions/mod.rs +1 -1
  34. package/dist-engine-src/src/functions/provider.rs +4 -4
  35. package/dist-engine-src/src/functions/state.rs +39 -66
  36. package/dist-engine-src/src/functions/types.rs +1 -1
  37. package/dist-engine-src/src/init.rs +204 -151
  38. package/dist-engine-src/src/json_store/context.rs +354 -60
  39. package/dist-engine-src/src/json_store/encoded.rs +6 -6
  40. package/dist-engine-src/src/json_store/mod.rs +4 -1
  41. package/dist-engine-src/src/json_store/store.rs +884 -11
  42. package/dist-engine-src/src/json_store/types.rs +166 -1
  43. package/dist-engine-src/src/lib.rs +10 -9
  44. package/dist-engine-src/src/live_state/context.rs +608 -830
  45. package/dist-engine-src/src/live_state/mod.rs +3 -3
  46. package/dist-engine-src/src/live_state/overlay.rs +7 -7
  47. package/dist-engine-src/src/live_state/reader.rs +5 -5
  48. package/dist-engine-src/src/live_state/types.rs +19 -36
  49. package/dist-engine-src/src/live_state/visibility.rs +19 -14
  50. package/dist-engine-src/src/plugin/archive.rs +3 -6
  51. package/dist-engine-src/src/plugin/install.rs +0 -18
  52. package/dist-engine-src/src/plugin/plugin_manifest.json +0 -1
  53. package/dist-engine-src/src/schema/annotations/defaults.rs +2 -7
  54. package/dist-engine-src/src/schema/builtin/lix_account.json +0 -1
  55. package/dist-engine-src/src/schema/builtin/lix_active_account.json +0 -1
  56. package/dist-engine-src/src/schema/builtin/lix_binary_blob_ref.json +0 -1
  57. package/dist-engine-src/src/schema/builtin/lix_change.json +11 -10
  58. package/dist-engine-src/src/schema/builtin/lix_change_author.json +0 -1
  59. package/dist-engine-src/src/schema/builtin/lix_commit.json +8 -46
  60. package/dist-engine-src/src/schema/builtin/lix_commit_edge.json +29 -22
  61. package/dist-engine-src/src/schema/builtin/lix_directory_descriptor.json +0 -1
  62. package/dist-engine-src/src/schema/builtin/lix_file_descriptor.json +0 -1
  63. package/dist-engine-src/src/schema/builtin/lix_key_value.json +0 -1
  64. package/dist-engine-src/src/schema/builtin/lix_label.json +10 -3
  65. package/dist-engine-src/src/schema/builtin/lix_label_assignment.json +74 -0
  66. package/dist-engine-src/src/schema/builtin/lix_registered_schema.json +2 -8
  67. package/dist-engine-src/src/schema/builtin/lix_version_descriptor.json +0 -1
  68. package/dist-engine-src/src/schema/builtin/lix_version_ref.json +0 -1
  69. package/dist-engine-src/src/schema/builtin/mod.rs +10 -59
  70. package/dist-engine-src/src/schema/compatibility.rs +787 -0
  71. package/dist-engine-src/src/schema/definition.json +47 -17
  72. package/dist-engine-src/src/schema/definition.rs +202 -96
  73. package/dist-engine-src/src/schema/key.rs +9 -77
  74. package/dist-engine-src/src/schema/mod.rs +4 -4
  75. package/dist-engine-src/src/schema/tests.rs +133 -92
  76. package/dist-engine-src/src/session/context.rs +40 -42
  77. package/dist-engine-src/src/session/create_version.rs +22 -14
  78. package/dist-engine-src/src/session/execute.rs +45 -14
  79. package/dist-engine-src/src/session/merge/apply.rs +4 -4
  80. package/dist-engine-src/src/session/merge/conflicts.rs +3 -2
  81. package/dist-engine-src/src/session/merge/stats.rs +1 -1
  82. package/dist-engine-src/src/session/merge/version.rs +35 -45
  83. package/dist-engine-src/src/session/mod.rs +4 -2
  84. package/dist-engine-src/src/session/optimization9_sql2_bench.rs +100 -0
  85. package/dist-engine-src/src/session/switch_version.rs +16 -28
  86. package/dist-engine-src/src/sql2/change_provider.rs +14 -20
  87. package/dist-engine-src/src/sql2/classify.rs +61 -26
  88. package/dist-engine-src/src/sql2/context.rs +22 -18
  89. package/dist-engine-src/src/sql2/directory_history_provider.rs +28 -20
  90. package/dist-engine-src/src/sql2/directory_provider.rs +131 -83
  91. package/dist-engine-src/src/sql2/entity_history_provider.rs +10 -14
  92. package/dist-engine-src/src/sql2/entity_provider.rs +680 -169
  93. package/dist-engine-src/src/sql2/error.rs +21 -1
  94. package/dist-engine-src/src/sql2/execute.rs +325 -264
  95. package/dist-engine-src/src/sql2/file_history_provider.rs +29 -21
  96. package/dist-engine-src/src/sql2/file_provider.rs +533 -108
  97. package/dist-engine-src/src/sql2/filesystem_planner.rs +58 -94
  98. package/dist-engine-src/src/sql2/filesystem_visibility.rs +37 -23
  99. package/dist-engine-src/src/sql2/history_projection.rs +3 -27
  100. package/dist-engine-src/src/sql2/history_provider.rs +11 -17
  101. package/dist-engine-src/src/sql2/history_route.rs +22 -8
  102. package/dist-engine-src/src/sql2/lix_state_provider.rs +178 -96
  103. package/dist-engine-src/src/sql2/mod.rs +6 -3
  104. package/dist-engine-src/src/sql2/predicate_typecheck.rs +246 -0
  105. package/dist-engine-src/src/sql2/public_bind/assignment.rs +46 -0
  106. package/dist-engine-src/src/sql2/public_bind/capability.rs +41 -0
  107. package/dist-engine-src/src/sql2/public_bind/dml.rs +166 -0
  108. package/dist-engine-src/src/sql2/public_bind/mod.rs +25 -0
  109. package/dist-engine-src/src/sql2/public_bind/table.rs +168 -0
  110. package/dist-engine-src/src/sql2/read_only.rs +10 -12
  111. package/dist-engine-src/src/sql2/session.rs +7 -10
  112. package/dist-engine-src/src/sql2/udfs/lix_timestamp.rs +76 -0
  113. package/dist-engine-src/src/sql2/udfs/mod.rs +8 -1
  114. package/dist-engine-src/src/sql2/udfs/public_call.rs +211 -0
  115. package/dist-engine-src/src/sql2/version_provider.rs +46 -31
  116. package/dist-engine-src/src/sql2/version_scope.rs +4 -4
  117. package/dist-engine-src/src/storage_bench.rs +1782 -325
  118. package/dist-engine-src/src/test_support.rs +183 -36
  119. package/dist-engine-src/src/tracked_state/by_file_index.rs +20 -24
  120. package/dist-engine-src/src/tracked_state/codec.rs +1519 -181
  121. package/dist-engine-src/src/tracked_state/context.rs +1155 -271
  122. package/dist-engine-src/src/tracked_state/diff.rs +249 -57
  123. package/dist-engine-src/src/tracked_state/materialization.rs +365 -103
  124. package/dist-engine-src/src/tracked_state/materializer.rs +488 -0
  125. package/dist-engine-src/src/tracked_state/merge.rs +37 -19
  126. package/dist-engine-src/src/tracked_state/mod.rs +8 -7
  127. package/dist-engine-src/src/tracked_state/storage.rs +138 -6
  128. package/dist-engine-src/src/tracked_state/tree.rs +695 -252
  129. package/dist-engine-src/src/tracked_state/types.rs +176 -6
  130. package/dist-engine-src/src/transaction/commit.rs +695 -435
  131. package/dist-engine-src/src/transaction/context.rs +551 -310
  132. package/dist-engine-src/src/transaction/live_state_overlay.rs +9 -8
  133. package/dist-engine-src/src/transaction/mod.rs +2 -0
  134. package/dist-engine-src/src/transaction/normalization.rs +311 -447
  135. package/dist-engine-src/src/transaction/prep.rs +37 -0
  136. package/dist-engine-src/src/transaction/schema_resolver.rs +93 -71
  137. package/dist-engine-src/src/transaction/staging.rs +701 -406
  138. package/dist-engine-src/src/transaction/types.rs +231 -122
  139. package/dist-engine-src/src/transaction/validation.rs +2717 -1698
  140. package/dist-engine-src/src/untracked_state/codec.rs +40 -96
  141. package/dist-engine-src/src/untracked_state/context.rs +21 -5
  142. package/dist-engine-src/src/untracked_state/materialization.rs +10 -104
  143. package/dist-engine-src/src/untracked_state/mod.rs +3 -5
  144. package/dist-engine-src/src/untracked_state/storage.rs +105 -57
  145. package/dist-engine-src/src/untracked_state/types.rs +63 -13
  146. package/dist-engine-src/src/version/context.rs +1 -13
  147. package/dist-engine-src/src/version/lifecycle.rs +221 -0
  148. package/dist-engine-src/src/version/mod.rs +3 -2
  149. package/dist-engine-src/src/version/refs.rs +12 -103
  150. package/dist-engine-src/src/version/stage_rows.rs +15 -19
  151. package/package.json +1 -1
  152. package/dist-engine-src/src/changelog/codec.rs +0 -321
  153. package/dist-engine-src/src/changelog/context.rs +0 -92
  154. package/dist-engine-src/src/changelog/materialization.rs +0 -121
  155. package/dist-engine-src/src/changelog/mod.rs +0 -13
  156. package/dist-engine-src/src/changelog/reader.rs +0 -20
  157. package/dist-engine-src/src/changelog/storage.rs +0 -220
  158. package/dist-engine-src/src/changelog/types.rs +0 -38
  159. package/dist-engine-src/src/schema/builtin/lix_change_set.json +0 -18
  160. package/dist-engine-src/src/schema/builtin/lix_change_set_element.json +0 -75
  161. package/dist-engine-src/src/schema/builtin/lix_entity_label.json +0 -63
  162. package/dist-engine-src/src/schema_registry.rs +0 -294
  163. package/dist-engine-src/src/sql2/commit_derived_provider.rs +0 -591
  164. package/dist-engine-src/src/tracked_state/rebuild.rs +0 -771
  165. package/dist-engine-src/src/tracked_state/tree_types.rs +0 -176
@@ -0,0 +1,100 @@
1
+ use crate::functions::FunctionContext;
2
+ use crate::session::context::{SessionContext, SessionSqlExecutionContext};
3
+ use crate::sql2::{self, SqlLogicalPlan};
4
+ use crate::storage::StorageReadScope;
5
+ use crate::transaction::open_transaction;
6
+ use crate::{LixError, SqlQueryResult, Value};
7
+
8
+ /// Opaque read plan used by the Optimization 9 SQL2 diagnostic benchmark.
9
+ ///
10
+ /// This module is gated behind `storage-benches` and exists only to split SQL2
11
+ /// planning cost from SQL2 execution cost without widening the normal session
12
+ /// API.
13
+ pub struct PreparedReadPlan {
14
+ plan: SqlLogicalPlan,
15
+ read_scope:
16
+ StorageReadScope<Box<dyn crate::storage::StorageReadTransaction + Send + Sync + 'static>>,
17
+ runtime_functions: FunctionContext,
18
+ }
19
+
20
+ pub async fn plan_read_only(session: &SessionContext, sql: &str) -> Result<(), LixError> {
21
+ let prepared = prepare_read_plan(session, sql).await?;
22
+ drop(prepared.plan);
23
+ drop(prepared.runtime_functions);
24
+ prepared.read_scope.rollback().await
25
+ }
26
+
27
+ pub async fn plan_write_only(session: &SessionContext, sql: &str) -> Result<(), LixError> {
28
+ session.ensure_open()?;
29
+ let opened = open_transaction(
30
+ &session.mode,
31
+ session.storage.clone(),
32
+ std::sync::Arc::clone(&session.live_state),
33
+ std::sync::Arc::clone(&session.tracked_state),
34
+ std::sync::Arc::clone(&session.binary_cas),
35
+ std::sync::Arc::clone(&session.commit_store),
36
+ std::sync::Arc::clone(&session.version_ctx),
37
+ std::sync::Arc::clone(&session.catalog_context),
38
+ )
39
+ .await?;
40
+ let mut transaction = opened.transaction;
41
+ let runtime_functions = opened.runtime_functions;
42
+ let plan = sql2::create_write_logical_plan(&mut transaction, sql).await?;
43
+ drop(plan);
44
+ drop(runtime_functions);
45
+ transaction.rollback().await
46
+ }
47
+
48
+ pub async fn prepare_read_plan(
49
+ session: &SessionContext,
50
+ sql: &str,
51
+ ) -> Result<PreparedReadPlan, LixError> {
52
+ session.ensure_open()?;
53
+ let read_scope = StorageReadScope::new(session.storage.begin_read_transaction().await?);
54
+ let mut read_store = read_scope.store();
55
+ let live_state: std::sync::Arc<dyn crate::live_state::LiveStateReader> =
56
+ std::sync::Arc::new(session.live_state.reader(read_store.clone()));
57
+ let runtime_functions = FunctionContext::prepare(live_state.as_ref()).await?;
58
+ let functions = runtime_functions.provider();
59
+ let active_version_id = session
60
+ .active_version_id_from_reader(&mut read_store)
61
+ .await?;
62
+ let visible_schemas = session
63
+ .catalog_context
64
+ .schema_jsons_for_sql_read_planning(live_state.as_ref(), &active_version_id)
65
+ .await?;
66
+ let ctx = SessionSqlExecutionContext {
67
+ active_version_id: &active_version_id,
68
+ read_store,
69
+ live_state: std::sync::Arc::clone(&session.live_state),
70
+ binary_cas: std::sync::Arc::clone(&session.binary_cas),
71
+ commit_store: std::sync::Arc::clone(&session.commit_store),
72
+ version_ctx: std::sync::Arc::clone(&session.version_ctx),
73
+ visible_schemas,
74
+ functions: functions.clone(),
75
+ };
76
+ let plan = sql2::create_logical_plan(&ctx, sql).await?;
77
+ drop(ctx);
78
+ drop(live_state);
79
+
80
+ Ok(PreparedReadPlan {
81
+ plan,
82
+ read_scope,
83
+ runtime_functions,
84
+ })
85
+ }
86
+
87
+ pub async fn execute_read_plan(
88
+ prepared: PreparedReadPlan,
89
+ params: &[Value],
90
+ ) -> Result<SqlQueryResult, LixError> {
91
+ let PreparedReadPlan {
92
+ plan,
93
+ read_scope,
94
+ runtime_functions,
95
+ } = prepared;
96
+ let result = sql2::execute_logical_plan(plan, params).await;
97
+ read_scope.rollback().await?;
98
+ drop(runtime_functions);
99
+ result
100
+ }
@@ -2,15 +2,14 @@ use std::sync::Arc;
2
2
 
3
3
  use serde_json::json;
4
4
 
5
- use crate::transaction::types::StageRow;
6
- use crate::version::VersionRefReader;
5
+ use crate::transaction::types::{TransactionJson, TransactionWriteRow};
6
+ use crate::version::{VersionLifecycle, VersionOperation, VersionReferenceRole};
7
7
  use crate::LixError;
8
8
  use crate::GLOBAL_VERSION_ID;
9
9
 
10
10
  use super::context::{SessionContext, SessionMode, WORKSPACE_VERSION_KEY};
11
11
 
12
12
  const KEY_VALUE_SCHEMA_KEY: &str = "lix_key_value";
13
- const KEY_VALUE_SCHEMA_VERSION: &str = "1";
14
13
 
15
14
  /// Options for switching a session to another version.
16
15
  #[derive(Debug, Clone, PartialEq, Eq)]
@@ -40,17 +39,16 @@ impl SessionContext {
40
39
  let next_mode = self
41
40
  .with_write_transaction(|transaction| {
42
41
  Box::pin(async move {
43
- let head = {
42
+ {
44
43
  let reader = transaction.version_ref_reader();
45
- reader.load_head_commit_id(&version_id).await?
44
+ VersionLifecycle::new(&reader)
45
+ .require_existing_commit_id(
46
+ &version_id,
47
+ VersionOperation::SwitchVersion,
48
+ VersionReferenceRole::Target,
49
+ )
50
+ .await?
46
51
  };
47
- if head.is_none() {
48
- return Err(LixError::version_not_found(
49
- version_id.clone(),
50
- "switch_version",
51
- "target",
52
- ));
53
- }
54
52
 
55
53
  match current_mode {
56
54
  SessionMode::Pinned { .. } => Ok(SessionMode::Pinned {
@@ -73,9 +71,9 @@ impl SessionContext {
73
71
  Arc::clone(&self.live_state),
74
72
  Arc::clone(&self.tracked_state),
75
73
  Arc::clone(&self.binary_cas),
76
- Arc::clone(&self.changelog),
74
+ Arc::clone(&self.commit_store),
77
75
  Arc::clone(&self.version_ctx),
78
- Arc::clone(&self.schema_registry),
76
+ Arc::clone(&self.catalog_context),
79
77
  self.closed_flag(),
80
78
  );
81
79
  Ok((
@@ -87,20 +85,19 @@ impl SessionContext {
87
85
  }
88
86
  }
89
87
 
90
- fn workspace_version_stage_row(version_id: &str) -> Result<StageRow, LixError> {
91
- Ok(StageRow {
88
+ fn workspace_version_stage_row(version_id: &str) -> Result<TransactionWriteRow, LixError> {
89
+ Ok(TransactionWriteRow {
92
90
  entity_id: Some(crate::entity_identity::EntityIdentity::single(
93
91
  WORKSPACE_VERSION_KEY,
94
92
  )),
95
93
  schema_key: KEY_VALUE_SCHEMA_KEY.to_string(),
96
94
  file_id: None,
97
- snapshot_content: Some(encode_snapshot(json!({
95
+ snapshot: Some(TransactionJson::from_value_unchecked(json!({
98
96
  "key": WORKSPACE_VERSION_KEY,
99
97
  "value": version_id,
100
- }))?),
98
+ }))),
101
99
  metadata: None,
102
100
  origin: None,
103
- schema_version: KEY_VALUE_SCHEMA_VERSION.to_string(),
104
101
  created_at: None,
105
102
  updated_at: None,
106
103
  global: true,
@@ -110,12 +107,3 @@ fn workspace_version_stage_row(version_id: &str) -> Result<StageRow, LixError> {
110
107
  version_id: GLOBAL_VERSION_ID.to_string(),
111
108
  })
112
109
  }
113
-
114
- fn encode_snapshot(value: serde_json::Value) -> Result<String, LixError> {
115
- serde_json::to_string(&value).map_err(|error| {
116
- LixError::new(
117
- "LIX_ERROR_UNKNOWN",
118
- format!("engine2 switch_version snapshot serialization failed: {error}"),
119
- )
120
- })
121
- }
@@ -18,17 +18,18 @@ use datafusion::physical_plan::{
18
18
  };
19
19
  use futures_util::stream;
20
20
 
21
- use crate::changelog::{materialize_change, ChangelogScanRequest, MaterializedCanonicalChange};
21
+ use crate::commit_store::ChangeScanRequest;
22
22
  use crate::serialize_row_metadata;
23
23
  use crate::LixError;
24
24
 
25
25
  use super::record_batch::record_batch_with_row_count;
26
26
  use super::result_metadata::json_field;
27
- use super::SqlChangelogQuerySource;
27
+ use super::SqlCommitStoreQuerySource;
28
+ use crate::commit_store::{materialize_change, MaterializedChange};
28
29
 
29
30
  pub(crate) async fn register_lix_change_provider(
30
31
  session: &datafusion::prelude::SessionContext,
31
- query_source: SqlChangelogQuerySource,
32
+ query_source: SqlCommitStoreQuerySource,
32
33
  ) -> Result<(), LixError> {
33
34
  session
34
35
  .register_table("lix_change", Arc::new(LixChangeProvider::new(query_source)))
@@ -38,7 +39,7 @@ pub(crate) async fn register_lix_change_provider(
38
39
 
39
40
  struct LixChangeProvider {
40
41
  schema: SchemaRef,
41
- query_source: SqlChangelogQuerySource,
42
+ query_source: SqlCommitStoreQuerySource,
42
43
  }
43
44
 
44
45
  impl std::fmt::Debug for LixChangeProvider {
@@ -48,7 +49,7 @@ impl std::fmt::Debug for LixChangeProvider {
48
49
  }
49
50
 
50
51
  impl LixChangeProvider {
51
- fn new(query_source: SqlChangelogQuerySource) -> Self {
52
+ fn new(query_source: SqlCommitStoreQuerySource) -> Self {
52
53
  Self {
53
54
  schema: lix_change_schema(),
54
55
  query_source,
@@ -97,7 +98,7 @@ impl TableProvider for LixChangeProvider {
97
98
  }
98
99
 
99
100
  struct LixChangeScanExec {
100
- query_source: SqlChangelogQuerySource,
101
+ query_source: SqlCommitStoreQuerySource,
101
102
  schema: SchemaRef,
102
103
  projection: Option<Vec<usize>>,
103
104
  limit: Option<usize>,
@@ -112,7 +113,7 @@ impl std::fmt::Debug for LixChangeScanExec {
112
113
 
113
114
  impl LixChangeScanExec {
114
115
  fn new(
115
- query_source: SqlChangelogQuerySource,
116
+ query_source: SqlCommitStoreQuerySource,
116
117
  schema: SchemaRef,
117
118
  projection: Option<Vec<usize>>,
118
119
  limit: Option<usize>,
@@ -191,8 +192,8 @@ impl ExecutionPlan for LixChangeScanExec {
191
192
  let stream = stream::once(async move {
192
193
  let mut json_reader = query_source.json_reader;
193
194
  let canonical_changes = query_source
194
- .changelog_reader
195
- .scan_changes(&ChangelogScanRequest { limit })
195
+ .commit_store_reader
196
+ .scan_changes(&ChangeScanRequest { limit })
196
197
  .await
197
198
  .map_err(lix_error_to_datafusion_error)?;
198
199
  let mut changes = Vec::with_capacity(canonical_changes.len());
@@ -214,7 +215,6 @@ enum ChangeColumn {
214
215
  Id,
215
216
  EntityId,
216
217
  SchemaKey,
217
- SchemaVersion,
218
218
  FileId,
219
219
  Metadata,
220
220
  CreatedAt,
@@ -224,9 +224,8 @@ enum ChangeColumn {
224
224
  fn lix_change_schema() -> SchemaRef {
225
225
  Arc::new(Schema::new(vec![
226
226
  Field::new("id", DataType::Utf8, false),
227
- Field::new("entity_id", DataType::Utf8, false),
227
+ json_field("entity_id", false),
228
228
  Field::new("schema_key", DataType::Utf8, false),
229
- Field::new("schema_version", DataType::Utf8, false),
230
229
  Field::new("file_id", DataType::Utf8, true),
231
230
  json_field("metadata", true),
232
231
  Field::new("created_at", DataType::Utf8, false),
@@ -239,7 +238,6 @@ fn change_projection_for_scan(projection: Option<&Vec<usize>>) -> Vec<ChangeColu
239
238
  ChangeColumn::Id,
240
239
  ChangeColumn::EntityId,
241
240
  ChangeColumn::SchemaKey,
242
- ChangeColumn::SchemaVersion,
243
241
  ChangeColumn::FileId,
244
242
  ChangeColumn::Metadata,
245
243
  ChangeColumn::CreatedAt,
@@ -262,7 +260,7 @@ fn projected_schema(schema: &SchemaRef, projection: Option<&Vec<usize>>) -> Sche
262
260
 
263
261
  fn change_record_batch(
264
262
  projection: &[ChangeColumn],
265
- changes: &[MaterializedCanonicalChange],
263
+ changes: &[MaterializedChange],
266
264
  ) -> Result<RecordBatch> {
267
265
  let arrays = projection
268
266
  .iter()
@@ -274,7 +272,7 @@ fn change_record_batch(
274
272
  .map(|row| {
275
273
  Some(
276
274
  row.entity_id
277
- .as_string()
275
+ .as_json_array_text()
278
276
  .expect("canonical change entity identity should project"),
279
277
  )
280
278
  })
@@ -283,9 +281,6 @@ fn change_record_batch(
283
281
  ChangeColumn::SchemaKey => {
284
282
  string_array(changes.iter().map(|row| Some(row.schema_key.as_str())))
285
283
  }
286
- ChangeColumn::SchemaVersion => {
287
- string_array(changes.iter().map(|row| Some(row.schema_version.as_str())))
288
- }
289
284
  ChangeColumn::FileId => string_array(changes.iter().map(|row| row.file_id.as_deref())),
290
285
  ChangeColumn::Metadata => Arc::new(StringArray::from(
291
286
  changes
@@ -312,9 +307,8 @@ fn change_schema(projection: &[ChangeColumn]) -> SchemaRef {
312
307
  .iter()
313
308
  .map(|column| match column {
314
309
  ChangeColumn::Id => Field::new("id", DataType::Utf8, false),
315
- ChangeColumn::EntityId => Field::new("entity_id", DataType::Utf8, false),
310
+ ChangeColumn::EntityId => json_field("entity_id", false),
316
311
  ChangeColumn::SchemaKey => Field::new("schema_key", DataType::Utf8, false),
317
- ChangeColumn::SchemaVersion => Field::new("schema_version", DataType::Utf8, false),
318
312
  ChangeColumn::FileId => Field::new("file_id", DataType::Utf8, true),
319
313
  ChangeColumn::Metadata => json_field("metadata", true),
320
314
  ChangeColumn::CreatedAt => Field::new("created_at", DataType::Utf8, false),
@@ -1,5 +1,7 @@
1
+ use datafusion::sql::parser::Statement as DataFusionStatement;
1
2
  use datafusion::sql::sqlparser::ast::{
2
- FromTable, ObjectName, Query, SetExpr, Statement, TableFactor, TableObject, TableWithJoins,
3
+ FromTable, ObjectName, Query, SetExpr, Statement as SqlStatement, TableFactor, TableObject,
4
+ TableWithJoins,
3
5
  };
4
6
  use datafusion::sql::sqlparser::dialect::GenericDialect;
5
7
  use datafusion::sql::sqlparser::parser::Parser;
@@ -31,17 +33,29 @@ pub(crate) fn validate_supported_statement_ast(sql: &str) -> Result<(), LixError
31
33
  validate_supported_ast_statement(statement)
32
34
  }
33
35
 
34
- pub(crate) fn dml_target_table_names(sql: &str) -> Result<Vec<String>, LixError> {
35
- let statements = parse_sql_statements(sql)?;
36
- let [statement] = statements.as_slice() else {
37
- return Ok(Vec::new());
38
- };
36
+ pub(crate) fn validate_supported_datafusion_statement_ast(
37
+ statement: &DataFusionStatement,
38
+ ) -> Result<(), LixError> {
39
+ match statement {
40
+ DataFusionStatement::Statement(statement) => validate_supported_ast_statement(statement),
41
+ DataFusionStatement::Explain(explain) => {
42
+ validate_supported_datafusion_statement_ast(explain.statement.as_ref())
43
+ }
44
+ _ => Err(unsupported_sql_error(format!(
45
+ "SQL statement is not supported by Lix SQL: {statement}"
46
+ ))),
47
+ }
48
+ }
49
+
50
+ pub(crate) fn datafusion_statement_dml_target_table_names(
51
+ statement: &DataFusionStatement,
52
+ ) -> Vec<String> {
39
53
  let mut targets = Vec::new();
40
- collect_dml_target_table_names(statement, &mut targets);
41
- Ok(targets)
54
+ collect_datafusion_statement_dml_target_table_names(statement, &mut targets);
55
+ targets
42
56
  }
43
57
 
44
- fn parse_sql_statements(sql: &str) -> Result<Vec<Statement>, LixError> {
58
+ fn parse_sql_statements(sql: &str) -> Result<Vec<SqlStatement>, LixError> {
45
59
  Parser::parse_sql(&GenericDialect {}, sql).map_err(|error| {
46
60
  LixError::new(
47
61
  LixError::CODE_PARSE_ERROR,
@@ -50,19 +64,37 @@ fn parse_sql_statements(sql: &str) -> Result<Vec<Statement>, LixError> {
50
64
  })
51
65
  }
52
66
 
53
- fn collect_dml_target_table_names(statement: &Statement, targets: &mut Vec<String>) {
67
+ fn collect_datafusion_statement_dml_target_table_names(
68
+ statement: &DataFusionStatement,
69
+ targets: &mut Vec<String>,
70
+ ) {
71
+ match statement {
72
+ DataFusionStatement::Statement(statement) => {
73
+ collect_dml_target_table_names(statement, targets);
74
+ }
75
+ DataFusionStatement::Explain(explain) => {
76
+ collect_datafusion_statement_dml_target_table_names(
77
+ explain.statement.as_ref(),
78
+ targets,
79
+ );
80
+ }
81
+ _ => {}
82
+ }
83
+ }
84
+
85
+ fn collect_dml_target_table_names(statement: &SqlStatement, targets: &mut Vec<String>) {
54
86
  match statement {
55
- Statement::Insert(insert) => {
87
+ SqlStatement::Insert(insert) => {
56
88
  if let TableObject::TableName(name) = &insert.table {
57
89
  if let Some(table_name) = object_name_table_part(name) {
58
90
  targets.push(table_name);
59
91
  }
60
92
  }
61
93
  }
62
- Statement::Update(update) => {
94
+ SqlStatement::Update(update) => {
63
95
  collect_table_with_joins_target(&update.table, targets);
64
96
  }
65
- Statement::Delete(delete) => {
97
+ SqlStatement::Delete(delete) => {
66
98
  let tables = match &delete.from {
67
99
  FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables,
68
100
  };
@@ -70,7 +102,7 @@ fn collect_dml_target_table_names(statement: &Statement, targets: &mut Vec<Strin
70
102
  collect_table_with_joins_target(table, targets);
71
103
  }
72
104
  }
73
- Statement::Explain { statement, .. } => {
105
+ SqlStatement::Explain { statement, .. } => {
74
106
  collect_dml_target_table_names(statement.as_ref(), targets);
75
107
  }
76
108
  _ => {}
@@ -86,28 +118,31 @@ fn collect_table_with_joins_target(table: &TableWithJoins, targets: &mut Vec<Str
86
118
  }
87
119
 
88
120
  fn object_name_table_part(name: &ObjectName) -> Option<String> {
89
- name.0
90
- .last()
91
- .and_then(|part| part.as_ident())
92
- .map(|ident| ident.value.clone())
121
+ name.0.last().and_then(|part| part.as_ident()).map(|ident| {
122
+ if ident.quote_style.is_some() {
123
+ ident.value.clone()
124
+ } else {
125
+ ident.value.to_ascii_lowercase()
126
+ }
127
+ })
93
128
  }
94
129
 
95
- fn classify_ast_statement(statement: &Statement) -> SqlStatementKind {
130
+ fn classify_ast_statement(statement: &SqlStatement) -> SqlStatementKind {
96
131
  match statement {
97
- Statement::Insert(_) | Statement::Update(_) | Statement::Delete(_) => {
132
+ SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => {
98
133
  SqlStatementKind::Write
99
134
  }
100
- Statement::Query(_) => SqlStatementKind::Read,
101
- Statement::Explain { statement, .. } => classify_ast_statement(statement.as_ref()),
135
+ SqlStatement::Query(_) => SqlStatementKind::Read,
136
+ SqlStatement::Explain { statement, .. } => classify_ast_statement(statement.as_ref()),
102
137
  _ => SqlStatementKind::Other,
103
138
  }
104
139
  }
105
140
 
106
- fn validate_supported_ast_statement(statement: &Statement) -> Result<(), LixError> {
141
+ fn validate_supported_ast_statement(statement: &SqlStatement) -> Result<(), LixError> {
107
142
  match statement {
108
- Statement::Query(query) => validate_supported_query(query),
109
- Statement::Insert(_) | Statement::Update(_) | Statement::Delete(_) => Ok(()),
110
- Statement::Explain { statement, .. } => validate_supported_ast_statement(statement),
143
+ SqlStatement::Query(query) => validate_supported_query(query),
144
+ SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => Ok(()),
145
+ SqlStatement::Explain { statement, .. } => validate_supported_ast_statement(statement),
111
146
  _ => Err(unsupported_sql_error(format!(
112
147
  "SQL statement is not supported by Lix SQL: {statement}"
113
148
  ))),
@@ -6,26 +6,27 @@ use serde_json::Value as JsonValue;
6
6
  use tokio::sync::Mutex;
7
7
 
8
8
  use crate::binary_cas::{BlobBytesBatch, BlobDataReader, BlobHash};
9
- use crate::changelog::ChangelogReader;
10
9
  use crate::commit_graph::CommitGraphReader;
10
+ use crate::commit_store::CommitStoreReader;
11
11
  use crate::functions::FunctionProviderHandle;
12
12
  use crate::json_store::JsonStoreReader;
13
13
  use crate::live_state::{
14
- LiveStateFilter, LiveStateReader, LiveStateRow, LiveStateRowRequest, LiveStateScanRequest,
14
+ LiveStateFilter, LiveStateReader, LiveStateRowRequest, LiveStateScanRequest,
15
+ MaterializedLiveStateRow,
15
16
  };
16
17
  use crate::storage::{ScopedStorageReader, StorageReadTransaction};
17
- use crate::transaction::types::{StageWrite, StageWriteOutcome};
18
+ use crate::transaction::types::{TransactionWrite, TransactionWriteOutcome};
18
19
  use crate::version::{VersionHead, VersionRefReader};
19
20
  use crate::LixError;
20
21
 
21
22
  pub(crate) type SqlReadStore =
22
23
  ScopedStorageReader<Box<dyn StorageReadTransaction + Send + Sync + 'static>>;
23
- pub(crate) type SqlChangelogQuerySource = ChangelogQuerySource<SqlReadStore>;
24
+ pub(crate) type SqlCommitStoreQuerySource = CommitStoreQuerySource<SqlReadStore>;
24
25
  pub(crate) type SqlJsonReader = JsonStoreReader<ScopedStorageReader<SqlReadStore>>;
25
26
 
26
27
  #[derive(Clone)]
27
- pub(crate) struct ChangelogQuerySource<S> {
28
- pub(crate) changelog_reader: Arc<dyn ChangelogReader>,
28
+ pub(crate) struct CommitStoreQuerySource<S> {
29
+ pub(crate) commit_store_reader: Arc<CommitStoreReader<ScopedStorageReader<S>>>,
29
30
  pub(crate) json_reader: JsonStoreReader<ScopedStorageReader<S>>,
30
31
  }
31
32
 
@@ -43,7 +44,7 @@ pub(crate) trait SqlExecutionContext {
43
44
  fn active_version_id(&self) -> &str;
44
45
  fn live_state(&self) -> Arc<dyn LiveStateReader>;
45
46
  fn functions(&self) -> FunctionProviderHandle;
46
- fn changelog_query_source(&self) -> SqlChangelogQuerySource;
47
+ fn commit_store_query_source(&self) -> SqlCommitStoreQuerySource;
47
48
  fn commit_graph(&self) -> Box<dyn CommitGraphReader>;
48
49
  fn version_ref(&self) -> Arc<dyn VersionRefReader>;
49
50
  fn blob_reader(&self) -> Arc<dyn BlobDataReader>;
@@ -52,9 +53,9 @@ pub(crate) trait SqlExecutionContext {
52
53
 
53
54
  /// Write-capable SQL runtime boundary.
54
55
  ///
55
- /// Providers that mutate engine2 state should target this shape instead of
56
+ /// Providers that mutate engine state should target this shape instead of
56
57
  /// reaching through session/backend escape hatches. The request and write
57
- /// payloads stay in the existing engine2 forms so this boundary centralizes
58
+ /// payloads stay in the existing engine forms so this boundary centralizes
58
59
  /// authority without adding another translation layer.
59
60
  #[async_trait]
60
61
  #[allow(dead_code)]
@@ -68,11 +69,14 @@ pub(crate) trait SqlWriteExecutionContext {
68
69
  async fn scan_live_state(
69
70
  &mut self,
70
71
  request: &LiveStateScanRequest,
71
- ) -> Result<Vec<LiveStateRow>, LixError>;
72
+ ) -> Result<Vec<MaterializedLiveStateRow>, LixError>;
72
73
 
73
74
  async fn load_version_head(&mut self, version_id: &str) -> Result<Option<String>, LixError>;
74
75
 
75
- async fn stage_write(&mut self, write: StageWrite) -> Result<StageWriteOutcome, LixError>;
76
+ async fn stage_write(
77
+ &mut self,
78
+ write: TransactionWrite,
79
+ ) -> Result<TransactionWriteOutcome, LixError>;
76
80
  }
77
81
 
78
82
  #[derive(Clone)]
@@ -123,7 +127,7 @@ impl SqlWriteContext {
123
127
  pub(crate) async fn scan_live_state(
124
128
  &self,
125
129
  request: &LiveStateScanRequest,
126
- ) -> Result<Vec<LiveStateRow>, LixError> {
130
+ ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
127
131
  let _guard = self.gate.lock().await;
128
132
  unsafe {
129
133
  self.ptr
@@ -170,8 +174,8 @@ impl SqlWriteContext {
170
174
 
171
175
  pub(crate) async fn stage_write(
172
176
  &self,
173
- write: StageWrite,
174
- ) -> Result<StageWriteOutcome, LixError> {
177
+ write: TransactionWrite,
178
+ ) -> Result<TransactionWriteOutcome, LixError> {
175
179
  let _guard = self.gate.lock().await;
176
180
  unsafe {
177
181
  self.ptr
@@ -219,12 +223,12 @@ impl WriteAccess {
219
223
 
220
224
  pub(crate) fn require_write(
221
225
  &self,
222
- operation: &str,
226
+ action: &str,
223
227
  ) -> Result<SqlWriteContext, datafusion::error::DataFusionError> {
224
228
  match self {
225
229
  Self::Write { ctx } => Ok(ctx.clone()),
226
230
  Self::ReadOnly => Err(datafusion::error::DataFusionError::Execution(format!(
227
- "{operation} requires a write transaction"
231
+ "{action} requires a write transaction"
228
232
  ))),
229
233
  }
230
234
  }
@@ -249,14 +253,14 @@ impl LiveStateReader for WriteContextLiveStateReader {
249
253
  async fn scan_rows(
250
254
  &self,
251
255
  request: &LiveStateScanRequest,
252
- ) -> Result<Vec<LiveStateRow>, LixError> {
256
+ ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
253
257
  self.ctx.scan_live_state(request).await
254
258
  }
255
259
 
256
260
  async fn load_row(
257
261
  &self,
258
262
  request: &LiveStateRowRequest,
259
- ) -> Result<Option<LiveStateRow>, LixError> {
263
+ ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
260
264
  let mut rows = self
261
265
  .ctx
262
266
  .scan_live_state(&LiveStateScanRequest {