@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/SKILL.md +4 -5
  2. package/dist/engine-wasm/wasm/lix_engine.js +1 -1
  3. package/dist/engine-wasm/wasm/lix_engine.wasm +0 -0
  4. package/dist/generated/builtin-schemas.d.ts +87 -162
  5. package/dist/generated/builtin-schemas.js +139 -236
  6. package/dist/open-lix.d.ts +1 -1
  7. package/dist-engine-src/src/binary_cas/types.rs +0 -6
  8. package/dist-engine-src/src/catalog/context.rs +412 -0
  9. package/dist-engine-src/src/catalog/mod.rs +10 -0
  10. package/dist-engine-src/src/catalog/schema.rs +4 -0
  11. package/dist-engine-src/src/catalog/snapshot.rs +1114 -0
  12. package/dist-engine-src/src/cel/mod.rs +1 -1
  13. package/dist-engine-src/src/cel/provider.rs +1 -1
  14. package/dist-engine-src/src/commit_graph/context.rs +328 -1015
  15. package/dist-engine-src/src/commit_graph/mod.rs +2 -3
  16. package/dist-engine-src/src/commit_graph/types.rs +7 -43
  17. package/dist-engine-src/src/commit_graph/walker.rs +57 -81
  18. package/dist-engine-src/src/commit_store/codec.rs +887 -0
  19. package/dist-engine-src/src/commit_store/context.rs +944 -0
  20. package/dist-engine-src/src/commit_store/materialization.rs +84 -0
  21. package/dist-engine-src/src/commit_store/mod.rs +16 -0
  22. package/dist-engine-src/src/commit_store/storage.rs +600 -0
  23. package/dist-engine-src/src/commit_store/types.rs +215 -0
  24. package/dist-engine-src/src/common/identity.rs +15 -5
  25. package/dist-engine-src/src/common/json_pointer.rs +67 -0
  26. package/dist-engine-src/src/common/metadata.rs +17 -12
  27. package/dist-engine-src/src/common/mod.rs +5 -5
  28. package/dist-engine-src/src/domain.rs +324 -0
  29. package/dist-engine-src/src/engine.rs +29 -43
  30. package/dist-engine-src/src/entity_identity.rs +238 -118
  31. package/dist-engine-src/src/functions/context.rs +17 -52
  32. package/dist-engine-src/src/functions/deterministic.rs +1 -1
  33. package/dist-engine-src/src/functions/mod.rs +1 -1
  34. package/dist-engine-src/src/functions/provider.rs +4 -4
  35. package/dist-engine-src/src/functions/state.rs +39 -66
  36. package/dist-engine-src/src/functions/types.rs +1 -1
  37. package/dist-engine-src/src/init.rs +204 -151
  38. package/dist-engine-src/src/json_store/context.rs +354 -60
  39. package/dist-engine-src/src/json_store/encoded.rs +6 -6
  40. package/dist-engine-src/src/json_store/mod.rs +4 -1
  41. package/dist-engine-src/src/json_store/store.rs +884 -11
  42. package/dist-engine-src/src/json_store/types.rs +166 -1
  43. package/dist-engine-src/src/lib.rs +10 -9
  44. package/dist-engine-src/src/live_state/context.rs +608 -830
  45. package/dist-engine-src/src/live_state/mod.rs +3 -3
  46. package/dist-engine-src/src/live_state/overlay.rs +7 -7
  47. package/dist-engine-src/src/live_state/reader.rs +5 -5
  48. package/dist-engine-src/src/live_state/types.rs +19 -36
  49. package/dist-engine-src/src/live_state/visibility.rs +19 -14
  50. package/dist-engine-src/src/plugin/archive.rs +3 -6
  51. package/dist-engine-src/src/plugin/install.rs +0 -18
  52. package/dist-engine-src/src/plugin/plugin_manifest.json +0 -1
  53. package/dist-engine-src/src/schema/annotations/defaults.rs +2 -7
  54. package/dist-engine-src/src/schema/builtin/lix_account.json +0 -1
  55. package/dist-engine-src/src/schema/builtin/lix_active_account.json +0 -1
  56. package/dist-engine-src/src/schema/builtin/lix_binary_blob_ref.json +0 -1
  57. package/dist-engine-src/src/schema/builtin/lix_change.json +11 -10
  58. package/dist-engine-src/src/schema/builtin/lix_change_author.json +0 -1
  59. package/dist-engine-src/src/schema/builtin/lix_commit.json +8 -46
  60. package/dist-engine-src/src/schema/builtin/lix_commit_edge.json +29 -22
  61. package/dist-engine-src/src/schema/builtin/lix_directory_descriptor.json +0 -1
  62. package/dist-engine-src/src/schema/builtin/lix_file_descriptor.json +0 -1
  63. package/dist-engine-src/src/schema/builtin/lix_key_value.json +0 -1
  64. package/dist-engine-src/src/schema/builtin/lix_label.json +10 -3
  65. package/dist-engine-src/src/schema/builtin/lix_label_assignment.json +74 -0
  66. package/dist-engine-src/src/schema/builtin/lix_registered_schema.json +2 -8
  67. package/dist-engine-src/src/schema/builtin/lix_version_descriptor.json +0 -1
  68. package/dist-engine-src/src/schema/builtin/lix_version_ref.json +0 -1
  69. package/dist-engine-src/src/schema/builtin/mod.rs +10 -59
  70. package/dist-engine-src/src/schema/compatibility.rs +787 -0
  71. package/dist-engine-src/src/schema/definition.json +47 -17
  72. package/dist-engine-src/src/schema/definition.rs +202 -96
  73. package/dist-engine-src/src/schema/key.rs +9 -77
  74. package/dist-engine-src/src/schema/mod.rs +4 -4
  75. package/dist-engine-src/src/schema/tests.rs +133 -92
  76. package/dist-engine-src/src/session/context.rs +40 -42
  77. package/dist-engine-src/src/session/create_version.rs +22 -14
  78. package/dist-engine-src/src/session/execute.rs +45 -14
  79. package/dist-engine-src/src/session/merge/apply.rs +4 -4
  80. package/dist-engine-src/src/session/merge/conflicts.rs +3 -2
  81. package/dist-engine-src/src/session/merge/stats.rs +1 -1
  82. package/dist-engine-src/src/session/merge/version.rs +35 -45
  83. package/dist-engine-src/src/session/mod.rs +4 -2
  84. package/dist-engine-src/src/session/optimization9_sql2_bench.rs +100 -0
  85. package/dist-engine-src/src/session/switch_version.rs +16 -28
  86. package/dist-engine-src/src/sql2/change_provider.rs +14 -20
  87. package/dist-engine-src/src/sql2/classify.rs +61 -26
  88. package/dist-engine-src/src/sql2/context.rs +22 -18
  89. package/dist-engine-src/src/sql2/directory_history_provider.rs +28 -20
  90. package/dist-engine-src/src/sql2/directory_provider.rs +131 -83
  91. package/dist-engine-src/src/sql2/entity_history_provider.rs +10 -14
  92. package/dist-engine-src/src/sql2/entity_provider.rs +680 -169
  93. package/dist-engine-src/src/sql2/error.rs +21 -1
  94. package/dist-engine-src/src/sql2/execute.rs +325 -264
  95. package/dist-engine-src/src/sql2/file_history_provider.rs +29 -21
  96. package/dist-engine-src/src/sql2/file_provider.rs +533 -108
  97. package/dist-engine-src/src/sql2/filesystem_planner.rs +58 -94
  98. package/dist-engine-src/src/sql2/filesystem_visibility.rs +37 -23
  99. package/dist-engine-src/src/sql2/history_projection.rs +3 -27
  100. package/dist-engine-src/src/sql2/history_provider.rs +11 -17
  101. package/dist-engine-src/src/sql2/history_route.rs +22 -8
  102. package/dist-engine-src/src/sql2/lix_state_provider.rs +178 -96
  103. package/dist-engine-src/src/sql2/mod.rs +6 -3
  104. package/dist-engine-src/src/sql2/predicate_typecheck.rs +246 -0
  105. package/dist-engine-src/src/sql2/public_bind/assignment.rs +46 -0
  106. package/dist-engine-src/src/sql2/public_bind/capability.rs +41 -0
  107. package/dist-engine-src/src/sql2/public_bind/dml.rs +166 -0
  108. package/dist-engine-src/src/sql2/public_bind/mod.rs +25 -0
  109. package/dist-engine-src/src/sql2/public_bind/table.rs +168 -0
  110. package/dist-engine-src/src/sql2/read_only.rs +10 -12
  111. package/dist-engine-src/src/sql2/session.rs +7 -10
  112. package/dist-engine-src/src/sql2/udfs/lix_timestamp.rs +76 -0
  113. package/dist-engine-src/src/sql2/udfs/mod.rs +8 -1
  114. package/dist-engine-src/src/sql2/udfs/public_call.rs +211 -0
  115. package/dist-engine-src/src/sql2/version_provider.rs +46 -31
  116. package/dist-engine-src/src/sql2/version_scope.rs +4 -4
  117. package/dist-engine-src/src/storage_bench.rs +1782 -325
  118. package/dist-engine-src/src/test_support.rs +183 -36
  119. package/dist-engine-src/src/tracked_state/by_file_index.rs +20 -24
  120. package/dist-engine-src/src/tracked_state/codec.rs +1519 -181
  121. package/dist-engine-src/src/tracked_state/context.rs +1155 -271
  122. package/dist-engine-src/src/tracked_state/diff.rs +249 -57
  123. package/dist-engine-src/src/tracked_state/materialization.rs +365 -103
  124. package/dist-engine-src/src/tracked_state/materializer.rs +488 -0
  125. package/dist-engine-src/src/tracked_state/merge.rs +37 -19
  126. package/dist-engine-src/src/tracked_state/mod.rs +8 -7
  127. package/dist-engine-src/src/tracked_state/storage.rs +138 -6
  128. package/dist-engine-src/src/tracked_state/tree.rs +695 -252
  129. package/dist-engine-src/src/tracked_state/types.rs +176 -6
  130. package/dist-engine-src/src/transaction/commit.rs +695 -435
  131. package/dist-engine-src/src/transaction/context.rs +551 -310
  132. package/dist-engine-src/src/transaction/live_state_overlay.rs +9 -8
  133. package/dist-engine-src/src/transaction/mod.rs +2 -0
  134. package/dist-engine-src/src/transaction/normalization.rs +311 -447
  135. package/dist-engine-src/src/transaction/prep.rs +37 -0
  136. package/dist-engine-src/src/transaction/schema_resolver.rs +93 -71
  137. package/dist-engine-src/src/transaction/staging.rs +701 -406
  138. package/dist-engine-src/src/transaction/types.rs +231 -122
  139. package/dist-engine-src/src/transaction/validation.rs +2717 -1698
  140. package/dist-engine-src/src/untracked_state/codec.rs +40 -96
  141. package/dist-engine-src/src/untracked_state/context.rs +21 -5
  142. package/dist-engine-src/src/untracked_state/materialization.rs +10 -104
  143. package/dist-engine-src/src/untracked_state/mod.rs +3 -5
  144. package/dist-engine-src/src/untracked_state/storage.rs +105 -57
  145. package/dist-engine-src/src/untracked_state/types.rs +63 -13
  146. package/dist-engine-src/src/version/context.rs +1 -13
  147. package/dist-engine-src/src/version/lifecycle.rs +221 -0
  148. package/dist-engine-src/src/version/mod.rs +3 -2
  149. package/dist-engine-src/src/version/refs.rs +12 -103
  150. package/dist-engine-src/src/version/stage_rows.rs +15 -19
  151. package/package.json +1 -1
  152. package/dist-engine-src/src/changelog/codec.rs +0 -321
  153. package/dist-engine-src/src/changelog/context.rs +0 -92
  154. package/dist-engine-src/src/changelog/materialization.rs +0 -121
  155. package/dist-engine-src/src/changelog/mod.rs +0 -13
  156. package/dist-engine-src/src/changelog/reader.rs +0 -20
  157. package/dist-engine-src/src/changelog/storage.rs +0 -220
  158. package/dist-engine-src/src/changelog/types.rs +0 -38
  159. package/dist-engine-src/src/schema/builtin/lix_change_set.json +0 -18
  160. package/dist-engine-src/src/schema/builtin/lix_change_set_element.json +0 -75
  161. package/dist-engine-src/src/schema/builtin/lix_entity_label.json +0 -63
  162. package/dist-engine-src/src/schema_registry.rs +0 -294
  163. package/dist-engine-src/src/sql2/commit_derived_provider.rs +0 -591
  164. package/dist-engine-src/src/tracked_state/rebuild.rs +0 -771
  165. package/dist-engine-src/src/tracked_state/tree_types.rs +0 -176
@@ -1,29 +1,53 @@
1
1
  use crate::binary_cas::{BinaryCasContext, BlobHash, BlobWrite};
2
- use crate::changelog::{
3
- canonicalize_materialized_change, CanonicalChange, ChangelogContext, ChangelogScanRequest,
4
- MaterializedCanonicalChange,
2
+ use crate::catalog::CatalogContext;
3
+ use crate::commit_graph::CommitGraphChangeHistoryRequest;
4
+ use crate::commit_store::{
5
+ Change, ChangeScanRequest, CommitDraftRef, CommitStoreContext, MaterializedChange,
5
6
  };
6
7
  use crate::entity_identity::EntityIdentity;
7
- use crate::entity_identity::EntityIdentityPart;
8
8
  use crate::json_store::context::JsonStoreContext;
9
- use crate::json_store::types::{JsonProjectionPath, JsonRef};
9
+ use crate::json_store::types::{
10
+ JsonLoadRequestRef, JsonProjectionLoadRequestRef, JsonProjectionPath, JsonReadScopeRef,
11
+ JsonRef, JsonWritePlacementRef, NormalizedJsonRef,
12
+ };
13
+ use crate::live_state::LiveStateContext;
14
+ use crate::session::SessionMode;
10
15
  use crate::storage::{
11
16
  KvGetGroup, KvGetRequest, KvScanRange, KvScanRequest, KvWriteBatch, StorageContext,
12
17
  StorageWriteSet,
13
18
  };
14
19
  use crate::tracked_state::{
15
- TrackedStateContext, TrackedStateDiffRequest, TrackedStateFilter, TrackedStateProjection,
16
- TrackedStateRow, TrackedStateRowRequest, TrackedStateScanRequest,
20
+ MaterializedTrackedStateRow, TrackedStateContext, TrackedStateDeltaRef,
21
+ TrackedStateDiffRequest, TrackedStateFilter, TrackedStateProjection, TrackedStateRowRequest,
22
+ TrackedStateScanRequest,
23
+ };
24
+ use crate::transaction::open_transaction;
25
+ use crate::transaction::types::{
26
+ TransactionJson, TransactionWrite, TransactionWriteMode, TransactionWriteRow,
17
27
  };
18
28
  use crate::untracked_state::{
19
- canonicalize_materialized_row, MaterializedUntrackedStateRow, UntrackedStateContext,
20
- UntrackedStateFilter, UntrackedStateProjection, UntrackedStateRowRequest,
21
- UntrackedStateScanRequest,
29
+ MaterializedUntrackedStateRow, UntrackedStateContext, UntrackedStateFilter,
30
+ UntrackedStateProjection, UntrackedStateRowRequest, UntrackedStateScanRequest,
22
31
  };
32
+ use crate::version::VersionContext;
23
33
  use crate::{Backend, LixError, NullableKeyFilter};
34
+ use std::collections::{BTreeMap, HashSet};
35
+ use std::sync::atomic::{AtomicUsize, Ordering};
24
36
  use std::sync::Arc;
37
+ use std::sync::Mutex;
38
+ use std::sync::OnceLock;
25
39
  use std::time::{Duration, Instant};
26
40
 
41
+ fn prepare_json_ref(document: &[u8]) -> Result<JsonRef, LixError> {
42
+ let text = std::str::from_utf8(document).map_err(|error| {
43
+ LixError::new(
44
+ LixError::CODE_UNKNOWN,
45
+ format!("benchmark JSON document is invalid UTF-8: {error}"),
46
+ )
47
+ })?;
48
+ Ok(JsonRef::for_content(text.as_bytes()))
49
+ }
50
+
27
51
  #[derive(Debug, Clone, Copy)]
28
52
  pub struct StorageBenchConfig {
29
53
  pub rows: usize,
@@ -115,13 +139,647 @@ pub struct StorageBenchReport {
115
139
  pub elapsed: Duration,
116
140
  }
117
141
 
142
+ #[derive(Debug, Clone, Default, PartialEq, Eq)]
143
+ pub struct TransactionBenchCounters {
144
+ pub rows_staged: usize,
145
+ pub untracked_rows: usize,
146
+ pub validation_version_count: usize,
147
+ pub schema_catalog_loads: usize,
148
+ pub json_store_stage_bytes_calls: usize,
149
+ pub unique_json_refs: usize,
150
+ }
151
+
152
+ #[derive(Debug, Clone, Default, PartialEq, Eq)]
153
+ pub struct TransactionAccountingReport {
154
+ pub counters: TransactionBenchCounters,
155
+ pub storage_write_batches: usize,
156
+ pub kv_puts_by_namespace: BTreeMap<String, usize>,
157
+ pub bytes_by_namespace: BTreeMap<String, usize>,
158
+ }
159
+
118
160
  pub struct StorageApiFixture {
119
161
  storage: StorageContext,
120
162
  rows: usize,
121
163
  }
122
164
 
165
+ pub struct TransactionBenchFixture {
166
+ storage: StorageContext,
167
+ live_state: Arc<LiveStateContext>,
168
+ tracked_state: Arc<TrackedStateContext>,
169
+ binary_cas: Arc<BinaryCasContext>,
170
+ commit_store: Arc<CommitStoreContext>,
171
+ version_ctx: Arc<VersionContext>,
172
+ catalog_context: Arc<CatalogContext>,
173
+ rows: Vec<TransactionWriteRow>,
174
+ }
175
+
176
+ pub struct TransactionCommitOnlyFixture {
177
+ runtime_functions: crate::functions::FunctionContext,
178
+ transaction: crate::transaction::Transaction,
179
+ rows: usize,
180
+ }
181
+
182
+ static TRANSACTION_ROWS_STAGED: AtomicUsize = AtomicUsize::new(0);
183
+ static TRANSACTION_UNTRACKED_ROWS: AtomicUsize = AtomicUsize::new(0);
184
+ static TRANSACTION_VALIDATION_VERSION_COUNT: AtomicUsize = AtomicUsize::new(0);
185
+ static TRANSACTION_SCHEMA_CATALOG_LOADS: AtomicUsize = AtomicUsize::new(0);
186
+ static JSON_STORE_STAGE_BYTES_CALLS: AtomicUsize = AtomicUsize::new(0);
187
+ static JSON_STORE_UNIQUE_REFS: OnceLock<Mutex<HashSet<[u8; 32]>>> = OnceLock::new();
188
+
123
189
  const STORAGE_API_NAMESPACE: &str = "bench.storage_api";
124
190
  const STORAGE_API_ALT_NAMESPACE: &str = "bench.storage_api.alt";
191
+ const TRANSACTION_BENCH_SCHEMA_KEY: &str = "bench_transaction_entity";
192
+
193
+ pub fn reset_transaction_bench_counters() {
194
+ TRANSACTION_ROWS_STAGED.store(0, Ordering::Relaxed);
195
+ TRANSACTION_UNTRACKED_ROWS.store(0, Ordering::Relaxed);
196
+ TRANSACTION_VALIDATION_VERSION_COUNT.store(0, Ordering::Relaxed);
197
+ TRANSACTION_SCHEMA_CATALOG_LOADS.store(0, Ordering::Relaxed);
198
+ JSON_STORE_STAGE_BYTES_CALLS.store(0, Ordering::Relaxed);
199
+ json_store_unique_refs()
200
+ .lock()
201
+ .expect("json store unique ref counter mutex should lock")
202
+ .clear();
203
+ }
204
+
205
+ pub fn transaction_bench_counters() -> TransactionBenchCounters {
206
+ TransactionBenchCounters {
207
+ rows_staged: TRANSACTION_ROWS_STAGED.load(Ordering::Relaxed),
208
+ untracked_rows: TRANSACTION_UNTRACKED_ROWS.load(Ordering::Relaxed),
209
+ validation_version_count: TRANSACTION_VALIDATION_VERSION_COUNT.load(Ordering::Relaxed),
210
+ schema_catalog_loads: TRANSACTION_SCHEMA_CATALOG_LOADS.load(Ordering::Relaxed),
211
+ json_store_stage_bytes_calls: JSON_STORE_STAGE_BYTES_CALLS.load(Ordering::Relaxed),
212
+ unique_json_refs: json_store_unique_refs()
213
+ .lock()
214
+ .expect("json store unique ref counter mutex should lock")
215
+ .len(),
216
+ }
217
+ }
218
+
219
+ pub(crate) fn record_transaction_rows_staged(rows: usize) {
220
+ TRANSACTION_ROWS_STAGED.fetch_add(rows, Ordering::Relaxed);
221
+ }
222
+
223
+ pub(crate) fn record_transaction_untracked_rows(rows: usize) {
224
+ TRANSACTION_UNTRACKED_ROWS.fetch_add(rows, Ordering::Relaxed);
225
+ }
226
+
227
+ pub(crate) fn record_transaction_validation_version() {
228
+ TRANSACTION_VALIDATION_VERSION_COUNT.fetch_add(1, Ordering::Relaxed);
229
+ }
230
+
231
+ pub(crate) fn record_transaction_schema_catalog_load() {
232
+ TRANSACTION_SCHEMA_CATALOG_LOADS.fetch_add(1, Ordering::Relaxed);
233
+ }
234
+
235
+ pub(crate) fn record_json_store_stage_bytes(hash: [u8; 32]) {
236
+ JSON_STORE_STAGE_BYTES_CALLS.fetch_add(1, Ordering::Relaxed);
237
+ json_store_unique_refs()
238
+ .lock()
239
+ .expect("json store unique ref counter mutex should lock")
240
+ .insert(hash);
241
+ }
242
+
243
+ fn json_store_unique_refs() -> &'static Mutex<HashSet<[u8; 32]>> {
244
+ JSON_STORE_UNIQUE_REFS.get_or_init(|| Mutex::new(HashSet::new()))
245
+ }
246
+
247
+ pub async fn prepare_transaction_commit_empty(
248
+ backend: Arc<dyn Backend + Send + Sync>,
249
+ ) -> Result<TransactionBenchFixture, LixError> {
250
+ prepare_transaction_fixture(backend, Vec::new()).await
251
+ }
252
+
253
+ pub async fn prepare_transaction_commit_schema_only(
254
+ backend: Arc<dyn Backend + Send + Sync>,
255
+ ) -> Result<TransactionBenchFixture, LixError> {
256
+ prepare_transaction_fixture(backend, vec![transaction_registered_schema_row()]).await
257
+ }
258
+
259
+ pub async fn prepare_transaction_commit_entities_no_payload(
260
+ backend: Arc<dyn Backend + Send + Sync>,
261
+ rows: usize,
262
+ ) -> Result<TransactionBenchFixture, LixError> {
263
+ prepare_transaction_fixture(
264
+ backend,
265
+ transaction_entity_rows(TransactionEntityRows {
266
+ rows,
267
+ payload_bytes: 0,
268
+ payload_pattern: TransactionPayloadPattern::Unique,
269
+ metadata_pattern: TransactionPayloadPattern::None,
270
+ untracked: false,
271
+ key_prefix: "entity-no-payload",
272
+ }),
273
+ )
274
+ .await
275
+ }
276
+
277
+ pub async fn prepare_transaction_commit_entities_payload_1k_unique(
278
+ backend: Arc<dyn Backend + Send + Sync>,
279
+ rows: usize,
280
+ ) -> Result<TransactionBenchFixture, LixError> {
281
+ prepare_transaction_payload_fixture(
282
+ backend,
283
+ rows,
284
+ 1024,
285
+ TransactionPayloadPattern::Unique,
286
+ false,
287
+ "entity-payload-1k-unique",
288
+ )
289
+ .await
290
+ }
291
+
292
+ pub async fn prepare_transaction_commit_entities_payload_1k_same(
293
+ backend: Arc<dyn Backend + Send + Sync>,
294
+ rows: usize,
295
+ ) -> Result<TransactionBenchFixture, LixError> {
296
+ prepare_transaction_payload_fixture(
297
+ backend,
298
+ rows,
299
+ 1024,
300
+ TransactionPayloadPattern::Same,
301
+ false,
302
+ "entity-payload-1k-same",
303
+ )
304
+ .await
305
+ }
306
+
307
+ pub async fn prepare_transaction_commit_entities_payload_1k_half_duplicate(
308
+ backend: Arc<dyn Backend + Send + Sync>,
309
+ rows: usize,
310
+ ) -> Result<TransactionBenchFixture, LixError> {
311
+ prepare_transaction_payload_fixture(
312
+ backend,
313
+ rows,
314
+ 1024,
315
+ TransactionPayloadPattern::HalfDuplicate,
316
+ false,
317
+ "entity-payload-1k-half-duplicate",
318
+ )
319
+ .await
320
+ }
321
+
322
+ pub async fn prepare_transaction_commit_entities_metadata_1k_same(
323
+ backend: Arc<dyn Backend + Send + Sync>,
324
+ rows: usize,
325
+ ) -> Result<TransactionBenchFixture, LixError> {
326
+ prepare_transaction_fixture(
327
+ backend,
328
+ transaction_entity_rows(TransactionEntityRows {
329
+ rows,
330
+ payload_bytes: 0,
331
+ payload_pattern: TransactionPayloadPattern::Unique,
332
+ metadata_pattern: TransactionPayloadPattern::Same,
333
+ untracked: false,
334
+ key_prefix: "entity-metadata-1k-same",
335
+ }),
336
+ )
337
+ .await
338
+ }
339
+
340
+ pub async fn prepare_transaction_commit_entities_payload_16k_unique(
341
+ backend: Arc<dyn Backend + Send + Sync>,
342
+ rows: usize,
343
+ ) -> Result<TransactionBenchFixture, LixError> {
344
+ prepare_transaction_payload_fixture(
345
+ backend,
346
+ rows,
347
+ 16 * 1024,
348
+ TransactionPayloadPattern::Unique,
349
+ false,
350
+ "entity-payload-16k-unique",
351
+ )
352
+ .await
353
+ }
354
+
355
+ pub async fn prepare_transaction_commit_untracked_payload_1k_same(
356
+ backend: Arc<dyn Backend + Send + Sync>,
357
+ rows: usize,
358
+ ) -> Result<TransactionBenchFixture, LixError> {
359
+ prepare_transaction_payload_fixture(
360
+ backend,
361
+ rows,
362
+ 1024,
363
+ TransactionPayloadPattern::Same,
364
+ true,
365
+ "untracked-payload-1k-same",
366
+ )
367
+ .await
368
+ }
369
+
370
+ pub async fn prepare_transaction_update_existing_payload_1k(
371
+ backend: Arc<dyn Backend + Send + Sync>,
372
+ root_rows: usize,
373
+ update_rows: usize,
374
+ ) -> Result<TransactionBenchFixture, LixError> {
375
+ let fixture = prepare_transaction_payload_fixture(
376
+ backend,
377
+ root_rows,
378
+ 1024,
379
+ TransactionPayloadPattern::Unique,
380
+ false,
381
+ "update-existing-root",
382
+ )
383
+ .await?;
384
+ transaction_commit_prepared(&fixture).await?;
385
+ let rows = transaction_entity_rows(TransactionEntityRows {
386
+ rows: update_rows,
387
+ payload_bytes: 1024,
388
+ payload_pattern: TransactionPayloadPattern::Unique,
389
+ metadata_pattern: TransactionPayloadPattern::None,
390
+ untracked: false,
391
+ key_prefix: "update-existing-root",
392
+ });
393
+ Ok(TransactionBenchFixture { rows, ..fixture })
394
+ }
395
+
396
+ pub async fn transaction_commit_prepared(
397
+ fixture: &TransactionBenchFixture,
398
+ ) -> Result<StorageBenchReport, LixError> {
399
+ let opened = open_transaction(
400
+ &SessionMode::Pinned {
401
+ version_id: crate::GLOBAL_VERSION_ID.to_string(),
402
+ },
403
+ fixture.storage.clone(),
404
+ Arc::clone(&fixture.live_state),
405
+ Arc::clone(&fixture.tracked_state),
406
+ Arc::clone(&fixture.binary_cas),
407
+ Arc::clone(&fixture.commit_store),
408
+ Arc::clone(&fixture.version_ctx),
409
+ Arc::clone(&fixture.catalog_context),
410
+ )
411
+ .await?;
412
+ let mut transaction = opened.transaction;
413
+ let runtime_functions = opened.runtime_functions;
414
+ let started_at = Instant::now();
415
+ if !fixture.rows.is_empty() {
416
+ transaction
417
+ .stage_write(TransactionWrite::Rows {
418
+ mode: TransactionWriteMode::Replace,
419
+ rows: fixture.rows.clone(),
420
+ })
421
+ .await?;
422
+ }
423
+ transaction.commit(&runtime_functions).await?;
424
+ Ok(StorageBenchReport {
425
+ measured_rows: fixture.rows.len(),
426
+ verified_rows: fixture.rows.len(),
427
+ elapsed: started_at.elapsed(),
428
+ })
429
+ }
430
+
431
+ pub async fn transaction_open_empty_prepared(
432
+ fixture: &TransactionBenchFixture,
433
+ ) -> Result<StorageBenchReport, LixError> {
434
+ let started_at = Instant::now();
435
+ let opened = open_transaction(
436
+ &SessionMode::Pinned {
437
+ version_id: crate::GLOBAL_VERSION_ID.to_string(),
438
+ },
439
+ fixture.storage.clone(),
440
+ Arc::clone(&fixture.live_state),
441
+ Arc::clone(&fixture.tracked_state),
442
+ Arc::clone(&fixture.binary_cas),
443
+ Arc::clone(&fixture.commit_store),
444
+ Arc::clone(&fixture.version_ctx),
445
+ Arc::clone(&fixture.catalog_context),
446
+ )
447
+ .await?;
448
+ let elapsed = started_at.elapsed();
449
+ opened.transaction.rollback().await?;
450
+ Ok(StorageBenchReport {
451
+ measured_rows: 0,
452
+ verified_rows: 0,
453
+ elapsed,
454
+ })
455
+ }
456
+
457
+ pub async fn transaction_stage_only_prepared(
458
+ fixture: &TransactionBenchFixture,
459
+ ) -> Result<StorageBenchReport, LixError> {
460
+ let opened = open_transaction(
461
+ &SessionMode::Pinned {
462
+ version_id: crate::GLOBAL_VERSION_ID.to_string(),
463
+ },
464
+ fixture.storage.clone(),
465
+ Arc::clone(&fixture.live_state),
466
+ Arc::clone(&fixture.tracked_state),
467
+ Arc::clone(&fixture.binary_cas),
468
+ Arc::clone(&fixture.commit_store),
469
+ Arc::clone(&fixture.version_ctx),
470
+ Arc::clone(&fixture.catalog_context),
471
+ )
472
+ .await?;
473
+ let mut transaction = opened.transaction;
474
+ let started_at = Instant::now();
475
+ if !fixture.rows.is_empty() {
476
+ transaction
477
+ .stage_write(TransactionWrite::Rows {
478
+ mode: TransactionWriteMode::Replace,
479
+ rows: fixture.rows.clone(),
480
+ })
481
+ .await?;
482
+ }
483
+ let elapsed = started_at.elapsed();
484
+ transaction.rollback().await?;
485
+ Ok(StorageBenchReport {
486
+ measured_rows: fixture.rows.len(),
487
+ verified_rows: fixture.rows.len(),
488
+ elapsed,
489
+ })
490
+ }
491
+
492
+ pub async fn prepare_transaction_commit_only(
493
+ fixture: TransactionBenchFixture,
494
+ ) -> Result<TransactionCommitOnlyFixture, LixError> {
495
+ let opened = open_transaction(
496
+ &SessionMode::Pinned {
497
+ version_id: crate::GLOBAL_VERSION_ID.to_string(),
498
+ },
499
+ fixture.storage.clone(),
500
+ Arc::clone(&fixture.live_state),
501
+ Arc::clone(&fixture.tracked_state),
502
+ Arc::clone(&fixture.binary_cas),
503
+ Arc::clone(&fixture.commit_store),
504
+ Arc::clone(&fixture.version_ctx),
505
+ Arc::clone(&fixture.catalog_context),
506
+ )
507
+ .await?;
508
+ let mut transaction = opened.transaction;
509
+ let runtime_functions = opened.runtime_functions;
510
+ let rows = fixture.rows.len();
511
+ if !fixture.rows.is_empty() {
512
+ transaction
513
+ .stage_write(TransactionWrite::Rows {
514
+ mode: TransactionWriteMode::Replace,
515
+ rows: fixture.rows,
516
+ })
517
+ .await?;
518
+ }
519
+ Ok(TransactionCommitOnlyFixture {
520
+ runtime_functions,
521
+ transaction,
522
+ rows,
523
+ })
524
+ }
525
+
526
+ pub async fn transaction_commit_only_prepared(
527
+ fixture: TransactionCommitOnlyFixture,
528
+ ) -> Result<StorageBenchReport, LixError> {
529
+ let rows = fixture.rows;
530
+ let started_at = Instant::now();
531
+ fixture
532
+ .transaction
533
+ .commit(&fixture.runtime_functions)
534
+ .await?;
535
+ Ok(StorageBenchReport {
536
+ measured_rows: rows,
537
+ verified_rows: rows,
538
+ elapsed: started_at.elapsed(),
539
+ })
540
+ }
541
+
542
+ async fn prepare_transaction_payload_fixture(
543
+ backend: Arc<dyn Backend + Send + Sync>,
544
+ rows: usize,
545
+ payload_bytes: usize,
546
+ payload_pattern: TransactionPayloadPattern,
547
+ untracked: bool,
548
+ key_prefix: &'static str,
549
+ ) -> Result<TransactionBenchFixture, LixError> {
550
+ prepare_transaction_fixture(
551
+ backend,
552
+ transaction_entity_rows(TransactionEntityRows {
553
+ rows,
554
+ payload_bytes,
555
+ payload_pattern,
556
+ metadata_pattern: TransactionPayloadPattern::None,
557
+ untracked,
558
+ key_prefix,
559
+ }),
560
+ )
561
+ .await
562
+ }
563
+
564
+ async fn prepare_transaction_fixture(
565
+ backend: Arc<dyn Backend + Send + Sync>,
566
+ rows: Vec<TransactionWriteRow>,
567
+ ) -> Result<TransactionBenchFixture, LixError> {
568
+ let storage = StorageContext::new(backend);
569
+ let tracked_state = Arc::new(TrackedStateContext::new());
570
+ let untracked_state = Arc::new(UntrackedStateContext::new());
571
+ let commit_store = Arc::new(CommitStoreContext::new());
572
+ let live_state = Arc::new(LiveStateContext::new(
573
+ tracked_state.as_ref().clone(),
574
+ untracked_state.as_ref().clone(),
575
+ crate::commit_graph::CommitGraphContext::new(),
576
+ ));
577
+ let binary_cas = Arc::new(BinaryCasContext::new());
578
+ let version_ctx = Arc::new(VersionContext::new(untracked_state));
579
+ let catalog_context = Arc::new(CatalogContext::new());
580
+ seed_transaction_visible_schema_rows(storage.clone()).await?;
581
+ Ok(TransactionBenchFixture {
582
+ storage,
583
+ live_state,
584
+ tracked_state,
585
+ binary_cas,
586
+ commit_store,
587
+ version_ctx,
588
+ catalog_context,
589
+ rows,
590
+ })
591
+ }
592
+
593
+ async fn seed_transaction_visible_schema_rows(storage: StorageContext) -> Result<(), LixError> {
594
+ let mut writes = StorageWriteSet::new();
595
+ let rows = crate::schema::seed_schema_definitions()
596
+ .into_iter()
597
+ .cloned()
598
+ .chain(std::iter::once(transaction_entity_schema_definition()))
599
+ .map(|schema| {
600
+ let key = crate::schema::schema_key_from_definition(&schema)
601
+ .expect("seed schema key should derive");
602
+ let snapshot_content = serde_json::json!({ "value": schema }).to_string();
603
+ Ok(crate::untracked_state::UntrackedStateRow {
604
+ entity_id: crate::schema::registered_schema_entity_id(&key.schema_key)
605
+ .expect("registered schema identity should derive"),
606
+ schema_key: "lix_registered_schema".to_string(),
607
+ file_id: None,
608
+ version_id: crate::GLOBAL_VERSION_ID.to_string(),
609
+ snapshot_content: Some(snapshot_content),
610
+ metadata: None,
611
+ created_at: "1970-01-01T00:00:00.000Z".to_string(),
612
+ updated_at: "1970-01-01T00:00:00.000Z".to_string(),
613
+ global: true,
614
+ })
615
+ })
616
+ .collect::<Result<Vec<_>, LixError>>()?;
617
+ let mut transaction = storage.begin_write_transaction().await?;
618
+ UntrackedStateContext::new()
619
+ .writer(&mut writes)
620
+ .stage_rows(rows.iter().map(|row| row.as_ref()))?;
621
+ writes.apply(&mut transaction.as_mut()).await?;
622
+ transaction.commit().await
623
+ }
624
+
625
+ fn transaction_entity_schema_definition() -> serde_json::Value {
626
+ serde_json::json!({
627
+ "x-lix-key": TRANSACTION_BENCH_SCHEMA_KEY,
628
+ "type": "object",
629
+ "properties": {
630
+ "value": {
631
+ "anyOf": [
632
+ { "type": "string" },
633
+ { "type": "object" },
634
+ { "type": "array" },
635
+ { "type": "number" },
636
+ { "type": "boolean" },
637
+ { "type": "null" }
638
+ ]
639
+ }
640
+ },
641
+ "required": ["value"],
642
+ "additionalProperties": false
643
+ })
644
+ }
645
+
646
+ #[derive(Debug, Clone, Copy)]
647
+ enum TransactionPayloadPattern {
648
+ None,
649
+ Unique,
650
+ Same,
651
+ HalfDuplicate,
652
+ }
653
+
654
+ struct TransactionEntityRows {
655
+ rows: usize,
656
+ payload_bytes: usize,
657
+ payload_pattern: TransactionPayloadPattern,
658
+ metadata_pattern: TransactionPayloadPattern,
659
+ untracked: bool,
660
+ key_prefix: &'static str,
661
+ }
662
+
663
+ fn transaction_entity_rows(config: TransactionEntityRows) -> Vec<TransactionWriteRow> {
664
+ (0..config.rows)
665
+ .map(|index| {
666
+ let key = format!("{}-{index:06}", config.key_prefix);
667
+ let value_index = payload_pattern_index(config.payload_pattern, index);
668
+ let metadata_index = payload_pattern_index(config.metadata_pattern, index);
669
+ TransactionWriteRow {
670
+ entity_id: Some(EntityIdentity::single(key.clone())),
671
+ schema_key: TRANSACTION_BENCH_SCHEMA_KEY.to_string(),
672
+ file_id: None,
673
+ snapshot: Some(transaction_snapshot_json(
674
+ &key,
675
+ value_index,
676
+ config.payload_bytes,
677
+ )),
678
+ metadata: transaction_metadata(config.metadata_pattern, metadata_index),
679
+ origin: None,
680
+ created_at: None,
681
+ updated_at: None,
682
+ global: true,
683
+ change_id: None,
684
+ commit_id: None,
685
+ untracked: config.untracked,
686
+ version_id: crate::GLOBAL_VERSION_ID.to_string(),
687
+ }
688
+ })
689
+ .collect()
690
+ }
691
+
692
+ fn transaction_registered_schema_row() -> TransactionWriteRow {
693
+ let schema = serde_json::json!({
694
+ "x-lix-key": "bench_transaction_schema",
695
+ "x-lix-primary-key": ["/id"],
696
+ "type": "object",
697
+ "properties": {
698
+ "id": { "type": "string" },
699
+ "value": { "type": "string" }
700
+ },
701
+ "required": ["id", "value"],
702
+ "additionalProperties": false
703
+ });
704
+ let key =
705
+ crate::schema::schema_key_from_definition(&schema).expect("seed schema key should derive");
706
+ TransactionWriteRow {
707
+ entity_id: Some(
708
+ crate::schema::registered_schema_entity_id(&key.schema_key)
709
+ .expect("registered schema identity should derive"),
710
+ ),
711
+ schema_key: "lix_registered_schema".to_string(),
712
+ file_id: None,
713
+ snapshot: Some(TransactionJson::from_value_unchecked(
714
+ serde_json::json!({ "value": schema }),
715
+ )),
716
+ metadata: None,
717
+ origin: None,
718
+ created_at: None,
719
+ updated_at: None,
720
+ global: true,
721
+ change_id: None,
722
+ commit_id: None,
723
+ untracked: false,
724
+ version_id: crate::GLOBAL_VERSION_ID.to_string(),
725
+ }
726
+ }
727
+
728
+ fn transaction_snapshot_json(
729
+ _key: &str,
730
+ payload_index: usize,
731
+ target_bytes: usize,
732
+ ) -> TransactionJson {
733
+ let base_value = format!("/entities/{payload_index}/value");
734
+ let value = if target_bytes == 0 {
735
+ base_value
736
+ } else {
737
+ let current = serde_json::json!({
738
+ "value": base_value,
739
+ })
740
+ .to_string()
741
+ .len();
742
+ let padding = target_bytes.saturating_sub(current);
743
+ format!("{base_value}:{}", "x".repeat(padding))
744
+ };
745
+ let mut object = serde_json::Map::new();
746
+ object.insert("value".to_string(), serde_json::Value::String(value));
747
+ TransactionJson::from_value_unchecked(serde_json::Value::Object(object))
748
+ }
749
+
750
+ fn transaction_metadata(
751
+ pattern: TransactionPayloadPattern,
752
+ metadata_index: usize,
753
+ ) -> Option<TransactionJson> {
754
+ match pattern {
755
+ TransactionPayloadPattern::None => None,
756
+ TransactionPayloadPattern::Unique
757
+ | TransactionPayloadPattern::Same
758
+ | TransactionPayloadPattern::HalfDuplicate => {
759
+ let mut object = serde_json::Map::new();
760
+ object.insert(
761
+ "source".to_string(),
762
+ serde_json::Value::String("transaction-bench".to_string()),
763
+ );
764
+ object.insert(
765
+ "metadata_index".to_string(),
766
+ serde_json::Value::String(metadata_index.to_string()),
767
+ );
768
+ pad_json_object(&mut object, 1024);
769
+ Some(TransactionJson::from_value_unchecked(
770
+ serde_json::Value::Object(object),
771
+ ))
772
+ }
773
+ }
774
+ }
775
+
776
+ fn payload_pattern_index(pattern: TransactionPayloadPattern, index: usize) -> usize {
777
+ match pattern {
778
+ TransactionPayloadPattern::None | TransactionPayloadPattern::Unique => index,
779
+ TransactionPayloadPattern::Same => 0,
780
+ TransactionPayloadPattern::HalfDuplicate => index % 2,
781
+ }
782
+ }
125
783
 
126
784
  pub async fn storage_api_write_kv_batch_puts(
127
785
  backend: Arc<dyn Backend + Send + Sync>,
@@ -689,7 +1347,7 @@ fn storage_api_updated_value(index: usize) -> Vec<u8> {
689
1347
 
690
1348
  pub struct TrackedStateWriteRootFixture {
691
1349
  context: TrackedStateContext,
692
- rows: Vec<TrackedStateRow>,
1350
+ rows: Vec<MaterializedTrackedStateRow>,
693
1351
  }
694
1352
 
695
1353
  pub struct TrackedStateReadFixture {
@@ -702,7 +1360,7 @@ pub struct TrackedStateReadFixture {
702
1360
 
703
1361
  pub struct TrackedStateUpdateFixture {
704
1362
  context: TrackedStateContext,
705
- rows: Vec<TrackedStateRow>,
1363
+ rows: Vec<MaterializedTrackedStateRow>,
706
1364
  }
707
1365
 
708
1366
  pub struct TrackedStateDiffFixture {
@@ -712,6 +1370,32 @@ pub struct TrackedStateDiffFixture {
712
1370
  expected_entries: usize,
713
1371
  }
714
1372
 
1373
+ pub struct TrackedStateMaterializeFixture {
1374
+ context: TrackedStateContext,
1375
+ commit_id: String,
1376
+ expected_rows: usize,
1377
+ }
1378
+
1379
+ #[derive(Clone)]
1380
+ pub struct JsonPointerStorageRow {
1381
+ pub path: String,
1382
+ pub value_json: String,
1383
+ pub updated_value_json: String,
1384
+ }
1385
+
1386
+ pub struct JsonPointerTrackedStateReadFixture {
1387
+ context: TrackedStateContext,
1388
+ rows: Vec<JsonPointerStorageRow>,
1389
+ commit_id: String,
1390
+ }
1391
+
1392
+ pub struct JsonPointerTrackedStateDiffFixture {
1393
+ context: TrackedStateContext,
1394
+ left_commit_id: String,
1395
+ right_commit_id: String,
1396
+ expected_entries: usize,
1397
+ }
1398
+
715
1399
  pub struct UntrackedStateWriteFixture {
716
1400
  context: UntrackedStateContext,
717
1401
  rows: Vec<MaterializedUntrackedStateRow>,
@@ -725,20 +1409,25 @@ pub struct UntrackedStateReadFixture {
725
1409
  }
726
1410
 
727
1411
  pub struct ChangelogAppendFixture {
728
- context: ChangelogContext,
729
- changes: Vec<MaterializedCanonicalChange>,
1412
+ context: CommitStoreContext,
1413
+ changes: Vec<MaterializedChange>,
730
1414
  }
731
1415
 
732
1416
  pub struct ChangelogReadFixture {
733
- context: ChangelogContext,
1417
+ context: CommitStoreContext,
734
1418
  rows: usize,
735
1419
  }
736
1420
 
737
1421
  pub struct ChangelogCodecFixture {
738
- changes: Vec<CanonicalChange>,
1422
+ changes: Vec<Change>,
739
1423
  encoded_changes: Vec<Vec<u8>>,
740
1424
  }
741
1425
 
1426
+ pub struct CommitGraphReadFixture {
1427
+ head_commit_id: String,
1428
+ rows: usize,
1429
+ }
1430
+
742
1431
  pub struct BinaryCasWriteFixture {
743
1432
  context: BinaryCasContext,
744
1433
  file_ids: Vec<String>,
@@ -839,97 +1528,161 @@ pub async fn prepare_tracked_state_read_file_selective(
839
1528
  })
840
1529
  }
841
1530
 
842
- pub async fn tracked_state_read_point_hit_prepared(
843
- backend: &Arc<dyn Backend + Send + Sync>,
844
- fixture: &TrackedStateReadFixture,
845
- ) -> Result<StorageBenchReport, LixError> {
846
- let mut verified_rows = 0;
847
- let mut reader = fixture
848
- .context
849
- .reader(StorageContext::new(Arc::clone(backend)));
850
- for index in 0..fixture.rows {
851
- if reader
852
- .load_row_at_commit(
853
- &fixture.commit_id,
854
- &TrackedStateRowRequest {
855
- schema_key: tracked_schema_key(index, StorageBenchSelectivity::Percent100),
856
- entity_id: EntityIdentity::single(entity_id(
857
- "tracked",
858
- index,
859
- fixture.key_pattern,
860
- )),
861
- file_id: NullableKeyFilter::Value("bench.json".to_string()),
862
- },
863
- )
864
- .await?
865
- .is_some()
866
- {
867
- verified_rows += 1;
868
- }
869
- }
870
- Ok(report(fixture.rows, verified_rows, Duration::ZERO))
871
- }
872
-
873
- pub async fn tracked_state_read_point_hit_constant_prepared(
1531
+ pub async fn prepare_tracked_state_read_after_update_rows(
874
1532
  backend: &Arc<dyn Backend + Send + Sync>,
875
- fixture: &TrackedStateReadFixture,
876
- measured_reads: usize,
1533
+ config: StorageBenchConfig,
1534
+ updated_rows: usize,
1535
+ ) -> Result<TrackedStateReadFixture, LixError> {
1536
+ let fixture = prepare_tracked_state_update_rows(backend, config, updated_rows).await?;
1537
+ tracked_state_update_existing_prepared(backend, &fixture).await?;
1538
+ Ok(TrackedStateReadFixture {
1539
+ context: fixture.context,
1540
+ rows: config.rows,
1541
+ commit_id: "bench-tracked-child".to_string(),
1542
+ key_pattern: config.key_pattern,
1543
+ selectivity: config.selectivity,
1544
+ })
1545
+ }
1546
+
1547
+ pub async fn prepare_tracked_state_read_delta_chain(
1548
+ backend: &Arc<dyn Backend + Send + Sync>,
1549
+ config: StorageBenchConfig,
1550
+ delta_commits: usize,
1551
+ updated_rows_per_commit: usize,
1552
+ ) -> Result<TrackedStateReadFixture, LixError> {
1553
+ let (context, final_commit_id) =
1554
+ write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
1555
+ Ok(TrackedStateReadFixture {
1556
+ context,
1557
+ rows: config.rows,
1558
+ commit_id: final_commit_id,
1559
+ key_pattern: config.key_pattern,
1560
+ selectivity: config.selectivity,
1561
+ })
1562
+ }
1563
+
1564
+ pub async fn prepare_tracked_state_read_materialized_delta_chain(
1565
+ backend: &Arc<dyn Backend + Send + Sync>,
1566
+ config: StorageBenchConfig,
1567
+ delta_commits: usize,
1568
+ updated_rows_per_commit: usize,
1569
+ ) -> Result<TrackedStateReadFixture, LixError> {
1570
+ let (context, final_commit_id) =
1571
+ write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
1572
+ materialize_tracked_root(backend, &context, &final_commit_id).await?;
1573
+ Ok(TrackedStateReadFixture {
1574
+ context,
1575
+ rows: config.rows,
1576
+ commit_id: final_commit_id,
1577
+ key_pattern: config.key_pattern,
1578
+ selectivity: config.selectivity,
1579
+ })
1580
+ }
1581
+
1582
+ pub async fn prepare_tracked_state_materialize_delta_chain(
1583
+ backend: &Arc<dyn Backend + Send + Sync>,
1584
+ config: StorageBenchConfig,
1585
+ delta_commits: usize,
1586
+ updated_rows_per_commit: usize,
1587
+ ) -> Result<TrackedStateMaterializeFixture, LixError> {
1588
+ let (context, final_commit_id) =
1589
+ write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
1590
+ Ok(TrackedStateMaterializeFixture {
1591
+ context,
1592
+ commit_id: final_commit_id,
1593
+ expected_rows: config.rows,
1594
+ })
1595
+ }
1596
+
1597
+ fn tracked_point_hit_requests(
1598
+ rows: usize,
1599
+ key_pattern: StorageBenchKeyPattern,
1600
+ ) -> Vec<TrackedStateRowRequest> {
1601
+ (0..rows)
1602
+ .map(|index| TrackedStateRowRequest {
1603
+ schema_key: tracked_schema_key(index, StorageBenchSelectivity::Percent100),
1604
+ entity_id: EntityIdentity::single(entity_id("tracked", index, key_pattern)),
1605
+ file_id: NullableKeyFilter::Value("bench.json".to_string()),
1606
+ })
1607
+ .collect()
1608
+ }
1609
+
1610
+ fn tracked_point_miss_requests(
1611
+ rows: usize,
1612
+ selectivity: StorageBenchSelectivity,
1613
+ ) -> Vec<TrackedStateRowRequest> {
1614
+ (0..rows)
1615
+ .map(|index| TrackedStateRowRequest {
1616
+ schema_key: tracked_schema_key(index, selectivity),
1617
+ entity_id: EntityIdentity::single(format!("missing-{index}")),
1618
+ file_id: NullableKeyFilter::Value("bench.json".to_string()),
1619
+ })
1620
+ .collect()
1621
+ }
1622
+
1623
+ fn tracked_point_miss_requests_for_schema(
1624
+ rows: usize,
1625
+ schema_key: &str,
1626
+ ) -> Vec<TrackedStateRowRequest> {
1627
+ (0..rows)
1628
+ .map(|index| TrackedStateRowRequest {
1629
+ schema_key: schema_key.to_string(),
1630
+ entity_id: EntityIdentity::single(format!("missing-{index}")),
1631
+ file_id: NullableKeyFilter::Value("bench.json".to_string()),
1632
+ })
1633
+ .collect()
1634
+ }
1635
+
1636
+ pub async fn tracked_state_read_point_hit_prepared(
1637
+ backend: &Arc<dyn Backend + Send + Sync>,
1638
+ fixture: &TrackedStateReadFixture,
877
1639
  ) -> Result<StorageBenchReport, LixError> {
878
- let mut verified_rows = 0;
879
1640
  let mut reader = fixture
880
1641
  .context
881
1642
  .reader(StorageContext::new(Arc::clone(backend)));
882
- for index in 0..measured_reads.min(fixture.rows) {
883
- if reader
884
- .load_row_at_commit(
885
- &fixture.commit_id,
886
- &TrackedStateRowRequest {
887
- schema_key: tracked_schema_key(index, StorageBenchSelectivity::Percent100),
888
- entity_id: EntityIdentity::single(entity_id(
889
- "tracked",
890
- index,
891
- fixture.key_pattern,
892
- )),
893
- file_id: NullableKeyFilter::Value("bench.json".to_string()),
894
- },
895
- )
896
- .await?
897
- .is_some()
898
- {
899
- verified_rows += 1;
900
- }
901
- }
902
- Ok(report(
903
- measured_reads.min(fixture.rows),
904
- verified_rows,
905
- Duration::ZERO,
906
- ))
1643
+ let requests = tracked_point_hit_requests(fixture.rows, fixture.key_pattern);
1644
+ let verified_rows = reader
1645
+ .load_rows_at_commit(&fixture.commit_id, &requests)
1646
+ .await?
1647
+ .into_iter()
1648
+ .filter(Option::is_some)
1649
+ .count();
1650
+ Ok(report(fixture.rows, verified_rows, Duration::ZERO))
1651
+ }
1652
+
1653
+ pub async fn tracked_state_read_point_hit_constant_prepared(
1654
+ backend: &Arc<dyn Backend + Send + Sync>,
1655
+ fixture: &TrackedStateReadFixture,
1656
+ measured_reads: usize,
1657
+ ) -> Result<StorageBenchReport, LixError> {
1658
+ let measured_rows = measured_reads.min(fixture.rows);
1659
+ let mut reader = fixture
1660
+ .context
1661
+ .reader(StorageContext::new(Arc::clone(backend)));
1662
+ let requests = tracked_point_hit_requests(measured_rows, fixture.key_pattern);
1663
+ let verified_rows = reader
1664
+ .load_rows_at_commit(&fixture.commit_id, &requests)
1665
+ .await?
1666
+ .into_iter()
1667
+ .filter(Option::is_some)
1668
+ .count();
1669
+ Ok(report(measured_rows, verified_rows, Duration::ZERO))
907
1670
  }
908
1671
 
909
1672
  pub async fn tracked_state_read_point_miss_prepared(
910
1673
  backend: &Arc<dyn Backend + Send + Sync>,
911
1674
  fixture: &TrackedStateReadFixture,
912
1675
  ) -> Result<StorageBenchReport, LixError> {
913
- let mut misses = 0;
914
1676
  let mut reader = fixture
915
1677
  .context
916
1678
  .reader(StorageContext::new(Arc::clone(backend)));
917
- for index in 0..fixture.rows {
918
- if reader
919
- .load_row_at_commit(
920
- &fixture.commit_id,
921
- &TrackedStateRowRequest {
922
- schema_key: "bench_tracked_entity".to_string(),
923
- entity_id: EntityIdentity::single(format!("missing-{index}")),
924
- file_id: NullableKeyFilter::Value("bench.json".to_string()),
925
- },
926
- )
927
- .await?
928
- .is_none()
929
- {
930
- misses += 1;
931
- }
932
- }
1679
+ let requests = tracked_point_miss_requests_for_schema(fixture.rows, TRACKED_MATCH_SCHEMA_KEY);
1680
+ let misses = reader
1681
+ .load_rows_at_commit(&fixture.commit_id, &requests)
1682
+ .await?
1683
+ .into_iter()
1684
+ .filter(Option::is_none)
1685
+ .count();
933
1686
  Ok(report(fixture.rows, misses, Duration::ZERO))
934
1687
  }
935
1688
 
@@ -1261,6 +2014,22 @@ pub async fn prepare_tracked_state_diff_update_rows(
1261
2014
  })
1262
2015
  }
1263
2016
 
2017
+ pub async fn prepare_tracked_state_diff_delta_chain(
2018
+ backend: &Arc<dyn Backend + Send + Sync>,
2019
+ config: StorageBenchConfig,
2020
+ delta_commits: usize,
2021
+ updated_rows_per_commit: usize,
2022
+ ) -> Result<TrackedStateDiffFixture, LixError> {
2023
+ let (context, final_commit_id) =
2024
+ write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
2025
+ Ok(TrackedStateDiffFixture {
2026
+ context,
2027
+ left_commit_id: "bench-tracked-base".to_string(),
2028
+ right_commit_id: final_commit_id,
2029
+ expected_entries: updated_rows_per_commit.min(config.rows),
2030
+ })
2031
+ }
2032
+
1264
2033
  pub async fn prepare_tracked_state_diff_tombstone_rows(
1265
2034
  backend: &Arc<dyn Backend + Send + Sync>,
1266
2035
  config: StorageBenchConfig,
@@ -1276,24 +2045,315 @@ pub async fn prepare_tracked_state_diff_tombstone_rows(
1276
2045
  })
1277
2046
  }
1278
2047
 
1279
- pub async fn prepare_tracked_state_diff_equal(
2048
+ pub async fn prepare_tracked_state_diff_equal(
2049
+ backend: &Arc<dyn Backend + Send + Sync>,
2050
+ config: StorageBenchConfig,
2051
+ ) -> Result<TrackedStateDiffFixture, LixError> {
2052
+ let context = TrackedStateContext::new();
2053
+ let rows = tracked_rows(config, "bench-tracked-parent");
2054
+ write_tracked_root(backend, &context, "bench-tracked-parent", None, &rows).await?;
2055
+ Ok(TrackedStateDiffFixture {
2056
+ context,
2057
+ left_commit_id: "bench-tracked-parent".to_string(),
2058
+ right_commit_id: "bench-tracked-parent".to_string(),
2059
+ expected_entries: 0,
2060
+ })
2061
+ }
2062
+
2063
+ pub async fn tracked_state_diff_commits_prepared(
2064
+ backend: &Arc<dyn Backend + Send + Sync>,
2065
+ fixture: &TrackedStateDiffFixture,
2066
+ ) -> Result<StorageBenchReport, LixError> {
2067
+ let mut reader = fixture
2068
+ .context
2069
+ .reader(StorageContext::new(Arc::clone(backend)));
2070
+ let diff = reader
2071
+ .diff_commits(
2072
+ &fixture.left_commit_id,
2073
+ &fixture.right_commit_id,
2074
+ &TrackedStateDiffRequest::default(),
2075
+ )
2076
+ .await?;
2077
+ Ok(report(
2078
+ fixture.expected_entries,
2079
+ diff.entries.len(),
2080
+ Duration::ZERO,
2081
+ ))
2082
+ }
2083
+
2084
+ pub async fn tracked_state_materialize_root_prepared(
2085
+ backend: &Arc<dyn Backend + Send + Sync>,
2086
+ fixture: &TrackedStateMaterializeFixture,
2087
+ ) -> Result<StorageBenchReport, LixError> {
2088
+ materialize_tracked_root(backend, &fixture.context, &fixture.commit_id).await?;
2089
+ Ok(report(
2090
+ fixture.expected_rows,
2091
+ fixture.expected_rows,
2092
+ Duration::ZERO,
2093
+ ))
2094
+ }
2095
+
2096
+ pub async fn prepare_json_pointer_tracked_state_write_root(
2097
+ rows: &[JsonPointerStorageRow],
2098
+ ) -> Result<TrackedStateWriteRootFixture, LixError> {
2099
+ Ok(TrackedStateWriteRootFixture {
2100
+ context: TrackedStateContext::new(),
2101
+ rows: json_pointer_tracked_rows(rows, "json-pointer-base", false),
2102
+ })
2103
+ }
2104
+
2105
+ pub async fn prepare_json_pointer_tracked_state_read(
2106
+ backend: &Arc<dyn Backend + Send + Sync>,
2107
+ rows: &[JsonPointerStorageRow],
2108
+ ) -> Result<JsonPointerTrackedStateReadFixture, LixError> {
2109
+ let context = TrackedStateContext::new();
2110
+ let materialized_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
2111
+ write_tracked_root(
2112
+ backend,
2113
+ &context,
2114
+ "json-pointer-base",
2115
+ None,
2116
+ &materialized_rows,
2117
+ )
2118
+ .await?;
2119
+ Ok(JsonPointerTrackedStateReadFixture {
2120
+ context,
2121
+ rows: rows.to_vec(),
2122
+ commit_id: "json-pointer-base".to_string(),
2123
+ })
2124
+ }
2125
+
2126
+ pub async fn prepare_json_pointer_tracked_state_diff_update_rows(
2127
+ backend: &Arc<dyn Backend + Send + Sync>,
2128
+ rows: &[JsonPointerStorageRow],
2129
+ updated_rows: usize,
2130
+ ) -> Result<JsonPointerTrackedStateDiffFixture, LixError> {
2131
+ let context = TrackedStateContext::new();
2132
+ let base_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
2133
+ write_tracked_root(backend, &context, "json-pointer-base", None, &base_rows).await?;
2134
+ let child_rows = json_pointer_tracked_rows(
2135
+ &rows[..updated_rows.min(rows.len())],
2136
+ "json-pointer-child",
2137
+ true,
2138
+ );
2139
+ write_tracked_root(
2140
+ backend,
2141
+ &context,
2142
+ "json-pointer-child",
2143
+ Some("json-pointer-base"),
2144
+ &child_rows,
2145
+ )
2146
+ .await?;
2147
+ Ok(JsonPointerTrackedStateDiffFixture {
2148
+ context,
2149
+ left_commit_id: "json-pointer-base".to_string(),
2150
+ right_commit_id: "json-pointer-child".to_string(),
2151
+ expected_entries: child_rows.len(),
2152
+ })
2153
+ }
2154
+
2155
+ pub async fn json_pointer_tracked_state_get_many_prepared(
2156
+ backend: &Arc<dyn Backend + Send + Sync>,
2157
+ fixture: &JsonPointerTrackedStateReadFixture,
2158
+ ) -> Result<StorageBenchReport, LixError> {
2159
+ let mut reader = fixture
2160
+ .context
2161
+ .reader(StorageContext::new(Arc::clone(backend)));
2162
+ let requests = fixture
2163
+ .rows
2164
+ .iter()
2165
+ .map(|row| TrackedStateRowRequest {
2166
+ schema_key: "json_pointer".to_string(),
2167
+ entity_id: EntityIdentity::single(row.path.as_str()),
2168
+ file_id: NullableKeyFilter::Null,
2169
+ })
2170
+ .collect::<Vec<_>>();
2171
+ let verified_rows = reader
2172
+ .load_rows_at_commit(&fixture.commit_id, &requests)
2173
+ .await?
2174
+ .into_iter()
2175
+ .filter(Option::is_some)
2176
+ .count();
2177
+ Ok(report(fixture.rows.len(), verified_rows, Duration::ZERO))
2178
+ }
2179
+
2180
+ pub async fn json_pointer_tracked_state_get_many_missing_prepared(
2181
+ backend: &Arc<dyn Backend + Send + Sync>,
2182
+ fixture: &JsonPointerTrackedStateReadFixture,
2183
+ ) -> Result<StorageBenchReport, LixError> {
2184
+ let mut reader = fixture
2185
+ .context
2186
+ .reader(StorageContext::new(Arc::clone(backend)));
2187
+ let requests = fixture
2188
+ .rows
2189
+ .iter()
2190
+ .map(|row| TrackedStateRowRequest {
2191
+ schema_key: "json_pointer".to_string(),
2192
+ entity_id: EntityIdentity::single(format!("missing{}", row.path)),
2193
+ file_id: NullableKeyFilter::Null,
2194
+ })
2195
+ .collect::<Vec<_>>();
2196
+ let verified_rows = reader
2197
+ .load_rows_at_commit(&fixture.commit_id, &requests)
2198
+ .await?
2199
+ .into_iter()
2200
+ .filter(Option::is_none)
2201
+ .count();
2202
+ Ok(report(fixture.rows.len(), verified_rows, Duration::ZERO))
2203
+ }
2204
+
2205
+ pub async fn json_pointer_tracked_state_scan_keys_only_prepared(
2206
+ backend: &Arc<dyn Backend + Send + Sync>,
2207
+ fixture: &JsonPointerTrackedStateReadFixture,
2208
+ ) -> Result<StorageBenchReport, LixError> {
2209
+ json_pointer_scan_with_projection(
2210
+ backend,
2211
+ fixture,
2212
+ TrackedStateProjection {
2213
+ columns: vec!["entity_id".to_string()],
2214
+ },
2215
+ )
2216
+ .await
2217
+ }
2218
+
2219
+ pub async fn json_pointer_tracked_state_scan_headers_only_prepared(
2220
+ backend: &Arc<dyn Backend + Send + Sync>,
2221
+ fixture: &JsonPointerTrackedStateReadFixture,
2222
+ ) -> Result<StorageBenchReport, LixError> {
2223
+ json_pointer_scan_with_projection(
2224
+ backend,
2225
+ fixture,
2226
+ TrackedStateProjection {
2227
+ columns: tracked_state_header_columns(),
2228
+ },
2229
+ )
2230
+ .await
2231
+ }
2232
+
2233
+ pub async fn json_pointer_tracked_state_scan_full_rows_prepared(
2234
+ backend: &Arc<dyn Backend + Send + Sync>,
2235
+ fixture: &JsonPointerTrackedStateReadFixture,
2236
+ ) -> Result<StorageBenchReport, LixError> {
2237
+ json_pointer_scan_with_projection(backend, fixture, TrackedStateProjection::default()).await
2238
+ }
2239
+
2240
+ pub async fn json_pointer_tracked_state_prefix_scan_schema_prepared(
2241
+ backend: &Arc<dyn Backend + Send + Sync>,
2242
+ fixture: &JsonPointerTrackedStateReadFixture,
2243
+ ) -> Result<StorageBenchReport, LixError> {
2244
+ json_pointer_scan_with_projection(backend, fixture, TrackedStateProjection::default()).await
2245
+ }
2246
+
2247
+ pub async fn json_pointer_tracked_state_prefix_scan_schema_file_null_prepared(
2248
+ backend: &Arc<dyn Backend + Send + Sync>,
2249
+ fixture: &JsonPointerTrackedStateReadFixture,
2250
+ ) -> Result<StorageBenchReport, LixError> {
2251
+ json_pointer_scan_with_projection(backend, fixture, TrackedStateProjection::default()).await
2252
+ }
2253
+
2254
+ async fn json_pointer_scan_with_projection(
2255
+ backend: &Arc<dyn Backend + Send + Sync>,
2256
+ fixture: &JsonPointerTrackedStateReadFixture,
2257
+ projection: TrackedStateProjection,
2258
+ ) -> Result<StorageBenchReport, LixError> {
2259
+ let mut reader = fixture
2260
+ .context
2261
+ .reader(StorageContext::new(Arc::clone(backend)));
2262
+ let verified_rows = reader
2263
+ .scan_rows_at_commit(
2264
+ &fixture.commit_id,
2265
+ &TrackedStateScanRequest {
2266
+ filter: TrackedStateFilter {
2267
+ schema_keys: vec!["json_pointer".to_string()],
2268
+ file_ids: vec![NullableKeyFilter::Null],
2269
+ ..Default::default()
2270
+ },
2271
+ projection,
2272
+ ..Default::default()
2273
+ },
2274
+ )
2275
+ .await?
2276
+ .len();
2277
+ Ok(report(fixture.rows.len(), verified_rows, Duration::ZERO))
2278
+ }
2279
+
2280
+ pub async fn prepare_json_pointer_tracked_state_update_rows(
2281
+ backend: &Arc<dyn Backend + Send + Sync>,
2282
+ rows: &[JsonPointerStorageRow],
2283
+ updated_rows: usize,
2284
+ ) -> Result<TrackedStateUpdateFixture, LixError> {
2285
+ let context = TrackedStateContext::new();
2286
+ let base_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
2287
+ write_tracked_root(backend, &context, "json-pointer-base", None, &base_rows).await?;
2288
+ let child_rows = json_pointer_tracked_rows(
2289
+ &rows[..updated_rows.min(rows.len())],
2290
+ "json-pointer-child",
2291
+ true,
2292
+ );
2293
+ Ok(TrackedStateUpdateFixture {
2294
+ context,
2295
+ rows: child_rows,
2296
+ })
2297
+ }
2298
+
2299
+ pub async fn prepare_json_pointer_tracked_state_tombstone_rows(
2300
+ backend: &Arc<dyn Backend + Send + Sync>,
2301
+ rows: &[JsonPointerStorageRow],
2302
+ tombstone_rows: usize,
2303
+ ) -> Result<TrackedStateUpdateFixture, LixError> {
2304
+ let context = TrackedStateContext::new();
2305
+ let base_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
2306
+ write_tracked_root(backend, &context, "json-pointer-base", None, &base_rows).await?;
2307
+ let mut child_rows = json_pointer_tracked_rows(
2308
+ &rows[..tombstone_rows.min(rows.len())],
2309
+ "json-pointer-child",
2310
+ true,
2311
+ );
2312
+ for row in &mut child_rows {
2313
+ row.snapshot_content = None;
2314
+ }
2315
+ Ok(TrackedStateUpdateFixture {
2316
+ context,
2317
+ rows: child_rows,
2318
+ })
2319
+ }
2320
+
2321
+ pub async fn prepare_json_pointer_tracked_state_diff_delta_chain(
2322
+ backend: &Arc<dyn Backend + Send + Sync>,
2323
+ rows: &[JsonPointerStorageRow],
2324
+ delta_commits: usize,
2325
+ updated_rows_per_commit: usize,
2326
+ ) -> Result<JsonPointerTrackedStateDiffFixture, LixError> {
2327
+ let (context, final_commit_id) =
2328
+ write_json_pointer_delta_chain(backend, rows, delta_commits, updated_rows_per_commit)
2329
+ .await?;
2330
+ Ok(JsonPointerTrackedStateDiffFixture {
2331
+ context,
2332
+ left_commit_id: "json-pointer-base".to_string(),
2333
+ right_commit_id: final_commit_id,
2334
+ expected_entries: updated_rows_per_commit.min(rows.len()),
2335
+ })
2336
+ }
2337
+
2338
+ pub async fn prepare_json_pointer_tracked_state_materialize_delta_chain(
1280
2339
  backend: &Arc<dyn Backend + Send + Sync>,
1281
- config: StorageBenchConfig,
1282
- ) -> Result<TrackedStateDiffFixture, LixError> {
1283
- let context = TrackedStateContext::new();
1284
- let rows = tracked_rows(config, "bench-tracked-parent");
1285
- write_tracked_root(backend, &context, "bench-tracked-parent", None, &rows).await?;
1286
- Ok(TrackedStateDiffFixture {
2340
+ rows: &[JsonPointerStorageRow],
2341
+ delta_commits: usize,
2342
+ updated_rows_per_commit: usize,
2343
+ ) -> Result<TrackedStateMaterializeFixture, LixError> {
2344
+ let (context, final_commit_id) =
2345
+ write_json_pointer_delta_chain(backend, rows, delta_commits, updated_rows_per_commit)
2346
+ .await?;
2347
+ Ok(TrackedStateMaterializeFixture {
1287
2348
  context,
1288
- left_commit_id: "bench-tracked-parent".to_string(),
1289
- right_commit_id: "bench-tracked-parent".to_string(),
1290
- expected_entries: 0,
2349
+ commit_id: final_commit_id,
2350
+ expected_rows: rows.len(),
1291
2351
  })
1292
2352
  }
1293
2353
 
1294
- pub async fn tracked_state_diff_commits_prepared(
2354
+ pub async fn json_pointer_tracked_state_changed_keys_prepared(
1295
2355
  backend: &Arc<dyn Backend + Send + Sync>,
1296
- fixture: &TrackedStateDiffFixture,
2356
+ fixture: &JsonPointerTrackedStateDiffFixture,
1297
2357
  ) -> Result<StorageBenchReport, LixError> {
1298
2358
  let mut reader = fixture
1299
2359
  .context
@@ -1617,7 +2677,7 @@ pub async fn prepare_changelog_append_changes(
1617
2677
  config: StorageBenchConfig,
1618
2678
  ) -> Result<ChangelogAppendFixture, LixError> {
1619
2679
  Ok(ChangelogAppendFixture {
1620
- context: ChangelogContext::new(),
2680
+ context: CommitStoreContext::new(),
1621
2681
  changes: changelog_materialized_changes(config),
1622
2682
  })
1623
2683
  }
@@ -1626,7 +2686,7 @@ pub async fn prepare_changelog_append_tombstones(
1626
2686
  config: StorageBenchConfig,
1627
2687
  ) -> Result<ChangelogAppendFixture, LixError> {
1628
2688
  Ok(ChangelogAppendFixture {
1629
- context: ChangelogContext::new(),
2689
+ context: CommitStoreContext::new(),
1630
2690
  changes: changelog_tombstone_changes(config),
1631
2691
  })
1632
2692
  }
@@ -1635,7 +2695,7 @@ pub async fn prepare_changelog_append_metadata(
1635
2695
  config: StorageBenchConfig,
1636
2696
  ) -> Result<ChangelogAppendFixture, LixError> {
1637
2697
  Ok(ChangelogAppendFixture {
1638
- context: ChangelogContext::new(),
2698
+ context: CommitStoreContext::new(),
1639
2699
  changes: changelog_metadata_changes(config),
1640
2700
  })
1641
2701
  }
@@ -1644,7 +2704,7 @@ pub async fn prepare_changelog_append_shared_payload(
1644
2704
  config: StorageBenchConfig,
1645
2705
  ) -> Result<ChangelogAppendFixture, LixError> {
1646
2706
  Ok(ChangelogAppendFixture {
1647
- context: ChangelogContext::new(),
2707
+ context: CommitStoreContext::new(),
1648
2708
  changes: changelog_shared_payload_changes(config),
1649
2709
  })
1650
2710
  }
@@ -1653,7 +2713,7 @@ pub async fn prepare_changelog_append_shared_metadata(
1653
2713
  config: StorageBenchConfig,
1654
2714
  ) -> Result<ChangelogAppendFixture, LixError> {
1655
2715
  Ok(ChangelogAppendFixture {
1656
- context: ChangelogContext::new(),
2716
+ context: CommitStoreContext::new(),
1657
2717
  changes: changelog_shared_metadata_changes(config),
1658
2718
  })
1659
2719
  }
@@ -1662,7 +2722,7 @@ pub async fn prepare_changelog_append_shared_payload_and_metadata(
1662
2722
  config: StorageBenchConfig,
1663
2723
  ) -> Result<ChangelogAppendFixture, LixError> {
1664
2724
  Ok(ChangelogAppendFixture {
1665
- context: ChangelogContext::new(),
2725
+ context: CommitStoreContext::new(),
1666
2726
  changes: changelog_shared_payload_and_metadata_changes(config),
1667
2727
  })
1668
2728
  }
@@ -1671,7 +2731,7 @@ pub async fn prepare_changelog_append_composite_entity_ids(
1671
2731
  config: StorageBenchConfig,
1672
2732
  ) -> Result<ChangelogAppendFixture, LixError> {
1673
2733
  Ok(ChangelogAppendFixture {
1674
- context: ChangelogContext::new(),
2734
+ context: CommitStoreContext::new(),
1675
2735
  changes: changelog_composite_entity_id_changes(config),
1676
2736
  })
1677
2737
  }
@@ -1682,7 +2742,7 @@ pub async fn prepare_changelog_codec(
1682
2742
  let changes = changelog_changes(config);
1683
2743
  let encoded_changes = changes
1684
2744
  .iter()
1685
- .map(crate::changelog::codec::encode_change)
2745
+ .map(|change| crate::commit_store::codec::encode_change_ref(change.as_ref()))
1686
2746
  .collect::<Result<Vec<_>, _>>()?;
1687
2747
  Ok(ChangelogCodecFixture {
1688
2748
  changes,
@@ -1699,7 +2759,7 @@ pub async fn changelog_append_changes_prepared(
1699
2759
  .context
1700
2760
  .reader(StorageContext::new(Arc::clone(backend)));
1701
2761
  let verified_rows = reader
1702
- .scan_changes(&ChangelogScanRequest::default())
2762
+ .scan_changes(&ChangeScanRequest::default())
1703
2763
  .await?
1704
2764
  .len();
1705
2765
  Ok(report(fixture.changes.len(), verified_rows, Duration::ZERO))
@@ -1709,7 +2769,7 @@ pub async fn prepare_changelog_read(
1709
2769
  backend: &Arc<dyn Backend + Send + Sync>,
1710
2770
  config: StorageBenchConfig,
1711
2771
  ) -> Result<ChangelogReadFixture, LixError> {
1712
- let context = ChangelogContext::new();
2772
+ let context = CommitStoreContext::new();
1713
2773
  let changes = changelog_materialized_changes(config);
1714
2774
  append_changelog_changes(backend, &context, &changes).await?;
1715
2775
  Ok(ChangelogReadFixture {
@@ -1722,7 +2782,7 @@ pub async fn prepare_changelog_read_with_selectivity(
1722
2782
  backend: &Arc<dyn Backend + Send + Sync>,
1723
2783
  config: StorageBenchConfig,
1724
2784
  ) -> Result<ChangelogReadFixture, LixError> {
1725
- let context = ChangelogContext::new();
2785
+ let context = CommitStoreContext::new();
1726
2786
  let changes = changelog_selective_changes(config);
1727
2787
  append_changelog_changes(backend, &context, &changes).await?;
1728
2788
  Ok(ChangelogReadFixture {
@@ -1735,7 +2795,7 @@ pub async fn prepare_changelog_read_entity_history(
1735
2795
  backend: &Arc<dyn Backend + Send + Sync>,
1736
2796
  config: StorageBenchConfig,
1737
2797
  ) -> Result<ChangelogReadFixture, LixError> {
1738
- let context = ChangelogContext::new();
2798
+ let context = CommitStoreContext::new();
1739
2799
  let changes = changelog_entity_history_changes(config);
1740
2800
  append_changelog_changes(backend, &context, &changes).await?;
1741
2801
  Ok(ChangelogReadFixture {
@@ -1750,7 +2810,7 @@ pub async fn changelog_encode_only_prepared(
1750
2810
  let mut verified_rows = 0;
1751
2811
  let mut encoded_bytes = 0;
1752
2812
  for change in &fixture.changes {
1753
- encoded_bytes += crate::changelog::codec::encode_change(change)?.len();
2813
+ encoded_bytes += crate::commit_store::codec::encode_change_ref(change.as_ref())?.len();
1754
2814
  verified_rows += 1;
1755
2815
  }
1756
2816
  Ok(report(
@@ -1766,7 +2826,7 @@ pub async fn changelog_decode_only_prepared(
1766
2826
  let mut verified_rows = 0;
1767
2827
  let mut decoded_bytes = 0;
1768
2828
  for bytes in &fixture.encoded_changes {
1769
- let change = crate::changelog::codec::decode_change(bytes)?;
2829
+ let change = crate::commit_store::codec::decode_change(bytes)?;
1770
2830
  decoded_bytes += change.schema_key.len();
1771
2831
  verified_rows += 1;
1772
2832
  }
@@ -1777,43 +2837,41 @@ pub async fn changelog_decode_only_prepared(
1777
2837
  ))
1778
2838
  }
1779
2839
 
1780
- pub async fn changelog_load_change_hit_prepared(
2840
+ pub async fn changelog_load_changes_hit_prepared(
1781
2841
  backend: &Arc<dyn Backend + Send + Sync>,
1782
2842
  fixture: &ChangelogReadFixture,
1783
2843
  ) -> Result<StorageBenchReport, LixError> {
1784
2844
  let reader = fixture
1785
2845
  .context
1786
2846
  .reader(StorageContext::new(Arc::clone(backend)));
1787
- let mut verified_rows = 0;
1788
- for index in 0..fixture.rows {
1789
- if reader
1790
- .load_change(&format!("bench-change-{index}"))
1791
- .await?
1792
- .is_some()
1793
- {
1794
- verified_rows += 1;
1795
- }
1796
- }
2847
+ let change_ids = (0..fixture.rows)
2848
+ .map(|index| format!("bench-change-{index}"))
2849
+ .collect::<Vec<_>>();
2850
+ let verified_rows = reader
2851
+ .load_changes(&change_ids)
2852
+ .await?
2853
+ .into_iter()
2854
+ .filter(Option::is_some)
2855
+ .count();
1797
2856
  Ok(report(fixture.rows, verified_rows, Duration::ZERO))
1798
2857
  }
1799
2858
 
1800
- pub async fn changelog_load_change_miss_prepared(
2859
+ pub async fn changelog_load_changes_miss_prepared(
1801
2860
  backend: &Arc<dyn Backend + Send + Sync>,
1802
2861
  fixture: &ChangelogReadFixture,
1803
2862
  ) -> Result<StorageBenchReport, LixError> {
1804
2863
  let reader = fixture
1805
2864
  .context
1806
2865
  .reader(StorageContext::new(Arc::clone(backend)));
1807
- let mut misses = 0;
1808
- for index in 0..fixture.rows {
1809
- if reader
1810
- .load_change(&format!("missing-change-{index}"))
1811
- .await?
1812
- .is_none()
1813
- {
1814
- misses += 1;
1815
- }
1816
- }
2866
+ let change_ids = (0..fixture.rows)
2867
+ .map(|index| format!("missing-change-{index}"))
2868
+ .collect::<Vec<_>>();
2869
+ let misses = reader
2870
+ .load_changes(&change_ids)
2871
+ .await?
2872
+ .into_iter()
2873
+ .filter(Option::is_none)
2874
+ .count();
1817
2875
  Ok(report(fixture.rows, misses, Duration::ZERO))
1818
2876
  }
1819
2877
 
@@ -1825,7 +2883,7 @@ pub async fn changelog_scan_all_prepared(
1825
2883
  .context
1826
2884
  .reader(StorageContext::new(Arc::clone(backend)));
1827
2885
  let verified_rows = reader
1828
- .scan_changes(&ChangelogScanRequest::default())
2886
+ .scan_changes(&ChangeScanRequest::default())
1829
2887
  .await?
1830
2888
  .len();
1831
2889
  Ok(report(fixture.rows, verified_rows, Duration::ZERO))
@@ -1847,7 +2905,7 @@ pub async fn changelog_scan_limit_100_prepared(
1847
2905
  .reader(StorageContext::new(Arc::clone(backend)));
1848
2906
  let expected = fixture.rows.min(100);
1849
2907
  let verified_rows = reader
1850
- .scan_changes(&ChangelogScanRequest {
2908
+ .scan_changes(&ChangeScanRequest {
1851
2909
  limit: Some(expected),
1852
2910
  })
1853
2911
  .await?
@@ -1855,6 +2913,25 @@ pub async fn changelog_scan_limit_100_prepared(
1855
2913
  Ok(report(expected, verified_rows, Duration::ZERO))
1856
2914
  }
1857
2915
 
2916
+ pub async fn changelog_scan_change_set_prepared(
2917
+ backend: &Arc<dyn Backend + Send + Sync>,
2918
+ fixture: &ChangelogReadFixture,
2919
+ ) -> Result<StorageBenchReport, LixError> {
2920
+ let reader = fixture
2921
+ .context
2922
+ .reader(StorageContext::new(Arc::clone(backend)));
2923
+ let change_ids = (0..fixture.rows)
2924
+ .map(|index| format!("bench-change-{index}"))
2925
+ .collect::<Vec<_>>();
2926
+ let verified_rows = reader
2927
+ .load_changes(&change_ids)
2928
+ .await?
2929
+ .into_iter()
2930
+ .filter(Option::is_some)
2931
+ .count();
2932
+ Ok(report(fixture.rows, verified_rows, Duration::ZERO))
2933
+ }
2934
+
1858
2935
  pub async fn changelog_scan_schema_prepared(
1859
2936
  backend: &Arc<dyn Backend + Send + Sync>,
1860
2937
  fixture: &ChangelogReadFixture,
@@ -1863,12 +2940,10 @@ pub async fn changelog_scan_schema_prepared(
1863
2940
  let reader = fixture
1864
2941
  .context
1865
2942
  .reader(StorageContext::new(Arc::clone(backend)));
1866
- let changes = reader
1867
- .scan_changes(&ChangelogScanRequest::default())
1868
- .await?;
2943
+ let changes = reader.scan_changes(&ChangeScanRequest::default()).await?;
1869
2944
  let verified_rows = changes
1870
2945
  .iter()
1871
- .filter(|change| change.schema_key == CHANGELOG_MATCH_SCHEMA_KEY)
2946
+ .filter(|change| change.record.schema_key == CHANGELOG_MATCH_SCHEMA_KEY)
1872
2947
  .count();
1873
2948
  Ok(report(
1874
2949
  selectivity.expected_rows(fixture.rows),
@@ -1884,13 +2959,11 @@ pub async fn changelog_scan_entity_history_prepared(
1884
2959
  let reader = fixture
1885
2960
  .context
1886
2961
  .reader(StorageContext::new(Arc::clone(backend)));
1887
- let changes = reader
1888
- .scan_changes(&ChangelogScanRequest::default())
1889
- .await?;
2962
+ let changes = reader.scan_changes(&ChangeScanRequest::default()).await?;
1890
2963
  let target = EntityIdentity::single(CHANGELOG_HISTORY_ENTITY_ID);
1891
2964
  let verified_rows = changes
1892
2965
  .iter()
1893
- .filter(|change| change.entity_id == target)
2966
+ .filter(|change| change.record.entity_id == target)
1894
2967
  .count();
1895
2968
  Ok(report(
1896
2969
  fixture.rows.div_ceil(10),
@@ -1899,6 +2972,41 @@ pub async fn changelog_scan_entity_history_prepared(
1899
2972
  ))
1900
2973
  }
1901
2974
 
2975
+ pub async fn prepare_commit_graph_read(
2976
+ backend: &Arc<dyn Backend + Send + Sync>,
2977
+ config: StorageBenchConfig,
2978
+ ) -> Result<CommitGraphReadFixture, LixError> {
2979
+ let changelog = CommitStoreContext::new();
2980
+ let mut changes = changelog_materialized_changes(config);
2981
+ let head_commit_id = "bench-commit-head".to_string();
2982
+ changes.push(commit_graph_materialized_commit_change(
2983
+ &head_commit_id,
2984
+ config.rows,
2985
+ ));
2986
+ append_changelog_changes(backend, &changelog, &changes).await?;
2987
+
2988
+ Ok(CommitGraphReadFixture {
2989
+ head_commit_id,
2990
+ rows: config.rows,
2991
+ })
2992
+ }
2993
+
2994
+ pub async fn commit_graph_change_history_from_commit_prepared(
2995
+ backend: &Arc<dyn Backend + Send + Sync>,
2996
+ fixture: &CommitGraphReadFixture,
2997
+ ) -> Result<StorageBenchReport, LixError> {
2998
+ let graph = crate::commit_graph::CommitGraphContext::new();
2999
+ let mut reader = graph.reader(StorageContext::new(Arc::clone(backend)));
3000
+ let verified_rows = reader
3001
+ .change_history_from_commit(
3002
+ &fixture.head_commit_id,
3003
+ &CommitGraphChangeHistoryRequest::default(),
3004
+ )
3005
+ .await?
3006
+ .len();
3007
+ Ok(report(fixture.rows, verified_rows, Duration::ZERO))
3008
+ }
3009
+
1902
3010
  pub async fn prepare_binary_cas_write_blobs(
1903
3011
  config: StorageBenchConfig,
1904
3012
  ) -> Result<BinaryCasWriteFixture, LixError> {
@@ -2029,9 +3137,24 @@ pub async fn json_store_write_prepared(
2029
3137
  {
2030
3138
  let mut writes = StorageWriteSet::new();
2031
3139
  let mut writer = fixture.context.writer();
2032
- for document in &fixture.documents {
2033
- writer.stage_bytes(&mut writes, document)?;
2034
- }
3140
+ writer.stage_batch(
3141
+ &mut writes,
3142
+ JsonWritePlacementRef::OutOfBand,
3143
+ fixture
3144
+ .documents
3145
+ .iter()
3146
+ .map(|document| {
3147
+ std::str::from_utf8(document)
3148
+ .map(NormalizedJsonRef::new)
3149
+ .map_err(|error| {
3150
+ LixError::new(
3151
+ LixError::CODE_UNKNOWN,
3152
+ format!("benchmark JSON document is invalid UTF-8: {error}"),
3153
+ )
3154
+ })
3155
+ })
3156
+ .collect::<Result<Vec<_>, _>>()?,
3157
+ )?;
2035
3158
  writes.apply(&mut transaction.as_mut()).await?;
2036
3159
  }
2037
3160
  transaction.commit().await?;
@@ -2070,9 +3193,26 @@ pub async fn prepare_json_store_projection_read(
2070
3193
  {
2071
3194
  let mut writes = StorageWriteSet::new();
2072
3195
  let mut writer = context.writer();
2073
- for document in documents {
2074
- refs.push(writer.stage_bytes(&mut writes, &document)?);
3196
+ for document in &documents {
3197
+ refs.push(prepare_json_ref(document)?);
2075
3198
  }
3199
+ writer.stage_batch(
3200
+ &mut writes,
3201
+ JsonWritePlacementRef::OutOfBand,
3202
+ documents
3203
+ .iter()
3204
+ .map(|document| {
3205
+ std::str::from_utf8(document)
3206
+ .map(NormalizedJsonRef::new)
3207
+ .map_err(|error| {
3208
+ LixError::new(
3209
+ LixError::CODE_UNKNOWN,
3210
+ format!("benchmark JSON document is invalid UTF-8: {error}"),
3211
+ )
3212
+ })
3213
+ })
3214
+ .collect::<Result<Vec<_>, _>>()?,
3215
+ )?;
2076
3216
  writes.apply(&mut transaction.as_mut()).await?;
2077
3217
  }
2078
3218
  transaction.commit().await?;
@@ -2091,8 +3231,14 @@ pub async fn json_store_read_bytes_prepared(
2091
3231
  let mut reader = fixture
2092
3232
  .context
2093
3233
  .reader(StorageContext::new(Arc::clone(backend)));
2094
- for json_ref in &fixture.refs {
2095
- if reader.load_bytes(json_ref).await?.is_some() {
3234
+ let batch = reader
3235
+ .load_bytes_many(JsonLoadRequestRef {
3236
+ refs: &fixture.refs,
3237
+ scope: JsonReadScopeRef::OutOfBand,
3238
+ })
3239
+ .await?;
3240
+ for value in batch.values() {
3241
+ if value.is_some() {
2096
3242
  verified_rows += 1;
2097
3243
  }
2098
3244
  }
@@ -2107,8 +3253,14 @@ pub async fn json_store_read_value_prepared(
2107
3253
  let mut reader = fixture
2108
3254
  .context
2109
3255
  .reader(StorageContext::new(Arc::clone(backend)));
2110
- for json_ref in &fixture.refs {
2111
- if reader.load_json_value(json_ref).await?.is_some() {
3256
+ let batch = reader
3257
+ .load_values_many(JsonLoadRequestRef {
3258
+ refs: &fixture.refs,
3259
+ scope: JsonReadScopeRef::OutOfBand,
3260
+ })
3261
+ .await?;
3262
+ for value in batch.values() {
3263
+ if value.is_some() {
2112
3264
  verified_rows += 1;
2113
3265
  }
2114
3266
  }
@@ -2123,12 +3275,15 @@ pub async fn json_store_read_projection_prepared(
2123
3275
  let mut reader = fixture
2124
3276
  .context
2125
3277
  .reader(StorageContext::new(Arc::clone(backend)));
2126
- for json_ref in &fixture.refs {
2127
- if reader
2128
- .load_json_projection(json_ref, &fixture.paths)
2129
- .await?
2130
- .is_some()
2131
- {
3278
+ let batch = reader
3279
+ .load_projections_many(JsonProjectionLoadRequestRef {
3280
+ refs: &fixture.refs,
3281
+ scope: JsonReadScopeRef::OutOfBand,
3282
+ paths: &fixture.paths,
3283
+ })
3284
+ .await?;
3285
+ for value in batch.values() {
3286
+ if value.is_some() {
2132
3287
  verified_rows += 1;
2133
3288
  }
2134
3289
  }
@@ -2162,9 +3317,26 @@ async fn prepare_json_store_base_update(
2162
3317
  {
2163
3318
  let mut writes = StorageWriteSet::new();
2164
3319
  let mut writer = context.writer();
2165
- for document in documents {
2166
- refs.push(writer.stage_bytes(&mut writes, &document)?);
3320
+ for document in &documents {
3321
+ refs.push(prepare_json_ref(document)?);
2167
3322
  }
3323
+ writer.stage_batch(
3324
+ &mut writes,
3325
+ JsonWritePlacementRef::OutOfBand,
3326
+ documents
3327
+ .iter()
3328
+ .map(|document| {
3329
+ std::str::from_utf8(document)
3330
+ .map(NormalizedJsonRef::new)
3331
+ .map_err(|error| {
3332
+ LixError::new(
3333
+ LixError::CODE_UNKNOWN,
3334
+ format!("benchmark JSON document is invalid UTF-8: {error}"),
3335
+ )
3336
+ })
3337
+ })
3338
+ .collect::<Result<Vec<_>, _>>()?,
3339
+ )?;
2168
3340
  writes.apply(&mut transaction.as_mut()).await?;
2169
3341
  }
2170
3342
  transaction.commit().await?;
@@ -2205,10 +3377,29 @@ async fn json_store_write_against_base_prepared(
2205
3377
  {
2206
3378
  let mut writes = StorageWriteSet::new();
2207
3379
  let mut writer = fixture.context.writer();
3380
+ let mut updated_documents = Vec::with_capacity(fixture.refs.len());
2208
3381
  for (index, _json_ref) in fixture.refs.iter().enumerate() {
2209
3382
  let updated = updated_json_document(shape, index);
2210
- writer.stage_bytes(&mut writes, &updated)?;
3383
+ prepare_json_ref(&updated)?;
3384
+ updated_documents.push(updated);
2211
3385
  }
3386
+ writer.stage_batch(
3387
+ &mut writes,
3388
+ JsonWritePlacementRef::OutOfBand,
3389
+ updated_documents
3390
+ .iter()
3391
+ .map(|document| {
3392
+ std::str::from_utf8(document)
3393
+ .map(NormalizedJsonRef::new)
3394
+ .map_err(|error| {
3395
+ LixError::new(
3396
+ LixError::CODE_UNKNOWN,
3397
+ format!("benchmark JSON document is invalid UTF-8: {error}"),
3398
+ )
3399
+ })
3400
+ })
3401
+ .collect::<Result<Vec<_>, _>>()?,
3402
+ )?;
2212
3403
  writes.apply(&mut transaction.as_mut()).await?;
2213
3404
  }
2214
3405
  transaction.commit().await?;
@@ -2243,28 +3434,14 @@ pub async fn tracked_state_read_point_hit(
2243
3434
  write_tracked_root(backend, &context, "bench-tracked-commit", None, &rows).await?;
2244
3435
 
2245
3436
  let started = Instant::now();
2246
- let mut verified_rows = 0;
2247
3437
  let mut reader = context.reader(StorageContext::new(Arc::clone(backend)));
2248
- for index in 0..config.rows {
2249
- if reader
2250
- .load_row_at_commit(
2251
- "bench-tracked-commit",
2252
- &TrackedStateRowRequest {
2253
- schema_key: tracked_schema_key(index, StorageBenchSelectivity::Percent100),
2254
- entity_id: EntityIdentity::single(entity_id(
2255
- "tracked",
2256
- index,
2257
- config.key_pattern,
2258
- )),
2259
- file_id: NullableKeyFilter::Value("bench.json".to_string()),
2260
- },
2261
- )
2262
- .await?
2263
- .is_some()
2264
- {
2265
- verified_rows += 1;
2266
- }
2267
- }
3438
+ let requests = tracked_point_hit_requests(config.rows, config.key_pattern);
3439
+ let verified_rows = reader
3440
+ .load_rows_at_commit("bench-tracked-commit", &requests)
3441
+ .await?
3442
+ .into_iter()
3443
+ .filter(Option::is_some)
3444
+ .count();
2268
3445
  Ok(report(config.rows, verified_rows, started.elapsed()))
2269
3446
  }
2270
3447
 
@@ -2277,24 +3454,14 @@ pub async fn tracked_state_read_point_miss(
2277
3454
  write_tracked_root(backend, &context, "bench-tracked-commit", None, &rows).await?;
2278
3455
 
2279
3456
  let started = Instant::now();
2280
- let mut misses = 0;
2281
3457
  let mut reader = context.reader(StorageContext::new(Arc::clone(backend)));
2282
- for index in 0..config.rows {
2283
- if reader
2284
- .load_row_at_commit(
2285
- "bench-tracked-commit",
2286
- &TrackedStateRowRequest {
2287
- schema_key: tracked_schema_key(index, StorageBenchSelectivity::Percent100),
2288
- entity_id: EntityIdentity::single(format!("missing-{index}")),
2289
- file_id: NullableKeyFilter::Value("bench.json".to_string()),
2290
- },
2291
- )
2292
- .await?
2293
- .is_none()
2294
- {
2295
- misses += 1;
2296
- }
2297
- }
3458
+ let requests = tracked_point_miss_requests(config.rows, StorageBenchSelectivity::Percent100);
3459
+ let misses = reader
3460
+ .load_rows_at_commit("bench-tracked-commit", &requests)
3461
+ .await?
3462
+ .into_iter()
3463
+ .filter(Option::is_none)
3464
+ .count();
2298
3465
  Ok(report(config.rows, misses, started.elapsed()))
2299
3466
  }
2300
3467
 
@@ -2559,61 +3726,59 @@ pub async fn changelog_append_changes(
2559
3726
  config: StorageBenchConfig,
2560
3727
  ) -> Result<StorageBenchReport, LixError> {
2561
3728
  let changes = changelog_materialized_changes(config);
2562
- let context = ChangelogContext::new();
3729
+ let context = CommitStoreContext::new();
2563
3730
  let started = Instant::now();
2564
3731
  append_changelog_changes(backend, &context, &changes).await?;
2565
3732
  let elapsed = started.elapsed();
2566
3733
  let reader = context.reader(StorageContext::new(Arc::clone(backend)));
2567
3734
  let verified_rows = reader
2568
- .scan_changes(&ChangelogScanRequest::default())
3735
+ .scan_changes(&ChangeScanRequest::default())
2569
3736
  .await?
2570
3737
  .len();
2571
3738
  Ok(report(changes.len(), verified_rows, elapsed))
2572
3739
  }
2573
3740
 
2574
- pub async fn changelog_load_change_hit(
3741
+ pub async fn changelog_load_changes_hit(
2575
3742
  backend: &Arc<dyn Backend + Send + Sync>,
2576
3743
  config: StorageBenchConfig,
2577
3744
  ) -> Result<StorageBenchReport, LixError> {
2578
- let context = ChangelogContext::new();
3745
+ let context = CommitStoreContext::new();
2579
3746
  let changes = changelog_materialized_changes(config);
2580
3747
  append_changelog_changes(backend, &context, &changes).await?;
2581
3748
  let reader = context.reader(StorageContext::new(Arc::clone(backend)));
2582
3749
 
2583
3750
  let started = Instant::now();
2584
- let mut verified_rows = 0;
2585
- for index in 0..config.rows {
2586
- if reader
2587
- .load_change(&format!("bench-change-{index}"))
2588
- .await?
2589
- .is_some()
2590
- {
2591
- verified_rows += 1;
2592
- }
2593
- }
3751
+ let change_ids = (0..config.rows)
3752
+ .map(|index| format!("bench-change-{index}"))
3753
+ .collect::<Vec<_>>();
3754
+ let verified_rows = reader
3755
+ .load_changes(&change_ids)
3756
+ .await?
3757
+ .into_iter()
3758
+ .filter(Option::is_some)
3759
+ .count();
2594
3760
  Ok(report(config.rows, verified_rows, started.elapsed()))
2595
3761
  }
2596
3762
 
2597
- pub async fn changelog_load_change_miss(
3763
+ pub async fn changelog_load_changes_miss(
2598
3764
  backend: &Arc<dyn Backend + Send + Sync>,
2599
3765
  config: StorageBenchConfig,
2600
3766
  ) -> Result<StorageBenchReport, LixError> {
2601
- let context = ChangelogContext::new();
3767
+ let context = CommitStoreContext::new();
2602
3768
  let changes = changelog_materialized_changes(config);
2603
3769
  append_changelog_changes(backend, &context, &changes).await?;
2604
3770
  let reader = context.reader(StorageContext::new(Arc::clone(backend)));
2605
3771
 
2606
3772
  let started = Instant::now();
2607
- let mut misses = 0;
2608
- for index in 0..config.rows {
2609
- if reader
2610
- .load_change(&format!("missing-change-{index}"))
2611
- .await?
2612
- .is_none()
2613
- {
2614
- misses += 1;
2615
- }
2616
- }
3773
+ let change_ids = (0..config.rows)
3774
+ .map(|index| format!("missing-change-{index}"))
3775
+ .collect::<Vec<_>>();
3776
+ let misses = reader
3777
+ .load_changes(&change_ids)
3778
+ .await?
3779
+ .into_iter()
3780
+ .filter(Option::is_none)
3781
+ .count();
2617
3782
  Ok(report(config.rows, misses, started.elapsed()))
2618
3783
  }
2619
3784
 
@@ -2621,14 +3786,14 @@ pub async fn changelog_scan_all(
2621
3786
  backend: &Arc<dyn Backend + Send + Sync>,
2622
3787
  config: StorageBenchConfig,
2623
3788
  ) -> Result<StorageBenchReport, LixError> {
2624
- let context = ChangelogContext::new();
3789
+ let context = CommitStoreContext::new();
2625
3790
  let changes = changelog_materialized_changes(config);
2626
3791
  append_changelog_changes(backend, &context, &changes).await?;
2627
3792
  let reader = context.reader(StorageContext::new(Arc::clone(backend)));
2628
3793
 
2629
3794
  let started = Instant::now();
2630
3795
  let verified_rows = reader
2631
- .scan_changes(&ChangelogScanRequest::default())
3796
+ .scan_changes(&ChangeScanRequest::default())
2632
3797
  .await?
2633
3798
  .len();
2634
3799
  Ok(report(config.rows, verified_rows, started.elapsed()))
@@ -2638,7 +3803,7 @@ pub async fn changelog_scan_limit_100(
2638
3803
  backend: &Arc<dyn Backend + Send + Sync>,
2639
3804
  config: StorageBenchConfig,
2640
3805
  ) -> Result<StorageBenchReport, LixError> {
2641
- let context = ChangelogContext::new();
3806
+ let context = CommitStoreContext::new();
2642
3807
  let changes = changelog_materialized_changes(config);
2643
3808
  append_changelog_changes(backend, &context, &changes).await?;
2644
3809
  let reader = context.reader(StorageContext::new(Arc::clone(backend)));
@@ -2646,7 +3811,7 @@ pub async fn changelog_scan_limit_100(
2646
3811
 
2647
3812
  let started = Instant::now();
2648
3813
  let verified_rows = reader
2649
- .scan_changes(&ChangelogScanRequest {
3814
+ .scan_changes(&ChangeScanRequest {
2650
3815
  limit: Some(expected),
2651
3816
  })
2652
3817
  .await?
@@ -2747,36 +3912,181 @@ async fn write_tracked_root(
2747
3912
  context: &TrackedStateContext,
2748
3913
  commit_id: &str,
2749
3914
  parent_commit_id: Option<&str>,
2750
- rows: &[TrackedStateRow],
3915
+ rows: &[MaterializedTrackedStateRow],
2751
3916
  ) -> Result<(), LixError> {
2752
3917
  let storage = StorageContext::new(Arc::clone(backend));
2753
3918
  let mut transaction = storage.begin_write_transaction().await?;
2754
- {
2755
- let mut writes = StorageWriteSet::new();
2756
- {
2757
- let mut json_writer = JsonStoreContext::new().writer();
2758
- context
2759
- .writer()
2760
- .stage_root(
2761
- &mut transaction.as_mut(),
2762
- &mut writes,
2763
- &mut json_writer,
2764
- commit_id,
2765
- parent_commit_id,
2766
- rows,
2767
- )
2768
- .await?;
3919
+ let mut writes = StorageWriteSet::new();
3920
+ let changes = rows
3921
+ .iter()
3922
+ .map(tracked_bench_change_from_materialized)
3923
+ .collect::<Result<Vec<_>, _>>()?;
3924
+ let payloads = tracked_bench_json_payloads(rows, &changes);
3925
+ let json_report = JsonStoreContext::new().writer().stage_batch_report(
3926
+ &mut writes,
3927
+ JsonWritePlacementRef::CommitPack {
3928
+ commit_id,
3929
+ pack_id: 0,
3930
+ },
3931
+ payloads.iter().map(|(payload, json_ref)| match json_ref {
3932
+ Some(json_ref) => NormalizedJsonRef::trusted_prehashed(payload.as_str(), *json_ref),
3933
+ None => NormalizedJsonRef::new(payload.as_str()),
3934
+ }),
3935
+ )?;
3936
+
3937
+ let parent_ids = parent_commit_id
3938
+ .map(|parent| vec![parent.to_string()])
3939
+ .unwrap_or_default();
3940
+ let commit_change_id = format!("{commit_id}:commit");
3941
+ let commit = CommitDraftRef {
3942
+ id: commit_id,
3943
+ change_id: &commit_change_id,
3944
+ parent_ids: &parent_ids,
3945
+ author_account_ids: &[],
3946
+ created_at: rows
3947
+ .first()
3948
+ .map(|row| row.updated_at.as_str())
3949
+ .unwrap_or("1970-01-01T00:00:00.000Z"),
3950
+ };
3951
+ let commit_store = CommitStoreContext::new();
3952
+ let authored_changes = changes.iter().map(Change::as_ref).collect::<Vec<_>>();
3953
+ let staged = commit_store
3954
+ .writer(&mut transaction.as_mut(), &mut writes)
3955
+ .stage_tracked_commit_draft(commit, authored_changes.clone(), Vec::new())
3956
+ .await?;
3957
+ let mut deltas = Vec::with_capacity(changes.len());
3958
+ deltas.extend(
3959
+ authored_changes
3960
+ .iter()
3961
+ .zip(&staged.authored_locators)
3962
+ .zip(rows)
3963
+ .map(|((change, locator), row)| TrackedStateDeltaRef {
3964
+ change: *change,
3965
+ locator: locator.as_ref(),
3966
+ created_at: row.created_at.as_str(),
3967
+ updated_at: row.updated_at.as_str(),
3968
+ }),
3969
+ );
3970
+ context
3971
+ .writer(&mut transaction.as_mut(), &mut writes)
3972
+ .stage_delta_with_json_pack_indexes(
3973
+ commit_id,
3974
+ parent_commit_id,
3975
+ &deltas,
3976
+ crate::tracked_state::DeltaJsonPackIndexesRef {
3977
+ commit_id,
3978
+ pack_id: 0,
3979
+ indexes: &json_report.pack_indexes,
3980
+ },
3981
+ )
3982
+ .await?;
3983
+ writes.apply(&mut transaction.as_mut()).await?;
3984
+ transaction.commit().await
3985
+ }
3986
+
3987
+ async fn materialize_tracked_root(
3988
+ backend: &Arc<dyn Backend + Send + Sync>,
3989
+ context: &TrackedStateContext,
3990
+ commit_id: &str,
3991
+ ) -> Result<(), LixError> {
3992
+ let storage = StorageContext::new(Arc::clone(backend));
3993
+ let mut transaction = storage.begin_write_transaction().await?;
3994
+ let mut writes = StorageWriteSet::new();
3995
+ let commit_store = CommitStoreContext::new();
3996
+ context
3997
+ .materializer(&mut transaction.as_mut(), &mut writes, &commit_store)
3998
+ .materialize_root_at(commit_id)
3999
+ .await?;
4000
+ writes.apply(&mut transaction.as_mut()).await?;
4001
+ transaction.commit().await
4002
+ }
4003
+
4004
+ async fn write_tracked_delta_chain(
4005
+ backend: &Arc<dyn Backend + Send + Sync>,
4006
+ config: StorageBenchConfig,
4007
+ delta_commits: usize,
4008
+ updated_rows_per_commit: usize,
4009
+ ) -> Result<(TrackedStateContext, String), LixError> {
4010
+ let context = TrackedStateContext::new();
4011
+ let base_commit_id = "bench-tracked-base";
4012
+ let rows = tracked_rows(config, base_commit_id);
4013
+ write_tracked_root(backend, &context, base_commit_id, None, &rows).await?;
4014
+
4015
+ let mut parent_commit_id = base_commit_id.to_string();
4016
+ for delta_index in 0..delta_commits {
4017
+ let commit_id = format!("bench-tracked-delta-{delta_index}");
4018
+ let mut updated_rows = tracked_rows(
4019
+ config.with_rows(updated_rows_per_commit.min(config.rows)),
4020
+ &commit_id,
4021
+ );
4022
+ for (row_index, row) in updated_rows.iter_mut().enumerate() {
4023
+ row.snapshot_content = Some(delta_chain_snapshot_content(
4024
+ delta_index,
4025
+ row_index,
4026
+ config.state_payload_bytes,
4027
+ ));
4028
+ row.updated_at = timestamp(config.rows + delta_index * config.rows + row_index);
2769
4029
  }
2770
- writes.apply(&mut transaction.as_mut()).await?;
4030
+ write_tracked_root(
4031
+ backend,
4032
+ &context,
4033
+ &commit_id,
4034
+ Some(parent_commit_id.as_str()),
4035
+ &updated_rows,
4036
+ )
4037
+ .await?;
4038
+ parent_commit_id = commit_id;
2771
4039
  }
2772
- transaction.commit().await
4040
+
4041
+ Ok((context, parent_commit_id))
4042
+ }
4043
+
4044
+ fn tracked_bench_change_from_materialized(
4045
+ row: &MaterializedTrackedStateRow,
4046
+ ) -> Result<Change, LixError> {
4047
+ Ok(Change {
4048
+ id: row.change_id.clone(),
4049
+ entity_id: row.entity_id.clone(),
4050
+ schema_key: row.schema_key.clone(),
4051
+ file_id: row.file_id.clone(),
4052
+ snapshot_ref: row
4053
+ .snapshot_content
4054
+ .as_deref()
4055
+ .map(|value| prepare_json_ref(value.as_bytes()))
4056
+ .transpose()?,
4057
+ metadata_ref: row
4058
+ .metadata
4059
+ .as_ref()
4060
+ .map(|value| {
4061
+ let serialized = crate::serialize_row_metadata(value);
4062
+ prepare_json_ref(serialized.as_bytes())
4063
+ })
4064
+ .transpose()?,
4065
+ created_at: row.created_at.clone(),
4066
+ })
4067
+ }
4068
+
4069
+ fn tracked_bench_json_payloads(
4070
+ rows: &[MaterializedTrackedStateRow],
4071
+ changes: &[Change],
4072
+ ) -> Vec<(String, Option<JsonRef>)> {
4073
+ let mut payloads = Vec::new();
4074
+ for (row, change) in rows.iter().zip(changes) {
4075
+ if let Some(snapshot) = row.snapshot_content.as_deref() {
4076
+ payloads.push((snapshot.to_string(), change.snapshot_ref));
4077
+ }
4078
+ if let Some(metadata) = row.metadata.as_ref() {
4079
+ payloads.push((crate::serialize_row_metadata(metadata), change.metadata_ref));
4080
+ }
4081
+ }
4082
+ payloads
2773
4083
  }
2774
4084
 
2775
4085
  async fn scan_tracked(
2776
4086
  backend: &Arc<dyn Backend + Send + Sync>,
2777
4087
  context: &TrackedStateContext,
2778
4088
  commit_id: &str,
2779
- ) -> Result<Vec<TrackedStateRow>, LixError> {
4089
+ ) -> Result<Vec<MaterializedTrackedStateRow>, LixError> {
2780
4090
  let mut reader = context.reader(StorageContext::new(Arc::clone(backend)));
2781
4091
  reader
2782
4092
  .scan_rows_at_commit(commit_id, &TrackedStateScanRequest::default())
@@ -2792,14 +4102,12 @@ async fn write_untracked_rows(
2792
4102
  let mut transaction = storage.begin_write_transaction().await?;
2793
4103
  {
2794
4104
  let mut writes = StorageWriteSet::new();
2795
- let canonical_rows = {
2796
- let mut json_writer = JsonStoreContext::new().writer();
2797
- rows.iter()
2798
- .map(|row| canonicalize_materialized_row(&mut writes, &mut json_writer, row))
2799
- .collect::<Result<Vec<_>, _>>()?
2800
- };
4105
+ let canonical_rows = rows
4106
+ .iter()
4107
+ .map(|row| crate::test_support::untracked_state_row_from_materialized(&mut writes, row))
4108
+ .collect::<Result<Vec<_>, _>>()?;
2801
4109
  let mut writer = context.writer(&mut writes);
2802
- writer.stage_rows(&canonical_rows)?;
4110
+ writer.stage_rows(canonical_rows.iter().map(|row| row.as_ref()))?;
2803
4111
  writes.apply(&mut transaction.as_mut()).await?;
2804
4112
  }
2805
4113
  transaction.commit().await
@@ -2816,24 +4124,47 @@ async fn scan_untracked(
2816
4124
 
2817
4125
  async fn append_changelog_changes(
2818
4126
  backend: &Arc<dyn Backend + Send + Sync>,
2819
- context: &ChangelogContext,
2820
- changes: &[MaterializedCanonicalChange],
4127
+ context: &CommitStoreContext,
4128
+ changes: &[MaterializedChange],
2821
4129
  ) -> Result<(), LixError> {
2822
4130
  let storage = StorageContext::new(Arc::clone(backend));
2823
4131
  let mut transaction = storage.begin_write_transaction().await?;
2824
4132
  {
2825
4133
  let mut writes = StorageWriteSet::new();
2826
- let canonical_changes = {
2827
- let mut json_writer = JsonStoreContext::new().writer();
2828
- changes
4134
+ let canonical_changes = changes
4135
+ .iter()
4136
+ .map(canonical_changelog_bench_change)
4137
+ .collect::<Result<Vec<_>, _>>()?;
4138
+ let payloads = changelog_bench_json_payloads(changes);
4139
+ JsonStoreContext::new().writer().stage_batch(
4140
+ &mut writes,
4141
+ JsonWritePlacementRef::OutOfBand,
4142
+ payloads
2829
4143
  .iter()
2830
- .map(|change| {
2831
- canonicalize_materialized_change(&mut writes, &mut json_writer, change)
2832
- })
2833
- .collect::<Result<Vec<_>, _>>()?
2834
- };
2835
- let mut writer = context.writer(&mut writes);
2836
- writer.stage_changes(&canonical_changes)?;
4144
+ .map(|payload| NormalizedJsonRef::new(payload.as_str())),
4145
+ )?;
4146
+ let parent_ids = Vec::new();
4147
+ let author_account_ids = vec!["bench-author".to_string()];
4148
+ {
4149
+ let mut transaction_ref = transaction.as_mut();
4150
+ let mut writer = context.writer(&mut transaction_ref, &mut writes);
4151
+ writer
4152
+ .stage_commit_draft(
4153
+ CommitDraftRef {
4154
+ id: "bench-changelog-commit-0",
4155
+ change_id: "bench-changelog-header-change-0",
4156
+ parent_ids: &parent_ids,
4157
+ author_account_ids: &author_account_ids,
4158
+ created_at: "2024-01-01T00:00:00.000Z",
4159
+ },
4160
+ canonical_changes
4161
+ .iter()
4162
+ .map(|change| change.as_ref())
4163
+ .collect(),
4164
+ Vec::new(),
4165
+ )
4166
+ .await?;
4167
+ }
2837
4168
  writes.apply(&mut transaction.as_mut()).await?;
2838
4169
  }
2839
4170
  transaction.commit().await
@@ -2879,29 +4210,101 @@ const CHANGELOG_MATCH_SCHEMA_KEY: &str = "bench_changelog_entity";
2879
4210
  const CHANGELOG_OTHER_SCHEMA_KEY: &str = "bench_changelog_other_entity";
2880
4211
  const CHANGELOG_HISTORY_ENTITY_ID: &str = "change-entity-history-target";
2881
4212
 
2882
- fn tracked_rows(config: StorageBenchConfig, commit_id: &str) -> Vec<TrackedStateRow> {
4213
+ fn tracked_rows(config: StorageBenchConfig, commit_id: &str) -> Vec<MaterializedTrackedStateRow> {
2883
4214
  (0..config.rows)
2884
- .map(|index| TrackedStateRow {
4215
+ .map(|index| MaterializedTrackedStateRow {
2885
4216
  entity_id: EntityIdentity::single(entity_id("tracked", index, config.key_pattern)),
2886
4217
  schema_key: tracked_schema_key(index, config.selectivity),
2887
4218
  file_id: Some("bench.json".to_string()),
2888
4219
  snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
2889
4220
  metadata: None,
2890
- schema_version: "1".to_string(),
4221
+ deleted: false,
2891
4222
  created_at: timestamp(index),
2892
4223
  updated_at: timestamp(index),
2893
- change_id: format!("tracked-change-{index}"),
4224
+ change_id: tracked_change_id(commit_id, index),
2894
4225
  commit_id: commit_id.to_string(),
2895
4226
  })
2896
4227
  .collect()
2897
4228
  }
2898
4229
 
4230
+ fn json_pointer_tracked_rows(
4231
+ rows: &[JsonPointerStorageRow],
4232
+ commit_id: &str,
4233
+ updated: bool,
4234
+ ) -> Vec<MaterializedTrackedStateRow> {
4235
+ rows.iter()
4236
+ .enumerate()
4237
+ .map(|(index, row)| {
4238
+ let value_json = if updated {
4239
+ row.updated_value_json.as_str()
4240
+ } else {
4241
+ row.value_json.as_str()
4242
+ };
4243
+ let value = serde_json::from_str::<serde_json::Value>(value_json)
4244
+ .unwrap_or_else(|_| serde_json::Value::String(value_json.to_string()));
4245
+ let snapshot = serde_json::json!({
4246
+ "path": row.path,
4247
+ "value": value,
4248
+ })
4249
+ .to_string();
4250
+ MaterializedTrackedStateRow {
4251
+ entity_id: EntityIdentity::single(row.path.as_str()),
4252
+ schema_key: "json_pointer".to_string(),
4253
+ file_id: None,
4254
+ snapshot_content: Some(snapshot),
4255
+ metadata: None,
4256
+ deleted: false,
4257
+ created_at: timestamp(index),
4258
+ updated_at: timestamp(index),
4259
+ change_id: tracked_change_id(commit_id, index),
4260
+ commit_id: commit_id.to_string(),
4261
+ }
4262
+ })
4263
+ .collect()
4264
+ }
4265
+
4266
+ async fn write_json_pointer_delta_chain(
4267
+ backend: &Arc<dyn Backend + Send + Sync>,
4268
+ rows: &[JsonPointerStorageRow],
4269
+ delta_commits: usize,
4270
+ updated_rows_per_commit: usize,
4271
+ ) -> Result<(TrackedStateContext, String), LixError> {
4272
+ let context = TrackedStateContext::new();
4273
+ let base_commit_id = "json-pointer-base";
4274
+ let base_rows = json_pointer_tracked_rows(rows, base_commit_id, false);
4275
+ write_tracked_root(backend, &context, base_commit_id, None, &base_rows).await?;
4276
+
4277
+ let mut parent_commit_id = base_commit_id.to_string();
4278
+ for delta_index in 0..delta_commits {
4279
+ let commit_id = format!("json-pointer-delta-{delta_index}");
4280
+ let mut child_rows = json_pointer_tracked_rows(
4281
+ &rows[..updated_rows_per_commit.min(rows.len())],
4282
+ &commit_id,
4283
+ true,
4284
+ );
4285
+ for row in &mut child_rows {
4286
+ row.updated_at = timestamp(rows.len() + delta_index);
4287
+ }
4288
+ write_tracked_root(
4289
+ backend,
4290
+ &context,
4291
+ &commit_id,
4292
+ Some(parent_commit_id.as_str()),
4293
+ &child_rows,
4294
+ )
4295
+ .await?;
4296
+ parent_commit_id = commit_id;
4297
+ }
4298
+
4299
+ Ok((context, parent_commit_id))
4300
+ }
4301
+
2899
4302
  fn tracked_rows_file_selective(
2900
4303
  config: StorageBenchConfig,
2901
4304
  commit_id: &str,
2902
- ) -> Vec<TrackedStateRow> {
4305
+ ) -> Vec<MaterializedTrackedStateRow> {
2903
4306
  (0..config.rows)
2904
- .map(|index| TrackedStateRow {
4307
+ .map(|index| MaterializedTrackedStateRow {
2905
4308
  entity_id: EntityIdentity::single(entity_id("tracked", index, config.key_pattern)),
2906
4309
  schema_key: TRACKED_MATCH_SCHEMA_KEY.to_string(),
2907
4310
  file_id: Some(
@@ -2914,15 +4317,19 @@ fn tracked_rows_file_selective(
2914
4317
  ),
2915
4318
  snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
2916
4319
  metadata: None,
2917
- schema_version: "1".to_string(),
4320
+ deleted: false,
2918
4321
  created_at: timestamp(index),
2919
4322
  updated_at: timestamp(index),
2920
- change_id: format!("tracked-change-{index}"),
4323
+ change_id: tracked_change_id(commit_id, index),
2921
4324
  commit_id: commit_id.to_string(),
2922
4325
  })
2923
4326
  .collect()
2924
4327
  }
2925
4328
 
4329
+ fn tracked_change_id(commit_id: &str, index: usize) -> String {
4330
+ format!("{commit_id}:tracked-change-{index}")
4331
+ }
4332
+
2926
4333
  fn untracked_rows(config: StorageBenchConfig) -> Vec<MaterializedUntrackedStateRow> {
2927
4334
  (0..config.rows)
2928
4335
  .map(|index| MaterializedUntrackedStateRow {
@@ -2931,7 +4338,7 @@ fn untracked_rows(config: StorageBenchConfig) -> Vec<MaterializedUntrackedStateR
2931
4338
  file_id: Some("bench.json".to_string()),
2932
4339
  snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
2933
4340
  metadata: None,
2934
- schema_version: "1".to_string(),
4341
+ deleted: false,
2935
4342
  created_at: timestamp(index),
2936
4343
  updated_at: timestamp(index),
2937
4344
  global: false,
@@ -2940,16 +4347,16 @@ fn untracked_rows(config: StorageBenchConfig) -> Vec<MaterializedUntrackedStateR
2940
4347
  .collect()
2941
4348
  }
2942
4349
 
2943
- fn changelog_changes(config: StorageBenchConfig) -> Vec<CanonicalChange> {
4350
+ fn changelog_changes(config: StorageBenchConfig) -> Vec<Change> {
2944
4351
  changelog_materialized_changes(config)
2945
4352
  .into_iter()
2946
- .map(canonical_changelog_bench_change)
4353
+ .map(changelog_bench_change_ref_only)
2947
4354
  .collect()
2948
4355
  }
2949
4356
 
2950
- fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<MaterializedCanonicalChange> {
4357
+ fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
2951
4358
  (0..config.rows)
2952
- .map(|index| MaterializedCanonicalChange {
4359
+ .map(|index| MaterializedChange {
2953
4360
  id: format!("bench-change-{index}"),
2954
4361
  entity_id: EntityIdentity::single(entity_id(
2955
4362
  "change-entity",
@@ -2957,7 +4364,6 @@ fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<Materialize
2957
4364
  config.key_pattern,
2958
4365
  )),
2959
4366
  schema_key: "bench_changelog_entity".to_string(),
2960
- schema_version: "1".to_string(),
2961
4367
  file_id: Some("bench.json".to_string()),
2962
4368
  snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
2963
4369
  metadata: None,
@@ -2966,21 +4372,72 @@ fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<Materialize
2966
4372
  .collect()
2967
4373
  }
2968
4374
 
2969
- fn canonical_changelog_bench_change(change: MaterializedCanonicalChange) -> CanonicalChange {
4375
+ fn commit_graph_materialized_commit_change(commit_id: &str, rows: usize) -> MaterializedChange {
4376
+ let snapshot_content = serde_json::json!({
4377
+ "id": commit_id,
4378
+ })
4379
+ .to_string();
4380
+
4381
+ MaterializedChange {
4382
+ id: format!("bench-commit-change-{commit_id}"),
4383
+ entity_id: EntityIdentity::single(commit_id.to_string()),
4384
+ schema_key: "lix_commit".to_string(),
4385
+ file_id: None,
4386
+ snapshot_content: Some(snapshot_content),
4387
+ metadata: None,
4388
+ created_at: timestamp(rows),
4389
+ }
4390
+ }
4391
+
4392
+ fn canonical_changelog_bench_change(change: &MaterializedChange) -> Result<Change, LixError> {
4393
+ let snapshot_ref = change
4394
+ .snapshot_content
4395
+ .as_ref()
4396
+ .map(|value| prepare_json_ref(value.as_bytes()))
4397
+ .transpose()?;
4398
+ let metadata_ref = change
4399
+ .metadata
4400
+ .as_ref()
4401
+ .map(|value| prepare_json_ref(value.as_bytes()))
4402
+ .transpose()?;
4403
+ Ok(Change {
4404
+ id: change.id.clone(),
4405
+ entity_id: change.entity_id.clone(),
4406
+ schema_key: change.schema_key.clone(),
4407
+ file_id: change.file_id.clone(),
4408
+ snapshot_ref,
4409
+ metadata_ref,
4410
+ created_at: change.created_at.clone(),
4411
+ })
4412
+ }
4413
+
4414
+ fn changelog_bench_json_payloads(changes: &[MaterializedChange]) -> Vec<String> {
4415
+ changes
4416
+ .iter()
4417
+ .flat_map(|change| {
4418
+ change
4419
+ .snapshot_content
4420
+ .iter()
4421
+ .chain(change.metadata.iter())
4422
+ .cloned()
4423
+ .collect::<Vec<_>>()
4424
+ })
4425
+ .collect()
4426
+ }
4427
+
4428
+ fn changelog_bench_change_ref_only(change: MaterializedChange) -> Change {
2970
4429
  let snapshot_ref = change
2971
4430
  .snapshot_content
2972
4431
  .as_ref()
2973
4432
  .map(|value| JsonRef::from_hash(blake3::hash(value.as_bytes())));
2974
- let metadata_ref = change.metadata.as_ref().map(|value| {
2975
- let bytes =
2976
- serde_json::to_vec(value).expect("bench metadata should serialize to JSON bytes");
2977
- JsonRef::from_hash(blake3::hash(&bytes))
2978
- });
2979
- CanonicalChange {
4433
+ let metadata_ref = change
4434
+ .metadata
4435
+ .as_ref()
4436
+ .map(|value| JsonRef::from_hash(blake3::hash(value.as_bytes())));
4437
+ Change {
2980
4438
  id: change.id,
2981
4439
  entity_id: change.entity_id,
2982
4440
  schema_key: change.schema_key,
2983
- schema_version: change.schema_version,
2984
4441
  file_id: change.file_id,
2985
4442
  snapshot_ref,
2986
4443
  metadata_ref,
@@ -2988,7 +4445,7 @@ fn canonical_changelog_bench_change(change: MaterializedCanonicalChange) -> Cano
2988
4445
  }
2989
4446
  }
2990
4447
 
2991
- fn changelog_tombstone_changes(config: StorageBenchConfig) -> Vec<MaterializedCanonicalChange> {
4448
+ fn changelog_tombstone_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
2992
4449
  changelog_materialized_changes(config)
2993
4450
  .into_iter()
2994
4451
  .map(|mut change| {
@@ -2999,7 +4456,7 @@ fn changelog_tombstone_changes(config: StorageBenchConfig) -> Vec<MaterializedCa
2999
4456
  .collect()
3000
4457
  }
3001
4458
 
3002
- fn changelog_metadata_changes(config: StorageBenchConfig) -> Vec<MaterializedCanonicalChange> {
4459
+ fn changelog_metadata_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
3003
4460
  changelog_materialized_changes(config)
3004
4461
  .into_iter()
3005
4462
  .enumerate()
@@ -3010,9 +4467,7 @@ fn changelog_metadata_changes(config: StorageBenchConfig) -> Vec<MaterializedCan
3010
4467
  .collect()
3011
4468
  }
3012
4469
 
3013
- fn changelog_shared_payload_changes(
3014
- config: StorageBenchConfig,
3015
- ) -> Vec<MaterializedCanonicalChange> {
4470
+ fn changelog_shared_payload_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
3016
4471
  let shared_snapshot_content = snapshot_content(0, config.state_payload_bytes);
3017
4472
  changelog_materialized_changes(config)
3018
4473
  .into_iter()
@@ -3023,9 +4478,7 @@ fn changelog_shared_payload_changes(
3023
4478
  .collect()
3024
4479
  }
3025
4480
 
3026
- fn changelog_shared_metadata_changes(
3027
- config: StorageBenchConfig,
3028
- ) -> Vec<MaterializedCanonicalChange> {
4481
+ fn changelog_shared_metadata_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
3029
4482
  let shared_metadata = snapshot_metadata(0, config.state_payload_bytes);
3030
4483
  changelog_materialized_changes(config)
3031
4484
  .into_iter()
@@ -3039,7 +4492,7 @@ fn changelog_shared_metadata_changes(
3039
4492
 
3040
4493
  fn changelog_shared_payload_and_metadata_changes(
3041
4494
  config: StorageBenchConfig,
3042
- ) -> Vec<MaterializedCanonicalChange> {
4495
+ ) -> Vec<MaterializedChange> {
3043
4496
  let shared_snapshot_content = snapshot_content(0, config.state_payload_bytes);
3044
4497
  let shared_metadata = snapshot_metadata(1, config.state_payload_bytes);
3045
4498
  changelog_materialized_changes(config)
@@ -3052,22 +4505,16 @@ fn changelog_shared_payload_and_metadata_changes(
3052
4505
  .collect()
3053
4506
  }
3054
4507
 
3055
- fn changelog_composite_entity_id_changes(
3056
- config: StorageBenchConfig,
3057
- ) -> Vec<MaterializedCanonicalChange> {
4508
+ fn changelog_composite_entity_id_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
3058
4509
  changelog_materialized_changes(config)
3059
4510
  .into_iter()
3060
4511
  .enumerate()
3061
4512
  .map(|(index, mut change)| {
3062
4513
  change.entity_id = EntityIdentity {
3063
4514
  parts: vec![
3064
- EntityIdentityPart::String(entity_id(
3065
- "change-composite",
3066
- index,
3067
- config.key_pattern,
3068
- )),
3069
- EntityIdentityPart::Number(index.to_string()),
3070
- EntityIdentityPart::Bool(index % 2 == 0),
4515
+ entity_id("change-composite", index, config.key_pattern),
4516
+ index.to_string(),
4517
+ (index % 2 == 0).to_string(),
3071
4518
  ],
3072
4519
  };
3073
4520
  change
@@ -3075,7 +4522,7 @@ fn changelog_composite_entity_id_changes(
3075
4522
  .collect()
3076
4523
  }
3077
4524
 
3078
- fn changelog_selective_changes(config: StorageBenchConfig) -> Vec<MaterializedCanonicalChange> {
4525
+ fn changelog_selective_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
3079
4526
  changelog_materialized_changes(config)
3080
4527
  .into_iter()
3081
4528
  .enumerate()
@@ -3086,9 +4533,7 @@ fn changelog_selective_changes(config: StorageBenchConfig) -> Vec<MaterializedCa
3086
4533
  .collect()
3087
4534
  }
3088
4535
 
3089
- fn changelog_entity_history_changes(
3090
- config: StorageBenchConfig,
3091
- ) -> Vec<MaterializedCanonicalChange> {
4536
+ fn changelog_entity_history_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
3092
4537
  changelog_materialized_changes(config)
3093
4538
  .into_iter()
3094
4539
  .enumerate()
@@ -3187,16 +4632,14 @@ fn snapshot_content(index: usize, target_bytes: usize) -> String {
3187
4632
  value.to_string()
3188
4633
  }
3189
4634
 
3190
- fn snapshot_metadata(index: usize, target_bytes: usize) -> serde_json::Value {
3191
- serde_json::from_str(&snapshot_content(index, target_bytes))
3192
- .expect("bench snapshot content should be valid JSON metadata")
4635
+ fn snapshot_metadata(index: usize, target_bytes: usize) -> String {
4636
+ snapshot_content(index, target_bytes)
3193
4637
  }
3194
4638
 
3195
4639
  fn tracked_state_header_columns() -> Vec<String> {
3196
4640
  [
3197
4641
  "entity_id",
3198
4642
  "schema_key",
3199
- "schema_version",
3200
4643
  "file_id",
3201
4644
  "metadata",
3202
4645
  "created_at",
@@ -3213,7 +4656,6 @@ fn untracked_state_header_columns() -> Vec<String> {
3213
4656
  [
3214
4657
  "entity_id",
3215
4658
  "schema_key",
3216
- "schema_version",
3217
4659
  "file_id",
3218
4660
  "metadata",
3219
4661
  "created_at",
@@ -3247,6 +4689,21 @@ fn partial_updated_snapshot_content(index: usize, target_bytes: usize) -> String
3247
4689
  value.to_string()
3248
4690
  }
3249
4691
 
4692
+ fn delta_chain_snapshot_content(
4693
+ delta_index: usize,
4694
+ row_index: usize,
4695
+ target_bytes: usize,
4696
+ ) -> String {
4697
+ let mut value = serde_json::json!({
4698
+ "id": format!("entity-{row_index}"),
4699
+ "value": format!("delta-{delta_index}-{row_index}"),
4700
+ "index": row_index,
4701
+ "delta": delta_index
4702
+ });
4703
+ pad_snapshot_content(&mut value, target_bytes);
4704
+ value.to_string()
4705
+ }
4706
+
3250
4707
  fn pad_snapshot_content(value: &mut serde_json::Value, target_bytes: usize) {
3251
4708
  let current = value.to_string().len();
3252
4709
  if target_bytes <= current {