@lix-js/sdk 0.6.0-preview.5 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. package/README.md +76 -4
  2. package/dist/errors.d.ts +7 -0
  3. package/dist/errors.js +19 -0
  4. package/dist/index.d.ts +4 -5
  5. package/dist/index.js +3 -3
  6. package/dist/native.d.ts +1 -0
  7. package/dist/native.js +47 -0
  8. package/dist/open-lix.d.ts +38 -207
  9. package/dist/open-lix.js +59 -284
  10. package/dist/result.d.ts +18 -0
  11. package/dist/result.js +48 -0
  12. package/dist/types.d.ts +114 -1
  13. package/dist/value.d.ts +28 -0
  14. package/dist/value.js +245 -0
  15. package/package.json +38 -71
  16. package/SKILL.md +0 -507
  17. package/dist/builtin-schemas.d.ts +0 -1
  18. package/dist/builtin-schemas.js +0 -1
  19. package/dist/engine-wasm/index.d.ts +0 -87
  20. package/dist/engine-wasm/index.js +0 -339
  21. package/dist/engine-wasm/wasm/lix_engine.d.ts +0 -79
  22. package/dist/engine-wasm/wasm/lix_engine.js +0 -833
  23. package/dist/engine-wasm/wasm/lix_engine.wasm +0 -0
  24. package/dist/engine-wasm/wasm/lix_engine.wasm.d.ts +0 -27
  25. package/dist/generated/builtin-schemas.d.ts +0 -427
  26. package/dist/generated/builtin-schemas.js +0 -643
  27. package/dist/sqlite/index.d.ts +0 -12
  28. package/dist/sqlite/index.js +0 -359
  29. package/dist-engine-src/README.md +0 -18
  30. package/dist-engine-src/src/backend/capabilities.rs +0 -67
  31. package/dist-engine-src/src/backend/conformance/baseline.rs +0 -1127
  32. package/dist-engine-src/src/backend/conformance/factory.rs +0 -93
  33. package/dist-engine-src/src/backend/conformance/failure_tests.rs +0 -608
  34. package/dist-engine-src/src/backend/conformance/fixtures.rs +0 -26
  35. package/dist-engine-src/src/backend/conformance/mod.rs +0 -75
  36. package/dist-engine-src/src/backend/conformance/model.rs +0 -28
  37. package/dist-engine-src/src/backend/conformance/model_based.rs +0 -257
  38. package/dist-engine-src/src/backend/conformance/persistence.rs +0 -204
  39. package/dist-engine-src/src/backend/conformance/projection.rs +0 -21
  40. package/dist-engine-src/src/backend/conformance/pushdown.rs +0 -24
  41. package/dist-engine-src/src/backend/conformance/runner.rs +0 -90
  42. package/dist-engine-src/src/backend/conformance/scan.rs +0 -24
  43. package/dist-engine-src/src/backend/conformance/write.rs +0 -16
  44. package/dist-engine-src/src/backend/error.rs +0 -94
  45. package/dist-engine-src/src/backend/in_memory.rs +0 -670
  46. package/dist-engine-src/src/backend/mod.rs +0 -39
  47. package/dist-engine-src/src/backend/predicate.rs +0 -80
  48. package/dist-engine-src/src/backend/traits.rs +0 -260
  49. package/dist-engine-src/src/backend/types.rs +0 -239
  50. package/dist-engine-src/src/binary_cas/chunking.rs +0 -31
  51. package/dist-engine-src/src/binary_cas/codec.rs +0 -346
  52. package/dist-engine-src/src/binary_cas/context.rs +0 -139
  53. package/dist-engine-src/src/binary_cas/kv.rs +0 -1038
  54. package/dist-engine-src/src/binary_cas/mod.rs +0 -11
  55. package/dist-engine-src/src/binary_cas/types.rs +0 -121
  56. package/dist-engine-src/src/branch/context.rs +0 -40
  57. package/dist-engine-src/src/branch/lifecycle.rs +0 -221
  58. package/dist-engine-src/src/branch/mod.rs +0 -13
  59. package/dist-engine-src/src/branch/refs.rs +0 -321
  60. package/dist-engine-src/src/branch/stage_rows.rs +0 -67
  61. package/dist-engine-src/src/branch/types.rs +0 -21
  62. package/dist-engine-src/src/catalog/context.rs +0 -412
  63. package/dist-engine-src/src/catalog/mod.rs +0 -10
  64. package/dist-engine-src/src/catalog/schema.rs +0 -4
  65. package/dist-engine-src/src/catalog/snapshot.rs +0 -1114
  66. package/dist-engine-src/src/cel/context.rs +0 -86
  67. package/dist-engine-src/src/cel/error.rs +0 -19
  68. package/dist-engine-src/src/cel/mod.rs +0 -8
  69. package/dist-engine-src/src/cel/provider.rs +0 -9
  70. package/dist-engine-src/src/cel/runtime.rs +0 -167
  71. package/dist-engine-src/src/cel/value.rs +0 -50
  72. package/dist-engine-src/src/changelog/bench_support.rs +0 -785
  73. package/dist-engine-src/src/changelog/change.rs +0 -1
  74. package/dist-engine-src/src/changelog/codec.rs +0 -497
  75. package/dist-engine-src/src/changelog/commit.rs +0 -1
  76. package/dist-engine-src/src/changelog/context.rs +0 -1614
  77. package/dist-engine-src/src/changelog/mod.rs +0 -29
  78. package/dist-engine-src/src/changelog/store.rs +0 -163
  79. package/dist-engine-src/src/changelog/test_support.rs +0 -54
  80. package/dist-engine-src/src/changelog/types.rs +0 -213
  81. package/dist-engine-src/src/commit_graph/context.rs +0 -944
  82. package/dist-engine-src/src/commit_graph/mod.rs +0 -9
  83. package/dist-engine-src/src/commit_graph/types.rs +0 -89
  84. package/dist-engine-src/src/commit_graph/walker.rs +0 -786
  85. package/dist-engine-src/src/common/error.rs +0 -347
  86. package/dist-engine-src/src/common/fingerprint.rs +0 -3
  87. package/dist-engine-src/src/common/fs_path.rs +0 -1336
  88. package/dist-engine-src/src/common/identity.rs +0 -145
  89. package/dist-engine-src/src/common/json_pointer.rs +0 -67
  90. package/dist-engine-src/src/common/metadata.rs +0 -40
  91. package/dist-engine-src/src/common/mod.rs +0 -23
  92. package/dist-engine-src/src/common/types.rs +0 -105
  93. package/dist-engine-src/src/common/wire.rs +0 -222
  94. package/dist-engine-src/src/domain.rs +0 -320
  95. package/dist-engine-src/src/engine.rs +0 -203
  96. package/dist-engine-src/src/entity_pk.rs +0 -402
  97. package/dist-engine-src/src/functions/context.rs +0 -296
  98. package/dist-engine-src/src/functions/deterministic.rs +0 -113
  99. package/dist-engine-src/src/functions/mod.rs +0 -18
  100. package/dist-engine-src/src/functions/provider.rs +0 -130
  101. package/dist-engine-src/src/functions/state.rs +0 -335
  102. package/dist-engine-src/src/functions/types.rs +0 -37
  103. package/dist-engine-src/src/init.rs +0 -692
  104. package/dist-engine-src/src/json_store/compression.rs +0 -77
  105. package/dist-engine-src/src/json_store/context.rs +0 -172
  106. package/dist-engine-src/src/json_store/encoded.rs +0 -15
  107. package/dist-engine-src/src/json_store/mod.rs +0 -38
  108. package/dist-engine-src/src/json_store/store.rs +0 -494
  109. package/dist-engine-src/src/json_store/types.rs +0 -212
  110. package/dist-engine-src/src/lib.rs +0 -92
  111. package/dist-engine-src/src/live_state/context.rs +0 -1883
  112. package/dist-engine-src/src/live_state/mod.rs +0 -21
  113. package/dist-engine-src/src/live_state/overlay.rs +0 -75
  114. package/dist-engine-src/src/live_state/reader.rs +0 -23
  115. package/dist-engine-src/src/live_state/types.rs +0 -231
  116. package/dist-engine-src/src/live_state/visibility.rs +0 -666
  117. package/dist-engine-src/src/plugin/archive.rs +0 -438
  118. package/dist-engine-src/src/plugin/component.rs +0 -183
  119. package/dist-engine-src/src/plugin/install.rs +0 -619
  120. package/dist-engine-src/src/plugin/manifest.rs +0 -516
  121. package/dist-engine-src/src/plugin/materializer.rs +0 -202
  122. package/dist-engine-src/src/plugin/mod.rs +0 -33
  123. package/dist-engine-src/src/plugin/plugin_manifest.json +0 -119
  124. package/dist-engine-src/src/plugin/storage.rs +0 -74
  125. package/dist-engine-src/src/schema/annotations/defaults.rs +0 -275
  126. package/dist-engine-src/src/schema/annotations/mod.rs +0 -1
  127. package/dist-engine-src/src/schema/builtin/lix_account.json +0 -21
  128. package/dist-engine-src/src/schema/builtin/lix_active_account.json +0 -29
  129. package/dist-engine-src/src/schema/builtin/lix_binary_blob_ref.json +0 -29
  130. package/dist-engine-src/src/schema/builtin/lix_branch_descriptor.json +0 -34
  131. package/dist-engine-src/src/schema/builtin/lix_branch_ref.json +0 -48
  132. package/dist-engine-src/src/schema/builtin/lix_change.json +0 -63
  133. package/dist-engine-src/src/schema/builtin/lix_change_author.json +0 -45
  134. package/dist-engine-src/src/schema/builtin/lix_commit.json +0 -24
  135. package/dist-engine-src/src/schema/builtin/lix_commit_edge.json +0 -53
  136. package/dist-engine-src/src/schema/builtin/lix_directory_descriptor.json +0 -52
  137. package/dist-engine-src/src/schema/builtin/lix_file_descriptor.json +0 -52
  138. package/dist-engine-src/src/schema/builtin/lix_key_value.json +0 -40
  139. package/dist-engine-src/src/schema/builtin/lix_label.json +0 -29
  140. package/dist-engine-src/src/schema/builtin/lix_label_assignment.json +0 -74
  141. package/dist-engine-src/src/schema/builtin/lix_registered_schema.json +0 -25
  142. package/dist-engine-src/src/schema/builtin/mod.rs +0 -220
  143. package/dist-engine-src/src/schema/compatibility.rs +0 -787
  144. package/dist-engine-src/src/schema/definition.json +0 -187
  145. package/dist-engine-src/src/schema/definition.rs +0 -742
  146. package/dist-engine-src/src/schema/key.rs +0 -138
  147. package/dist-engine-src/src/schema/mod.rs +0 -20
  148. package/dist-engine-src/src/schema/seed.rs +0 -14
  149. package/dist-engine-src/src/schema/tests.rs +0 -780
  150. package/dist-engine-src/src/session/context.rs +0 -1059
  151. package/dist-engine-src/src/session/create_branch.rs +0 -94
  152. package/dist-engine-src/src/session/execute.rs +0 -681
  153. package/dist-engine-src/src/session/merge/analysis.rs +0 -108
  154. package/dist-engine-src/src/session/merge/branch.rs +0 -417
  155. package/dist-engine-src/src/session/merge/conflicts.rs +0 -63
  156. package/dist-engine-src/src/session/merge/mod.rs +0 -10
  157. package/dist-engine-src/src/session/merge/stats.rs +0 -61
  158. package/dist-engine-src/src/session/mod.rs +0 -30
  159. package/dist-engine-src/src/session/switch_branch.rs +0 -113
  160. package/dist-engine-src/src/session/transaction.rs +0 -557
  161. package/dist-engine-src/src/sql2/bind/classify.rs +0 -102
  162. package/dist-engine-src/src/sql2/bind/error.rs +0 -5
  163. package/dist-engine-src/src/sql2/bind/expr.rs +0 -29
  164. package/dist-engine-src/src/sql2/bind/mod.rs +0 -12
  165. package/dist-engine-src/src/sql2/bind/public_udf.rs +0 -306
  166. package/dist-engine-src/src/sql2/bind/read.rs +0 -65
  167. package/dist-engine-src/src/sql2/bind/statement.rs +0 -2236
  168. package/dist-engine-src/src/sql2/bind/table.rs +0 -273
  169. package/dist-engine-src/src/sql2/bind/write.rs +0 -86
  170. package/dist-engine-src/src/sql2/branch_scope.rs +0 -436
  171. package/dist-engine-src/src/sql2/catalog/capability.rs +0 -20
  172. package/dist-engine-src/src/sql2/catalog/entity_surface.rs +0 -296
  173. package/dist-engine-src/src/sql2/catalog/mod.rs +0 -15
  174. package/dist-engine-src/src/sql2/catalog/registry.rs +0 -556
  175. package/dist-engine-src/src/sql2/catalog/schema.rs +0 -88
  176. package/dist-engine-src/src/sql2/catalog/surface.rs +0 -41
  177. package/dist-engine-src/src/sql2/change_materialization.rs +0 -122
  178. package/dist-engine-src/src/sql2/context.rs +0 -317
  179. package/dist-engine-src/src/sql2/dml.rs +0 -148
  180. package/dist-engine-src/src/sql2/error.rs +0 -215
  181. package/dist-engine-src/src/sql2/exec/bound_public_write.rs +0 -1593
  182. package/dist-engine-src/src/sql2/exec/datafusion.rs +0 -5266
  183. package/dist-engine-src/src/sql2/exec/fast_write.rs +0 -82
  184. package/dist-engine-src/src/sql2/exec/mod.rs +0 -24
  185. package/dist-engine-src/src/sql2/exec/write.rs +0 -661
  186. package/dist-engine-src/src/sql2/filesystem_planner.rs +0 -1485
  187. package/dist-engine-src/src/sql2/filesystem_predicates.rs +0 -159
  188. package/dist-engine-src/src/sql2/filesystem_visibility.rs +0 -383
  189. package/dist-engine-src/src/sql2/history_projection.rs +0 -56
  190. package/dist-engine-src/src/sql2/history_route.rs +0 -661
  191. package/dist-engine-src/src/sql2/mod.rs +0 -52
  192. package/dist-engine-src/src/sql2/optimize/datafusion.rs +0 -1
  193. package/dist-engine-src/src/sql2/optimize/mod.rs +0 -2
  194. package/dist-engine-src/src/sql2/optimize/simple_write.rs +0 -116
  195. package/dist-engine-src/src/sql2/parse/mod.rs +0 -69
  196. package/dist-engine-src/src/sql2/parse/normalize.rs +0 -1
  197. package/dist-engine-src/src/sql2/plan/branch_scope.rs +0 -24
  198. package/dist-engine-src/src/sql2/plan/mod.rs +0 -5
  199. package/dist-engine-src/src/sql2/plan/predicate.rs +0 -22
  200. package/dist-engine-src/src/sql2/plan/write.rs +0 -147
  201. package/dist-engine-src/src/sql2/predicate_typecheck.rs +0 -504
  202. package/dist-engine-src/src/sql2/providers/branch.rs +0 -1206
  203. package/dist-engine-src/src/sql2/providers/change.rs +0 -445
  204. package/dist-engine-src/src/sql2/providers/directory.rs +0 -2422
  205. package/dist-engine-src/src/sql2/providers/directory_history.rs +0 -645
  206. package/dist-engine-src/src/sql2/providers/entity.rs +0 -1484
  207. package/dist-engine-src/src/sql2/providers/entity_history.rs +0 -452
  208. package/dist-engine-src/src/sql2/providers/file.rs +0 -3686
  209. package/dist-engine-src/src/sql2/providers/file_history.rs +0 -924
  210. package/dist-engine-src/src/sql2/providers/history.rs +0 -426
  211. package/dist-engine-src/src/sql2/providers/lix_state.rs +0 -2542
  212. package/dist-engine-src/src/sql2/providers/mod.rs +0 -508
  213. package/dist-engine-src/src/sql2/read_only.rs +0 -63
  214. package/dist-engine-src/src/sql2/record_batch.rs +0 -17
  215. package/dist-engine-src/src/sql2/result_metadata.rs +0 -29
  216. package/dist-engine-src/src/sql2/runtime.rs +0 -60
  217. package/dist-engine-src/src/sql2/session.rs +0 -83
  218. package/dist-engine-src/src/sql2/storage/constraints.rs +0 -1
  219. package/dist-engine-src/src/sql2/storage/mod.rs +0 -1
  220. package/dist-engine-src/src/sql2/test_support/differential.rs +0 -712
  221. package/dist-engine-src/src/sql2/test_support/generators.rs +0 -354
  222. package/dist-engine-src/src/sql2/test_support/mod.rs +0 -2
  223. package/dist-engine-src/src/sql2/udfs/common.rs +0 -295
  224. package/dist-engine-src/src/sql2/udfs/lix_active_branch_commit_id.rs +0 -53
  225. package/dist-engine-src/src/sql2/udfs/lix_empty_blob.rs +0 -47
  226. package/dist-engine-src/src/sql2/udfs/lix_json.rs +0 -100
  227. package/dist-engine-src/src/sql2/udfs/lix_json_get.rs +0 -99
  228. package/dist-engine-src/src/sql2/udfs/lix_json_get_text.rs +0 -99
  229. package/dist-engine-src/src/sql2/udfs/lix_text_decode.rs +0 -82
  230. package/dist-engine-src/src/sql2/udfs/lix_text_encode.rs +0 -85
  231. package/dist-engine-src/src/sql2/udfs/lix_timestamp.rs +0 -76
  232. package/dist-engine-src/src/sql2/udfs/lix_uuid_v7.rs +0 -76
  233. package/dist-engine-src/src/sql2/udfs/mod.rs +0 -86
  234. package/dist-engine-src/src/sql2/write_normalization.rs +0 -368
  235. package/dist-engine-src/src/storage/conformance.rs +0 -399
  236. package/dist-engine-src/src/storage/context.rs +0 -620
  237. package/dist-engine-src/src/storage/mod.rs +0 -52
  238. package/dist-engine-src/src/storage/point.rs +0 -440
  239. package/dist-engine-src/src/storage/read_scope.rs +0 -67
  240. package/dist-engine-src/src/storage/reader.rs +0 -867
  241. package/dist-engine-src/src/storage/scan.rs +0 -784
  242. package/dist-engine-src/src/storage/spaces.rs +0 -236
  243. package/dist-engine-src/src/storage/stats.rs +0 -80
  244. package/dist-engine-src/src/storage/write_set.rs +0 -962
  245. package/dist-engine-src/src/storage_bench.rs +0 -171
  246. package/dist-engine-src/src/test_support.rs +0 -450
  247. package/dist-engine-src/src/tracked_state/bench_support.rs +0 -394
  248. package/dist-engine-src/src/tracked_state/codec.rs +0 -1183
  249. package/dist-engine-src/src/tracked_state/commit_root_rebuild.rs +0 -358
  250. package/dist-engine-src/src/tracked_state/context.rs +0 -2801
  251. package/dist-engine-src/src/tracked_state/diff.rs +0 -2140
  252. package/dist-engine-src/src/tracked_state/merge.rs +0 -478
  253. package/dist-engine-src/src/tracked_state/mod.rs +0 -35
  254. package/dist-engine-src/src/tracked_state/row_materialization.rs +0 -275
  255. package/dist-engine-src/src/tracked_state/storage.rs +0 -427
  256. package/dist-engine-src/src/tracked_state/tree.rs +0 -3063
  257. package/dist-engine-src/src/tracked_state/types.rs +0 -238
  258. package/dist-engine-src/src/transaction/bench_support.rs +0 -407
  259. package/dist-engine-src/src/transaction/commit.rs +0 -1592
  260. package/dist-engine-src/src/transaction/context.rs +0 -1653
  261. package/dist-engine-src/src/transaction/mod.rs +0 -24
  262. package/dist-engine-src/src/transaction/normalization.rs +0 -877
  263. package/dist-engine-src/src/transaction/prep.rs +0 -37
  264. package/dist-engine-src/src/transaction/schema_resolver.rs +0 -163
  265. package/dist-engine-src/src/transaction/staging.rs +0 -1525
  266. package/dist-engine-src/src/transaction/types.rs +0 -403
  267. package/dist-engine-src/src/transaction/validation.rs +0 -5766
  268. package/dist-engine-src/src/untracked_state/codec.rs +0 -615
  269. package/dist-engine-src/src/untracked_state/context.rs +0 -98
  270. package/dist-engine-src/src/untracked_state/materialization.rs +0 -63
  271. package/dist-engine-src/src/untracked_state/mod.rs +0 -15
  272. package/dist-engine-src/src/untracked_state/storage.rs +0 -898
  273. package/dist-engine-src/src/untracked_state/types.rs +0 -146
  274. package/dist-engine-src/src/wasm/mod.rs +0 -60
@@ -1,3063 +0,0 @@
1
- use std::{
2
- collections::{BTreeMap, VecDeque},
3
- future::Future,
4
- ops::Range,
5
- pin::Pin,
6
- };
7
-
8
- use crate::storage::{StorageRead, StorageWriteSet};
9
- use crate::tracked_state::codec::{
10
- boundary_trigger, child_summary_from_node, decode_key, decode_key_with_trusted_prefix,
11
- decode_node, decode_node_ref, decode_value, decode_visible_value, encode_internal_node,
12
- encode_internal_node_refs, encode_key, encode_leaf_node, encode_leaf_node_refs,
13
- encode_schema_file_prefix, encode_schema_key_prefix, ChildSummary, ChildSummaryRef,
14
- DecodedLeafNodeRef, DecodedNode, DecodedNodeRef, EncodedLeafEntry, EncodedLeafEntryRef,
15
- PendingChunkWrite,
16
- };
17
- use crate::tracked_state::storage;
18
- use crate::tracked_state::types::{
19
- TrackedStateApplyResult, TrackedStateIndexValue, TrackedStateKey, TrackedStateMutation,
20
- TrackedStateRootId, TrackedStateTreeDiffEntry, TrackedStateTreeScanRequest,
21
- TRACKED_STATE_HASH_BYTES,
22
- };
23
- use crate::{LixError, NullableKeyFilter};
24
-
25
- #[derive(Debug, Clone, PartialEq, Eq)]
26
- pub(crate) struct TrackedStateTreeOptions {
27
- pub(crate) target_chunk_bytes: usize,
28
- pub(crate) min_chunk_bytes: usize,
29
- pub(crate) max_chunk_bytes: usize,
30
- }
31
-
32
- enum MutationApply<T> {
33
- Applied(TrackedStateApplyResult),
34
- Fallback(T),
35
- }
36
-
37
- impl Default for TrackedStateTreeOptions {
38
- fn default() -> Self {
39
- Self {
40
- target_chunk_bytes: 4 * 1024,
41
- min_chunk_bytes: 512,
42
- max_chunk_bytes: 16 * 1024,
43
- }
44
- }
45
- }
46
-
47
- /// Content-addressed tracked-state tree operations.
48
- ///
49
- /// This type owns tracked-state tree mechanics only. Branch refs, untracked overlay,
50
- /// and SQL visibility remain outside the tree.
51
- #[derive(Debug, Clone)]
52
- pub(crate) struct TrackedStateTree {
53
- options: TrackedStateTreeOptions,
54
- }
55
-
56
- impl TrackedStateTree {
57
- pub(crate) fn new() -> Self {
58
- Self {
59
- options: TrackedStateTreeOptions::default(),
60
- }
61
- }
62
-
63
- #[cfg(test)]
64
- pub(crate) fn with_options(options: TrackedStateTreeOptions) -> Self {
65
- Self { options }
66
- }
67
-
68
- pub(crate) async fn load_root(
69
- &self,
70
- store: &(impl StorageRead + Send + Sync + ?Sized),
71
- commit_id: &str,
72
- ) -> Result<Option<TrackedStateRootId>, LixError> {
73
- storage::load_root(store, commit_id).await
74
- }
75
-
76
- #[cfg(test)]
77
- pub(crate) async fn get(
78
- &self,
79
- store: &(impl StorageRead + Send + Sync),
80
- root_id: &TrackedStateRootId,
81
- key: &TrackedStateKey,
82
- ) -> Result<Option<TrackedStateIndexValue>, LixError> {
83
- let encoded_key = encode_key(key);
84
- let mut current = *root_id.as_bytes();
85
- loop {
86
- match self.load_node(store, &current).await? {
87
- DecodedNode::Leaf(leaf) => {
88
- let entry = leaf
89
- .entries()
90
- .binary_search_by(|entry| entry.key.as_slice().cmp(&encoded_key))
91
- .ok()
92
- .map(|index| &leaf.entries()[index]);
93
- return entry.map(|entry| decode_value(&entry.value)).transpose();
94
- }
95
- DecodedNode::Internal(internal) => {
96
- let child = internal
97
- .children()
98
- .iter()
99
- .find(|child| child.last_key.as_slice() >= encoded_key.as_slice())
100
- .or_else(|| internal.children().last())
101
- .ok_or_else(|| {
102
- LixError::new(
103
- "LIX_ERROR_UNKNOWN",
104
- "tracked-state tree internal node has no children",
105
- )
106
- })?;
107
- current = child.child_hash;
108
- }
109
- }
110
- }
111
- }
112
-
113
- pub(crate) async fn get_many(
114
- &self,
115
- store: &(impl StorageRead + Send + Sync + ?Sized),
116
- root_id: &TrackedStateRootId,
117
- keys: &[TrackedStateKey],
118
- ) -> Result<Vec<Option<TrackedStateIndexValue>>, LixError> {
119
- if keys.is_empty() {
120
- return Ok(Vec::new());
121
- }
122
-
123
- let mut encoded_keys = keys
124
- .iter()
125
- .enumerate()
126
- .map(|(index, key)| (index, encode_key(key)))
127
- .collect::<Vec<_>>();
128
- encoded_keys.sort_by(|left, right| left.1.cmp(&right.1));
129
-
130
- let mut values = vec![None; keys.len()];
131
- self.get_many_node(store, *root_id.as_bytes(), &encoded_keys, &mut values)
132
- .await?;
133
- Ok(values)
134
- }
135
-
136
- pub(crate) async fn scan(
137
- &self,
138
- store: &(impl StorageRead + Send + Sync + ?Sized),
139
- root_id: &TrackedStateRootId,
140
- request: &TrackedStateTreeScanRequest,
141
- ) -> Result<Vec<(TrackedStateKey, TrackedStateIndexValue)>, LixError> {
142
- if request.limit == Some(0) {
143
- return Ok(Vec::new());
144
- }
145
-
146
- let ranges = scan_ranges(request);
147
- let key_decode_hint = scan_key_decode_hint(request, &ranges);
148
- let mut rows = Vec::new();
149
- self.scan_node(
150
- store,
151
- *root_id.as_bytes(),
152
- request,
153
- &ranges,
154
- key_decode_hint,
155
- &mut rows,
156
- )
157
- .await?;
158
- Ok(rows)
159
- }
160
-
161
- pub(crate) async fn diff(
162
- &self,
163
- store: &(impl StorageRead + Send + Sync),
164
- left_root: Option<&TrackedStateRootId>,
165
- right_root: Option<&TrackedStateRootId>,
166
- request: &TrackedStateTreeScanRequest,
167
- ) -> Result<Vec<TrackedStateTreeDiffEntry>, LixError> {
168
- match (left_root, right_root) {
169
- (None, None) => Ok(Vec::new()),
170
- (Some(left), Some(right)) if left == right => Ok(Vec::new()),
171
- (Some(left), Some(right)) => {
172
- let mut out = Vec::new();
173
- self.diff_nodes(
174
- store,
175
- *left.as_bytes(),
176
- *right.as_bytes(),
177
- request,
178
- &mut out,
179
- )
180
- .await?;
181
- Ok(out)
182
- }
183
- (Some(left), None) => Ok(self
184
- .collect_filtered_entries(store, left, request)
185
- .await?
186
- .into_iter()
187
- .map(|(key, value)| TrackedStateTreeDiffEntry {
188
- before: Some((key, value)),
189
- after: None,
190
- })
191
- .collect()),
192
- (None, Some(right)) => Ok(self
193
- .collect_filtered_entries(store, right, request)
194
- .await?
195
- .into_iter()
196
- .map(|(key, value)| TrackedStateTreeDiffEntry {
197
- before: None,
198
- after: Some((key, value)),
199
- })
200
- .collect()),
201
- }
202
- }
203
-
204
- #[cfg(test)]
205
- pub(crate) async fn apply_mutations(
206
- &self,
207
- store: &(impl StorageRead + Send + Sync + ?Sized),
208
- writes: &mut StorageWriteSet,
209
- base_root: Option<&TrackedStateRootId>,
210
- mutations: Vec<TrackedStateMutation>,
211
- commit_id: Option<&str>,
212
- ) -> Result<TrackedStateApplyResult, LixError> {
213
- let mut overlay = storage::TrackedStateChunkOverlay::new();
214
- self.apply_mutations_with_overlay(
215
- store,
216
- writes,
217
- &mut overlay,
218
- base_root,
219
- mutations,
220
- commit_id,
221
- )
222
- .await
223
- }
224
-
225
- pub(crate) async fn apply_mutations_with_overlay(
226
- &self,
227
- store: &(impl StorageRead + Send + Sync + ?Sized),
228
- writes: &mut StorageWriteSet,
229
- overlay: &mut storage::TrackedStateChunkOverlay,
230
- base_root: Option<&TrackedStateRootId>,
231
- mut mutations: Vec<TrackedStateMutation>,
232
- commit_id: Option<&str>,
233
- ) -> Result<TrackedStateApplyResult, LixError> {
234
- if let Some(root_id) = base_root {
235
- if mutations.len() == 1 {
236
- let mutation = mutations.pop().expect("single mutation should exist");
237
- match self
238
- .apply_single_mutation(store, writes, overlay, root_id, mutation, commit_id)
239
- .await?
240
- {
241
- MutationApply::Applied(result) => return Ok(result),
242
- MutationApply::Fallback(mutation) => mutations = vec![mutation],
243
- }
244
- } else if mutations.len() > 1 {
245
- match self
246
- .apply_sorted_mutations_chunker(
247
- store, writes, overlay, root_id, mutations, commit_id,
248
- )
249
- .await?
250
- {
251
- MutationApply::Applied(result) => return Ok(result),
252
- MutationApply::Fallback(fallback_mutations) => mutations = fallback_mutations,
253
- }
254
- }
255
- }
256
-
257
- let mut entries = match base_root {
258
- Some(root_id) => self
259
- .collect_leaf_entries_with_overlay(store, overlay, root_id)
260
- .await?
261
- .into_iter()
262
- .map(|entry| (entry.key, entry.value))
263
- .collect::<BTreeMap<_, _>>(),
264
- None => BTreeMap::new(),
265
- };
266
-
267
- // Apply in caller order so repeated writes to the same key behave like
268
- // normal transaction staging: the latest mutation wins.
269
- for mutation in mutations {
270
- entries.insert(mutation.encoded_key, mutation.encoded_value);
271
- }
272
-
273
- let built = self.build_tree_from_entries(
274
- entries
275
- .into_iter()
276
- .map(|(key, value)| EncodedLeafEntry { key, value })
277
- .collect(),
278
- )?;
279
- overlay.stage_chunks(writes, &built.chunks);
280
-
281
- Ok(TrackedStateApplyResult {
282
- root_id: built.root_id,
283
- row_count: built.row_count,
284
- tree_height: built.tree_height,
285
- chunk_count: built.chunks.len(),
286
- chunk_bytes: built.chunk_bytes,
287
- })
288
- }
289
-
290
- async fn apply_single_mutation(
291
- &self,
292
- store: &(impl StorageRead + Send + Sync + ?Sized),
293
- writes: &mut StorageWriteSet,
294
- overlay: &mut storage::TrackedStateChunkOverlay,
295
- root_id: &TrackedStateRootId,
296
- mutation: TrackedStateMutation,
297
- commit_id: Option<&str>,
298
- ) -> Result<MutationApply<TrackedStateMutation>, LixError> {
299
- let mutation = match self
300
- .apply_single_mutation_from_seek_path(
301
- store, writes, overlay, root_id, mutation, commit_id,
302
- )
303
- .await?
304
- {
305
- MutationApply::Applied(result) => return Ok(MutationApply::Applied(result)),
306
- MutationApply::Fallback(mutation) => mutation,
307
- };
308
-
309
- let TrackedStateMutation {
310
- encoded_key,
311
- encoded_value,
312
- } = mutation;
313
-
314
- let levels = self
315
- .collect_summary_levels_with_overlay(store, overlay, root_id)
316
- .await?;
317
- let Some(leaves) = levels.first() else {
318
- return Ok(MutationApply::Fallback(TrackedStateMutation {
319
- encoded_key,
320
- encoded_value,
321
- }));
322
- };
323
- let target_leaf_index = leaves
324
- .iter()
325
- .position(|leaf| leaf.last_key.as_slice() >= encoded_key.as_slice())
326
- .unwrap_or_else(|| leaves.len().saturating_sub(1));
327
- let Some(target_leaf) = leaves.get(target_leaf_index).cloned() else {
328
- return Ok(MutationApply::Fallback(TrackedStateMutation {
329
- encoded_key,
330
- encoded_value,
331
- }));
332
- };
333
-
334
- let mut entries = self
335
- .load_leaf_entries_with_overlay(store, overlay, &target_leaf.child_hash)
336
- .await?;
337
- let mutation_entry_index = match entries
338
- .binary_search_by(|entry| entry.key.as_slice().cmp(encoded_key.as_slice()))
339
- {
340
- Ok(index) => {
341
- if entries[index].value.as_slice() == encoded_value.as_slice() {
342
- return Ok(MutationApply::Fallback(TrackedStateMutation {
343
- encoded_key,
344
- encoded_value,
345
- }));
346
- }
347
- entries[index].value = encoded_value;
348
- index
349
- }
350
- Err(index) => {
351
- entries.insert(
352
- index,
353
- EncodedLeafEntry {
354
- key: encoded_key,
355
- value: encoded_value,
356
- },
357
- );
358
- index
359
- }
360
- };
361
-
362
- let mut chunks = BTreeMap::new();
363
- let mut suffix_entries = entries;
364
- let mut next_leaf_index = target_leaf_index + 1;
365
- let mut replacement_leaves;
366
- let old_leaf_count;
367
-
368
- // Rechunk from the edited leaf until a generated leaf matches an
369
- // existing post-mutation leaf, then reuse the rest of the old suffix.
370
- loop {
371
- let mut candidate_chunks = BTreeMap::new();
372
- let candidate_summaries = self.build_leaf_level_from_refs(
373
- suffix_entries.iter().map(EncodedLeafEntry::as_ref),
374
- &mut candidate_chunks,
375
- );
376
-
377
- if let Some((generated_resync_index, existing_resync_index)) = first_resync_index(
378
- &candidate_summaries,
379
- &leaves[target_leaf_index..],
380
- suffix_entries[mutation_entry_index].key.as_slice(),
381
- ) {
382
- for summary in &candidate_summaries[..generated_resync_index] {
383
- if let Some(chunk) = candidate_chunks.remove(&summary.child_hash) {
384
- chunks.entry(chunk.hash).or_insert(chunk);
385
- }
386
- }
387
- replacement_leaves = candidate_summaries
388
- .into_iter()
389
- .take(generated_resync_index)
390
- .collect();
391
- old_leaf_count = existing_resync_index;
392
- break;
393
- }
394
-
395
- if next_leaf_index >= leaves.len() {
396
- chunks.extend(candidate_chunks);
397
- replacement_leaves = candidate_summaries;
398
- old_leaf_count = leaves.len() - target_leaf_index;
399
- break;
400
- }
401
-
402
- suffix_entries.extend(
403
- self.load_leaf_entries_with_overlay(
404
- store,
405
- overlay,
406
- &leaves[next_leaf_index].child_hash,
407
- )
408
- .await?,
409
- );
410
- next_leaf_index += 1;
411
- }
412
-
413
- let built = self.build_tree_from_leaf_patch(
414
- &levels,
415
- target_leaf_index,
416
- old_leaf_count,
417
- std::mem::take(&mut replacement_leaves),
418
- chunks,
419
- suffix_entries[mutation_entry_index].key.as_slice(),
420
- )?;
421
- overlay.stage_chunks(writes, &built.chunks);
422
-
423
- Ok(MutationApply::Applied(TrackedStateApplyResult {
424
- root_id: built.root_id,
425
- row_count: built.row_count,
426
- tree_height: built.tree_height,
427
- chunk_count: built.chunks.len(),
428
- chunk_bytes: built.chunk_bytes,
429
- }))
430
- }
431
-
432
- fn diff_nodes<'a, S>(
433
- &'a self,
434
- store: &'a S,
435
- left_hash: [u8; TRACKED_STATE_HASH_BYTES],
436
- right_hash: [u8; TRACKED_STATE_HASH_BYTES],
437
- request: &'a TrackedStateTreeScanRequest,
438
- out: &'a mut Vec<TrackedStateTreeDiffEntry>,
439
- ) -> Pin<Box<dyn Future<Output = Result<(), LixError>> + 'a>>
440
- where
441
- S: StorageRead + Send + Sync + 'a,
442
- {
443
- Box::pin(async move {
444
- if left_hash == right_hash {
445
- return Ok(());
446
- }
447
-
448
- let left = self.load_node(store, &left_hash).await?;
449
- let right = self.load_node(store, &right_hash).await?;
450
- match (left, right) {
451
- (DecodedNode::Leaf(left), DecodedNode::Leaf(right)) => {
452
- self.diff_leaf_entries(left.entries(), right.entries(), request, out)?;
453
- }
454
- (DecodedNode::Internal(left), DecodedNode::Internal(right))
455
- if internal_boundaries_match(left.children(), right.children()) =>
456
- {
457
- for (left_child, right_child) in left.children().iter().zip(right.children()) {
458
- if left_child == right_child {
459
- continue;
460
- }
461
- self.diff_nodes(
462
- store,
463
- left_child.child_hash,
464
- right_child.child_hash,
465
- request,
466
- out,
467
- )
468
- .await?;
469
- }
470
- }
471
- _ => {
472
- self.diff_leaf_summary_cursors(store, left_hash, right_hash, request, out)
473
- .await?;
474
- }
475
- }
476
- Ok(())
477
- })
478
- }
479
-
480
- async fn diff_leaf_summary_cursors(
481
- &self,
482
- store: &(impl StorageRead + Send + Sync),
483
- left_hash: [u8; TRACKED_STATE_HASH_BYTES],
484
- right_hash: [u8; TRACKED_STATE_HASH_BYTES],
485
- request: &TrackedStateTreeScanRequest,
486
- out: &mut Vec<TrackedStateTreeDiffEntry>,
487
- ) -> Result<(), LixError> {
488
- let mut left = LeafSummaryCursor::new(self, store, left_hash).await?;
489
- let mut right = LeafSummaryCursor::new(self, store, right_hash).await?;
490
- let mut left_window = Vec::new();
491
- let mut right_window = Vec::new();
492
-
493
- loop {
494
- match (left.current(), right.current()) {
495
- (Some(left_leaf), Some(right_leaf)) if left_leaf == right_leaf => {
496
- self.diff_leaf_summary_window(store, &left_window, &right_window, request, out)
497
- .await?;
498
- left_window.clear();
499
- right_window.clear();
500
- left.advance(self, store).await?;
501
- right.advance(self, store).await?;
502
- }
503
- (Some(left_leaf), Some(right_leaf)) => {
504
- match left_leaf.last_key.cmp(&right_leaf.last_key) {
505
- std::cmp::Ordering::Less => {
506
- left_window.push(left_leaf.clone());
507
- left.advance(self, store).await?;
508
- }
509
- std::cmp::Ordering::Greater => {
510
- right_window.push(right_leaf.clone());
511
- right.advance(self, store).await?;
512
- }
513
- std::cmp::Ordering::Equal => {
514
- left_window.push(left_leaf.clone());
515
- right_window.push(right_leaf.clone());
516
- left.advance(self, store).await?;
517
- right.advance(self, store).await?;
518
- }
519
- }
520
- }
521
- (Some(left_leaf), None) => {
522
- left_window.push(left_leaf.clone());
523
- left.advance(self, store).await?;
524
- }
525
- (None, Some(right_leaf)) => {
526
- right_window.push(right_leaf.clone());
527
- right.advance(self, store).await?;
528
- }
529
- (None, None) => {
530
- self.diff_leaf_summary_window(store, &left_window, &right_window, request, out)
531
- .await?;
532
- return Ok(());
533
- }
534
- }
535
- }
536
- }
537
-
538
- async fn diff_leaf_summary_window(
539
- &self,
540
- store: &(impl StorageRead + Send + Sync),
541
- left_leaves: &[ChildSummary],
542
- right_leaves: &[ChildSummary],
543
- request: &TrackedStateTreeScanRequest,
544
- out: &mut Vec<TrackedStateTreeDiffEntry>,
545
- ) -> Result<(), LixError> {
546
- if left_leaves.is_empty() && right_leaves.is_empty() {
547
- return Ok(());
548
- }
549
- let left_entries = self
550
- .collect_entries_from_leaf_summaries(store, left_leaves)
551
- .await?;
552
- let right_entries = self
553
- .collect_entries_from_leaf_summaries(store, right_leaves)
554
- .await?;
555
- self.diff_leaf_entries(&left_entries, &right_entries, request, out)
556
- }
557
-
558
- fn diff_leaf_entries(
559
- &self,
560
- left: &[EncodedLeafEntry],
561
- right: &[EncodedLeafEntry],
562
- request: &TrackedStateTreeScanRequest,
563
- out: &mut Vec<TrackedStateTreeDiffEntry>,
564
- ) -> Result<(), LixError> {
565
- let mut left_index = 0usize;
566
- let mut right_index = 0usize;
567
- while left_index < left.len() && right_index < right.len() {
568
- match left[left_index].key.cmp(&right[right_index].key) {
569
- std::cmp::Ordering::Less => {
570
- self.push_removed_diff(&left[left_index], request, out)?;
571
- left_index += 1;
572
- }
573
- std::cmp::Ordering::Greater => {
574
- self.push_added_diff(&right[right_index], request, out)?;
575
- right_index += 1;
576
- }
577
- std::cmp::Ordering::Equal => {
578
- if left[left_index].value != right[right_index].value {
579
- self.push_modified_diff(
580
- &left[left_index],
581
- &right[right_index],
582
- request,
583
- out,
584
- )?;
585
- }
586
- left_index += 1;
587
- right_index += 1;
588
- }
589
- }
590
- }
591
- for entry in &left[left_index..] {
592
- self.push_removed_diff(entry, request, out)?;
593
- }
594
- for entry in &right[right_index..] {
595
- self.push_added_diff(entry, request, out)?;
596
- }
597
- Ok(())
598
- }
599
-
600
- fn push_removed_diff(
601
- &self,
602
- entry: &EncodedLeafEntry,
603
- request: &TrackedStateTreeScanRequest,
604
- out: &mut Vec<TrackedStateTreeDiffEntry>,
605
- ) -> Result<(), LixError> {
606
- let (key, value) = decode_entry(entry)?;
607
- if request.matches(&key, &value) {
608
- out.push(TrackedStateTreeDiffEntry {
609
- before: Some((key, value)),
610
- after: None,
611
- });
612
- }
613
- Ok(())
614
- }
615
-
616
- fn push_added_diff(
617
- &self,
618
- entry: &EncodedLeafEntry,
619
- request: &TrackedStateTreeScanRequest,
620
- out: &mut Vec<TrackedStateTreeDiffEntry>,
621
- ) -> Result<(), LixError> {
622
- let (key, value) = decode_entry(entry)?;
623
- if request.matches(&key, &value) {
624
- out.push(TrackedStateTreeDiffEntry {
625
- before: None,
626
- after: Some((key, value)),
627
- });
628
- }
629
- Ok(())
630
- }
631
-
632
- fn push_modified_diff(
633
- &self,
634
- left: &EncodedLeafEntry,
635
- right: &EncodedLeafEntry,
636
- request: &TrackedStateTreeScanRequest,
637
- out: &mut Vec<TrackedStateTreeDiffEntry>,
638
- ) -> Result<(), LixError> {
639
- let (left_key, left_value) = decode_entry(left)?;
640
- let (right_key, right_value) = decode_entry(right)?;
641
- if request.matches(&left_key, &left_value) || request.matches(&right_key, &right_value) {
642
- out.push(TrackedStateTreeDiffEntry {
643
- before: Some((left_key, left_value)),
644
- after: Some((right_key, right_value)),
645
- });
646
- }
647
- Ok(())
648
- }
649
-
650
- async fn apply_sorted_mutations_chunker(
651
- &self,
652
- store: &(impl StorageRead + Send + Sync + ?Sized),
653
- writes: &mut StorageWriteSet,
654
- overlay: &mut storage::TrackedStateChunkOverlay,
655
- root_id: &TrackedStateRootId,
656
- mutations: Vec<TrackedStateMutation>,
657
- commit_id: Option<&str>,
658
- ) -> Result<MutationApply<Vec<TrackedStateMutation>>, LixError> {
659
- let mut mutation_map = BTreeMap::new();
660
- for mutation in mutations {
661
- mutation_map.insert(mutation.encoded_key, mutation.encoded_value);
662
- }
663
- if mutation_map.is_empty() {
664
- return Ok(MutationApply::Fallback(Vec::new()));
665
- }
666
-
667
- let levels = self
668
- .collect_summary_levels_with_overlay(store, overlay, root_id)
669
- .await?;
670
- let Some(leaves) = levels.first() else {
671
- return Ok(MutationApply::Fallback(
672
- mutation_map
673
- .into_iter()
674
- .map(|(encoded_key, encoded_value)| TrackedStateMutation {
675
- encoded_key,
676
- encoded_value,
677
- })
678
- .collect(),
679
- ));
680
- };
681
-
682
- let base_row_count = leaves
683
- .iter()
684
- .map(|leaf| leaf.subtree_count as usize)
685
- .sum::<usize>();
686
- let first_mutation_key = mutation_map
687
- .keys()
688
- .next()
689
- .expect("non-empty mutation map should have first key");
690
- let append_only = leaves
691
- .last()
692
- .is_some_and(|leaf| first_mutation_key.as_slice() > leaf.last_key.as_slice());
693
- if !append_only && mutation_map.len() * 2 > base_row_count {
694
- return Ok(MutationApply::Fallback(
695
- mutation_map
696
- .into_iter()
697
- .map(|(encoded_key, encoded_value)| TrackedStateMutation {
698
- encoded_key,
699
- encoded_value,
700
- })
701
- .collect(),
702
- ));
703
- }
704
-
705
- let mut mutations = mutation_map.into_iter().collect::<VecDeque<_>>();
706
- let mut output_leaves = Vec::new();
707
- let mut chunks = BTreeMap::new();
708
- let mut leaf_index = 0usize;
709
-
710
- while leaf_index < leaves.len() {
711
- let current_leaf_has_mutation = mutations
712
- .front()
713
- .is_some_and(|(key, _)| key.as_slice() <= leaves[leaf_index].last_key.as_slice());
714
- if !current_leaf_has_mutation {
715
- output_leaves.push(leaves[leaf_index].clone());
716
- leaf_index += 1;
717
- continue;
718
- }
719
-
720
- let window_start = leaf_index;
721
- let mut window_entries = BTreeMap::new();
722
- let mut window_mutation_ceiling = mutations
723
- .front()
724
- .map(|(key, _)| key.clone())
725
- .expect("window with mutation should have front mutation");
726
-
727
- loop {
728
- if leaf_index < leaves.len() {
729
- let leaf = &leaves[leaf_index];
730
- for entry in self
731
- .load_leaf_entries_with_overlay(store, overlay, &leaf.child_hash)
732
- .await?
733
- {
734
- window_entries.insert(entry.key, entry.value);
735
- }
736
-
737
- while mutations
738
- .front()
739
- .is_some_and(|(key, _)| key.as_slice() <= leaf.last_key.as_slice())
740
- {
741
- let (key, value) = mutations
742
- .pop_front()
743
- .expect("front mutation should be present");
744
- window_mutation_ceiling = key.clone();
745
- window_entries.insert(key, value);
746
- }
747
- leaf_index += 1;
748
- }
749
-
750
- while let Some((key, _)) = mutations.front() {
751
- if leaf_index < leaves.len()
752
- && key.as_slice() >= leaves[leaf_index].first_key.as_slice()
753
- {
754
- break;
755
- }
756
- let (key, value) = mutations
757
- .pop_front()
758
- .expect("front mutation should be present");
759
- window_mutation_ceiling = key.clone();
760
- window_entries.insert(key, value);
761
- }
762
-
763
- if leaf_index < leaves.len()
764
- && mutations.front().is_some_and(|(key, _)| {
765
- key.as_slice() <= leaves[leaf_index].last_key.as_slice()
766
- })
767
- {
768
- continue;
769
- }
770
-
771
- let mut candidate_chunks = BTreeMap::new();
772
- let candidate_leaves = self.build_leaf_level_from_refs(
773
- window_entries
774
- .iter()
775
- .map(|(key, value)| EncodedLeafEntryRef { key, value }),
776
- &mut candidate_chunks,
777
- );
778
-
779
- if let Some((generated_resync_index, existing_resync_index)) = first_resync_index(
780
- &candidate_leaves,
781
- &leaves[window_start..],
782
- &window_mutation_ceiling,
783
- ) {
784
- for summary in &candidate_leaves[..generated_resync_index] {
785
- if let Some(chunk) = candidate_chunks.remove(&summary.child_hash) {
786
- chunks.entry(chunk.hash).or_insert(chunk);
787
- }
788
- }
789
- output_leaves.extend(candidate_leaves.into_iter().take(generated_resync_index));
790
- leaf_index = window_start + existing_resync_index;
791
- break;
792
- }
793
-
794
- if leaf_index >= leaves.len() {
795
- chunks.extend(candidate_chunks);
796
- output_leaves.extend(candidate_leaves);
797
- break;
798
- }
799
- }
800
- }
801
-
802
- if !mutations.is_empty() {
803
- let entries = mutations
804
- .into_iter()
805
- .map(|(key, value)| EncodedLeafEntry { key, value })
806
- .collect();
807
- output_leaves.extend(self.build_leaf_level(entries, &mut chunks));
808
- }
809
-
810
- let built = self.build_tree_from_leaf_summaries(output_leaves, chunks)?;
811
- Ok(MutationApply::Applied(
812
- self.persist_built_tree(writes, overlay, built, commit_id)
813
- .await?,
814
- ))
815
- }
816
-
817
- async fn apply_single_mutation_from_seek_path(
818
- &self,
819
- store: &(impl StorageRead + Send + Sync + ?Sized),
820
- writes: &mut StorageWriteSet,
821
- overlay: &mut storage::TrackedStateChunkOverlay,
822
- root_id: &TrackedStateRootId,
823
- mutation: TrackedStateMutation,
824
- commit_id: Option<&str>,
825
- ) -> Result<MutationApply<TrackedStateMutation>, LixError> {
826
- let TrackedStateMutation {
827
- encoded_key,
828
- encoded_value,
829
- } = mutation;
830
- let mut current = *root_id.as_bytes();
831
- let mut path = Vec::new();
832
- let mut entries = loop {
833
- match self
834
- .load_node_with_overlay(store, overlay, &current)
835
- .await?
836
- {
837
- DecodedNode::Leaf(leaf) => break leaf.entries().to_vec(),
838
- DecodedNode::Internal(internal) => {
839
- let children = internal.children().to_vec();
840
- let child_index = children
841
- .iter()
842
- .position(|child| child.last_key.as_slice() >= encoded_key.as_slice())
843
- .or_else(|| (!children.is_empty()).then_some(children.len() - 1))
844
- .ok_or_else(|| {
845
- LixError::new(
846
- "LIX_ERROR_UNKNOWN",
847
- "tracked-state tree internal node has no children",
848
- )
849
- })?;
850
- current = children[child_index].child_hash;
851
- path.push(SeekPathFrame {
852
- children,
853
- child_index,
854
- });
855
- }
856
- }
857
- };
858
-
859
- let mutation_entry_index = match entries
860
- .binary_search_by(|entry| entry.key.as_slice().cmp(encoded_key.as_slice()))
861
- {
862
- Ok(index) => {
863
- if entries[index].value.as_slice() == encoded_value.as_slice() {
864
- return Ok(MutationApply::Fallback(TrackedStateMutation {
865
- encoded_key,
866
- encoded_value,
867
- }));
868
- }
869
- entries[index].value = encoded_value;
870
- index
871
- }
872
- Err(index) => {
873
- entries.insert(
874
- index,
875
- EncodedLeafEntry {
876
- key: encoded_key,
877
- value: encoded_value,
878
- },
879
- );
880
- index
881
- }
882
- };
883
-
884
- let mut chunks = BTreeMap::new();
885
- let mut replacement_children;
886
- let mut old_child_count;
887
-
888
- let Some(leaf_parent) = path.pop() else {
889
- let built = self.build_tree_from_entries(entries)?;
890
- return Ok(MutationApply::Applied(
891
- self.persist_built_tree(writes, overlay, built, commit_id)
892
- .await?,
893
- ));
894
- };
895
- let mutation_is_right_edge = leaf_parent.child_index + 1 == leaf_parent.children.len()
896
- && path
897
- .iter()
898
- .all(|frame| frame.child_index + 1 == frame.children.len());
899
-
900
- let mut leaf_entries = entries;
901
- let mut next_leaf_index = leaf_parent.child_index + 1;
902
- loop {
903
- let mut candidate_chunks = BTreeMap::new();
904
- let candidate_leaves = self.build_leaf_level_from_refs(
905
- leaf_entries.iter().map(EncodedLeafEntry::as_ref),
906
- &mut candidate_chunks,
907
- );
908
- if let Some((generated_resync_index, existing_resync_index)) = first_resync_index(
909
- &candidate_leaves,
910
- &leaf_parent.children[leaf_parent.child_index..],
911
- leaf_entries[mutation_entry_index].key.as_slice(),
912
- ) {
913
- for summary in &candidate_leaves[..generated_resync_index] {
914
- if let Some(chunk) = candidate_chunks.remove(&summary.child_hash) {
915
- chunks.entry(chunk.hash).or_insert(chunk);
916
- }
917
- }
918
- replacement_children = candidate_leaves
919
- .into_iter()
920
- .take(generated_resync_index)
921
- .collect();
922
- old_child_count = existing_resync_index;
923
- break;
924
- }
925
-
926
- if next_leaf_index >= leaf_parent.children.len() {
927
- if !mutation_is_right_edge {
928
- let entry = leaf_entries.remove(mutation_entry_index);
929
- return Ok(MutationApply::Fallback(TrackedStateMutation {
930
- encoded_key: entry.key,
931
- encoded_value: entry.value,
932
- }));
933
- }
934
- chunks.extend(candidate_chunks);
935
- replacement_children = candidate_leaves;
936
- old_child_count = leaf_parent.children.len() - leaf_parent.child_index;
937
- break;
938
- }
939
-
940
- leaf_entries.extend(
941
- self.load_leaf_entries_with_overlay(
942
- store,
943
- overlay,
944
- &leaf_parent.children[next_leaf_index].child_hash,
945
- )
946
- .await?,
947
- );
948
- next_leaf_index += 1;
949
- }
950
-
951
- let mut child_index = leaf_parent.child_index;
952
- let mut children = leaf_parent.children;
953
- let mut parent_level = 1usize;
954
- loop {
955
- children.splice(
956
- child_index..child_index + old_child_count,
957
- replacement_children,
958
- );
959
- replacement_children = self.build_internal_level(children, parent_level, &mut chunks);
960
- old_child_count = 1;
961
-
962
- let Some(frame) = path.pop() else {
963
- let mut summaries = replacement_children;
964
- let mut tree_height = parent_level + 1;
965
- while summaries.len() > 1 {
966
- summaries = self.build_internal_level(summaries, tree_height, &mut chunks);
967
- tree_height += 1;
968
- }
969
- let root = summaries.pop().ok_or_else(|| {
970
- LixError::new(
971
- "LIX_ERROR_UNKNOWN",
972
- "tracked-state seek-path mutation produced no root",
973
- )
974
- })?;
975
- let chunks = chunks.into_values().collect::<Vec<_>>();
976
- let chunk_bytes = chunks.iter().map(|chunk| chunk.data.len()).sum();
977
- let built = BuiltTree {
978
- root_id: TrackedStateRootId::new(root.child_hash),
979
- chunks,
980
- row_count: root.subtree_count as usize,
981
- tree_height,
982
- chunk_bytes,
983
- };
984
- return Ok(MutationApply::Applied(
985
- self.persist_built_tree(writes, overlay, built, commit_id)
986
- .await?,
987
- ));
988
- };
989
-
990
- child_index = frame.child_index;
991
- children = frame.children;
992
- parent_level += 1;
993
- }
994
- }
995
-
996
- async fn persist_built_tree(
997
- &self,
998
- writes: &mut StorageWriteSet,
999
- overlay: &mut storage::TrackedStateChunkOverlay,
1000
- built: BuiltTree,
1001
- _commit_id: Option<&str>,
1002
- ) -> Result<TrackedStateApplyResult, LixError> {
1003
- overlay.stage_chunks(writes, &built.chunks);
1004
- Ok(TrackedStateApplyResult {
1005
- root_id: built.root_id,
1006
- row_count: built.row_count,
1007
- tree_height: built.tree_height,
1008
- chunk_count: built.chunks.len(),
1009
- chunk_bytes: built.chunk_bytes,
1010
- })
1011
- }
1012
-
1013
- fn build_tree_from_entries(
1014
- &self,
1015
- entries: Vec<EncodedLeafEntry>,
1016
- ) -> Result<BuiltTree, LixError> {
1017
- let row_count = entries.len();
1018
- let mut chunks = BTreeMap::<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>::new();
1019
- let mut summaries = self.build_leaf_level(entries, &mut chunks);
1020
- let mut tree_height = 1usize;
1021
- while summaries.len() > 1 {
1022
- summaries = self.build_internal_level(summaries, tree_height, &mut chunks);
1023
- tree_height += 1;
1024
- }
1025
- let root = summaries.pop().ok_or_else(|| {
1026
- LixError::new(
1027
- "LIX_ERROR_UNKNOWN",
1028
- "tracked-state tree tree build produced no root",
1029
- )
1030
- })?;
1031
- let chunks = chunks.into_values().collect::<Vec<_>>();
1032
- let chunk_bytes = chunks.iter().map(|chunk| chunk.data.len()).sum();
1033
- Ok(BuiltTree {
1034
- root_id: TrackedStateRootId::new(root.child_hash),
1035
- chunks,
1036
- row_count,
1037
- tree_height,
1038
- chunk_bytes,
1039
- })
1040
- }
1041
-
1042
- fn build_tree_from_leaf_summaries(
1043
- &self,
1044
- leaf_summaries: Vec<ChildSummary>,
1045
- mut chunks: BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
1046
- ) -> Result<BuiltTree, LixError> {
1047
- let row_count = leaf_summaries
1048
- .iter()
1049
- .map(|summary| summary.subtree_count as usize)
1050
- .sum();
1051
- let mut summaries = leaf_summaries;
1052
- let mut tree_height = 1usize;
1053
- while summaries.len() > 1 {
1054
- summaries = self.build_internal_level(summaries, tree_height, &mut chunks);
1055
- tree_height += 1;
1056
- }
1057
- let root = summaries.pop().ok_or_else(|| {
1058
- LixError::new(
1059
- "LIX_ERROR_UNKNOWN",
1060
- "tracked-state tree build from leaves produced no root",
1061
- )
1062
- })?;
1063
- let chunks = chunks.into_values().collect::<Vec<_>>();
1064
- let chunk_bytes = chunks.iter().map(|chunk| chunk.data.len()).sum();
1065
- Ok(BuiltTree {
1066
- root_id: TrackedStateRootId::new(root.child_hash),
1067
- chunks,
1068
- row_count,
1069
- tree_height,
1070
- chunk_bytes,
1071
- })
1072
- }
1073
-
1074
- fn build_tree_from_leaf_patch(
1075
- &self,
1076
- levels: &[Vec<ChildSummary>],
1077
- leaf_start: usize,
1078
- old_leaf_count: usize,
1079
- replacement_leaves: Vec<ChildSummary>,
1080
- mut chunks: BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
1081
- mutation_key: &[u8],
1082
- ) -> Result<BuiltTree, LixError> {
1083
- if levels.len() <= 1 {
1084
- let mut leaves = levels.first().cloned().unwrap_or_default();
1085
- leaves.splice(leaf_start..leaf_start + old_leaf_count, replacement_leaves);
1086
- return self.build_tree_from_leaf_summaries(leaves, chunks);
1087
- }
1088
-
1089
- let mut child_start = leaf_start;
1090
- let mut old_child_count = old_leaf_count;
1091
- let mut replacement_children = replacement_leaves;
1092
-
1093
- for level in 0..levels.len() - 1 {
1094
- let patch = self.patch_parent_level(
1095
- &levels[level],
1096
- &levels[level + 1],
1097
- child_start,
1098
- old_child_count,
1099
- replacement_children,
1100
- level + 1,
1101
- &mut chunks,
1102
- mutation_key,
1103
- )?;
1104
- child_start = patch.parent_start;
1105
- old_child_count = patch.old_parent_count;
1106
- replacement_children = patch.replacement_parents;
1107
- }
1108
-
1109
- let mut summaries = replacement_children;
1110
- let mut tree_height = levels.len();
1111
- while summaries.len() > 1 {
1112
- summaries = self.build_internal_level(summaries, tree_height, &mut chunks);
1113
- tree_height += 1;
1114
- }
1115
- let root = summaries.pop().ok_or_else(|| {
1116
- LixError::new(
1117
- "LIX_ERROR_UNKNOWN",
1118
- "tracked-state patched tree produced no root",
1119
- )
1120
- })?;
1121
- let chunks = chunks.into_values().collect::<Vec<_>>();
1122
- let chunk_bytes = chunks.iter().map(|chunk| chunk.data.len()).sum();
1123
- Ok(BuiltTree {
1124
- root_id: TrackedStateRootId::new(root.child_hash),
1125
- chunks,
1126
- row_count: root.subtree_count as usize,
1127
- tree_height,
1128
- chunk_bytes,
1129
- })
1130
- }
1131
-
1132
- fn patch_parent_level(
1133
- &self,
1134
- old_children: &[ChildSummary],
1135
- old_parents: &[ChildSummary],
1136
- child_start: usize,
1137
- old_child_count: usize,
1138
- replacement_children: Vec<ChildSummary>,
1139
- parent_level: usize,
1140
- chunks: &mut BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
1141
- mutation_key: &[u8],
1142
- ) -> Result<ParentLevelPatch, LixError> {
1143
- if old_parents.is_empty() {
1144
- return Ok(ParentLevelPatch {
1145
- parent_start: 0,
1146
- old_parent_count: 0,
1147
- replacement_parents: self.build_internal_level(
1148
- replacement_children,
1149
- parent_level,
1150
- chunks,
1151
- ),
1152
- });
1153
- }
1154
-
1155
- let parent_start = parent_index_for_child_index(old_children, old_parents, child_start);
1156
- let parent_child_range = child_range_for_parent(old_children, &old_parents[parent_start])?;
1157
- let old_child_end = child_start + old_child_count;
1158
- let parent_end = if old_child_count == 0 {
1159
- parent_start
1160
- } else {
1161
- parent_index_for_child_index(old_children, old_parents, old_child_end - 1)
1162
- };
1163
- let parent_end_child_range =
1164
- child_range_for_parent(old_children, &old_parents[parent_end])?;
1165
- let mut window_children = Vec::new();
1166
- window_children.extend(
1167
- old_children[parent_child_range.start..child_start]
1168
- .iter()
1169
- .map(ChildSummary::as_ref),
1170
- );
1171
- window_children.extend(replacement_children.iter().map(ChildSummary::as_ref));
1172
- window_children.extend(
1173
- old_children[old_child_end..parent_end_child_range.end]
1174
- .iter()
1175
- .map(ChildSummary::as_ref),
1176
- );
1177
- let mut next_parent_index = parent_end + 1;
1178
-
1179
- loop {
1180
- let mut candidate_chunks = BTreeMap::new();
1181
- let candidate_parents = self.build_internal_level_from_refs(
1182
- window_children.iter().copied(),
1183
- parent_level,
1184
- &mut candidate_chunks,
1185
- );
1186
-
1187
- if let Some((generated_resync_index, existing_resync_index)) = first_resync_index(
1188
- &candidate_parents,
1189
- &old_parents[parent_start..],
1190
- mutation_key,
1191
- ) {
1192
- for summary in &candidate_parents[..generated_resync_index] {
1193
- if let Some(chunk) = candidate_chunks.remove(&summary.child_hash) {
1194
- chunks.entry(chunk.hash).or_insert(chunk);
1195
- }
1196
- }
1197
- return Ok(ParentLevelPatch {
1198
- parent_start,
1199
- old_parent_count: existing_resync_index,
1200
- replacement_parents: candidate_parents
1201
- .into_iter()
1202
- .take(generated_resync_index)
1203
- .collect(),
1204
- });
1205
- }
1206
-
1207
- if next_parent_index >= old_parents.len() {
1208
- chunks.extend(candidate_chunks);
1209
- return Ok(ParentLevelPatch {
1210
- parent_start,
1211
- old_parent_count: old_parents.len() - parent_start,
1212
- replacement_parents: candidate_parents,
1213
- });
1214
- }
1215
-
1216
- let next_range = child_range_for_parent(old_children, &old_parents[next_parent_index])?;
1217
- window_children.extend(old_children[next_range].iter().map(ChildSummary::as_ref));
1218
- next_parent_index += 1;
1219
- }
1220
- }
1221
-
1222
- fn build_leaf_level(
1223
- &self,
1224
- entries: Vec<EncodedLeafEntry>,
1225
- chunks: &mut BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
1226
- ) -> Vec<ChildSummary> {
1227
- let groups = chunk_leaf_entries(entries, &self.options);
1228
- groups
1229
- .into_iter()
1230
- .map(|group| {
1231
- let subtree_count = group.entries.len() as u64;
1232
- let first_key = group
1233
- .entries
1234
- .first()
1235
- .map(|entry| entry.key.clone())
1236
- .unwrap_or_default();
1237
- let last_key = group
1238
- .entries
1239
- .last()
1240
- .map(|entry| entry.key.clone())
1241
- .unwrap_or_default();
1242
- let node = encode_leaf_node(&group.entries);
1243
- let (chunk, summary) =
1244
- child_summary_from_node(node, first_key, last_key, subtree_count);
1245
- chunks.entry(chunk.hash).or_insert(chunk);
1246
- summary
1247
- })
1248
- .collect()
1249
- }
1250
-
1251
- fn build_leaf_level_from_refs<'a>(
1252
- &self,
1253
- entries: impl IntoIterator<Item = EncodedLeafEntryRef<'a>>,
1254
- chunks: &mut BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
1255
- ) -> Vec<ChildSummary> {
1256
- let groups = chunk_leaf_entry_refs(entries, &self.options);
1257
- groups
1258
- .into_iter()
1259
- .map(|group| {
1260
- let subtree_count = group.entries.len() as u64;
1261
- let first_key = group
1262
- .entries
1263
- .first()
1264
- .map(|entry| entry.key.to_vec())
1265
- .unwrap_or_default();
1266
- let last_key = group
1267
- .entries
1268
- .last()
1269
- .map(|entry| entry.key.to_vec())
1270
- .unwrap_or_default();
1271
- let node = encode_leaf_node_refs(&group.entries);
1272
- let (chunk, summary) =
1273
- child_summary_from_node(node, first_key, last_key, subtree_count);
1274
- chunks.entry(chunk.hash).or_insert(chunk);
1275
- summary
1276
- })
1277
- .collect()
1278
- }
1279
-
1280
- fn build_internal_level(
1281
- &self,
1282
- children: Vec<ChildSummary>,
1283
- level: usize,
1284
- chunks: &mut BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
1285
- ) -> Vec<ChildSummary> {
1286
- let groups = chunk_internal_entries(children, &self.options, level);
1287
- groups
1288
- .into_iter()
1289
- .map(|group| {
1290
- let subtree_count = group.children.iter().map(|child| child.subtree_count).sum();
1291
- let first_key = group
1292
- .children
1293
- .first()
1294
- .map(|child| child.first_key.clone())
1295
- .unwrap_or_default();
1296
- let last_key = group
1297
- .children
1298
- .last()
1299
- .map(|child| child.last_key.clone())
1300
- .unwrap_or_default();
1301
- let node = encode_internal_node(&group.children);
1302
- let (chunk, summary) =
1303
- child_summary_from_node(node, first_key, last_key, subtree_count);
1304
- chunks.entry(chunk.hash).or_insert(chunk);
1305
- summary
1306
- })
1307
- .collect()
1308
- }
1309
-
1310
- fn build_internal_level_from_refs<'a>(
1311
- &self,
1312
- children: impl IntoIterator<Item = ChildSummaryRef<'a>>,
1313
- level: usize,
1314
- chunks: &mut BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
1315
- ) -> Vec<ChildSummary> {
1316
- let groups = chunk_internal_entry_refs(children, &self.options, level);
1317
- groups
1318
- .into_iter()
1319
- .map(|group| {
1320
- let subtree_count = group.children.iter().map(|child| child.subtree_count).sum();
1321
- let first_key = group
1322
- .children
1323
- .first()
1324
- .map(|child| child.first_key.to_vec())
1325
- .unwrap_or_default();
1326
- let last_key = group
1327
- .children
1328
- .last()
1329
- .map(|child| child.last_key.to_vec())
1330
- .unwrap_or_default();
1331
- let node = encode_internal_node_refs(&group.children);
1332
- let (chunk, summary) =
1333
- child_summary_from_node(node, first_key, last_key, subtree_count);
1334
- chunks.entry(chunk.hash).or_insert(chunk);
1335
- summary
1336
- })
1337
- .collect()
1338
- }
1339
-
1340
- #[cfg(test)]
1341
- async fn collect_leaf_entries(
1342
- &self,
1343
- store: &(impl StorageRead + Send + Sync + ?Sized),
1344
- root_id: &TrackedStateRootId,
1345
- ) -> Result<Vec<EncodedLeafEntry>, LixError> {
1346
- let overlay = storage::TrackedStateChunkOverlay::new();
1347
- self.collect_leaf_entries_with_overlay(store, &overlay, root_id)
1348
- .await
1349
- }
1350
-
1351
- async fn collect_leaf_entries_with_overlay(
1352
- &self,
1353
- store: &(impl StorageRead + Send + Sync + ?Sized),
1354
- overlay: &storage::TrackedStateChunkOverlay,
1355
- root_id: &TrackedStateRootId,
1356
- ) -> Result<Vec<EncodedLeafEntry>, LixError> {
1357
- let mut out = Vec::new();
1358
- let mut current = vec![*root_id.as_bytes()];
1359
- while !current.is_empty() {
1360
- let mut next = Vec::new();
1361
- for hash in current {
1362
- match self.load_node_with_overlay(store, overlay, &hash).await? {
1363
- DecodedNode::Leaf(leaf) => out.extend(leaf.entries().iter().cloned()),
1364
- DecodedNode::Internal(internal) => {
1365
- next.extend(internal.children().iter().map(|child| child.child_hash));
1366
- }
1367
- }
1368
- }
1369
- current = next;
1370
- }
1371
- Ok(out)
1372
- }
1373
-
1374
- async fn collect_filtered_entries(
1375
- &self,
1376
- store: &(impl StorageRead + Send + Sync + ?Sized),
1377
- root_id: &TrackedStateRootId,
1378
- request: &TrackedStateTreeScanRequest,
1379
- ) -> Result<Vec<(TrackedStateKey, TrackedStateIndexValue)>, LixError> {
1380
- self.scan(store, root_id, request).await
1381
- }
1382
-
1383
- fn scan_node<'a, S>(
1384
- &'a self,
1385
- store: &'a S,
1386
- hash: [u8; TRACKED_STATE_HASH_BYTES],
1387
- request: &'a TrackedStateTreeScanRequest,
1388
- ranges: &'a [EncodedScanRange],
1389
- key_decode_hint: Option<ScanKeyDecodeHint<'a>>,
1390
- rows: &'a mut Vec<(TrackedStateKey, TrackedStateIndexValue)>,
1391
- ) -> Pin<Box<dyn Future<Output = Result<(), LixError>> + Send + 'a>>
1392
- where
1393
- S: StorageRead + Send + Sync + ?Sized + 'a,
1394
- {
1395
- Box::pin(async move {
1396
- let bytes = self.load_node_bytes(store, &hash).await?;
1397
- match decode_node_ref(&bytes)? {
1398
- DecodedNodeRef::Leaf(leaf) => {
1399
- for index in 0..leaf.len() {
1400
- if scan_limit_reached(request, rows.len()) {
1401
- break;
1402
- }
1403
- let entry = leaf.entry(index)?.ok_or_else(|| {
1404
- LixError::new(
1405
- "LIX_ERROR_UNKNOWN",
1406
- "tracked-state leaf entry disappeared during scan",
1407
- )
1408
- })?;
1409
- if !encoded_key_in_scan_ranges(entry.key, ranges) {
1410
- continue;
1411
- }
1412
- let key = match key_decode_hint {
1413
- Some(hint) => decode_key_with_trusted_prefix(
1414
- entry.key,
1415
- hint.schema_key,
1416
- hint.file_id,
1417
- hint.prefix_len,
1418
- )?,
1419
- None => decode_key(entry.key)?,
1420
- };
1421
- if key_decode_hint.is_none() && !key_matches_scan_filters(request, &key) {
1422
- continue;
1423
- }
1424
- let Some(value) =
1425
- decode_visible_value(entry.value, request.include_tombstones)?
1426
- else {
1427
- continue;
1428
- };
1429
- if key_decode_hint.is_some() || request.matches(&key, &value) {
1430
- rows.push((key, value));
1431
- }
1432
- }
1433
- }
1434
- DecodedNodeRef::Internal(internal) => {
1435
- for child in internal.children() {
1436
- if scan_limit_reached(request, rows.len()) {
1437
- break;
1438
- }
1439
- if child_summary_overlaps_scan_ranges(child, ranges) {
1440
- self.scan_node(
1441
- store,
1442
- child.child_hash,
1443
- request,
1444
- ranges,
1445
- key_decode_hint,
1446
- rows,
1447
- )
1448
- .await?;
1449
- }
1450
- }
1451
- }
1452
- }
1453
- Ok(())
1454
- })
1455
- }
1456
-
1457
- fn get_many_node<'a, S>(
1458
- &'a self,
1459
- store: &'a S,
1460
- hash: [u8; TRACKED_STATE_HASH_BYTES],
1461
- encoded_keys: &'a [(usize, Vec<u8>)],
1462
- values: &'a mut [Option<TrackedStateIndexValue>],
1463
- ) -> Pin<Box<dyn Future<Output = Result<(), LixError>> + Send + 'a>>
1464
- where
1465
- S: StorageRead + Send + Sync + ?Sized + 'a,
1466
- {
1467
- Box::pin(async move {
1468
- if encoded_keys.is_empty() {
1469
- return Ok(());
1470
- }
1471
-
1472
- let bytes = self.load_node_bytes(store, &hash).await?;
1473
- match decode_node_ref(&bytes)? {
1474
- DecodedNodeRef::Leaf(leaf) => {
1475
- for (original_index, encoded_key) in encoded_keys {
1476
- if let Some(entry_index) = binary_search_leaf_key(&leaf, encoded_key)? {
1477
- let entry = leaf.entry(entry_index)?.ok_or_else(|| {
1478
- LixError::new(
1479
- "LIX_ERROR_UNKNOWN",
1480
- "tracked-state leaf entry disappeared during get_many",
1481
- )
1482
- })?;
1483
- values[*original_index] = Some(decode_value(entry.value)?);
1484
- }
1485
- }
1486
- }
1487
- DecodedNodeRef::Internal(internal) => {
1488
- let mut start = 0usize;
1489
- let children = internal.children();
1490
- for (child_index, child) in children.iter().enumerate() {
1491
- if start >= encoded_keys.len() {
1492
- break;
1493
- }
1494
-
1495
- let mut end = start;
1496
- if child_index + 1 == children.len() {
1497
- end = encoded_keys.len();
1498
- } else {
1499
- while end < encoded_keys.len()
1500
- && encoded_keys[end].1.as_slice() <= child.last_key.as_slice()
1501
- {
1502
- end += 1;
1503
- }
1504
- }
1505
-
1506
- if start < end {
1507
- self.get_many_node(
1508
- store,
1509
- child.child_hash,
1510
- &encoded_keys[start..end],
1511
- values,
1512
- )
1513
- .await?;
1514
- }
1515
- start = end;
1516
- }
1517
- }
1518
- }
1519
- Ok(())
1520
- })
1521
- }
1522
-
1523
- async fn collect_entries_from_leaf_summaries(
1524
- &self,
1525
- store: &(impl StorageRead + Send + Sync),
1526
- leaves: &[ChildSummary],
1527
- ) -> Result<Vec<EncodedLeafEntry>, LixError> {
1528
- let mut entries = Vec::new();
1529
- for leaf in leaves {
1530
- entries.extend(self.load_leaf_entries(store, &leaf.child_hash).await?);
1531
- }
1532
- Ok(entries)
1533
- }
1534
-
1535
- async fn collect_summary_levels_with_overlay(
1536
- &self,
1537
- store: &(impl StorageRead + Send + Sync + ?Sized),
1538
- overlay: &storage::TrackedStateChunkOverlay,
1539
- root_id: &TrackedStateRootId,
1540
- ) -> Result<Vec<Vec<ChildSummary>>, LixError> {
1541
- let mut levels = Vec::new();
1542
- self.collect_summary_levels_for_node_with_overlay(
1543
- store,
1544
- overlay,
1545
- *root_id.as_bytes(),
1546
- &mut levels,
1547
- )
1548
- .await?;
1549
- Ok(levels)
1550
- }
1551
-
1552
- fn collect_summary_levels_for_node_with_overlay<'a, S>(
1553
- &'a self,
1554
- store: &'a S,
1555
- overlay: &'a storage::TrackedStateChunkOverlay,
1556
- hash: [u8; TRACKED_STATE_HASH_BYTES],
1557
- levels: &'a mut Vec<Vec<ChildSummary>>,
1558
- ) -> Pin<Box<dyn Future<Output = Result<(ChildSummary, usize), LixError>> + 'a>>
1559
- where
1560
- S: StorageRead + Send + Sync + ?Sized + 'a,
1561
- {
1562
- Box::pin(async move {
1563
- match self.load_node_with_overlay(store, overlay, &hash).await? {
1564
- DecodedNode::Leaf(leaf) => {
1565
- let summary = leaf_summary(hash, leaf.entries());
1566
- push_level_summary(levels, 0, summary.clone());
1567
- Ok((summary, 0))
1568
- }
1569
- DecodedNode::Internal(internal) => {
1570
- let children = internal.children().to_vec();
1571
- let child_height = match children.first() {
1572
- Some(child) => match self
1573
- .load_node_with_overlay(store, overlay, &child.child_hash)
1574
- .await?
1575
- {
1576
- DecodedNode::Leaf(_) => {
1577
- if levels.is_empty() {
1578
- levels.push(Vec::new());
1579
- }
1580
- levels[0].extend(children.iter().cloned());
1581
- 0
1582
- }
1583
- DecodedNode::Internal(_) => {
1584
- let mut child_height = None;
1585
- for child in &children {
1586
- let (_, height) = self
1587
- .collect_summary_levels_for_node_with_overlay(
1588
- store,
1589
- overlay,
1590
- child.child_hash,
1591
- levels,
1592
- )
1593
- .await?;
1594
- child_height = Some(height);
1595
- }
1596
- child_height.unwrap_or(0)
1597
- }
1598
- },
1599
- None => 0,
1600
- };
1601
- let height = child_height + 1;
1602
- let summary = internal_summary(hash, &children)?;
1603
- push_level_summary(levels, height, summary.clone());
1604
- Ok((summary, height))
1605
- }
1606
- }
1607
- })
1608
- }
1609
-
1610
- async fn load_leaf_entries(
1611
- &self,
1612
- store: &(impl StorageRead + Send + Sync + ?Sized),
1613
- hash: &[u8; TRACKED_STATE_HASH_BYTES],
1614
- ) -> Result<Vec<EncodedLeafEntry>, LixError> {
1615
- match self.load_node(store, hash).await? {
1616
- DecodedNode::Leaf(leaf) => Ok(leaf.entries().to_vec()),
1617
- DecodedNode::Internal(_) => Err(LixError::new(
1618
- "LIX_ERROR_UNKNOWN",
1619
- "tracked-state expected leaf chunk but found internal node",
1620
- )),
1621
- }
1622
- }
1623
-
1624
- async fn load_leaf_entries_with_overlay(
1625
- &self,
1626
- store: &(impl StorageRead + Send + Sync + ?Sized),
1627
- overlay: &storage::TrackedStateChunkOverlay,
1628
- hash: &[u8; TRACKED_STATE_HASH_BYTES],
1629
- ) -> Result<Vec<EncodedLeafEntry>, LixError> {
1630
- match self.load_node_with_overlay(store, overlay, hash).await? {
1631
- DecodedNode::Leaf(leaf) => Ok(leaf.entries().to_vec()),
1632
- DecodedNode::Internal(_) => Err(LixError::new(
1633
- "LIX_ERROR_UNKNOWN",
1634
- "tracked-state expected leaf chunk but found internal node",
1635
- )),
1636
- }
1637
- }
1638
-
1639
- async fn load_node(
1640
- &self,
1641
- store: &(impl StorageRead + Send + Sync + ?Sized),
1642
- hash: &[u8; TRACKED_STATE_HASH_BYTES],
1643
- ) -> Result<DecodedNode, LixError> {
1644
- let bytes = self.load_node_bytes(store, hash).await?;
1645
- decode_node(&bytes)
1646
- }
1647
-
1648
- async fn load_node_bytes(
1649
- &self,
1650
- store: &(impl StorageRead + Send + Sync + ?Sized),
1651
- hash: &[u8; TRACKED_STATE_HASH_BYTES],
1652
- ) -> Result<Vec<u8>, LixError> {
1653
- let bytes = storage::read_chunk(store, hash).await?.ok_or_else(|| {
1654
- LixError::new("LIX_ERROR_UNKNOWN", "tracked-state tree chunk is missing")
1655
- })?;
1656
- storage::verify_chunk_hash(hash, &bytes)?;
1657
- Ok(bytes)
1658
- }
1659
-
1660
- async fn load_node_with_overlay(
1661
- &self,
1662
- store: &(impl StorageRead + Send + Sync + ?Sized),
1663
- overlay: &storage::TrackedStateChunkOverlay,
1664
- hash: &[u8; TRACKED_STATE_HASH_BYTES],
1665
- ) -> Result<DecodedNode, LixError> {
1666
- let bytes = overlay.read_chunk(store, hash).await?.ok_or_else(|| {
1667
- LixError::new("LIX_ERROR_UNKNOWN", "tracked-state tree chunk is missing")
1668
- })?;
1669
- storage::verify_chunk_hash(hash, &bytes)?;
1670
- decode_node(&bytes)
1671
- }
1672
- }
1673
-
1674
- #[derive(Debug)]
1675
- struct BuiltTree {
1676
- root_id: TrackedStateRootId,
1677
- chunks: Vec<PendingChunkWrite>,
1678
- row_count: usize,
1679
- tree_height: usize,
1680
- chunk_bytes: usize,
1681
- }
1682
-
1683
- struct ParentLevelPatch {
1684
- parent_start: usize,
1685
- old_parent_count: usize,
1686
- replacement_parents: Vec<ChildSummary>,
1687
- }
1688
-
1689
- struct SeekPathFrame {
1690
- children: Vec<ChildSummary>,
1691
- child_index: usize,
1692
- }
1693
-
1694
- #[derive(Debug, Clone)]
1695
- struct EncodedScanRange {
1696
- start: Vec<u8>,
1697
- end: Option<Vec<u8>>,
1698
- }
1699
-
1700
- #[derive(Debug, Clone, Copy)]
1701
- struct ScanKeyDecodeHint<'a> {
1702
- schema_key: &'a str,
1703
- file_id: Option<&'a str>,
1704
- prefix_len: usize,
1705
- }
1706
-
1707
- fn binary_search_leaf_key(
1708
- leaf: &DecodedLeafNodeRef<'_>,
1709
- encoded_key: &[u8],
1710
- ) -> Result<Option<usize>, LixError> {
1711
- let mut low = 0usize;
1712
- let mut high = leaf.len();
1713
- while low < high {
1714
- let mid = low + (high - low) / 2;
1715
- let key = leaf.key(mid)?.ok_or_else(|| {
1716
- LixError::new(
1717
- "LIX_ERROR_UNKNOWN",
1718
- "tracked-state leaf key disappeared during binary search",
1719
- )
1720
- })?;
1721
- match key.cmp(encoded_key) {
1722
- std::cmp::Ordering::Less => low = mid + 1,
1723
- std::cmp::Ordering::Equal => return Ok(Some(mid)),
1724
- std::cmp::Ordering::Greater => high = mid,
1725
- }
1726
- }
1727
- Ok(None)
1728
- }
1729
-
1730
- struct LeafSummaryCursor {
1731
- stack: Vec<LeafSummaryCursorFrame>,
1732
- current: Option<ChildSummary>,
1733
- }
1734
-
1735
- struct LeafSummaryCursorFrame {
1736
- children: Vec<ChildSummary>,
1737
- next_index: usize,
1738
- children_are_leaves: bool,
1739
- }
1740
-
1741
- impl LeafSummaryCursor {
1742
- async fn new(
1743
- tree: &TrackedStateTree,
1744
- store: &(impl StorageRead + Send + Sync),
1745
- root_hash: [u8; TRACKED_STATE_HASH_BYTES],
1746
- ) -> Result<Self, LixError> {
1747
- let mut cursor = Self {
1748
- stack: Vec::new(),
1749
- current: None,
1750
- };
1751
- match tree.load_node(store, &root_hash).await? {
1752
- DecodedNode::Leaf(leaf) => {
1753
- cursor.current = Some(leaf_summary(root_hash, leaf.entries()));
1754
- }
1755
- DecodedNode::Internal(internal) => {
1756
- let children = internal.children().to_vec();
1757
- let children_are_leaves =
1758
- child_summaries_are_leaves(tree, store, &children).await?;
1759
- cursor.stack.push(LeafSummaryCursorFrame {
1760
- children,
1761
- next_index: 0,
1762
- children_are_leaves,
1763
- });
1764
- cursor.advance(tree, store).await?;
1765
- }
1766
- }
1767
- Ok(cursor)
1768
- }
1769
-
1770
- fn current(&self) -> Option<&ChildSummary> {
1771
- self.current.as_ref()
1772
- }
1773
-
1774
- async fn advance(
1775
- &mut self,
1776
- tree: &TrackedStateTree,
1777
- store: &(impl StorageRead + Send + Sync),
1778
- ) -> Result<(), LixError> {
1779
- self.current = None;
1780
- while let Some(frame) = self.stack.last_mut() {
1781
- if frame.next_index >= frame.children.len() {
1782
- self.stack.pop();
1783
- continue;
1784
- }
1785
-
1786
- let next = frame.children[frame.next_index].clone();
1787
- let next_is_leaf = frame.children_are_leaves;
1788
- frame.next_index += 1;
1789
- if next_is_leaf {
1790
- self.current = Some(next);
1791
- return Ok(());
1792
- }
1793
- self.descend_to_leaf(tree, store, next).await?;
1794
- return Ok(());
1795
- }
1796
- Ok(())
1797
- }
1798
-
1799
- async fn descend_to_leaf(
1800
- &mut self,
1801
- tree: &TrackedStateTree,
1802
- store: &(impl StorageRead + Send + Sync),
1803
- mut summary: ChildSummary,
1804
- ) -> Result<(), LixError> {
1805
- loop {
1806
- match tree.load_node(store, &summary.child_hash).await? {
1807
- DecodedNode::Leaf(_) => {
1808
- self.current = Some(summary);
1809
- return Ok(());
1810
- }
1811
- DecodedNode::Internal(internal) => {
1812
- let children = internal.children().to_vec();
1813
- let children_are_leaves =
1814
- child_summaries_are_leaves(tree, store, &children).await?;
1815
- let Some(first_child) = children.first().cloned() else {
1816
- return Err(LixError::new(
1817
- "LIX_ERROR_UNKNOWN",
1818
- "tracked-state internal node has no children",
1819
- ));
1820
- };
1821
- self.stack.push(LeafSummaryCursorFrame {
1822
- children,
1823
- next_index: 1,
1824
- children_are_leaves,
1825
- });
1826
- if children_are_leaves {
1827
- self.current = Some(first_child);
1828
- return Ok(());
1829
- } else {
1830
- summary = first_child;
1831
- }
1832
- }
1833
- }
1834
- }
1835
- }
1836
- }
1837
-
1838
- #[derive(Debug, Default)]
1839
- struct LeafChunkAccumulator {
1840
- entries: Vec<EncodedLeafEntry>,
1841
- key_bytes: usize,
1842
- }
1843
-
1844
- #[derive(Debug, Default)]
1845
- struct LeafChunkRefAccumulator<'a> {
1846
- entries: Vec<EncodedLeafEntryRef<'a>>,
1847
- key_bytes: usize,
1848
- }
1849
-
1850
- #[derive(Debug, Default)]
1851
- struct InternalChunkAccumulator {
1852
- children: Vec<ChildSummary>,
1853
- first_key_bytes: usize,
1854
- last_key_bytes: usize,
1855
- }
1856
-
1857
- #[derive(Debug, Default)]
1858
- struct InternalChunkRefAccumulator<'a> {
1859
- children: Vec<ChildSummaryRef<'a>>,
1860
- first_key_bytes: usize,
1861
- last_key_bytes: usize,
1862
- }
1863
-
1864
- fn chunk_leaf_entries(
1865
- entries: Vec<EncodedLeafEntry>,
1866
- options: &TrackedStateTreeOptions,
1867
- ) -> Vec<LeafChunkAccumulator> {
1868
- if entries.is_empty() {
1869
- return vec![LeafChunkAccumulator::default()];
1870
- }
1871
- let mut groups = Vec::new();
1872
- let mut current = LeafChunkAccumulator::default();
1873
- for entry in entries {
1874
- let item_size = estimate_leaf_boundary_entry_size(entry.key.len());
1875
- let projected_size = estimate_leaf_boundary_chunk_size(
1876
- current.entries.len() + 1,
1877
- current.key_bytes + entry.key.len(),
1878
- );
1879
- if !current.entries.is_empty() && projected_size > options.max_chunk_bytes {
1880
- groups.push(std::mem::take(&mut current));
1881
- }
1882
-
1883
- current.key_bytes += entry.key.len();
1884
- current.entries.push(entry);
1885
- let current_size =
1886
- estimate_leaf_boundary_chunk_size(current.entries.len(), current.key_bytes);
1887
- if current_size >= options.min_chunk_bytes
1888
- && (current_size >= options.max_chunk_bytes
1889
- || current.entries.last().is_some_and(|entry| {
1890
- boundary_trigger(
1891
- &entry.key,
1892
- 0,
1893
- current_size,
1894
- item_size,
1895
- options.target_chunk_bytes,
1896
- )
1897
- }))
1898
- {
1899
- groups.push(std::mem::take(&mut current));
1900
- }
1901
- }
1902
- if !current.entries.is_empty() {
1903
- groups.push(current);
1904
- }
1905
- groups
1906
- }
1907
-
1908
- fn chunk_leaf_entry_refs<'a>(
1909
- entries: impl IntoIterator<Item = EncodedLeafEntryRef<'a>>,
1910
- options: &TrackedStateTreeOptions,
1911
- ) -> Vec<LeafChunkRefAccumulator<'a>> {
1912
- let mut iter = entries.into_iter().peekable();
1913
- if iter.peek().is_none() {
1914
- return vec![LeafChunkRefAccumulator::default()];
1915
- }
1916
- let mut groups = Vec::new();
1917
- let mut current = LeafChunkRefAccumulator::default();
1918
- for entry in iter {
1919
- let item_size = estimate_leaf_boundary_entry_size(entry.key.len());
1920
- let projected_size = estimate_leaf_boundary_chunk_size(
1921
- current.entries.len() + 1,
1922
- current.key_bytes + entry.key.len(),
1923
- );
1924
- if !current.entries.is_empty() && projected_size > options.max_chunk_bytes {
1925
- groups.push(std::mem::take(&mut current));
1926
- }
1927
-
1928
- current.key_bytes += entry.key.len();
1929
- current.entries.push(entry);
1930
- let current_size =
1931
- estimate_leaf_boundary_chunk_size(current.entries.len(), current.key_bytes);
1932
- if current_size >= options.min_chunk_bytes
1933
- && (current_size >= options.max_chunk_bytes
1934
- || current.entries.last().is_some_and(|entry| {
1935
- boundary_trigger(
1936
- entry.key,
1937
- 0,
1938
- current_size,
1939
- item_size,
1940
- options.target_chunk_bytes,
1941
- )
1942
- }))
1943
- {
1944
- groups.push(std::mem::take(&mut current));
1945
- }
1946
- }
1947
- if !current.entries.is_empty() {
1948
- groups.push(current);
1949
- }
1950
- groups
1951
- }
1952
-
1953
- fn chunk_internal_entries(
1954
- children: Vec<ChildSummary>,
1955
- options: &TrackedStateTreeOptions,
1956
- level: usize,
1957
- ) -> Vec<InternalChunkAccumulator> {
1958
- let mut groups = Vec::new();
1959
- let mut current = InternalChunkAccumulator::default();
1960
- for child in children {
1961
- let item_size = child.first_key.len()
1962
- + child.last_key.len()
1963
- + TRACKED_STATE_HASH_BYTES
1964
- + std::mem::size_of::<u64>();
1965
- let projected_size = estimate_internal_chunk_size(
1966
- current.children.len() + 1,
1967
- current.first_key_bytes + child.first_key.len(),
1968
- current.last_key_bytes + child.last_key.len(),
1969
- );
1970
- if !current.children.is_empty() && projected_size > options.max_chunk_bytes {
1971
- groups.push(std::mem::take(&mut current));
1972
- }
1973
-
1974
- current.first_key_bytes += child.first_key.len();
1975
- current.last_key_bytes += child.last_key.len();
1976
- current.children.push(child);
1977
- let current_size = estimate_internal_chunk_size(
1978
- current.children.len(),
1979
- current.first_key_bytes,
1980
- current.last_key_bytes,
1981
- );
1982
- if current_size >= options.min_chunk_bytes
1983
- && (current_size >= options.max_chunk_bytes
1984
- || current.children.last().is_some_and(|child| {
1985
- boundary_trigger(
1986
- &child.first_key,
1987
- level,
1988
- current_size,
1989
- item_size,
1990
- options.target_chunk_bytes,
1991
- )
1992
- }))
1993
- {
1994
- groups.push(std::mem::take(&mut current));
1995
- }
1996
- }
1997
- if !current.children.is_empty() {
1998
- groups.push(current);
1999
- }
2000
- groups
2001
- }
2002
-
2003
- fn chunk_internal_entry_refs<'a>(
2004
- children: impl IntoIterator<Item = ChildSummaryRef<'a>>,
2005
- options: &TrackedStateTreeOptions,
2006
- level: usize,
2007
- ) -> Vec<InternalChunkRefAccumulator<'a>> {
2008
- let mut groups = Vec::new();
2009
- let mut current = InternalChunkRefAccumulator::default();
2010
- for child in children {
2011
- let item_size = child.first_key.len()
2012
- + child.last_key.len()
2013
- + TRACKED_STATE_HASH_BYTES
2014
- + std::mem::size_of::<u64>();
2015
- let projected_size = estimate_internal_chunk_size(
2016
- current.children.len() + 1,
2017
- current.first_key_bytes + child.first_key.len(),
2018
- current.last_key_bytes + child.last_key.len(),
2019
- );
2020
- if !current.children.is_empty() && projected_size > options.max_chunk_bytes {
2021
- groups.push(std::mem::take(&mut current));
2022
- }
2023
-
2024
- current.first_key_bytes += child.first_key.len();
2025
- current.last_key_bytes += child.last_key.len();
2026
- current.children.push(child);
2027
- let current_size = estimate_internal_chunk_size(
2028
- current.children.len(),
2029
- current.first_key_bytes,
2030
- current.last_key_bytes,
2031
- );
2032
- if current_size >= options.min_chunk_bytes
2033
- && (current_size >= options.max_chunk_bytes
2034
- || current.children.last().is_some_and(|child| {
2035
- boundary_trigger(
2036
- child.first_key,
2037
- level,
2038
- current_size,
2039
- item_size,
2040
- options.target_chunk_bytes,
2041
- )
2042
- }))
2043
- {
2044
- groups.push(std::mem::take(&mut current));
2045
- }
2046
- }
2047
- if !current.children.is_empty() {
2048
- groups.push(current);
2049
- }
2050
- groups
2051
- }
2052
-
2053
- fn estimate_leaf_chunk_size(entry_count: usize, key_bytes: usize, value_bytes: usize) -> usize {
2054
- 10 + entry_count * 12 + key_bytes + value_bytes
2055
- }
2056
-
2057
- fn estimate_leaf_boundary_chunk_size(entry_count: usize, key_bytes: usize) -> usize {
2058
- estimate_leaf_chunk_size(entry_count, key_bytes, 0)
2059
- }
2060
-
2061
- fn estimate_leaf_boundary_entry_size(key_bytes: usize) -> usize {
2062
- 12 + key_bytes
2063
- }
2064
-
2065
- fn estimate_internal_chunk_size(
2066
- child_count: usize,
2067
- first_key_bytes: usize,
2068
- last_key_bytes: usize,
2069
- ) -> usize {
2070
- 16 + child_count * (8 + TRACKED_STATE_HASH_BYTES + std::mem::size_of::<u64>())
2071
- + first_key_bytes
2072
- + last_key_bytes
2073
- }
2074
-
2075
- fn first_resync_index(
2076
- generated: &[ChildSummary],
2077
- existing: &[ChildSummary],
2078
- mutation_key: &[u8],
2079
- ) -> Option<(usize, usize)> {
2080
- for (generated_index, generated) in generated.iter().enumerate() {
2081
- // A matching old chunk before the mutation key is only unchanged
2082
- // prefix; resync is only valid after the mutation has been emitted.
2083
- if generated.first_key.as_slice() <= mutation_key {
2084
- continue;
2085
- }
2086
- if let Some(existing_index) = existing.iter().position(|existing| generated == existing) {
2087
- return Some((generated_index, existing_index));
2088
- }
2089
- }
2090
- None
2091
- }
2092
-
2093
- fn internal_boundaries_match(left: &[ChildSummary], right: &[ChildSummary]) -> bool {
2094
- left.len() == right.len()
2095
- && left.iter().zip(right).all(|(left, right)| {
2096
- left.first_key == right.first_key && left.last_key == right.last_key
2097
- })
2098
- }
2099
-
2100
- async fn child_summaries_are_leaves(
2101
- tree: &TrackedStateTree,
2102
- store: &(impl StorageRead + Send + Sync),
2103
- children: &[ChildSummary],
2104
- ) -> Result<bool, LixError> {
2105
- let Some(first_child) = children.first() else {
2106
- return Ok(false);
2107
- };
2108
- Ok(matches!(
2109
- tree.load_node(store, &first_child.child_hash).await?,
2110
- DecodedNode::Leaf(_)
2111
- ))
2112
- }
2113
-
2114
- fn decode_entry(
2115
- entry: &EncodedLeafEntry,
2116
- ) -> Result<(TrackedStateKey, TrackedStateIndexValue), LixError> {
2117
- Ok((decode_key(&entry.key)?, decode_value(&entry.value)?))
2118
- }
2119
-
2120
- fn parent_index_for_child_index(
2121
- old_children: &[ChildSummary],
2122
- old_parents: &[ChildSummary],
2123
- child_index: usize,
2124
- ) -> usize {
2125
- let key = if child_index < old_children.len() {
2126
- old_children[child_index].first_key.as_slice()
2127
- } else {
2128
- old_children
2129
- .last()
2130
- .map(|child| child.last_key.as_slice())
2131
- .unwrap_or_default()
2132
- };
2133
- old_parents
2134
- .iter()
2135
- .position(|parent| parent.last_key.as_slice() >= key)
2136
- .unwrap_or_else(|| old_parents.len().saturating_sub(1))
2137
- }
2138
-
2139
- fn child_range_for_parent(
2140
- old_children: &[ChildSummary],
2141
- parent: &ChildSummary,
2142
- ) -> Result<Range<usize>, LixError> {
2143
- let start = old_children
2144
- .iter()
2145
- .position(|child| child.last_key.as_slice() >= parent.first_key.as_slice())
2146
- .ok_or_else(|| {
2147
- LixError::new(
2148
- "LIX_ERROR_UNKNOWN",
2149
- "tracked-state parent summary does not overlap child summaries",
2150
- )
2151
- })?;
2152
- let end = old_children[start..]
2153
- .iter()
2154
- .position(|child| child.last_key == parent.last_key)
2155
- .map(|offset| start + offset + 1)
2156
- .ok_or_else(|| {
2157
- LixError::new(
2158
- "LIX_ERROR_UNKNOWN",
2159
- "tracked-state parent summary end does not match child summaries",
2160
- )
2161
- })?;
2162
- Ok(start..end)
2163
- }
2164
-
2165
- fn leaf_summary(
2166
- hash: [u8; TRACKED_STATE_HASH_BYTES],
2167
- entries: &[EncodedLeafEntry],
2168
- ) -> ChildSummary {
2169
- ChildSummary {
2170
- first_key: entries
2171
- .first()
2172
- .map(|entry| entry.key.clone())
2173
- .unwrap_or_default(),
2174
- last_key: entries
2175
- .last()
2176
- .map(|entry| entry.key.clone())
2177
- .unwrap_or_default(),
2178
- child_hash: hash,
2179
- subtree_count: entries.len() as u64,
2180
- }
2181
- }
2182
-
2183
- fn internal_summary(
2184
- hash: [u8; TRACKED_STATE_HASH_BYTES],
2185
- children: &[ChildSummary],
2186
- ) -> Result<ChildSummary, LixError> {
2187
- let first_key = children
2188
- .first()
2189
- .map(|child| child.first_key.clone())
2190
- .ok_or_else(|| {
2191
- LixError::new(
2192
- "LIX_ERROR_UNKNOWN",
2193
- "tracked-state internal node has no children",
2194
- )
2195
- })?;
2196
- let last_key = children
2197
- .last()
2198
- .map(|child| child.last_key.clone())
2199
- .ok_or_else(|| {
2200
- LixError::new(
2201
- "LIX_ERROR_UNKNOWN",
2202
- "tracked-state internal node has no children",
2203
- )
2204
- })?;
2205
- Ok(ChildSummary {
2206
- first_key,
2207
- last_key,
2208
- child_hash: hash,
2209
- subtree_count: children.iter().map(|child| child.subtree_count).sum(),
2210
- })
2211
- }
2212
-
2213
- fn push_level_summary(levels: &mut Vec<Vec<ChildSummary>>, level: usize, summary: ChildSummary) {
2214
- while levels.len() <= level {
2215
- levels.push(Vec::new());
2216
- }
2217
- levels[level].push(summary);
2218
- }
2219
-
2220
- fn scan_ranges(request: &TrackedStateTreeScanRequest) -> Vec<EncodedScanRange> {
2221
- if request.schema_keys.is_empty() {
2222
- return Vec::new();
2223
- }
2224
-
2225
- let can_bind_entity = !request.entity_pks.is_empty()
2226
- && !request.file_ids.is_empty()
2227
- && request
2228
- .file_ids
2229
- .iter()
2230
- .all(|filter| !matches!(filter, NullableKeyFilter::Any));
2231
-
2232
- let mut ranges = Vec::new();
2233
- for schema_key in &request.schema_keys {
2234
- if can_bind_entity {
2235
- for file_filter in &request.file_ids {
2236
- let file_id = match file_filter {
2237
- NullableKeyFilter::Null => None,
2238
- NullableKeyFilter::Value(file_id) => Some(file_id.clone()),
2239
- NullableKeyFilter::Any => unreachable!("filtered above"),
2240
- };
2241
- for entity_pk in &request.entity_pks {
2242
- let key = TrackedStateKey {
2243
- schema_key: schema_key.clone(),
2244
- file_id: file_id.clone(),
2245
- entity_pk: entity_pk.clone(),
2246
- };
2247
- ranges.push(exact_scan_range(encode_key(&key)));
2248
- }
2249
- }
2250
- continue;
2251
- }
2252
-
2253
- if request.file_ids.is_empty()
2254
- || request
2255
- .file_ids
2256
- .iter()
2257
- .any(|filter| matches!(filter, NullableKeyFilter::Any))
2258
- {
2259
- ranges.push(prefix_scan_range(encode_schema_key_prefix(schema_key)));
2260
- continue;
2261
- }
2262
-
2263
- for file_filter in &request.file_ids {
2264
- let prefix = match file_filter {
2265
- NullableKeyFilter::Null => encode_schema_file_prefix(schema_key, None),
2266
- NullableKeyFilter::Value(file_id) => {
2267
- encode_schema_file_prefix(schema_key, Some(file_id))
2268
- }
2269
- NullableKeyFilter::Any => unreachable!("handled above"),
2270
- };
2271
- ranges.push(prefix_scan_range(prefix));
2272
- }
2273
- }
2274
- ranges
2275
- }
2276
-
2277
- fn scan_key_decode_hint<'a>(
2278
- request: &'a TrackedStateTreeScanRequest,
2279
- ranges: &[EncodedScanRange],
2280
- ) -> Option<ScanKeyDecodeHint<'a>> {
2281
- if ranges.len() != 1 || request.schema_keys.len() != 1 || request.file_ids.len() != 1 {
2282
- return None;
2283
- }
2284
- if !request.entity_pks.is_empty() {
2285
- return None;
2286
- }
2287
- let file_id = match request.file_ids.first()? {
2288
- NullableKeyFilter::Null => None,
2289
- NullableKeyFilter::Value(file_id) => Some(file_id.as_str()),
2290
- NullableKeyFilter::Any => return None,
2291
- };
2292
- Some(ScanKeyDecodeHint {
2293
- schema_key: request.schema_keys.first()?.as_str(),
2294
- file_id,
2295
- prefix_len: ranges.first()?.start.len(),
2296
- })
2297
- }
2298
-
2299
- fn prefix_scan_range(prefix: Vec<u8>) -> EncodedScanRange {
2300
- EncodedScanRange {
2301
- end: lexicographic_successor(&prefix),
2302
- start: prefix,
2303
- }
2304
- }
2305
-
2306
- fn exact_scan_range(key: Vec<u8>) -> EncodedScanRange {
2307
- EncodedScanRange {
2308
- end: lexicographic_successor(&key),
2309
- start: key,
2310
- }
2311
- }
2312
-
2313
- fn lexicographic_successor(bytes: &[u8]) -> Option<Vec<u8>> {
2314
- let mut out = bytes.to_vec();
2315
- for index in (0..out.len()).rev() {
2316
- if out[index] != u8::MAX {
2317
- out[index] += 1;
2318
- out.truncate(index + 1);
2319
- return Some(out);
2320
- }
2321
- }
2322
- None
2323
- }
2324
-
2325
- fn child_summary_overlaps_scan_ranges(child: &ChildSummary, ranges: &[EncodedScanRange]) -> bool {
2326
- ranges.is_empty()
2327
- || ranges.iter().any(|range| {
2328
- child.last_key.as_slice() >= range.start.as_slice()
2329
- && range
2330
- .end
2331
- .as_ref()
2332
- .is_none_or(|end| child.first_key.as_slice() < end.as_slice())
2333
- })
2334
- }
2335
-
2336
- fn encoded_key_in_scan_ranges(key: &[u8], ranges: &[EncodedScanRange]) -> bool {
2337
- ranges.is_empty()
2338
- || ranges.iter().any(|range| {
2339
- key >= range.start.as_slice()
2340
- && range.end.as_ref().is_none_or(|end| key < end.as_slice())
2341
- })
2342
- }
2343
-
2344
- fn key_matches_scan_filters(request: &TrackedStateTreeScanRequest, key: &TrackedStateKey) -> bool {
2345
- if !request.schema_keys.is_empty() && !request.schema_keys.contains(&key.schema_key) {
2346
- return false;
2347
- }
2348
- if !request.entity_pks.is_empty() && !request.entity_pks.contains(&key.entity_pk) {
2349
- return false;
2350
- }
2351
- if !request.file_ids.is_empty()
2352
- && !request
2353
- .file_ids
2354
- .iter()
2355
- .any(|filter| filter.matches(key.file_id.as_ref()))
2356
- {
2357
- return false;
2358
- }
2359
- true
2360
- }
2361
-
2362
- fn scan_limit_reached(request: &TrackedStateTreeScanRequest, row_count: usize) -> bool {
2363
- request.limit.is_some_and(|limit| row_count >= limit)
2364
- }
2365
-
2366
- #[cfg(test)]
2367
- mod tests {
2368
- use super::*;
2369
- use crate::entity_pk::EntityPk;
2370
- use crate::storage::StorageContext;
2371
- use crate::storage::{InMemoryStorageBackend, StorageReadOptions, StorageWriteOptions};
2372
- use crate::tracked_state::codec::encode_value;
2373
-
2374
- #[tokio::test]
2375
- async fn exact_read_roundtrips_from_applied_root() {
2376
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2377
- let tree = TrackedStateTree::new();
2378
- let key = key("schema", None, "entity");
2379
- let value = value("change-1", Some("{}"));
2380
- let result =
2381
- apply_mutations_for_test(&tree, &storage, None, vec![mutation(&key, &value)], None)
2382
- .await
2383
- .expect("mutations should apply");
2384
-
2385
- let store = storage
2386
- .begin_read(StorageReadOptions::default())
2387
- .expect("read should open");
2388
- assert_eq!(
2389
- tree.get(&store, &result.root_id, &key)
2390
- .await
2391
- .expect("row should load"),
2392
- Some(value)
2393
- );
2394
- }
2395
-
2396
- #[tokio::test]
2397
- async fn latest_mutation_for_key_wins() {
2398
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2399
- let tree = TrackedStateTree::new();
2400
- let key = key("schema", None, "entity");
2401
- let old_value = value("change-old", Some("{\"v\":1}"));
2402
- let new_value = value("change-new", Some("{\"v\":2}"));
2403
- let result = apply_mutations_for_test(
2404
- &tree,
2405
- &storage,
2406
- None,
2407
- vec![mutation(&key, &old_value), mutation(&key, &new_value)],
2408
- None,
2409
- )
2410
- .await
2411
- .expect("mutations should apply");
2412
-
2413
- let store = storage
2414
- .begin_read(StorageReadOptions::default())
2415
- .expect("read should open");
2416
- let loaded = tree
2417
- .get(&store, &result.root_id, &key)
2418
- .await
2419
- .expect("row should load")
2420
- .expect("row should exist");
2421
- assert_eq!(loaded.change_id, "change-new");
2422
- assert_eq!(loaded.commit_id, "commit");
2423
- }
2424
-
2425
- #[tokio::test]
2426
- async fn scan_filters_by_index_key_without_materializing_tombstones() {
2427
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2428
- let tree = TrackedStateTree::new();
2429
- let result = apply_mutations_for_test(
2430
- &tree,
2431
- &storage,
2432
- None,
2433
- vec![
2434
- mutation_owned(key("schema-a", None, "visible"), value("c1", Some("{}"))),
2435
- mutation_owned(key("schema-a", None, "deleted"), value("c2", None)),
2436
- mutation_owned(key("schema-b", None, "other"), value("c3", Some("{}"))),
2437
- ],
2438
- None,
2439
- )
2440
- .await
2441
- .expect("mutations should apply");
2442
-
2443
- let store = storage
2444
- .begin_read(StorageReadOptions::default())
2445
- .expect("read should open");
2446
- let rows = tree
2447
- .scan(
2448
- &store,
2449
- &result.root_id,
2450
- &TrackedStateTreeScanRequest {
2451
- schema_keys: vec!["schema-a".to_string()],
2452
- ..Default::default()
2453
- },
2454
- )
2455
- .await
2456
- .expect("scan should succeed");
2457
- assert_eq!(rows.len(), 2);
2458
- let identities = rows
2459
- .iter()
2460
- .map(|(key, _)| key.entity_pk.as_single_string_owned().expect("identity"))
2461
- .collect::<Vec<_>>();
2462
- assert_eq!(identities, vec!["deleted", "visible"]);
2463
-
2464
- let live_rows = tree
2465
- .scan(
2466
- &store,
2467
- &result.root_id,
2468
- &TrackedStateTreeScanRequest {
2469
- schema_keys: vec!["schema-a".to_string()],
2470
- include_tombstones: false,
2471
- ..Default::default()
2472
- },
2473
- )
2474
- .await
2475
- .expect("live scan should succeed");
2476
- let live_identities = live_rows
2477
- .iter()
2478
- .map(|(key, _)| key.entity_pk.as_single_string_owned().expect("identity"))
2479
- .collect::<Vec<_>>();
2480
- assert_eq!(live_identities, vec!["visible"]);
2481
- }
2482
-
2483
- #[tokio::test]
2484
- async fn scan_filters_by_schema_entity_and_file() {
2485
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2486
- let tree = TrackedStateTree::new();
2487
- let result = apply_mutations_for_test(
2488
- &tree,
2489
- &storage,
2490
- None,
2491
- vec![
2492
- mutation_owned(
2493
- key("schema-a", Some("file-a"), "entity-a"),
2494
- value("c1", Some("{}")),
2495
- ),
2496
- mutation_owned(
2497
- key("schema-a", Some("file-b"), "entity-a"),
2498
- value("c2", Some("{}")),
2499
- ),
2500
- mutation_owned(
2501
- key("schema-a", Some("file-a"), "entity-b"),
2502
- value("c3", Some("{}")),
2503
- ),
2504
- mutation_owned(
2505
- key("schema-b", Some("file-a"), "entity-a"),
2506
- value("c4", Some("{}")),
2507
- ),
2508
- ],
2509
- None,
2510
- )
2511
- .await
2512
- .expect("mutations should apply");
2513
-
2514
- let store = storage
2515
- .begin_read(StorageReadOptions::default())
2516
- .expect("read should open");
2517
- let rows = tree
2518
- .scan(
2519
- &store,
2520
- &result.root_id,
2521
- &TrackedStateTreeScanRequest {
2522
- schema_keys: vec!["schema-a".to_string()],
2523
- entity_pks: vec![crate::entity_pk::EntityPk::single("entity-a")],
2524
- file_ids: vec![crate::NullableKeyFilter::Value("file-a".to_string())],
2525
- ..Default::default()
2526
- },
2527
- )
2528
- .await
2529
- .expect("scan should succeed");
2530
-
2531
- assert_eq!(rows.len(), 1);
2532
- assert_eq!(rows[0].0.schema_key, "schema-a");
2533
- assert_eq!(
2534
- rows[0]
2535
- .0
2536
- .entity_pk
2537
- .as_single_string_owned()
2538
- .expect("identity"),
2539
- "entity-a"
2540
- );
2541
- assert_eq!(rows[0].0.file_id.as_deref(), Some("file-a"));
2542
- }
2543
-
2544
- #[tokio::test]
2545
- async fn scan_schema_file_prefix_honors_tombstones_and_limit() {
2546
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2547
- let tree = TrackedStateTree::new();
2548
- let result = apply_mutations_for_test(
2549
- &tree,
2550
- &storage,
2551
- None,
2552
- vec![
2553
- mutation_owned(
2554
- key("schema-a", Some("file-a"), "entity-a"),
2555
- value("c1", Some("{}")),
2556
- ),
2557
- mutation_owned(
2558
- key("schema-a", Some("file-a"), "entity-b"),
2559
- value("c2", None),
2560
- ),
2561
- mutation_owned(
2562
- key("schema-a", Some("file-a"), "entity-c"),
2563
- value("c3", Some("{}")),
2564
- ),
2565
- mutation_owned(
2566
- key("schema-a", Some("file-b"), "entity-d"),
2567
- value("c4", Some("{}")),
2568
- ),
2569
- ],
2570
- None,
2571
- )
2572
- .await
2573
- .expect("mutations should apply");
2574
-
2575
- let store = storage
2576
- .begin_read(StorageReadOptions::default())
2577
- .expect("read should open");
2578
- let rows = tree
2579
- .scan(
2580
- &store,
2581
- &result.root_id,
2582
- &TrackedStateTreeScanRequest {
2583
- schema_keys: vec!["schema-a".to_string()],
2584
- file_ids: vec![crate::NullableKeyFilter::Value("file-a".to_string())],
2585
- include_tombstones: false,
2586
- limit: Some(2),
2587
- ..Default::default()
2588
- },
2589
- )
2590
- .await
2591
- .expect("scan should succeed");
2592
-
2593
- assert_eq!(rows.len(), 2);
2594
- assert!(rows.iter().all(
2595
- |(key, _)| key.schema_key == "schema-a" && key.file_id.as_deref() == Some("file-a")
2596
- ));
2597
- assert_eq!(
2598
- rows.iter()
2599
- .map(|(key, _)| key.entity_pk.as_single_string_owned().expect("identity"))
2600
- .collect::<Vec<_>>(),
2601
- vec!["entity-a", "entity-c"]
2602
- );
2603
- }
2604
-
2605
- #[tokio::test]
2606
- async fn applying_to_base_root_reuses_existing_rows_and_overwrites_changed_rows() {
2607
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2608
- let tree = TrackedStateTree::new();
2609
- let unchanged_key = key("schema", None, "unchanged");
2610
- let changed_key = key("schema", None, "changed");
2611
- let unchanged_value = value("c1", Some("{}"));
2612
- let old_changed_value = value("c2", Some("{\"old\":true}"));
2613
- let new_changed_value = value("c3", Some("{\"new\":true}"));
2614
- let base = apply_mutations_for_test(
2615
- &tree,
2616
- &storage,
2617
- None,
2618
- vec![
2619
- mutation(&unchanged_key, &unchanged_value),
2620
- mutation(&changed_key, &old_changed_value),
2621
- ],
2622
- None,
2623
- )
2624
- .await
2625
- .expect("base should build");
2626
- let next = apply_mutations_for_test(
2627
- &tree,
2628
- &storage,
2629
- Some(&base.root_id),
2630
- vec![mutation(&changed_key, &new_changed_value)],
2631
- None,
2632
- )
2633
- .await
2634
- .expect("next should build");
2635
-
2636
- let store = storage
2637
- .begin_read(StorageReadOptions::default())
2638
- .expect("read should open");
2639
- assert_eq!(
2640
- tree.get(&store, &next.root_id, &unchanged_key)
2641
- .await
2642
- .expect("unchanged read")
2643
- .expect("unchanged exists")
2644
- .change_id,
2645
- "c1"
2646
- );
2647
- assert_eq!(
2648
- tree.get(&store, &next.root_id, &changed_key)
2649
- .await
2650
- .expect("changed read")
2651
- .expect("changed exists")
2652
- .change_id,
2653
- "c3"
2654
- );
2655
- }
2656
-
2657
- #[tokio::test]
2658
- async fn two_commit_roots_can_share_unchanged_rows() {
2659
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2660
- let tree = TrackedStateTree::new();
2661
- let shared_key = key("schema", None, "shared");
2662
- let branch_a_key = key("schema", None, "branch-a");
2663
- let branch_b_key = key("schema", None, "branch-b");
2664
- let shared_value = value("shared-change", Some("{\"shared\":true}"));
2665
- let branch_a_value = value("branch-a-change", Some("{\"branch\":\"a\"}"));
2666
- let branch_b_value = value("branch-b-change", Some("{\"branch\":\"b\"}"));
2667
- let base = apply_mutations_for_test(
2668
- &tree,
2669
- &storage,
2670
- None,
2671
- vec![mutation(&shared_key, &shared_value)],
2672
- Some("commit-base"),
2673
- )
2674
- .await
2675
- .expect("base root should build");
2676
- let branch_a = apply_mutations_for_test(
2677
- &tree,
2678
- &storage,
2679
- Some(&base.root_id),
2680
- vec![mutation(&branch_a_key, &branch_a_value)],
2681
- Some("commit-a"),
2682
- )
2683
- .await
2684
- .expect("branch a root should build");
2685
- let branch_b = apply_mutations_for_test(
2686
- &tree,
2687
- &storage,
2688
- Some(&base.root_id),
2689
- vec![mutation(&branch_b_key, &branch_b_value)],
2690
- Some("commit-b"),
2691
- )
2692
- .await
2693
- .expect("branch b root should build");
2694
-
2695
- assert_ne!(branch_a.root_id, branch_b.root_id);
2696
- let store = storage
2697
- .begin_read(StorageReadOptions::default())
2698
- .expect("read should open");
2699
- assert_eq!(
2700
- tree.get(&store, &branch_a.root_id, &shared_key)
2701
- .await
2702
- .expect("branch a shared row should load"),
2703
- Some(value("shared-change", Some("{\"shared\":true}")))
2704
- );
2705
- assert_eq!(
2706
- tree.get(&store, &branch_b.root_id, &shared_key)
2707
- .await
2708
- .expect("branch b shared row should load"),
2709
- Some(value("shared-change", Some("{\"shared\":true}")))
2710
- );
2711
- assert!(tree
2712
- .get(&store, &branch_a.root_id, &branch_b_key)
2713
- .await
2714
- .expect("branch a should read")
2715
- .is_none());
2716
- assert!(tree
2717
- .get(&store, &branch_b.root_id, &branch_a_key)
2718
- .await
2719
- .expect("branch b should read")
2720
- .is_none());
2721
- }
2722
-
2723
- #[tokio::test]
2724
- async fn single_update_matches_full_canonical_rebuild() {
2725
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2726
- let tree = TrackedStateTree::with_options(TrackedStateTreeOptions {
2727
- target_chunk_bytes: 128,
2728
- min_chunk_bytes: 64,
2729
- max_chunk_bytes: 256,
2730
- });
2731
- let rows = (0..100)
2732
- .map(|index| {
2733
- mutation_owned(
2734
- key("schema", None, &format!("entity-{index:03}")),
2735
- value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
2736
- )
2737
- })
2738
- .collect::<Vec<_>>();
2739
- let changed_key = key("schema", None, "entity-000");
2740
- let changed_value = value("changed", Some("{\"v\":\"changed\"}"));
2741
- let base = apply_mutations_for_test(&tree, &storage, None, rows, None)
2742
- .await
2743
- .expect("base should build");
2744
- let fast = apply_mutations_for_test(
2745
- &tree,
2746
- &storage,
2747
- Some(&base.root_id),
2748
- vec![mutation(&changed_key, &changed_value)],
2749
- None,
2750
- )
2751
- .await
2752
- .expect("fast path should apply");
2753
- let read = storage
2754
- .begin_read(StorageReadOptions::default())
2755
- .expect("read should open");
2756
- let mut canonical_entries = tree
2757
- .collect_leaf_entries(&read, &base.root_id)
2758
- .await
2759
- .expect("base entries should collect");
2760
- assert!(canonical_entries
2761
- .windows(2)
2762
- .all(|window| window[0].key < window[1].key));
2763
- let encoded_changed_key = encode_key(&changed_key);
2764
- let encoded_changed_value = encode_value(&changed_value);
2765
- let index = canonical_entries
2766
- .binary_search_by(|entry| entry.key.as_slice().cmp(&encoded_changed_key))
2767
- .expect("changed key should exist");
2768
- canonical_entries[index].value = encoded_changed_value;
2769
- let canonical = tree
2770
- .build_tree_from_entries(canonical_entries)
2771
- .expect("canonical root should build");
2772
-
2773
- assert_eq!(fast.root_id, canonical.root_id);
2774
- }
2775
-
2776
- #[tokio::test]
2777
- async fn single_insert_matches_full_canonical_rebuild() {
2778
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2779
- let tree = TrackedStateTree::with_options(TrackedStateTreeOptions {
2780
- target_chunk_bytes: 128,
2781
- min_chunk_bytes: 64,
2782
- max_chunk_bytes: 256,
2783
- });
2784
- let rows = (0..100)
2785
- .map(|index| {
2786
- mutation_owned(
2787
- key("schema", None, &format!("entity-{index:03}")),
2788
- value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
2789
- )
2790
- })
2791
- .collect::<Vec<_>>();
2792
- let inserted_key = key("schema", None, "entity-050a");
2793
- let inserted_value = value("inserted", Some("{\"v\":\"inserted\"}"));
2794
- let base = apply_mutations_for_test(&tree, &storage, None, rows, None)
2795
- .await
2796
- .expect("base should build");
2797
- let fast = apply_mutations_for_test(
2798
- &tree,
2799
- &storage,
2800
- Some(&base.root_id),
2801
- vec![mutation(&inserted_key, &inserted_value)],
2802
- None,
2803
- )
2804
- .await
2805
- .expect("fast path should apply");
2806
- let read = storage
2807
- .begin_read(StorageReadOptions::default())
2808
- .expect("read should open");
2809
- let mut canonical_entries = tree
2810
- .collect_leaf_entries(&read, &base.root_id)
2811
- .await
2812
- .expect("base entries should collect");
2813
- let encoded_inserted_key = encode_key(&inserted_key);
2814
- let encoded_inserted_value = encode_value(&inserted_value);
2815
- let index = canonical_entries
2816
- .binary_search_by(|entry| entry.key.as_slice().cmp(&encoded_inserted_key))
2817
- .expect_err("inserted key should not exist");
2818
- canonical_entries.insert(
2819
- index,
2820
- EncodedLeafEntry {
2821
- key: encoded_inserted_key,
2822
- value: encoded_inserted_value,
2823
- },
2824
- );
2825
- let canonical = tree
2826
- .build_tree_from_entries(canonical_entries)
2827
- .expect("canonical root should build");
2828
-
2829
- assert_eq!(fast.root_id, canonical.root_id);
2830
- }
2831
-
2832
- #[tokio::test]
2833
- async fn batch_update_matches_full_canonical_rebuild() {
2834
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2835
- let tree = TrackedStateTree::with_options(TrackedStateTreeOptions {
2836
- target_chunk_bytes: 128,
2837
- min_chunk_bytes: 64,
2838
- max_chunk_bytes: 256,
2839
- });
2840
- let rows = (0..100)
2841
- .map(|index| {
2842
- mutation_owned(
2843
- key("schema", None, &format!("entity-{index:03}")),
2844
- value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
2845
- )
2846
- })
2847
- .collect::<Vec<_>>();
2848
- let updates = (10..25)
2849
- .map(|index| {
2850
- (
2851
- key("schema", None, &format!("entity-{index:03}")),
2852
- value(
2853
- &format!("changed-{index}"),
2854
- Some(&format!("{{\"changed\":{index}}}")),
2855
- ),
2856
- )
2857
- })
2858
- .collect::<Vec<_>>();
2859
- let base = apply_mutations_for_test(&tree, &storage, None, rows, None)
2860
- .await
2861
- .expect("base should build");
2862
- let fast = apply_mutations_for_test(
2863
- &tree,
2864
- &storage,
2865
- Some(&base.root_id),
2866
- updates
2867
- .iter()
2868
- .map(|(key, value)| mutation(key, value))
2869
- .collect(),
2870
- None,
2871
- )
2872
- .await
2873
- .expect("batch path should apply");
2874
- let read = storage
2875
- .begin_read(StorageReadOptions::default())
2876
- .expect("read should open");
2877
- let mut canonical_entries = tree
2878
- .collect_leaf_entries(&read, &base.root_id)
2879
- .await
2880
- .expect("base entries should collect");
2881
- for (key, value) in updates {
2882
- let encoded_key = encode_key(&key);
2883
- let encoded_value = encode_value(&value);
2884
- let index = canonical_entries
2885
- .binary_search_by(|entry| entry.key.as_slice().cmp(&encoded_key))
2886
- .expect("updated key should exist");
2887
- canonical_entries[index].value = encoded_value;
2888
- }
2889
- let canonical = tree
2890
- .build_tree_from_entries(canonical_entries)
2891
- .expect("canonical root should build");
2892
-
2893
- assert_eq!(fast.root_id, canonical.root_id);
2894
- }
2895
-
2896
- #[tokio::test]
2897
- async fn batch_insert_matches_full_canonical_rebuild() {
2898
- let storage = StorageContext::new(InMemoryStorageBackend::new());
2899
- let tree = TrackedStateTree::with_options(TrackedStateTreeOptions {
2900
- target_chunk_bytes: 128,
2901
- min_chunk_bytes: 64,
2902
- max_chunk_bytes: 256,
2903
- });
2904
- let rows = (0..100)
2905
- .map(|index| {
2906
- mutation_owned(
2907
- key("schema", None, &format!("entity-{index:03}")),
2908
- value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
2909
- )
2910
- })
2911
- .collect::<Vec<_>>();
2912
- let inserts = ["entity-050a", "entity-050b", "entity-050c"]
2913
- .into_iter()
2914
- .enumerate()
2915
- .map(|(index, entity_pk)| {
2916
- (
2917
- key("schema", None, entity_pk),
2918
- value(
2919
- &format!("inserted-{index}"),
2920
- Some(&format!("{{\"inserted\":{index}}}")),
2921
- ),
2922
- )
2923
- })
2924
- .collect::<Vec<_>>();
2925
- let base = apply_mutations_for_test(&tree, &storage, None, rows, None)
2926
- .await
2927
- .expect("base should build");
2928
- let fast = apply_mutations_for_test(
2929
- &tree,
2930
- &storage,
2931
- Some(&base.root_id),
2932
- inserts
2933
- .iter()
2934
- .map(|(key, value)| mutation(key, value))
2935
- .collect(),
2936
- None,
2937
- )
2938
- .await
2939
- .expect("batch path should apply");
2940
- let read = storage
2941
- .begin_read(StorageReadOptions::default())
2942
- .expect("read should open");
2943
- let mut canonical_entries = tree
2944
- .collect_leaf_entries(&read, &base.root_id)
2945
- .await
2946
- .expect("base entries should collect");
2947
- for (key, value) in inserts {
2948
- let encoded_key = encode_key(&key);
2949
- let encoded_value = encode_value(&value);
2950
- let index = canonical_entries
2951
- .binary_search_by(|entry| entry.key.as_slice().cmp(&encoded_key))
2952
- .expect_err("inserted key should not exist");
2953
- canonical_entries.insert(
2954
- index,
2955
- EncodedLeafEntry {
2956
- key: encoded_key,
2957
- value: encoded_value,
2958
- },
2959
- );
2960
- }
2961
- let canonical = tree
2962
- .build_tree_from_entries(canonical_entries)
2963
- .expect("canonical root should build");
2964
-
2965
- assert_eq!(fast.root_id, canonical.root_id);
2966
- }
2967
-
2968
- #[test]
2969
- fn leaf_chunk_boundaries_ignore_value_bytes() {
2970
- let options = TrackedStateTreeOptions {
2971
- target_chunk_bytes: 64,
2972
- min_chunk_bytes: 32,
2973
- max_chunk_bytes: 96,
2974
- };
2975
- let short_entries = encoded_entries_with_change_id("c");
2976
- let large_entries = encoded_entries_with_change_id(&"c".repeat(4096));
2977
-
2978
- assert_eq!(
2979
- leaf_chunk_boundary_keys(chunk_leaf_entries(short_entries, &options)),
2980
- leaf_chunk_boundary_keys(chunk_leaf_entries(large_entries, &options))
2981
- );
2982
- }
2983
-
2984
- async fn apply_mutations_for_test(
2985
- tree: &TrackedStateTree,
2986
- storage: &StorageContext,
2987
- base_root: Option<&TrackedStateRootId>,
2988
- mutations: Vec<TrackedStateMutation>,
2989
- commit_id: Option<&str>,
2990
- ) -> Result<TrackedStateApplyResult, LixError> {
2991
- let read = storage
2992
- .begin_read(StorageReadOptions::default())
2993
- .expect("read should open");
2994
- let mut writes = storage.new_write_set();
2995
- let result = tree
2996
- .apply_mutations(&read, &mut writes, base_root, mutations, commit_id)
2997
- .await?;
2998
- storage.commit_write_set(writes, StorageWriteOptions::default())?;
2999
- Ok(result)
3000
- }
3001
-
3002
- fn mutation(key: &TrackedStateKey, value: &TrackedStateIndexValue) -> TrackedStateMutation {
3003
- TrackedStateMutation::put_encoded(encode_key(key), encode_value(value))
3004
- }
3005
-
3006
- fn mutation_owned(key: TrackedStateKey, value: TrackedStateIndexValue) -> TrackedStateMutation {
3007
- mutation(&key, &value)
3008
- }
3009
-
3010
- fn encoded_entries_with_change_id(change_id: &str) -> Vec<EncodedLeafEntry> {
3011
- (0..64)
3012
- .map(|index| {
3013
- let key = key("schema", None, &format!("entity-{index:03}"));
3014
- EncodedLeafEntry {
3015
- key: encode_key(&key),
3016
- value: encode_value(&value(change_id, Some("{}"))),
3017
- }
3018
- })
3019
- .collect()
3020
- }
3021
-
3022
- fn leaf_chunk_boundary_keys(
3023
- groups: Vec<LeafChunkAccumulator>,
3024
- ) -> Vec<(Vec<u8>, Vec<u8>, usize)> {
3025
- groups
3026
- .into_iter()
3027
- .map(|group| {
3028
- let first_key = group
3029
- .entries
3030
- .first()
3031
- .map(|entry| entry.key.clone())
3032
- .unwrap_or_default();
3033
- let last_key = group
3034
- .entries
3035
- .last()
3036
- .map(|entry| entry.key.clone())
3037
- .unwrap_or_default();
3038
- (first_key, last_key, group.entries.len())
3039
- })
3040
- .collect()
3041
- }
3042
-
3043
- fn key(schema_key: &str, file_id: Option<&str>, entity_pk: &str) -> TrackedStateKey {
3044
- TrackedStateKey {
3045
- schema_key: schema_key.to_string(),
3046
- file_id: file_id.map(str::to_string),
3047
- entity_pk: EntityPk::single(entity_pk),
3048
- }
3049
- }
3050
-
3051
- fn value(change_id: &str, snapshot_content: Option<&str>) -> TrackedStateIndexValue {
3052
- TrackedStateIndexValue {
3053
- change_id: change_id.to_string(),
3054
- commit_id: "commit".to_string(),
3055
- deleted: snapshot_content.is_none(),
3056
- snapshot_ref: snapshot_content
3057
- .map(|content| crate::json_store::JsonRef::for_content(content.as_bytes())),
3058
- metadata_ref: None,
3059
- created_at: "2026-01-01T00:00:00Z".to_string(),
3060
- updated_at: "2026-01-01T00:00:00Z".to_string(),
3061
- }
3062
- }
3063
- }