@lix-js/sdk 0.6.0-preview.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. package/README.md +76 -4
  2. package/dist/errors.d.ts +7 -0
  3. package/dist/errors.js +19 -0
  4. package/dist/index.d.ts +4 -5
  5. package/dist/index.js +3 -3
  6. package/dist/native.d.ts +1 -0
  7. package/dist/native.js +47 -0
  8. package/dist/open-lix.d.ts +38 -207
  9. package/dist/open-lix.js +59 -284
  10. package/dist/result.d.ts +18 -0
  11. package/dist/result.js +48 -0
  12. package/dist/types.d.ts +114 -1
  13. package/dist/value.d.ts +28 -0
  14. package/dist/value.js +245 -0
  15. package/package.json +38 -71
  16. package/SKILL.md +0 -507
  17. package/dist/builtin-schemas.d.ts +0 -1
  18. package/dist/builtin-schemas.js +0 -1
  19. package/dist/engine-wasm/index.d.ts +0 -87
  20. package/dist/engine-wasm/index.js +0 -339
  21. package/dist/engine-wasm/wasm/lix_engine.d.ts +0 -79
  22. package/dist/engine-wasm/wasm/lix_engine.js +0 -833
  23. package/dist/engine-wasm/wasm/lix_engine.wasm +0 -0
  24. package/dist/engine-wasm/wasm/lix_engine.wasm.d.ts +0 -27
  25. package/dist/generated/builtin-schemas.d.ts +0 -427
  26. package/dist/generated/builtin-schemas.js +0 -643
  27. package/dist/sqlite/index.d.ts +0 -12
  28. package/dist/sqlite/index.js +0 -359
  29. package/dist-engine-src/README.md +0 -18
  30. package/dist-engine-src/src/backend/capabilities.rs +0 -67
  31. package/dist-engine-src/src/backend/conformance/baseline.rs +0 -1127
  32. package/dist-engine-src/src/backend/conformance/factory.rs +0 -93
  33. package/dist-engine-src/src/backend/conformance/failure_tests.rs +0 -608
  34. package/dist-engine-src/src/backend/conformance/fixtures.rs +0 -26
  35. package/dist-engine-src/src/backend/conformance/mod.rs +0 -75
  36. package/dist-engine-src/src/backend/conformance/model.rs +0 -28
  37. package/dist-engine-src/src/backend/conformance/model_based.rs +0 -257
  38. package/dist-engine-src/src/backend/conformance/persistence.rs +0 -204
  39. package/dist-engine-src/src/backend/conformance/projection.rs +0 -21
  40. package/dist-engine-src/src/backend/conformance/pushdown.rs +0 -24
  41. package/dist-engine-src/src/backend/conformance/runner.rs +0 -90
  42. package/dist-engine-src/src/backend/conformance/scan.rs +0 -24
  43. package/dist-engine-src/src/backend/conformance/write.rs +0 -16
  44. package/dist-engine-src/src/backend/error.rs +0 -94
  45. package/dist-engine-src/src/backend/in_memory.rs +0 -670
  46. package/dist-engine-src/src/backend/mod.rs +0 -39
  47. package/dist-engine-src/src/backend/predicate.rs +0 -80
  48. package/dist-engine-src/src/backend/traits.rs +0 -260
  49. package/dist-engine-src/src/backend/types.rs +0 -239
  50. package/dist-engine-src/src/binary_cas/chunking.rs +0 -31
  51. package/dist-engine-src/src/binary_cas/codec.rs +0 -346
  52. package/dist-engine-src/src/binary_cas/context.rs +0 -139
  53. package/dist-engine-src/src/binary_cas/kv.rs +0 -1038
  54. package/dist-engine-src/src/binary_cas/mod.rs +0 -11
  55. package/dist-engine-src/src/binary_cas/types.rs +0 -121
  56. package/dist-engine-src/src/branch/context.rs +0 -40
  57. package/dist-engine-src/src/branch/lifecycle.rs +0 -221
  58. package/dist-engine-src/src/branch/mod.rs +0 -13
  59. package/dist-engine-src/src/branch/refs.rs +0 -321
  60. package/dist-engine-src/src/branch/stage_rows.rs +0 -67
  61. package/dist-engine-src/src/branch/types.rs +0 -21
  62. package/dist-engine-src/src/catalog/context.rs +0 -412
  63. package/dist-engine-src/src/catalog/mod.rs +0 -10
  64. package/dist-engine-src/src/catalog/schema.rs +0 -4
  65. package/dist-engine-src/src/catalog/snapshot.rs +0 -1114
  66. package/dist-engine-src/src/cel/context.rs +0 -86
  67. package/dist-engine-src/src/cel/error.rs +0 -19
  68. package/dist-engine-src/src/cel/mod.rs +0 -8
  69. package/dist-engine-src/src/cel/provider.rs +0 -9
  70. package/dist-engine-src/src/cel/runtime.rs +0 -167
  71. package/dist-engine-src/src/cel/value.rs +0 -50
  72. package/dist-engine-src/src/changelog/bench_support.rs +0 -785
  73. package/dist-engine-src/src/changelog/change.rs +0 -1
  74. package/dist-engine-src/src/changelog/codec.rs +0 -497
  75. package/dist-engine-src/src/changelog/commit.rs +0 -1
  76. package/dist-engine-src/src/changelog/context.rs +0 -1614
  77. package/dist-engine-src/src/changelog/mod.rs +0 -29
  78. package/dist-engine-src/src/changelog/store.rs +0 -163
  79. package/dist-engine-src/src/changelog/test_support.rs +0 -54
  80. package/dist-engine-src/src/changelog/types.rs +0 -213
  81. package/dist-engine-src/src/commit_graph/context.rs +0 -944
  82. package/dist-engine-src/src/commit_graph/mod.rs +0 -9
  83. package/dist-engine-src/src/commit_graph/types.rs +0 -89
  84. package/dist-engine-src/src/commit_graph/walker.rs +0 -786
  85. package/dist-engine-src/src/common/error.rs +0 -347
  86. package/dist-engine-src/src/common/fingerprint.rs +0 -3
  87. package/dist-engine-src/src/common/fs_path.rs +0 -1336
  88. package/dist-engine-src/src/common/identity.rs +0 -145
  89. package/dist-engine-src/src/common/json_pointer.rs +0 -67
  90. package/dist-engine-src/src/common/metadata.rs +0 -40
  91. package/dist-engine-src/src/common/mod.rs +0 -23
  92. package/dist-engine-src/src/common/types.rs +0 -105
  93. package/dist-engine-src/src/common/wire.rs +0 -222
  94. package/dist-engine-src/src/domain.rs +0 -320
  95. package/dist-engine-src/src/engine.rs +0 -203
  96. package/dist-engine-src/src/entity_pk.rs +0 -402
  97. package/dist-engine-src/src/functions/context.rs +0 -296
  98. package/dist-engine-src/src/functions/deterministic.rs +0 -113
  99. package/dist-engine-src/src/functions/mod.rs +0 -18
  100. package/dist-engine-src/src/functions/provider.rs +0 -130
  101. package/dist-engine-src/src/functions/state.rs +0 -335
  102. package/dist-engine-src/src/functions/types.rs +0 -37
  103. package/dist-engine-src/src/init.rs +0 -692
  104. package/dist-engine-src/src/json_store/compression.rs +0 -77
  105. package/dist-engine-src/src/json_store/context.rs +0 -172
  106. package/dist-engine-src/src/json_store/encoded.rs +0 -15
  107. package/dist-engine-src/src/json_store/mod.rs +0 -38
  108. package/dist-engine-src/src/json_store/store.rs +0 -494
  109. package/dist-engine-src/src/json_store/types.rs +0 -212
  110. package/dist-engine-src/src/lib.rs +0 -92
  111. package/dist-engine-src/src/live_state/context.rs +0 -1883
  112. package/dist-engine-src/src/live_state/mod.rs +0 -21
  113. package/dist-engine-src/src/live_state/overlay.rs +0 -75
  114. package/dist-engine-src/src/live_state/reader.rs +0 -23
  115. package/dist-engine-src/src/live_state/types.rs +0 -231
  116. package/dist-engine-src/src/live_state/visibility.rs +0 -666
  117. package/dist-engine-src/src/plugin/archive.rs +0 -438
  118. package/dist-engine-src/src/plugin/component.rs +0 -183
  119. package/dist-engine-src/src/plugin/install.rs +0 -619
  120. package/dist-engine-src/src/plugin/manifest.rs +0 -516
  121. package/dist-engine-src/src/plugin/materializer.rs +0 -202
  122. package/dist-engine-src/src/plugin/mod.rs +0 -33
  123. package/dist-engine-src/src/plugin/plugin_manifest.json +0 -119
  124. package/dist-engine-src/src/plugin/storage.rs +0 -74
  125. package/dist-engine-src/src/schema/annotations/defaults.rs +0 -275
  126. package/dist-engine-src/src/schema/annotations/mod.rs +0 -1
  127. package/dist-engine-src/src/schema/builtin/lix_account.json +0 -21
  128. package/dist-engine-src/src/schema/builtin/lix_active_account.json +0 -29
  129. package/dist-engine-src/src/schema/builtin/lix_binary_blob_ref.json +0 -29
  130. package/dist-engine-src/src/schema/builtin/lix_branch_descriptor.json +0 -34
  131. package/dist-engine-src/src/schema/builtin/lix_branch_ref.json +0 -48
  132. package/dist-engine-src/src/schema/builtin/lix_change.json +0 -63
  133. package/dist-engine-src/src/schema/builtin/lix_change_author.json +0 -45
  134. package/dist-engine-src/src/schema/builtin/lix_commit.json +0 -24
  135. package/dist-engine-src/src/schema/builtin/lix_commit_edge.json +0 -53
  136. package/dist-engine-src/src/schema/builtin/lix_directory_descriptor.json +0 -52
  137. package/dist-engine-src/src/schema/builtin/lix_file_descriptor.json +0 -52
  138. package/dist-engine-src/src/schema/builtin/lix_key_value.json +0 -40
  139. package/dist-engine-src/src/schema/builtin/lix_label.json +0 -29
  140. package/dist-engine-src/src/schema/builtin/lix_label_assignment.json +0 -74
  141. package/dist-engine-src/src/schema/builtin/lix_registered_schema.json +0 -25
  142. package/dist-engine-src/src/schema/builtin/mod.rs +0 -220
  143. package/dist-engine-src/src/schema/compatibility.rs +0 -787
  144. package/dist-engine-src/src/schema/definition.json +0 -187
  145. package/dist-engine-src/src/schema/definition.rs +0 -742
  146. package/dist-engine-src/src/schema/key.rs +0 -138
  147. package/dist-engine-src/src/schema/mod.rs +0 -20
  148. package/dist-engine-src/src/schema/seed.rs +0 -14
  149. package/dist-engine-src/src/schema/tests.rs +0 -780
  150. package/dist-engine-src/src/session/context.rs +0 -1059
  151. package/dist-engine-src/src/session/create_branch.rs +0 -94
  152. package/dist-engine-src/src/session/execute.rs +0 -681
  153. package/dist-engine-src/src/session/merge/analysis.rs +0 -108
  154. package/dist-engine-src/src/session/merge/branch.rs +0 -417
  155. package/dist-engine-src/src/session/merge/conflicts.rs +0 -63
  156. package/dist-engine-src/src/session/merge/mod.rs +0 -10
  157. package/dist-engine-src/src/session/merge/stats.rs +0 -61
  158. package/dist-engine-src/src/session/mod.rs +0 -30
  159. package/dist-engine-src/src/session/switch_branch.rs +0 -113
  160. package/dist-engine-src/src/session/transaction.rs +0 -557
  161. package/dist-engine-src/src/sql2/bind/classify.rs +0 -102
  162. package/dist-engine-src/src/sql2/bind/error.rs +0 -5
  163. package/dist-engine-src/src/sql2/bind/expr.rs +0 -29
  164. package/dist-engine-src/src/sql2/bind/mod.rs +0 -12
  165. package/dist-engine-src/src/sql2/bind/public_udf.rs +0 -306
  166. package/dist-engine-src/src/sql2/bind/read.rs +0 -65
  167. package/dist-engine-src/src/sql2/bind/statement.rs +0 -2236
  168. package/dist-engine-src/src/sql2/bind/table.rs +0 -273
  169. package/dist-engine-src/src/sql2/bind/write.rs +0 -86
  170. package/dist-engine-src/src/sql2/branch_scope.rs +0 -436
  171. package/dist-engine-src/src/sql2/catalog/capability.rs +0 -20
  172. package/dist-engine-src/src/sql2/catalog/entity_surface.rs +0 -296
  173. package/dist-engine-src/src/sql2/catalog/mod.rs +0 -15
  174. package/dist-engine-src/src/sql2/catalog/registry.rs +0 -556
  175. package/dist-engine-src/src/sql2/catalog/schema.rs +0 -88
  176. package/dist-engine-src/src/sql2/catalog/surface.rs +0 -41
  177. package/dist-engine-src/src/sql2/change_materialization.rs +0 -122
  178. package/dist-engine-src/src/sql2/context.rs +0 -317
  179. package/dist-engine-src/src/sql2/dml.rs +0 -148
  180. package/dist-engine-src/src/sql2/error.rs +0 -215
  181. package/dist-engine-src/src/sql2/exec/bound_public_write.rs +0 -1593
  182. package/dist-engine-src/src/sql2/exec/datafusion.rs +0 -5266
  183. package/dist-engine-src/src/sql2/exec/fast_write.rs +0 -82
  184. package/dist-engine-src/src/sql2/exec/mod.rs +0 -24
  185. package/dist-engine-src/src/sql2/exec/write.rs +0 -661
  186. package/dist-engine-src/src/sql2/filesystem_planner.rs +0 -1485
  187. package/dist-engine-src/src/sql2/filesystem_predicates.rs +0 -159
  188. package/dist-engine-src/src/sql2/filesystem_visibility.rs +0 -383
  189. package/dist-engine-src/src/sql2/history_projection.rs +0 -56
  190. package/dist-engine-src/src/sql2/history_route.rs +0 -661
  191. package/dist-engine-src/src/sql2/mod.rs +0 -52
  192. package/dist-engine-src/src/sql2/optimize/datafusion.rs +0 -1
  193. package/dist-engine-src/src/sql2/optimize/mod.rs +0 -2
  194. package/dist-engine-src/src/sql2/optimize/simple_write.rs +0 -116
  195. package/dist-engine-src/src/sql2/parse/mod.rs +0 -69
  196. package/dist-engine-src/src/sql2/parse/normalize.rs +0 -1
  197. package/dist-engine-src/src/sql2/plan/branch_scope.rs +0 -24
  198. package/dist-engine-src/src/sql2/plan/mod.rs +0 -5
  199. package/dist-engine-src/src/sql2/plan/predicate.rs +0 -22
  200. package/dist-engine-src/src/sql2/plan/write.rs +0 -147
  201. package/dist-engine-src/src/sql2/predicate_typecheck.rs +0 -504
  202. package/dist-engine-src/src/sql2/providers/branch.rs +0 -1206
  203. package/dist-engine-src/src/sql2/providers/change.rs +0 -445
  204. package/dist-engine-src/src/sql2/providers/directory.rs +0 -2422
  205. package/dist-engine-src/src/sql2/providers/directory_history.rs +0 -645
  206. package/dist-engine-src/src/sql2/providers/entity.rs +0 -1484
  207. package/dist-engine-src/src/sql2/providers/entity_history.rs +0 -452
  208. package/dist-engine-src/src/sql2/providers/file.rs +0 -3686
  209. package/dist-engine-src/src/sql2/providers/file_history.rs +0 -924
  210. package/dist-engine-src/src/sql2/providers/history.rs +0 -426
  211. package/dist-engine-src/src/sql2/providers/lix_state.rs +0 -2542
  212. package/dist-engine-src/src/sql2/providers/mod.rs +0 -508
  213. package/dist-engine-src/src/sql2/read_only.rs +0 -63
  214. package/dist-engine-src/src/sql2/record_batch.rs +0 -17
  215. package/dist-engine-src/src/sql2/result_metadata.rs +0 -29
  216. package/dist-engine-src/src/sql2/runtime.rs +0 -60
  217. package/dist-engine-src/src/sql2/session.rs +0 -83
  218. package/dist-engine-src/src/sql2/storage/constraints.rs +0 -1
  219. package/dist-engine-src/src/sql2/storage/mod.rs +0 -1
  220. package/dist-engine-src/src/sql2/test_support/differential.rs +0 -712
  221. package/dist-engine-src/src/sql2/test_support/generators.rs +0 -354
  222. package/dist-engine-src/src/sql2/test_support/mod.rs +0 -2
  223. package/dist-engine-src/src/sql2/udfs/common.rs +0 -295
  224. package/dist-engine-src/src/sql2/udfs/lix_active_branch_commit_id.rs +0 -53
  225. package/dist-engine-src/src/sql2/udfs/lix_empty_blob.rs +0 -47
  226. package/dist-engine-src/src/sql2/udfs/lix_json.rs +0 -100
  227. package/dist-engine-src/src/sql2/udfs/lix_json_get.rs +0 -99
  228. package/dist-engine-src/src/sql2/udfs/lix_json_get_text.rs +0 -99
  229. package/dist-engine-src/src/sql2/udfs/lix_text_decode.rs +0 -82
  230. package/dist-engine-src/src/sql2/udfs/lix_text_encode.rs +0 -85
  231. package/dist-engine-src/src/sql2/udfs/lix_timestamp.rs +0 -76
  232. package/dist-engine-src/src/sql2/udfs/lix_uuid_v7.rs +0 -76
  233. package/dist-engine-src/src/sql2/udfs/mod.rs +0 -86
  234. package/dist-engine-src/src/sql2/write_normalization.rs +0 -368
  235. package/dist-engine-src/src/storage/conformance.rs +0 -399
  236. package/dist-engine-src/src/storage/context.rs +0 -620
  237. package/dist-engine-src/src/storage/mod.rs +0 -52
  238. package/dist-engine-src/src/storage/point.rs +0 -440
  239. package/dist-engine-src/src/storage/read_scope.rs +0 -67
  240. package/dist-engine-src/src/storage/reader.rs +0 -867
  241. package/dist-engine-src/src/storage/scan.rs +0 -784
  242. package/dist-engine-src/src/storage/spaces.rs +0 -236
  243. package/dist-engine-src/src/storage/stats.rs +0 -80
  244. package/dist-engine-src/src/storage/write_set.rs +0 -962
  245. package/dist-engine-src/src/storage_bench.rs +0 -171
  246. package/dist-engine-src/src/test_support.rs +0 -450
  247. package/dist-engine-src/src/tracked_state/bench_support.rs +0 -394
  248. package/dist-engine-src/src/tracked_state/codec.rs +0 -1183
  249. package/dist-engine-src/src/tracked_state/commit_root_rebuild.rs +0 -358
  250. package/dist-engine-src/src/tracked_state/context.rs +0 -2801
  251. package/dist-engine-src/src/tracked_state/diff.rs +0 -2140
  252. package/dist-engine-src/src/tracked_state/merge.rs +0 -478
  253. package/dist-engine-src/src/tracked_state/mod.rs +0 -35
  254. package/dist-engine-src/src/tracked_state/row_materialization.rs +0 -275
  255. package/dist-engine-src/src/tracked_state/storage.rs +0 -427
  256. package/dist-engine-src/src/tracked_state/tree.rs +0 -3063
  257. package/dist-engine-src/src/tracked_state/types.rs +0 -238
  258. package/dist-engine-src/src/transaction/bench_support.rs +0 -407
  259. package/dist-engine-src/src/transaction/commit.rs +0 -1592
  260. package/dist-engine-src/src/transaction/context.rs +0 -1653
  261. package/dist-engine-src/src/transaction/mod.rs +0 -24
  262. package/dist-engine-src/src/transaction/normalization.rs +0 -877
  263. package/dist-engine-src/src/transaction/prep.rs +0 -37
  264. package/dist-engine-src/src/transaction/schema_resolver.rs +0 -163
  265. package/dist-engine-src/src/transaction/staging.rs +0 -1525
  266. package/dist-engine-src/src/transaction/types.rs +0 -403
  267. package/dist-engine-src/src/transaction/validation.rs +0 -5766
  268. package/dist-engine-src/src/untracked_state/codec.rs +0 -615
  269. package/dist-engine-src/src/untracked_state/context.rs +0 -98
  270. package/dist-engine-src/src/untracked_state/materialization.rs +0 -63
  271. package/dist-engine-src/src/untracked_state/mod.rs +0 -15
  272. package/dist-engine-src/src/untracked_state/storage.rs +0 -898
  273. package/dist-engine-src/src/untracked_state/types.rs +0 -146
  274. package/dist-engine-src/src/wasm/mod.rs +0 -60
@@ -1,494 +0,0 @@
1
- use crate::json_store::compression::{compress_json_payload, decode_json_zstd_payload};
2
- use crate::json_store::encoded::{EncodedJson, JsonCodec};
3
- use crate::json_store::types::{JsonReadScopeRef, JsonRef};
4
- use crate::storage::{PointReadPlan, StorageRead, StorageSpace};
5
- use crate::storage::{StorageGetOptions, StorageKey, StorageProjectedValue, StorageSpaceId};
6
- use crate::LixError;
7
- use bytes::Bytes;
8
- use std::borrow::Cow;
9
- use std::collections::HashMap;
10
-
11
- pub(crate) const JSON_NAMESPACE: &str = "json_store.json";
12
- pub(crate) const JSON_SPACE: StorageSpace =
13
- StorageSpace::new(StorageSpaceId(0x0002_0001), JSON_NAMESPACE);
14
- const STORED_JSON_MAGIC: &[u8] = b"lix-json:v1";
15
- const STORED_JSON_HEADER_LEN: usize = STORED_JSON_MAGIC.len() + 1 + 8;
16
- const ZSTD_MIN_JSON_BYTES: usize = 16 * 1024;
17
- const MIN_ZSTD_SAVINGS_BYTES: usize = 128;
18
-
19
- struct StoredJsonPayload<'a> {
20
- codec: JsonCodec,
21
- uncompressed_len: usize,
22
- data: &'a [u8],
23
- }
24
-
25
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
26
- enum JsonHashCheck {
27
- /// Hot reads trust the local storage layer and pack directory. Content
28
- /// hashes are computed at write time; exhaustive verification belongs in
29
- /// explicit integrity-check/fsck callers rather than every row scan.
30
- TrustedHotRead,
31
- Verify,
32
- }
33
-
34
- fn raw_json_ref_for_content(json: &str) -> JsonRef {
35
- JsonRef::from_hash(blake3::hash(json.as_bytes()))
36
- }
37
-
38
- pub(crate) fn json_ref_for_content(bytes: &[u8]) -> JsonRef {
39
- JsonRef::for_content(bytes)
40
- }
41
-
42
- #[cfg(test)]
43
- fn encode_json(json: &str) -> Result<EncodedJson<'_>, LixError> {
44
- encode_json_for_storage(json)
45
- }
46
-
47
- fn encode_json_for_storage(json: &str) -> Result<EncodedJson<'_>, LixError> {
48
- let raw_ref = raw_json_ref_for_content(json);
49
- encode_json_for_storage_with_ref(json, raw_ref)
50
- }
51
-
52
- fn encode_json_for_storage_with_ref(
53
- json: &str,
54
- raw_ref: JsonRef,
55
- ) -> Result<EncodedJson<'_>, LixError> {
56
- let raw_data = json.as_bytes();
57
-
58
- if raw_data.len() >= ZSTD_MIN_JSON_BYTES {
59
- let compressed = compress_json_payload(raw_data)?;
60
- if raw_data.len().saturating_sub(compressed.len()) >= MIN_ZSTD_SAVINGS_BYTES {
61
- return Ok(EncodedJson {
62
- json_ref: raw_ref,
63
- codec: JsonCodec::Zstd,
64
- uncompressed_len: json.len(),
65
- data: Cow::Owned(compressed),
66
- });
67
- }
68
- }
69
-
70
- Ok(EncodedJson {
71
- json_ref: raw_ref,
72
- codec: JsonCodec::Raw,
73
- uncompressed_len: json.len(),
74
- data: Cow::Borrowed(raw_data),
75
- })
76
- }
77
-
78
- pub(crate) fn encode_json_str(json: &str) -> Result<EncodedJson<'_>, LixError> {
79
- encode_json_for_storage(json)
80
- }
81
-
82
- pub(crate) fn encode_json_str_with_ref(
83
- json: &str,
84
- json_ref: JsonRef,
85
- ) -> Result<EncodedJson<'_>, LixError> {
86
- debug_assert_eq!(JsonRef::for_content(json.as_bytes()), json_ref);
87
- encode_json_for_storage_with_ref(json, json_ref)
88
- }
89
-
90
- pub(crate) fn encode_direct_json_payload(encoded_json: &EncodedJson<'_>) -> Vec<u8> {
91
- encode_stored_json_payload(encoded_json)
92
- }
93
-
94
- pub(crate) fn encode_json_bytes_for_storage(bytes: &[u8]) -> Result<(JsonRef, Vec<u8>), LixError> {
95
- let json = std::str::from_utf8(bytes).map_err(|error| {
96
- LixError::new(
97
- "LIX_ERROR_UNKNOWN",
98
- format!("json bytes are invalid UTF-8: {error}"),
99
- )
100
- })?;
101
- let json_ref = JsonRef::from_hash(blake3::hash(bytes));
102
- encode_json_str_for_storage_with_ref(json, json_ref)
103
- }
104
-
105
- pub(crate) fn encode_json_str_for_storage_with_ref(
106
- json: &str,
107
- json_ref: JsonRef,
108
- ) -> Result<(JsonRef, Vec<u8>), LixError> {
109
- let encoded_json = encode_json_for_storage_with_ref(json, json_ref)?;
110
- let json_ref = encoded_json.json_ref.clone();
111
- Ok((json_ref, encode_stored_json_payload(&encoded_json)))
112
- }
113
-
114
- async fn load_json_bytes_direct(
115
- store: &(impl StorageRead + ?Sized),
116
- json_ref: &JsonRef,
117
- ) -> Result<Option<Vec<u8>>, LixError> {
118
- let result = load_values(store, JSON_SPACE, vec![json_ref.as_hash_bytes().to_vec()])?
119
- .into_iter()
120
- .next()
121
- .flatten();
122
- let Some(bytes) = result else {
123
- return Ok(None);
124
- };
125
- let stored_payload = decode_stored_json_payload(&bytes)?;
126
- let _ = store;
127
- decode_json_payload(json_ref, stored_payload, JsonHashCheck::TrustedHotRead).map(Some)
128
- }
129
-
130
- pub(crate) async fn load_json_bytes_many_in_scope(
131
- store: &(impl StorageRead + ?Sized),
132
- json_refs: &[JsonRef],
133
- scope: JsonReadScopeRef,
134
- ) -> Result<Vec<Option<Vec<u8>>>, LixError> {
135
- load_json_bytes_many_in_scope_with_hash_check(
136
- store,
137
- json_refs,
138
- scope,
139
- JsonHashCheck::TrustedHotRead,
140
- )
141
- .await
142
- }
143
-
144
- pub(crate) async fn verify_json_bytes_many_in_scope(
145
- store: &impl StorageRead,
146
- json_refs: &[JsonRef],
147
- scope: JsonReadScopeRef,
148
- ) -> Result<Vec<Option<Vec<u8>>>, LixError> {
149
- load_json_bytes_many_in_scope_with_hash_check(store, json_refs, scope, JsonHashCheck::Verify)
150
- .await
151
- }
152
-
153
- async fn load_json_bytes_many_in_scope_with_hash_check(
154
- store: &(impl StorageRead + ?Sized),
155
- json_refs: &[JsonRef],
156
- scope: JsonReadScopeRef,
157
- hash_check: JsonHashCheck,
158
- ) -> Result<Vec<Option<Vec<u8>>>, LixError> {
159
- if json_refs.is_empty() {
160
- return Ok(Vec::new());
161
- }
162
-
163
- let mut unique_keys = Vec::new();
164
- let mut unique_refs = Vec::new();
165
- let mut key_indexes = HashMap::<[u8; 32], usize>::new();
166
- let mut requested_indexes = Vec::with_capacity(json_refs.len());
167
- let mut has_duplicate_refs = false;
168
- for json_ref in json_refs {
169
- let hash = *json_ref.as_hash_array();
170
- let index = match key_indexes.get(&hash) {
171
- Some(index) => {
172
- has_duplicate_refs = true;
173
- *index
174
- }
175
- None => {
176
- let index = unique_keys.len();
177
- key_indexes.insert(hash, index);
178
- unique_keys.push(hash.to_vec());
179
- unique_refs.push(*json_ref);
180
- index
181
- }
182
- };
183
- requested_indexes.push(index);
184
- }
185
-
186
- let JsonReadScopeRef::OutOfBand = scope;
187
- let mut unique_values = vec![None; unique_refs.len()];
188
-
189
- let missing = unique_values
190
- .iter()
191
- .enumerate()
192
- .filter_map(|(index, value)| value.is_none().then_some(index))
193
- .collect::<Vec<_>>();
194
- if missing.is_empty() {
195
- return Ok(json_values_in_request_order(
196
- unique_values,
197
- requested_indexes,
198
- has_duplicate_refs,
199
- ));
200
- }
201
-
202
- let loaded = load_values(
203
- store,
204
- JSON_SPACE,
205
- missing
206
- .iter()
207
- .map(|&index| unique_keys[index].clone())
208
- .collect(),
209
- )?;
210
- if loaded.len() != missing.len() {
211
- return Err(LixError::new(
212
- LixError::CODE_INTERNAL_ERROR,
213
- format!(
214
- "json_store batch load returned {} values for {} requested refs",
215
- loaded.len(),
216
- missing.len()
217
- ),
218
- ));
219
- }
220
-
221
- for (index, stored_bytes) in loaded.into_iter().enumerate() {
222
- let unique_index = missing[index];
223
- let Some(stored_bytes) = stored_bytes else {
224
- continue;
225
- };
226
- let stored_payload = decode_stored_json_payload(&stored_bytes)?;
227
- let _ = store;
228
- unique_values[unique_index] = Some(decode_json_payload(
229
- &unique_refs[unique_index],
230
- stored_payload,
231
- hash_check,
232
- )?);
233
- }
234
-
235
- Ok(json_values_in_request_order(
236
- unique_values,
237
- requested_indexes,
238
- has_duplicate_refs,
239
- ))
240
- }
241
-
242
- fn json_values_in_request_order(
243
- unique_values: Vec<Option<Vec<u8>>>,
244
- requested_indexes: Vec<usize>,
245
- has_duplicate_refs: bool,
246
- ) -> Vec<Option<Vec<u8>>> {
247
- if !has_duplicate_refs {
248
- debug_assert_eq!(requested_indexes.len(), unique_values.len());
249
- debug_assert!(requested_indexes
250
- .iter()
251
- .copied()
252
- .enumerate()
253
- .all(|(request_index, unique_index)| request_index == unique_index));
254
- return unique_values;
255
- }
256
- requested_indexes
257
- .into_iter()
258
- .map(|index| unique_values[index].clone())
259
- .collect()
260
- }
261
-
262
- fn load_values(
263
- store: &(impl StorageRead + ?Sized),
264
- space: StorageSpace,
265
- keys: Vec<Vec<u8>>,
266
- ) -> Result<Vec<Option<Vec<u8>>>, LixError> {
267
- let keys = keys
268
- .into_iter()
269
- .map(|key| StorageKey(Bytes::from(key)))
270
- .collect::<Vec<_>>();
271
- let result =
272
- PointReadPlan::new(space, &keys).materialize(store, StorageGetOptions::default())?;
273
- Ok(result
274
- .value
275
- .into_iter()
276
- .map(|value| match value {
277
- Some(StorageProjectedValue::FullValue(bytes)) => Some(bytes.to_vec()),
278
- Some(StorageProjectedValue::KeyOnly) | None => None,
279
- })
280
- .collect())
281
- }
282
-
283
- fn encode_stored_json_payload(encoded_json: &EncodedJson<'_>) -> Vec<u8> {
284
- let mut out = Vec::with_capacity(STORED_JSON_HEADER_LEN + encoded_json.data.as_ref().len());
285
- out.extend_from_slice(STORED_JSON_MAGIC);
286
- out.push(json_codec_byte(encoded_json.codec));
287
- out.extend_from_slice(&(encoded_json.uncompressed_len as u64).to_be_bytes());
288
- out.extend_from_slice(encoded_json.data.as_ref());
289
- out
290
- }
291
-
292
- fn decode_stored_json_payload(bytes: &[u8]) -> Result<StoredJsonPayload<'_>, LixError> {
293
- if bytes.len() < STORED_JSON_HEADER_LEN {
294
- return Err(LixError::new(
295
- "LIX_ERROR_UNKNOWN",
296
- "stored JSON payload is truncated",
297
- ));
298
- }
299
- if &bytes[..STORED_JSON_MAGIC.len()] != STORED_JSON_MAGIC {
300
- return Err(LixError::new(
301
- "LIX_ERROR_UNKNOWN",
302
- "stored JSON payload has invalid header",
303
- ));
304
- }
305
- let codec = read_json_codec(bytes[STORED_JSON_MAGIC.len()])?;
306
- let len_start = STORED_JSON_MAGIC.len() + 1;
307
- let len_end = len_start + 8;
308
- let uncompressed_len = u64::from_be_bytes(
309
- bytes[len_start..len_end]
310
- .try_into()
311
- .expect("stored JSON length header is fixed size"),
312
- ) as usize;
313
- Ok(StoredJsonPayload {
314
- codec,
315
- uncompressed_len,
316
- data: &bytes[len_end..],
317
- })
318
- }
319
-
320
- fn json_codec_byte(codec: JsonCodec) -> u8 {
321
- match codec {
322
- JsonCodec::Raw => 0,
323
- JsonCodec::Zstd => 1,
324
- }
325
- }
326
-
327
- fn read_json_codec(byte: u8) -> Result<JsonCodec, LixError> {
328
- match byte {
329
- 0 => Ok(JsonCodec::Raw),
330
- 1 => Ok(JsonCodec::Zstd),
331
- _ => Err(LixError::new(
332
- "LIX_ERROR_UNKNOWN",
333
- format!("stored JSON payload has unknown codec byte {byte}"),
334
- )),
335
- }
336
- }
337
-
338
- fn decode_json_payload(
339
- json_ref: &JsonRef,
340
- stored_payload: StoredJsonPayload<'_>,
341
- hash_check: JsonHashCheck,
342
- ) -> Result<Vec<u8>, LixError> {
343
- let data = match stored_payload.codec {
344
- JsonCodec::Raw => Ok(stored_payload.data.to_vec()),
345
- JsonCodec::Zstd => decode_json_zstd_payload(
346
- stored_payload.data,
347
- stored_payload.uncompressed_len,
348
- &json_ref.to_hex(),
349
- ),
350
- }?;
351
- if data.len() != stored_payload.uncompressed_len {
352
- return Err(LixError::new(
353
- "LIX_ERROR_UNKNOWN",
354
- format!(
355
- "json ref '{}' decoded to {} bytes, expected {}",
356
- json_ref.to_hex(),
357
- data.len(),
358
- stored_payload.uncompressed_len
359
- ),
360
- ));
361
- }
362
- if hash_check == JsonHashCheck::Verify {
363
- let actual_hash = blake3::hash(&data);
364
- if actual_hash.as_bytes() != json_ref.as_hash_bytes() {
365
- return Err(LixError::new(
366
- "LIX_ERROR_UNKNOWN",
367
- format!("json ref '{}' hash mismatch", json_ref.to_hex()),
368
- ));
369
- }
370
- }
371
- Ok(data)
372
- }
373
-
374
- #[cfg(test)]
375
- mod tests {
376
- use super::*;
377
- use crate::storage::StorageContext;
378
- use crate::storage::{
379
- InMemoryStorageBackend, StorageKey, StorageReadOptions, StorageValue, StorageWriteOptions,
380
- };
381
-
382
- #[tokio::test]
383
- async fn json_roundtrips_raw_payload() {
384
- let storage = StorageContext::new(InMemoryStorageBackend::new());
385
- let json = "{\"value\":\"small\"}";
386
- let encoded = encode_json(json).expect("json should encode");
387
- assert_eq!(encoded.codec, JsonCodec::Raw);
388
-
389
- let mut writes = storage.new_write_set();
390
- writes.put(
391
- JSON_SPACE,
392
- StorageKey(Bytes::copy_from_slice(encoded.json_ref.as_hash_bytes())),
393
- StorageValue {
394
- bytes: Bytes::from(encode_stored_json_payload(&encoded)),
395
- },
396
- );
397
- storage
398
- .commit_write_set(writes, StorageWriteOptions::default())
399
- .expect("writes should commit");
400
-
401
- let store = storage
402
- .begin_read(StorageReadOptions::default())
403
- .expect("read should open");
404
- assert_eq!(
405
- load_json_bytes_direct(&store, &encoded.json_ref)
406
- .await
407
- .expect("json should load"),
408
- Some(json.as_bytes().to_vec())
409
- );
410
- }
411
-
412
- #[tokio::test]
413
- async fn json_batch_load_roundtrips_in_request_order() {
414
- let storage = StorageContext::new(InMemoryStorageBackend::new());
415
- let first = encode_json("{\"value\":\"first\"}").expect("first json should encode");
416
- let second = encode_json("{\"value\":\"second\"}").expect("second json should encode");
417
-
418
- let mut writes = storage.new_write_set();
419
- writes.put(
420
- JSON_SPACE,
421
- StorageKey(Bytes::copy_from_slice(first.json_ref.as_hash_bytes())),
422
- StorageValue {
423
- bytes: Bytes::from(encode_stored_json_payload(&first)),
424
- },
425
- );
426
- writes.put(
427
- JSON_SPACE,
428
- StorageKey(Bytes::copy_from_slice(second.json_ref.as_hash_bytes())),
429
- StorageValue {
430
- bytes: Bytes::from(encode_stored_json_payload(&second)),
431
- },
432
- );
433
- storage
434
- .commit_write_set(writes, StorageWriteOptions::default())
435
- .expect("writes should commit");
436
-
437
- let store = storage
438
- .begin_read(StorageReadOptions::default())
439
- .expect("read should open");
440
- let values = load_json_bytes_many_in_scope(
441
- &store,
442
- &[second.json_ref, first.json_ref, second.json_ref],
443
- JsonReadScopeRef::OutOfBand,
444
- )
445
- .await
446
- .expect("json batch should load");
447
-
448
- assert_eq!(
449
- values,
450
- vec![
451
- Some(second.data.as_ref().to_vec()),
452
- Some(first.data.as_ref().to_vec()),
453
- Some(second.data.as_ref().to_vec()),
454
- ]
455
- );
456
- }
457
-
458
- #[tokio::test]
459
- async fn verified_batch_load_rejects_hash_mismatch() {
460
- let storage = StorageContext::new(InMemoryStorageBackend::new());
461
- let requested_ref = JsonRef::for_content(br#"{"value":"requested"}"#);
462
- let stored = encode_json("{\"value\":\"different\"}").expect("stored json should encode");
463
-
464
- let mut writes = storage.new_write_set();
465
- writes.put(
466
- JSON_SPACE,
467
- StorageKey(Bytes::copy_from_slice(requested_ref.as_hash_bytes())),
468
- StorageValue {
469
- bytes: Bytes::from(encode_stored_json_payload(&stored)),
470
- },
471
- );
472
- storage
473
- .commit_write_set(writes, StorageWriteOptions::default())
474
- .expect("writes should commit");
475
-
476
- let store = storage
477
- .begin_read(StorageReadOptions::default())
478
- .expect("read should open");
479
- let trusted =
480
- load_json_bytes_many_in_scope(&store, &[requested_ref], JsonReadScopeRef::OutOfBand)
481
- .await
482
- .expect("trusted hot read should not hash-check");
483
- assert_eq!(trusted, vec![Some(stored.data.as_ref().to_vec())]);
484
-
485
- let error =
486
- verify_json_bytes_many_in_scope(&store, &[requested_ref], JsonReadScopeRef::OutOfBand)
487
- .await
488
- .expect_err("verified read should reject mismatched content address");
489
- assert!(
490
- error.to_string().contains("hash mismatch"),
491
- "error should mention hash mismatch: {error}"
492
- );
493
- }
494
- }
@@ -1,212 +0,0 @@
1
- use std::sync::Arc;
2
-
3
- use crate::LixError;
4
-
5
- #[derive(Debug, Clone, PartialEq, Eq)]
6
- pub(crate) struct NormalizedJson(Arc<str>);
7
-
8
- impl NormalizedJson {
9
- pub(crate) fn from_arc_unchecked(normalized: Arc<str>) -> Self {
10
- Self(normalized)
11
- }
12
-
13
- pub(crate) fn from_value(value: &serde_json::Value, context: &str) -> Result<Self, LixError> {
14
- let normalized: Arc<str> = serde_json::to_string(value)
15
- .map_err(|error| {
16
- LixError::new(
17
- LixError::CODE_UNKNOWN,
18
- format!("{context} failed to serialize as normalized JSON: {error}"),
19
- )
20
- })?
21
- .into();
22
- Ok(Self(normalized))
23
- }
24
-
25
- pub(crate) fn as_str(&self) -> &str {
26
- self.0.as_ref()
27
- }
28
-
29
- pub(crate) fn as_bytes(&self) -> &[u8] {
30
- self.as_str().as_bytes()
31
- }
32
- }
33
-
34
- #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
35
- pub(crate) struct JsonRef {
36
- hash: [u8; 32],
37
- }
38
-
39
- impl JsonRef {
40
- pub(crate) fn from_hash(hash: blake3::Hash) -> Self {
41
- Self {
42
- hash: *hash.as_bytes(),
43
- }
44
- }
45
-
46
- pub(crate) fn from_hash_bytes(hash: [u8; 32]) -> Self {
47
- Self { hash }
48
- }
49
-
50
- pub(crate) fn for_content(bytes: &[u8]) -> Self {
51
- Self::from_hash(blake3::hash(bytes))
52
- }
53
-
54
- pub(crate) fn as_hash_bytes(&self) -> &[u8] {
55
- &self.hash
56
- }
57
-
58
- pub(crate) fn as_hash_array(&self) -> &[u8; 32] {
59
- &self.hash
60
- }
61
-
62
- pub(crate) fn to_hex(&self) -> String {
63
- self.hash.iter().map(|byte| format!("{byte:02x}")).collect()
64
- }
65
- }
66
-
67
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
68
- pub(crate) struct NormalizedJsonRef<'a> {
69
- normalized: &'a str,
70
- trusted_json_ref: Option<JsonRef>,
71
- }
72
-
73
- impl<'a> NormalizedJsonRef<'a> {
74
- pub(crate) fn new(normalized: &'a str) -> Self {
75
- Self {
76
- normalized,
77
- trusted_json_ref: None,
78
- }
79
- }
80
-
81
- /// Uses a caller-owned invariant that `json_ref` was computed from
82
- /// `normalized`. This avoids rehashing JSON already normalized by the
83
- /// transaction staging boundary.
84
- pub(crate) fn trusted_prehashed(normalized: &'a str, json_ref: JsonRef) -> Self {
85
- Self {
86
- normalized,
87
- trusted_json_ref: Some(json_ref),
88
- }
89
- }
90
-
91
- pub(crate) fn normalized(&self) -> &'a str {
92
- self.normalized
93
- }
94
-
95
- pub(crate) fn trusted_json_ref(&self) -> Option<JsonRef> {
96
- self.trusted_json_ref
97
- }
98
- }
99
-
100
- impl<'a> From<&'a NormalizedJson> for NormalizedJsonRef<'a> {
101
- fn from(value: &'a NormalizedJson) -> Self {
102
- Self::new(value.as_str())
103
- }
104
- }
105
-
106
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
107
- pub(crate) enum JsonWritePlacementRef {
108
- OutOfBand,
109
- }
110
-
111
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
112
- pub(crate) enum JsonReadScopeRef {
113
- OutOfBand,
114
- }
115
-
116
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
117
- pub(crate) struct JsonLoadRequestRef<'a> {
118
- pub(crate) refs: &'a [JsonRef],
119
- pub(crate) scope: JsonReadScopeRef,
120
- }
121
-
122
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
123
- pub(crate) struct JsonProjectionLoadRequestRef<'a> {
124
- pub(crate) refs: &'a [JsonRef],
125
- pub(crate) scope: JsonReadScopeRef,
126
- pub(crate) paths: &'a [JsonProjectionPath],
127
- }
128
-
129
- #[derive(Debug, Clone, PartialEq, Eq)]
130
- pub(crate) struct JsonLoadBatch {
131
- values: Vec<Option<Vec<u8>>>,
132
- }
133
-
134
- impl JsonLoadBatch {
135
- pub(crate) fn new(values: Vec<Option<Vec<u8>>>) -> Self {
136
- Self { values }
137
- }
138
-
139
- pub(crate) fn values(&self) -> &[Option<Vec<u8>>] {
140
- &self.values
141
- }
142
-
143
- pub(crate) fn into_values(self) -> Vec<Option<Vec<u8>>> {
144
- self.values
145
- }
146
- }
147
-
148
- #[derive(Debug, Clone, PartialEq)]
149
- pub(crate) struct JsonValueBatch {
150
- values: Vec<Option<serde_json::Value>>,
151
- }
152
-
153
- impl JsonValueBatch {
154
- pub(crate) fn new(values: Vec<Option<serde_json::Value>>) -> Self {
155
- Self { values }
156
- }
157
-
158
- pub(crate) fn values(&self) -> &[Option<serde_json::Value>] {
159
- &self.values
160
- }
161
-
162
- pub(crate) fn into_values(self) -> Vec<Option<serde_json::Value>> {
163
- self.values
164
- }
165
- }
166
-
167
- #[derive(Debug, Clone, PartialEq, Eq)]
168
- pub(crate) struct JsonProjectionPath(String);
169
-
170
- impl JsonProjectionPath {
171
- pub(crate) fn new(pointer: impl Into<String>) -> Self {
172
- Self(pointer.into())
173
- }
174
-
175
- pub(crate) fn as_str(&self) -> &str {
176
- &self.0
177
- }
178
- }
179
-
180
- #[derive(Debug, Clone, PartialEq)]
181
- pub(crate) struct JsonProjection {
182
- values: Vec<Option<serde_json::Value>>,
183
- }
184
-
185
- impl JsonProjection {
186
- pub(crate) fn new(values: Vec<Option<serde_json::Value>>) -> Self {
187
- Self { values }
188
- }
189
-
190
- pub(crate) fn values(&self) -> &[Option<serde_json::Value>] {
191
- &self.values
192
- }
193
- }
194
-
195
- #[derive(Debug, Clone, PartialEq)]
196
- pub(crate) struct JsonProjectionBatch {
197
- values: Vec<Option<JsonProjection>>,
198
- }
199
-
200
- impl JsonProjectionBatch {
201
- pub(crate) fn new(values: Vec<Option<JsonProjection>>) -> Self {
202
- Self { values }
203
- }
204
-
205
- pub(crate) fn values(&self) -> &[Option<JsonProjection>] {
206
- &self.values
207
- }
208
-
209
- pub(crate) fn into_values(self) -> Vec<Option<JsonProjection>> {
210
- self.values
211
- }
212
- }