@elaraai/east 1.0.79 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/comparison.d.ts.map +1 -1
- package/dist/src/comparison.js +4 -5
- package/dist/src/comparison.js.map +1 -1
- package/dist/src/compile/builtins/array.d.ts +59 -0
- package/dist/src/compile/builtins/array.d.ts.map +1 -0
- package/dist/src/compile/builtins/array.js +656 -0
- package/dist/src/compile/builtins/array.js.map +1 -0
- package/dist/src/compile/builtins/blob.d.ts +18 -0
- package/dist/src/compile/builtins/blob.d.ts.map +1 -0
- package/dist/src/compile/builtins/blob.js +150 -0
- package/dist/src/compile/builtins/blob.js.map +1 -0
- package/dist/src/compile/builtins/compare.d.ts +18 -0
- package/dist/src/compile/builtins/compare.d.ts.map +1 -0
- package/dist/src/compile/builtins/compare.js +52 -0
- package/dist/src/compile/builtins/compare.js.map +1 -0
- package/dist/src/compile/builtins/datetime.d.ts +21 -0
- package/dist/src/compile/builtins/datetime.d.ts.map +1 -0
- package/dist/src/compile/builtins/datetime.js +46 -0
- package/dist/src/compile/builtins/datetime.js.map +1 -0
- package/dist/src/compile/builtins/dict.d.ts +46 -0
- package/dist/src/compile/builtins/dict.d.ts.map +1 -0
- package/dist/src/compile/builtins/dict.js +551 -0
- package/dist/src/compile/builtins/dict.js.map +1 -0
- package/dist/src/compile/builtins/index.d.ts +9 -0
- package/dist/src/compile/builtins/index.d.ts.map +1 -0
- package/dist/src/compile/builtins/index.js +28 -0
- package/dist/src/compile/builtins/index.js.map +1 -0
- package/dist/src/compile/builtins/matrix.d.ts +31 -0
- package/dist/src/compile/builtins/matrix.d.ts.map +1 -0
- package/dist/src/compile/builtins/matrix.js +310 -0
- package/dist/src/compile/builtins/matrix.js.map +1 -0
- package/dist/src/compile/builtins/primitives.d.ts +36 -0
- package/dist/src/compile/builtins/primitives.d.ts.map +1 -0
- package/dist/src/compile/builtins/primitives.js +67 -0
- package/dist/src/compile/builtins/primitives.js.map +1 -0
- package/dist/src/compile/builtins/ref.d.ts +11 -0
- package/dist/src/compile/builtins/ref.d.ts.map +1 -0
- package/dist/src/compile/builtins/ref.js +30 -0
- package/dist/src/compile/builtins/ref.js.map +1 -0
- package/dist/src/compile/builtins/set.d.ts +41 -0
- package/dist/src/compile/builtins/set.d.ts.map +1 -0
- package/dist/src/compile/builtins/set.js +385 -0
- package/dist/src/compile/builtins/set.js.map +1 -0
- package/dist/src/compile/builtins/sparse.d.ts +19 -0
- package/dist/src/compile/builtins/sparse.d.ts.map +1 -0
- package/dist/src/compile/builtins/sparse.js +143 -0
- package/dist/src/compile/builtins/sparse.js.map +1 -0
- package/dist/src/compile/builtins/string.d.ts +34 -0
- package/dist/src/compile/builtins/string.d.ts.map +1 -0
- package/dist/src/compile/builtins/string.js +238 -0
- package/dist/src/compile/builtins/string.js.map +1 -0
- package/dist/src/compile/builtins/vector.d.ts +44 -0
- package/dist/src/compile/builtins/vector.d.ts.map +1 -0
- package/dist/src/compile/builtins/vector.js +463 -0
- package/dist/src/compile/builtins/vector.js.map +1 -0
- package/dist/src/compile/constructors.d.ts +13 -0
- package/dist/src/compile/constructors.d.ts.map +1 -0
- package/dist/src/compile/constructors.js +179 -0
- package/dist/src/compile/constructors.js.map +1 -0
- package/dist/src/compile/control.d.ts +13 -0
- package/dist/src/compile/control.d.ts.map +1 -0
- package/dist/src/compile/control.js +513 -0
- package/dist/src/compile/control.js.map +1 -0
- package/dist/src/compile/functions.d.ts +13 -0
- package/dist/src/compile/functions.d.ts.map +1 -0
- package/dist/src/compile/functions.js +492 -0
- package/dist/src/compile/functions.js.map +1 -0
- package/dist/src/compile/ir.d.ts +19 -0
- package/dist/src/compile/ir.d.ts.map +1 -0
- package/dist/src/compile/ir.js +193 -0
- package/dist/src/compile/ir.js.map +1 -0
- package/dist/src/compile/runtime.d.ts +78 -0
- package/dist/src/compile/runtime.d.ts.map +1 -0
- package/dist/src/compile/runtime.js +111 -0
- package/dist/src/compile/runtime.js.map +1 -0
- package/dist/src/compile/type_parameters.d.ts +4 -0
- package/dist/src/compile/type_parameters.d.ts.map +1 -0
- package/dist/src/compile/type_parameters.js +55 -0
- package/dist/src/compile/type_parameters.js.map +1 -0
- package/dist/src/compile/typed_arrays.d.ts +28 -0
- package/dist/src/compile/typed_arrays.d.ts.map +1 -0
- package/dist/src/compile/typed_arrays.js +77 -0
- package/dist/src/compile/typed_arrays.js.map +1 -0
- package/dist/src/compile.d.ts +6 -41
- package/dist/src/compile.d.ts.map +1 -1
- package/dist/src/compile.js +8 -4415
- package/dist/src/compile.js.map +1 -1
- package/dist/src/compile.open_beast.spec.js +5 -4
- package/dist/src/compile.open_beast.spec.js.map +1 -1
- package/dist/src/containers/sortedmap.d.ts +2 -1
- package/dist/src/containers/sortedmap.d.ts.map +1 -1
- package/dist/src/containers/sortedmap.js +4 -2
- package/dist/src/containers/sortedmap.js.map +1 -1
- package/dist/src/containers/sortedset.d.ts +2 -1
- package/dist/src/containers/sortedset.d.ts.map +1 -1
- package/dist/src/containers/sortedset.js +4 -2
- package/dist/src/containers/sortedset.js.map +1 -1
- package/dist/src/error.d.ts +15 -0
- package/dist/src/error.d.ts.map +1 -1
- package/dist/src/error.js +16 -0
- package/dist/src/error.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/patch/index.d.ts +1 -1
- package/dist/src/patch/index.d.ts.map +1 -1
- package/dist/src/patch/index.js +1 -1
- package/dist/src/patch/index.js.map +1 -1
- package/dist/src/patch/type_of_patch.d.ts +25 -1
- package/dist/src/patch/type_of_patch.d.ts.map +1 -1
- package/dist/src/patch/type_of_patch.js +33 -10
- package/dist/src/patch/type_of_patch.js.map +1 -1
- package/dist/src/platform.spec.js +19 -1
- package/dist/src/platform.spec.js.map +1 -1
- package/dist/src/runner_protocol.d.ts +316 -0
- package/dist/src/runner_protocol.d.ts.map +1 -0
- package/dist/src/runner_protocol.js +135 -0
- package/dist/src/runner_protocol.js.map +1 -0
- package/dist/src/serialization/beast2/index.d.ts +21 -5
- package/dist/src/serialization/beast2/index.d.ts.map +1 -1
- package/dist/src/serialization/beast2/index.js +12 -2
- package/dist/src/serialization/beast2/index.js.map +1 -1
- package/dist/src/serialization/beast2/index.spec.js +84 -3
- package/dist/src/serialization/beast2/index.spec.js.map +1 -1
- package/dist/src/serialization/beast2/shared.d.ts +32 -3
- package/dist/src/serialization/beast2/shared.d.ts.map +1 -1
- package/dist/src/serialization/beast2/shared.js +135 -10
- package/dist/src/serialization/beast2/shared.js.map +1 -1
- package/dist/src/serialization/beast2/v4/container.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v4/container.js +5 -3
- package/dist/src/serialization/beast2/v4/container.js.map +1 -1
- package/dist/src/serialization/beast2/v5/aliasing.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/aliasing.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/aliasing.spec.js +75 -0
- package/dist/src/serialization/beast2/v5/aliasing.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/boundary.d.ts +176 -0
- package/dist/src/serialization/beast2/v5/boundary.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/boundary.js +297 -0
- package/dist/src/serialization/beast2/v5/boundary.js.map +1 -0
- package/dist/src/serialization/beast2/v5/boundary.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/boundary.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/boundary.spec.js +479 -0
- package/dist/src/serialization/beast2/v5/boundary.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/codec.d.ts +6 -5
- package/dist/src/serialization/beast2/v5/codec.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/codec.js +18 -3
- package/dist/src/serialization/beast2/v5/codec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/deflate.d.ts +24 -2
- package/dist/src/serialization/beast2/v5/deflate.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/deflate.js +47 -12
- package/dist/src/serialization/beast2/v5/deflate.js.map +1 -1
- package/dist/src/serialization/beast2/v5/deflate.spec.js +15 -1
- package/dist/src/serialization/beast2/v5/deflate.spec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/elements.d.ts +64 -0
- package/dist/src/serialization/beast2/v5/elements.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/elements.js +320 -0
- package/dist/src/serialization/beast2/v5/elements.js.map +1 -0
- package/dist/src/serialization/beast2/v5/elements.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/elements.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/elements.spec.js +161 -0
- package/dist/src/serialization/beast2/v5/elements.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/frame-pool.d.ts +62 -21
- package/dist/src/serialization/beast2/v5/frame-pool.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/frame-pool.js +138 -27
- package/dist/src/serialization/beast2/v5/frame-pool.js.map +1 -1
- package/dist/src/serialization/beast2/v5/frame-pool.spec.js +198 -71
- package/dist/src/serialization/beast2/v5/frame-pool.spec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/frame-worker.js +25 -15
- package/dist/src/serialization/beast2/v5/frame-worker.js.map +1 -1
- package/dist/src/serialization/beast2/v5/frames.d.ts +22 -12
- package/dist/src/serialization/beast2/v5/frames.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/frames.js +52 -27
- package/dist/src/serialization/beast2/v5/frames.js.map +1 -1
- package/dist/src/serialization/beast2/v5/geometry.d.ts +38 -1
- package/dist/src/serialization/beast2/v5/geometry.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/geometry.js +73 -1
- package/dist/src/serialization/beast2/v5/geometry.js.map +1 -1
- package/dist/src/serialization/beast2/v5/geometry.spec.js +34 -23
- package/dist/src/serialization/beast2/v5/geometry.spec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/index.spec.js +32 -28
- package/dist/src/serialization/beast2/v5/index.spec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/lazy.d.ts +42 -1
- package/dist/src/serialization/beast2/v5/lazy.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/lazy.js +143 -28
- package/dist/src/serialization/beast2/v5/lazy.js.map +1 -1
- package/dist/src/serialization/beast2/v5/lazy.spec.js +102 -24
- package/dist/src/serialization/beast2/v5/lazy.spec.js.map +1 -1
- package/dist/src/serialization/beast2/v5/manifest-writer.d.ts +110 -0
- package/dist/src/serialization/beast2/v5/manifest-writer.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/manifest-writer.js +118 -0
- package/dist/src/serialization/beast2/v5/manifest-writer.js.map +1 -0
- package/dist/src/serialization/beast2/v5/manifest-writer.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/manifest-writer.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/manifest-writer.spec.js +207 -0
- package/dist/src/serialization/beast2/v5/manifest-writer.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/manifest.d.ts +241 -0
- package/dist/src/serialization/beast2/v5/manifest.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/manifest.js +233 -0
- package/dist/src/serialization/beast2/v5/manifest.js.map +1 -0
- package/dist/src/serialization/beast2/v5/merge.d.ts +93 -0
- package/dist/src/serialization/beast2/v5/merge.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/merge.js +202 -0
- package/dist/src/serialization/beast2/v5/merge.js.map +1 -0
- package/dist/src/serialization/beast2/v5/merge.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/merge.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/merge.spec.js +118 -0
- package/dist/src/serialization/beast2/v5/merge.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/recut.d.ts +143 -0
- package/dist/src/serialization/beast2/v5/recut.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/recut.js +365 -0
- package/dist/src/serialization/beast2/v5/recut.js.map +1 -0
- package/dist/src/serialization/beast2/v5/recut.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/recut.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/recut.spec.js +215 -0
- package/dist/src/serialization/beast2/v5/recut.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/runs.d.ts +148 -0
- package/dist/src/serialization/beast2/v5/runs.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/runs.js +260 -0
- package/dist/src/serialization/beast2/v5/runs.js.map +1 -0
- package/dist/src/serialization/beast2/v5/runs.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/runs.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/runs.spec.js +182 -0
- package/dist/src/serialization/beast2/v5/runs.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/sha256.d.ts +19 -0
- package/dist/src/serialization/beast2/v5/sha256.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/sha256.js +103 -0
- package/dist/src/serialization/beast2/v5/sha256.js.map +1 -0
- package/dist/src/serialization/beast2/v5/sha256.spec.d.ts +6 -0
- package/dist/src/serialization/beast2/v5/sha256.spec.d.ts.map +1 -0
- package/dist/src/serialization/beast2/v5/sha256.spec.js +42 -0
- package/dist/src/serialization/beast2/v5/sha256.spec.js.map +1 -0
- package/dist/src/serialization/beast2/v5/stream.d.ts +273 -86
- package/dist/src/serialization/beast2/v5/stream.d.ts.map +1 -1
- package/dist/src/serialization/beast2/v5/stream.js +576 -244
- package/dist/src/serialization/beast2/v5/stream.js.map +1 -1
- package/dist/src/serialization/beast2/v5/sync-reader.spec.js +24 -10
- package/dist/src/serialization/beast2/v5/sync-reader.spec.js.map +1 -1
- package/dist/src/serialization/index.d.ts +1 -1
- package/dist/src/serialization/index.d.ts.map +1 -1
- package/dist/src/serialization/index.js +1 -1
- package/dist/src/serialization/index.js.map +1 -1
- package/dist/src/serialization/json.d.ts +28 -0
- package/dist/src/serialization/json.d.ts.map +1 -1
- package/dist/src/serialization/json.js +141 -30
- package/dist/src/serialization/json.js.map +1 -1
- package/dist/src/serialization/json.spec.js +47 -2
- package/dist/src/serialization/json.spec.js.map +1 -1
- package/dist/src/serialization/json_schema.d.ts +15 -11
- package/dist/src/serialization/json_schema.d.ts.map +1 -1
- package/dist/src/serialization/json_schema.js +21 -36
- package/dist/src/serialization/json_schema.js.map +1 -1
- package/dist/src/serialization/json_schema.spec.js +26 -32
- package/dist/src/serialization/json_schema.spec.js.map +1 -1
- package/dist/src/serialization/json_schema_to_type.d.ts +10 -5
- package/dist/src/serialization/json_schema_to_type.d.ts.map +1 -1
- package/dist/src/serialization/json_schema_to_type.js +13 -6
- package/dist/src/serialization/json_schema_to_type.js.map +1 -1
- package/dist/src/serialization/json_schema_to_type.spec.js +19 -0
- package/dist/src/serialization/json_schema_to_type.spec.js.map +1 -1
- package/dist/src/string_order.d.ts +36 -0
- package/dist/src/string_order.d.ts.map +1 -0
- package/dist/src/string_order.js +64 -0
- package/dist/src/string_order.js.map +1 -0
- package/dist/src/string_order.spec.d.ts +6 -0
- package/dist/src/string_order.spec.d.ts.map +1 -0
- package/dist/src/string_order.spec.js +79 -0
- package/dist/src/string_order.spec.js.map +1 -0
- package/dist/src/type_of_type.d.ts +17 -0
- package/dist/src/type_of_type.d.ts.map +1 -1
- package/dist/src/type_of_type.js +66 -0
- package/dist/src/type_of_type.js.map +1 -1
- package/dist/src/type_of_type.to_etv.spec.js +69 -1
- package/dist/src/type_of_type.to_etv.spec.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* The merge of sorted collections: a key several inputs hold folds in input
|
|
7
|
+
* order; the output is the canonical blob of the merged value, whatever the
|
|
8
|
+
* inputs' segments; a key range merges just its keys; and every refusal names
|
|
9
|
+
* the input it is about.
|
|
10
|
+
*/
|
|
11
|
+
import { describe, test } from "node:test";
|
|
12
|
+
import assert from "node:assert/strict";
|
|
13
|
+
import { ArrayType, DictType, IntegerType, SetType, StringType } from "../../../types.js";
|
|
14
|
+
import { compareFor } from "../../../comparison.js";
|
|
15
|
+
import { toEastTypeValue } from "../../../type_of_type.js";
|
|
16
|
+
import { SortedMap, SortedSet } from "../../../index.js";
|
|
17
|
+
import { COLLECTION_MANIFEST_KIND, carveBeast2, decodeBeast2For, encodeBeast2FenceFor, encodeBeast2For, encodeBeast2PagedFor, encodeBeast2SegmentsFor, mergeBeast2For, openBeast2PagesFor, readBeast2Extents, segmentRuleFor, } from "../index.js";
|
|
18
|
+
const DictSS = DictType(StringType, StringType);
|
|
19
|
+
const DictSI = DictType(StringType, IntegerType);
|
|
20
|
+
/** A sorted Dict from `[key, value]` pairs. */
|
|
21
|
+
const dict = (entries) => new SortedMap(entries, compareFor(StringType));
|
|
22
|
+
/** Merges `sources` and returns the output blob and what the merge came to. */
|
|
23
|
+
function mergeInto(type, sources, options) {
|
|
24
|
+
const chunks = [];
|
|
25
|
+
const stats = mergeBeast2For(type, options)(sources, (bytes) => chunks.push(bytes));
|
|
26
|
+
return { blob: new Uint8Array(Buffer.concat(chunks)), stats };
|
|
27
|
+
}
|
|
28
|
+
describe("beast2 v5 merge of sorted collections", () => {
|
|
29
|
+
const concat = { merge: (_key, acc, value) => acc + value };
|
|
30
|
+
test("folds a key several inputs hold, in input order", () => {
|
|
31
|
+
const inputs = [
|
|
32
|
+
encodeBeast2PagedFor(DictSS)(dict([["a", "1"], ["b", "x"]])),
|
|
33
|
+
encodeBeast2PagedFor(DictSS)(dict([["a", "2"]])),
|
|
34
|
+
encodeBeast2PagedFor(DictSS)(dict([["a", "3"], ["c", "y"]])),
|
|
35
|
+
];
|
|
36
|
+
const { blob, stats } = mergeInto(DictSS, inputs, concat);
|
|
37
|
+
assert.deepEqual([...decodeBeast2For(DictSS)(blob)], [["a", "123"], ["b", "x"], ["c", "y"]]);
|
|
38
|
+
assert.deepEqual(stats, { inputs: 3, entries: 3, folds: 2 });
|
|
39
|
+
});
|
|
40
|
+
test("writes the canonical blob of the merged value, whatever the inputs' segments", () => {
|
|
41
|
+
// Inputs cut into segments of the test's own size: the output is still the
|
|
42
|
+
// bytes a writer of the merged value writes.
|
|
43
|
+
const evens = dict(Array.from({ length: 6_000 }, (_, i) => [`k${String(2 * i).padStart(6, "0")}`, BigInt(i)]));
|
|
44
|
+
const odds = dict(Array.from({ length: 6_000 }, (_, i) => [`k${String(2 * i + 1).padStart(6, "0")}`, BigInt(-i)]));
|
|
45
|
+
const inSegmentsOf = (value, size) => {
|
|
46
|
+
const entries = [...value];
|
|
47
|
+
const batches = [];
|
|
48
|
+
for (let i = 0; i < entries.length; i += size)
|
|
49
|
+
batches.push(dict(entries.slice(i, i + size)));
|
|
50
|
+
return encodeBeast2SegmentsFor(DictSI)(batches);
|
|
51
|
+
};
|
|
52
|
+
const { blob } = mergeInto(DictSI, [inSegmentsOf(evens, 700), inSegmentsOf(odds, 1_300)]);
|
|
53
|
+
assert.deepEqual(blob, encodeBeast2PagedFor(DictSI)(dict([...evens, ...odds])));
|
|
54
|
+
});
|
|
55
|
+
test("merges just the keys of a range", () => {
|
|
56
|
+
const value = dict(Array.from({ length: 5_000 }, (_, i) => [`k${String(i).padStart(6, "0")}`, BigInt(i)]));
|
|
57
|
+
const blob = encodeBeast2PagedFor(DictSI)(value);
|
|
58
|
+
const inRange = (from, to) => dict([...value].filter(([k]) => (from === undefined || k >= from) && (to === undefined || k < to)));
|
|
59
|
+
for (const [from, to] of [["k001000", "k003500"], ["k004000", undefined], [undefined, "k000010"], ["k002500", "k002500"]]) {
|
|
60
|
+
const { blob: merged } = mergeInto(DictSI, [blob], { from, to });
|
|
61
|
+
assert.deepEqual(merged, encodeBeast2PagedFor(DictSI)(inRange(from, to)), `[${from}, ${to})`);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
test("keeps a Set's shared element once under union", () => {
|
|
65
|
+
const type = SetType(IntegerType);
|
|
66
|
+
const set = (xs) => new SortedSet(xs, compareFor(IntegerType));
|
|
67
|
+
const { blob, stats } = mergeInto(type, [encodeBeast2PagedFor(type)(set([1n, 3n])), encodeBeast2PagedFor(type)(set([2n, 3n]))], { union: true });
|
|
68
|
+
assert.deepEqual([...decodeBeast2For(type)(blob)], [1n, 2n, 3n]);
|
|
69
|
+
assert.equal(stats.folds, 1);
|
|
70
|
+
});
|
|
71
|
+
test("refuses a key several inputs hold without a fold, naming the key", () => {
|
|
72
|
+
const inputs = [encodeBeast2PagedFor(DictSS)(dict([["a", "1"]])), encodeBeast2PagedFor(DictSS)(dict([["a", "2"]]))];
|
|
73
|
+
assert.throws(() => mergeInto(DictSS, inputs), { message: `beast2 v5: duplicate Dict key emitted: "a" — Dict keys must be unique` });
|
|
74
|
+
});
|
|
75
|
+
test("reads inputs through ranged access and through manifests", () => {
|
|
76
|
+
const value = dict(Array.from({ length: 3_000 }, (_, i) => [`k${String(i).padStart(6, "0")}`, `v${i}`]));
|
|
77
|
+
const entries = [...value];
|
|
78
|
+
const batches = [];
|
|
79
|
+
for (let i = 0; i < entries.length; i += 400)
|
|
80
|
+
batches.push(dict(entries.slice(i, i + 400)));
|
|
81
|
+
const blob = encodeBeast2SegmentsFor(DictSS)(batches);
|
|
82
|
+
const extents = readBeast2Extents(blob);
|
|
83
|
+
const pages = openBeast2PagesFor(DictSS)(blob);
|
|
84
|
+
const fence = encodeBeast2FenceFor(StringType);
|
|
85
|
+
const segments = extents.offsets.map((_, i) => carveBeast2(blob, i, i + 1, extents));
|
|
86
|
+
const manifest = {
|
|
87
|
+
manifest: {
|
|
88
|
+
kind: COLLECTION_MANIFEST_KIND,
|
|
89
|
+
level: 0n,
|
|
90
|
+
type: toEastTypeValue(DictSS),
|
|
91
|
+
rule: segmentRuleFor(DictSS),
|
|
92
|
+
header: "",
|
|
93
|
+
entries: segments.map((segment, i) => ({ hash: `${i}`, fence: fence(pages.fence(i)), count: BigInt(extents.counts[i]), bytes: BigInt(segment.length) })),
|
|
94
|
+
},
|
|
95
|
+
segment: (i) => segments[i],
|
|
96
|
+
};
|
|
97
|
+
const reader = { size: blob.length, read: (offset, length) => blob.subarray(offset, offset + length) };
|
|
98
|
+
const expected = encodeBeast2PagedFor(DictSS)(value);
|
|
99
|
+
assert.deepEqual(mergeInto(DictSS, [reader]).blob, expected);
|
|
100
|
+
assert.deepEqual(mergeInto(DictSS, [manifest]).blob, expected);
|
|
101
|
+
assert.deepEqual(mergeInto(DictSS, [manifest], { from: "k001234", to: "k002345" }).blob, encodeBeast2PagedFor(DictSS)(dict(entries.filter(([k]) => k >= "k001234" && k < "k002345"))));
|
|
102
|
+
});
|
|
103
|
+
test("names the input a refusal is about", () => {
|
|
104
|
+
const good = encodeBeast2PagedFor(DictSS)(dict([["a", "1"]]));
|
|
105
|
+
const otherType = encodeBeast2PagedFor(DictSI)(dict([["a", 1n]]));
|
|
106
|
+
assert.throws(() => mergeInto(DictSS, [good, otherType], { labels: ["a.beast2", "b.beast2"] }), (err) => err.message.startsWith("merge: input 1 (b.beast2) has type ") && err.message.includes(", expected "));
|
|
107
|
+
// An input without an index cannot be read by segment.
|
|
108
|
+
const whole = encodeBeast2For(DictSS)(dict([["a", "1"]]));
|
|
109
|
+
assert.throws(() => mergeInto(DictSS, [whole]), { message: "merge: input 0: beast2 v5: blob carries no index — carve/splice need one (write with the index enabled, the default)" });
|
|
110
|
+
});
|
|
111
|
+
test("refuses no inputs, a root it cannot merge, and a fold that does not fit the root", () => {
|
|
112
|
+
assert.throws(() => mergeInto(DictSS, []), { message: "merge: at least one input is needed" });
|
|
113
|
+
assert.throws(() => mergeBeast2For(ArrayType(IntegerType)), { message: "merge: inputs must be Set or Dict blobs, got Array" });
|
|
114
|
+
assert.throws(() => mergeBeast2For(SetType(IntegerType), { merge: (_k, a) => a }), /folds a Dict/);
|
|
115
|
+
assert.throws(() => mergeBeast2For(DictSS, { union: true }), /collapses a Set/);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
//# sourceMappingURL=merge.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"merge.spec.js","sourceRoot":"","sources":["../../../../../src/serialization/beast2/v5/merge.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EACL,wBAAwB,EAIxB,WAAW,EACX,eAAe,EACf,oBAAoB,EACpB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;AAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;AAEjD,+CAA+C;AAC/C,MAAM,IAAI,GAAG,CAAI,OAAsB,EAAwB,EAAE,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;AAEjH,+EAA+E;AAC/E,SAAS,SAAS,CAAC,IAA0C,EAAE,OAAqC,EAAE,OAA4B;IAChI,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACpF,OAAO,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AAChE,CAAC;AAED,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;IACrD,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,CAAC,IAAY,EAAE,GAAW,EAAE,KAAa,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC;IAEpF,IAAI,CAAC,iDAAiD,EAAE,GAAG,EAAE;QAC3D,MAAM,MAAM,GAAG;YACb,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YAC5D,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YAChD,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;SAC7D,CAAC;QACF,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1D,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,8EAA8E,EAAE,GAAG,EAAE;QACxF,2EAA2E;QAC3E,6CAA6C;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAqB,CAAC,CAAC,CAAC;QACnI,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC,CAAC,CAAC;QACvI,MAAM,YAAY,GAAG,CAAC,KAAgC,EAAE,IAAY,EAAc,EAAE;YAClF,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;YAC3B,MAAM,OAAO,GAAgC,EAAE,CAAC;YAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9F,OAAO,uBAAuB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC;QAClD,CAAC,CAAC;QACF,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1F,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,iCAAiC,EAAE,GAAG,EAAE;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAqB,CAAC,CAAC,CAAC;QAC/H,MAAM,IAAI,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,CAAC,IAAwB,EAAE,EAAsB,EAA6B,EAAE,CAC9F,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACtG,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAU,EAAE,CAAC;YACnI,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,IAAI,KAAK,EAAE,GAAG,CAAC,CAAC;QAChG,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACzD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;QAClC,MAAM,GAAG,GAAG,CAAC,EAAY,EAAqB,EAAE,CAAC,IAAI,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;QAC5F,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjJ,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACjE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC5E,MAAM,MAAM,GAAG,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpH,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,uEAAuE,EAAE,CAAC,CAAC;IACvI,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,0DAA0D,EAAE,GAAG,EAAE;QACpE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAqB,CAAC,CAAC,CAAC;QAC7H,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAgC,EAAE,CAAC;QAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,IAAI,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACrF,MAAM,QAAQ,GAAyB;YACrC,QAAQ,EAAE;gBACR,IAAI,EAAE,wBAAwB;gBAC9B,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC;gBAC7B,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC;gBAC5B,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;aAC1J;YACD,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAE;SAC7B,CAAC;QACF,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,MAAc,EAAE,MAAc,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;QACvH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7D,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC/D,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,EACrF,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,SAAS,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAClG,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC9C,MAAM,IAAI,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,MAAM,CAAC,MAAM,CACX,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC,EAChF,CAAC,GAAU,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,qCAAqC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CACrH,CAAC;QACF,uDAAuD;QACvD,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,CAAC,MAAM,CACX,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,EAChC,EAAE,OAAO,EAAE,sHAAsH,EAAE,CACpI,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,kFAAkF,EAAE,GAAG,EAAE;QAC5F,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;QAC/F,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC,CAAC;QAC/H,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC;QACnG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Re-cutting — a collection's pieces in, in order, and the canonical whole out,
|
|
7
|
+
* segment by segment.
|
|
8
|
+
*
|
|
9
|
+
* The cut rule decides where a segment starts from the elements since the last
|
|
10
|
+
* boundary and nothing else. So a piece the Writer wrote — a partition's
|
|
11
|
+
* output, the segments of a record around an edit — cuts where the whole cuts
|
|
12
|
+
* from the first boundary the two share: once the whole starts a segment where
|
|
13
|
+
* the piece started one, every later segment of the piece is a segment of the
|
|
14
|
+
* whole, byte for byte, and is carried over without being read. Only where the
|
|
15
|
+
* two can disagree is anything re-cut: at a seam, where a piece's last segment
|
|
16
|
+
* ended only because the piece did, and through elements given one by one — an
|
|
17
|
+
* edited region — until the whole starts a segment where a piece does again.
|
|
18
|
+
*
|
|
19
|
+
* What comes out is what the Writer writes for the whole value, segment for
|
|
20
|
+
* segment; nothing in it records where the pieces were cut.
|
|
21
|
+
*/
|
|
22
|
+
import { type EastTypeValue } from "../../../type_of_type.js";
|
|
23
|
+
import type { EastType } from "../../../types.js";
|
|
24
|
+
import { type Beast2ElementOf, type Beast2ElementWriterOptions, type Beast2Segment } from "./stream.js";
|
|
25
|
+
/**
|
|
26
|
+
* A segment of a collection the Writer wrote, as {@link recutBeast2For} takes
|
|
27
|
+
* it: what a manifest entry records of it, and a way to read it.
|
|
28
|
+
*
|
|
29
|
+
* A re-cut reads a segment only where it has to look inside — at a seam, or to
|
|
30
|
+
* re-cut through it — so a segment it carries over is never read at all.
|
|
31
|
+
*/
|
|
32
|
+
export interface Beast2SegmentRef {
|
|
33
|
+
/** Elements (pairs, for a Dict) in the segment; at least one. */
|
|
34
|
+
readonly count: number;
|
|
35
|
+
/** Set/Dict: the segment's first key in its canonical bare encoding
|
|
36
|
+
* (`encodeBeast2FenceFor`) — a manifest entry's fence. Array: empty, and
|
|
37
|
+
* the first element is read when a seam needs it. */
|
|
38
|
+
readonly fence: Uint8Array;
|
|
39
|
+
/** The segment's logical size, when the caller knows it; otherwise it is
|
|
40
|
+
* read from the segment's frame when a seam after it needs it. */
|
|
41
|
+
readonly logicalBytes?: number;
|
|
42
|
+
/** Set/Dict: the fence of the segment that followed this one in the
|
|
43
|
+
* collection it was written in. The rule started a segment at that key
|
|
44
|
+
* after this one, so a seam where the same key follows again is decided
|
|
45
|
+
* without reading anything. */
|
|
46
|
+
readonly nextFence?: Uint8Array;
|
|
47
|
+
/**
|
|
48
|
+
* Reads the segment.
|
|
49
|
+
*
|
|
50
|
+
* @returns the segment as a standalone blob holding exactly it
|
|
51
|
+
*/
|
|
52
|
+
read(): Uint8Array | Promise<Uint8Array>;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* A piece of a collection, as {@link recutBeast2For} takes it: a run of
|
|
56
|
+
* consecutive segments of a collection the Writer wrote, under the header the
|
|
57
|
+
* re-cut writes, or elements.
|
|
58
|
+
*/
|
|
59
|
+
export type Beast2RecutPiece<T extends EastType = EastType, R extends Beast2SegmentRef = Beast2SegmentRef> = {
|
|
60
|
+
readonly segments: Iterable<R> | AsyncIterable<R>;
|
|
61
|
+
} | {
|
|
62
|
+
readonly elements: Iterable<Beast2ElementOf<T>> | AsyncIterable<Beast2ElementOf<T>>;
|
|
63
|
+
};
|
|
64
|
+
/** Where {@link recutBeast2For} puts the whole's segments, in order. */
|
|
65
|
+
export interface Beast2RecutSink<R extends Beast2SegmentRef = Beast2SegmentRef> {
|
|
66
|
+
/**
|
|
67
|
+
* Receives a segment the re-cut wrote.
|
|
68
|
+
*
|
|
69
|
+
* @param segment - the segment, as a standalone blob
|
|
70
|
+
*/
|
|
71
|
+
written(segment: Beast2Segment): void | Promise<void>;
|
|
72
|
+
/**
|
|
73
|
+
* Receives a segment of a piece that is a segment of the whole as it
|
|
74
|
+
* stands, never read.
|
|
75
|
+
*
|
|
76
|
+
* @param segment - the reference it came as
|
|
77
|
+
*/
|
|
78
|
+
carried(segment: R): void | Promise<void>;
|
|
79
|
+
}
|
|
80
|
+
/** Options accepted by {@link recutBeast2For}: those of the Writer the
|
|
81
|
+
* written segments go through. `headerPrefix` is the header the carried
|
|
82
|
+
* segments are under, when that is not the one the Writer builds for the
|
|
83
|
+
* type. */
|
|
84
|
+
export type Beast2RecutOptions = Beast2ElementWriterOptions;
|
|
85
|
+
/** What a re-cut came to. */
|
|
86
|
+
export type Beast2RecutStats = {
|
|
87
|
+
/** The header every segment of the whole is under — what a manifest of it
|
|
88
|
+
* names. */
|
|
89
|
+
header: Uint8Array;
|
|
90
|
+
/** Segments carried over from the pieces. */
|
|
91
|
+
carried: number;
|
|
92
|
+
/** Segments written. */
|
|
93
|
+
written: number;
|
|
94
|
+
};
|
|
95
|
+
/**
|
|
96
|
+
* Builds a re-cut: `recut(pieces, sink)` writes the canonical whole of a
|
|
97
|
+
* collection given as pieces in order, segment by segment.
|
|
98
|
+
*
|
|
99
|
+
* A piece is either a run of consecutive segments of a collection the Writer
|
|
100
|
+
* wrote — given by reference, and read only where the re-cut must look inside
|
|
101
|
+
* one — or elements. A segment of a piece that is a segment of the whole is
|
|
102
|
+
* carried over as it came; everything else is written, cut by the rule. The
|
|
103
|
+
* segments are exactly those the Writer writes for the whole value, so an edit
|
|
104
|
+
* is a re-cut of the edited elements between the segments around them, and
|
|
105
|
+
* costs what the edit touched plus at most a segment or two each side.
|
|
106
|
+
*
|
|
107
|
+
* @param type - the collection type (Array/Set/Dict)
|
|
108
|
+
* @param options - the Writer's options for the written segments; its
|
|
109
|
+
* `headerPrefix` is the header the carried segments are under
|
|
110
|
+
* @returns a function taking the pieces, in order, and the sink the whole's
|
|
111
|
+
* segments go to, and resolving to what the re-cut came to
|
|
112
|
+
* @throws {TypeError} When `type` is not an Array, Set or Dict type.
|
|
113
|
+
*
|
|
114
|
+
* @remarks
|
|
115
|
+
* The pieces' segments must be the Writer's own — cut by the current rule,
|
|
116
|
+
* with aliasing scoped per element and the same codec — or they are carried
|
|
117
|
+
* over as something the Writer would not write; a collection stored some other
|
|
118
|
+
* way is given as its elements. Elements in one piece must ascend in East
|
|
119
|
+
* order (a Set's elements, a Dict's keys), which the re-cut checks; the order
|
|
120
|
+
* across pieces is the caller's. The returned function rejects, having written
|
|
121
|
+
* part of the whole, when a piece is out of order, a segment's blob is not the
|
|
122
|
+
* one segment its reference describes, or the sink rejects.
|
|
123
|
+
*
|
|
124
|
+
* @example
|
|
125
|
+
* ```ts
|
|
126
|
+
* const type = ArrayType(IntegerType);
|
|
127
|
+
* const blob = encodeBeast2PagedFor(type)(values);
|
|
128
|
+
* const extents = readBeast2Extents(blob);
|
|
129
|
+
* const segments = extents.counts.map((count, i) => ({
|
|
130
|
+
* count,
|
|
131
|
+
* fence: new Uint8Array(0),
|
|
132
|
+
* read: () => carveBeast2(blob, i, i + 1, extents),
|
|
133
|
+
* }));
|
|
134
|
+
* const parts: Uint8Array[] = [];
|
|
135
|
+
* await recutBeast2For(type)([{ segments }, { elements: [42n] }], {
|
|
136
|
+
* written: (segment) => { parts.push(segment.blob); },
|
|
137
|
+
* carried: (segment) => { parts.push(segment.read() as Uint8Array); },
|
|
138
|
+
* });
|
|
139
|
+
* spliceBeast2(parts); // encodeBeast2PagedFor(type)([...values, 42n])
|
|
140
|
+
* ```
|
|
141
|
+
*/
|
|
142
|
+
export declare function recutBeast2For<T extends EastType, R extends Beast2SegmentRef = Beast2SegmentRef>(type: T | EastTypeValue, options?: Beast2RecutOptions): (pieces: Iterable<Beast2RecutPiece<T, R>> | AsyncIterable<Beast2RecutPiece<T, R>>, sink: Beast2RecutSink<R>) => Promise<Beast2RecutStats>;
|
|
143
|
+
//# sourceMappingURL=recut.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recut.d.ts","sourceRoot":"","sources":["../../../../../src/serialization/beast2/v5/recut.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,KAAK,aAAa,EAAuC,MAAM,0BAA0B,CAAC;AACnG,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAUlD,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,0BAA0B,EAC/B,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AAErB;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;0DAEsD;IACtD,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B;uEACmE;IACnE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;;;oCAGgC;IAChC,QAAQ,CAAC,SAAS,CAAC,EAAE,UAAU,CAAC;IAChC;;;;OAIG;IACH,IAAI,IAAI,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CAC1C;AAED;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,EAAE,CAAC,SAAS,gBAAgB,GAAG,gBAAgB,IACrG;IAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;CAAE,GACrD;IAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAA;CAAE,CAAC;AAE5F,wEAAwE;AACxE,MAAM,WAAW,eAAe,CAAC,CAAC,SAAS,gBAAgB,GAAG,gBAAgB;IAC5E;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD;;;;;OAKG;IACH,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3C;AAED;;;YAGY;AACZ,MAAM,MAAM,kBAAkB,GAAG,0BAA0B,CAAC;AAE5D,6BAA6B;AAC7B,MAAM,MAAM,gBAAgB,GAAG;IAC7B;iBACa;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AA4PF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,gBAAgB,GAAG,gBAAgB,EAC9F,IAAI,EAAE,CAAC,GAAG,aAAa,EACvB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,CAAC,MAAM,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAwC3I"}
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Re-cutting — a collection's pieces in, in order, and the canonical whole out,
|
|
7
|
+
* segment by segment.
|
|
8
|
+
*
|
|
9
|
+
* The cut rule decides where a segment starts from the elements since the last
|
|
10
|
+
* boundary and nothing else. So a piece the Writer wrote — a partition's
|
|
11
|
+
* output, the segments of a record around an edit — cuts where the whole cuts
|
|
12
|
+
* from the first boundary the two share: once the whole starts a segment where
|
|
13
|
+
* the piece started one, every later segment of the piece is a segment of the
|
|
14
|
+
* whole, byte for byte, and is carried over without being read. Only where the
|
|
15
|
+
* two can disagree is anything re-cut: at a seam, where a piece's last segment
|
|
16
|
+
* ended only because the piece did, and through elements given one by one — an
|
|
17
|
+
* edited region — until the whole starts a segment where a piece does again.
|
|
18
|
+
*
|
|
19
|
+
* What comes out is what the Writer writes for the whole value, segment for
|
|
20
|
+
* segment; nothing in it records where the pieces were cut.
|
|
21
|
+
*/
|
|
22
|
+
import { EastTypeValueType, isTypeValueEqual } from "../../../type_of_type.js";
|
|
23
|
+
import { printFor } from "../../east.js";
|
|
24
|
+
import { compareFor } from "../../../comparison.js";
|
|
25
|
+
import { BufferReader, BufferWriter } from "../../binary-utils.js";
|
|
26
|
+
import { buildPlatformContext } from "../shared.js";
|
|
27
|
+
import { asTypeValue, readTypeSection } from "./type-section.js";
|
|
28
|
+
import { MAGIC_BYTES_V5, buildV5Decoder, isSegmentedRoot, readSourceMapSectionV5 } from "./codec.js";
|
|
29
|
+
import { FrameReader } from "./frames.js";
|
|
30
|
+
import { readBeast2Extents } from "./geometry.js";
|
|
31
|
+
import { startsSegmentAfter } from "./boundary.js";
|
|
32
|
+
import { Beast2ElementWriter, elementEncoderFor, } from "./stream.js";
|
|
33
|
+
function elementParserFor(typeValue) {
|
|
34
|
+
const typeCtx = new Map();
|
|
35
|
+
if (typeValue.type === "Dict") {
|
|
36
|
+
const key = buildV5Decoder(typeValue.value.key, typeCtx);
|
|
37
|
+
const value = buildV5Decoder(typeValue.value.value, typeCtx);
|
|
38
|
+
return (reader, ctx) => {
|
|
39
|
+
const start = reader.offset;
|
|
40
|
+
key(reader, ctx);
|
|
41
|
+
const keyLength = reader.offset - start;
|
|
42
|
+
value(reader, ctx);
|
|
43
|
+
return keyLength;
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const elem = buildV5Decoder(typeValue.value, typeCtx);
|
|
47
|
+
const keyed = typeValue.type === "Set";
|
|
48
|
+
return (reader, ctx) => {
|
|
49
|
+
const start = reader.offset;
|
|
50
|
+
elem(reader, ctx);
|
|
51
|
+
return keyed ? reader.offset - start : 0;
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/** A segment read for a re-cut: its frame's logical bytes, and where its
|
|
55
|
+
* elements lie in them. */
|
|
56
|
+
class SegmentContent {
|
|
57
|
+
logical;
|
|
58
|
+
start;
|
|
59
|
+
count;
|
|
60
|
+
parse;
|
|
61
|
+
context;
|
|
62
|
+
constructor(
|
|
63
|
+
/** The frame's logical bytes: the element count, then the elements. */
|
|
64
|
+
logical,
|
|
65
|
+
/** Where the first element starts. */
|
|
66
|
+
start, count, parse,
|
|
67
|
+
/** A decode context with an empty definition table. */
|
|
68
|
+
context) {
|
|
69
|
+
this.logical = logical;
|
|
70
|
+
this.start = start;
|
|
71
|
+
this.count = count;
|
|
72
|
+
this.parse = parse;
|
|
73
|
+
this.context = context;
|
|
74
|
+
}
|
|
75
|
+
/** The elements' canonical bytes before compression — what the cut rule
|
|
76
|
+
* measures a segment by. */
|
|
77
|
+
get logicalBytes() {
|
|
78
|
+
return this.logical.length - this.start;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Each element's bytes and the length of its key, in order.
|
|
82
|
+
*
|
|
83
|
+
* Every element decodes against an empty definition table, so one that
|
|
84
|
+
* refers to a container outside itself — a segment written before aliasing
|
|
85
|
+
* was scoped per element — fails here, rather than being moved by byte copy
|
|
86
|
+
* away from what it refers to.
|
|
87
|
+
*/
|
|
88
|
+
*elements() {
|
|
89
|
+
const reader = new BufferReader(this.logical, this.start);
|
|
90
|
+
for (let i = 0; i < this.count; i++) {
|
|
91
|
+
const start = reader.offset;
|
|
92
|
+
const keyLength = this.parse(reader, this.context());
|
|
93
|
+
yield [this.logical.subarray(start, reader.offset), keyLength];
|
|
94
|
+
}
|
|
95
|
+
if (reader.offset !== this.logical.length) {
|
|
96
|
+
throw new Error(`beast2 v5: ${this.logical.length - reader.offset} logical bytes after a segment's last element`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** One re-cut in progress. */
|
|
101
|
+
class Recutter {
|
|
102
|
+
typeValue;
|
|
103
|
+
parse;
|
|
104
|
+
sink;
|
|
105
|
+
carried = 0;
|
|
106
|
+
written = 0;
|
|
107
|
+
keyed;
|
|
108
|
+
writer;
|
|
109
|
+
encodeElement;
|
|
110
|
+
platform = buildPlatformContext();
|
|
111
|
+
/** Segments the writer has written that the sink has not been given yet. */
|
|
112
|
+
queue = [];
|
|
113
|
+
/** A segment that starts where the whole starts one, held until what
|
|
114
|
+
* follows says whether the whole ends one where it ends: then it is carried
|
|
115
|
+
* over whole, and otherwise its elements join what follows. The writer's
|
|
116
|
+
* open segment is empty while one is held. */
|
|
117
|
+
pending = null;
|
|
118
|
+
/** What has been read of the segments in hand, so each is read once. */
|
|
119
|
+
contents = new Map();
|
|
120
|
+
scratch = new BufferWriter();
|
|
121
|
+
constructor(typeValue, parse, sink, options) {
|
|
122
|
+
this.typeValue = typeValue;
|
|
123
|
+
this.parse = parse;
|
|
124
|
+
this.sink = sink;
|
|
125
|
+
this.keyed = typeValue.type !== "Array";
|
|
126
|
+
this.writer = new Beast2ElementWriter(typeValue, { segment: (segment) => { this.queue.push(segment); } }, options);
|
|
127
|
+
this.encodeElement = elementEncoderFor(typeValue, options?.sourceMap ?? null);
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Takes the next segment of a piece.
|
|
131
|
+
*
|
|
132
|
+
* @param ref - the segment
|
|
133
|
+
* @param continues - whether the segment before it came from the same piece,
|
|
134
|
+
* whose writer cut between them
|
|
135
|
+
*/
|
|
136
|
+
async segment(ref, continues) {
|
|
137
|
+
if (!Number.isInteger(ref.count) || ref.count < 1) {
|
|
138
|
+
throw new Error(`beast2 v5: recut: a segment holds at least one element, and a reference says ${ref.count}`);
|
|
139
|
+
}
|
|
140
|
+
const pending = this.pending;
|
|
141
|
+
if (pending !== null) {
|
|
142
|
+
// The whole started `pending` where its piece did, so it cuts after it
|
|
143
|
+
// wherever the piece did.
|
|
144
|
+
if (continues || await this.cutsAfter(pending, await this.hashInputOf(ref))) {
|
|
145
|
+
await this.carry(pending);
|
|
146
|
+
this.pending = ref;
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
this.pending = null;
|
|
150
|
+
await this.dissolve(pending);
|
|
151
|
+
await this.dissolve(ref);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
if (this.writer.openCount === 0) {
|
|
155
|
+
// The whole's first element opens its first segment.
|
|
156
|
+
this.pending = ref;
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
if (startsSegmentAfter(this.writer.openCount, this.writer.openBytes, await this.hashInputOf(ref))) {
|
|
160
|
+
this.writer.closeSegment();
|
|
161
|
+
await this.drain();
|
|
162
|
+
this.pending = ref;
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
await this.dissolve(ref);
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Takes the next element.
|
|
169
|
+
*
|
|
170
|
+
* @param element - an Array or Set element, or a Dict's `[key, value]` pair
|
|
171
|
+
*/
|
|
172
|
+
async element(element) {
|
|
173
|
+
// An encode that threw left its bytes behind; this one starts clean.
|
|
174
|
+
this.scratch.pop();
|
|
175
|
+
const keyLength = this.encodeElement(element, this.scratch);
|
|
176
|
+
const bytes = this.scratch.toUint8Array();
|
|
177
|
+
const pending = this.pending;
|
|
178
|
+
if (pending !== null) {
|
|
179
|
+
this.pending = null;
|
|
180
|
+
if (await this.cutsAfter(pending, this.keyed ? bytes.subarray(0, keyLength) : bytes))
|
|
181
|
+
await this.carry(pending);
|
|
182
|
+
else
|
|
183
|
+
await this.dissolve(pending);
|
|
184
|
+
}
|
|
185
|
+
this.writer.addEncoded(bytes, keyLength);
|
|
186
|
+
if (this.queue.length > 0)
|
|
187
|
+
await this.drain();
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Writes what is left and hands the last segments to the sink.
|
|
191
|
+
*
|
|
192
|
+
* @returns what the re-cut came to
|
|
193
|
+
*/
|
|
194
|
+
async finish() {
|
|
195
|
+
const pending = this.pending;
|
|
196
|
+
this.pending = null;
|
|
197
|
+
// The whole's last segment holds whatever is left, so a held segment ends
|
|
198
|
+
// one wherever it ends.
|
|
199
|
+
if (pending !== null)
|
|
200
|
+
await this.carry(pending);
|
|
201
|
+
this.writer.finish();
|
|
202
|
+
await this.drain();
|
|
203
|
+
return { header: this.writer.header, carried: this.carried, written: this.written };
|
|
204
|
+
}
|
|
205
|
+
/** Whether the cut rule starts a segment after `held` at an element whose
|
|
206
|
+
* hash input is `hashInput`. */
|
|
207
|
+
async cutsAfter(held, hashInput) {
|
|
208
|
+
const next = held.nextFence;
|
|
209
|
+
if (this.keyed && next !== undefined && next.length === hashInput.length && next.every((byte, i) => byte === hashInput[i])) {
|
|
210
|
+
return true;
|
|
211
|
+
}
|
|
212
|
+
return startsSegmentAfter(held.count, held.logicalBytes ?? (await this.content(held)).logicalBytes, hashInput);
|
|
213
|
+
}
|
|
214
|
+
/** The bytes the cut rule hashes for a segment's first element: its fence
|
|
215
|
+
* for a Set or Dict, the element itself for an Array. */
|
|
216
|
+
async hashInputOf(ref) {
|
|
217
|
+
if (this.keyed)
|
|
218
|
+
return ref.fence;
|
|
219
|
+
const first = (await this.content(ref)).elements().next();
|
|
220
|
+
return first.value[0];
|
|
221
|
+
}
|
|
222
|
+
/** Hands a segment to the sink as it stands, after every segment written
|
|
223
|
+
* before it. */
|
|
224
|
+
async carry(ref) {
|
|
225
|
+
this.writer.settle();
|
|
226
|
+
await this.drain();
|
|
227
|
+
this.contents.delete(ref);
|
|
228
|
+
this.carried++;
|
|
229
|
+
await this.sink.carried(ref);
|
|
230
|
+
}
|
|
231
|
+
/** Gives a segment's elements to the writer one by one: the whole does not
|
|
232
|
+
* start a segment where it starts, so the writer cuts them. */
|
|
233
|
+
async dissolve(ref) {
|
|
234
|
+
const content = await this.content(ref);
|
|
235
|
+
this.contents.delete(ref);
|
|
236
|
+
for (const [bytes, keyLength] of content.elements()) {
|
|
237
|
+
this.writer.addEncoded(bytes, keyLength);
|
|
238
|
+
if (this.queue.length > 0)
|
|
239
|
+
await this.drain();
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
async drain() {
|
|
243
|
+
while (this.queue.length > 0) {
|
|
244
|
+
this.written++;
|
|
245
|
+
await this.sink.written(this.queue.shift());
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
/** Reads a segment once: its standalone blob, checked to hold exactly the
|
|
249
|
+
* one segment its reference describes, of this collection's type. */
|
|
250
|
+
async content(ref) {
|
|
251
|
+
const held = this.contents.get(ref);
|
|
252
|
+
if (held !== undefined)
|
|
253
|
+
return held;
|
|
254
|
+
const blob = await ref.read();
|
|
255
|
+
const extents = readBeast2Extents(blob);
|
|
256
|
+
if (!isTypeValueEqual(extents.typeValue, this.typeValue)) {
|
|
257
|
+
const printType = printFor(EastTypeValueType);
|
|
258
|
+
throw new Error(`beast2 v5: recut: a segment holds ${printType(extents.typeValue)}, not ${printType(this.typeValue)}`);
|
|
259
|
+
}
|
|
260
|
+
if (extents.offsets.length !== 1 || extents.counts[0] !== ref.count) {
|
|
261
|
+
const holds = extents.offsets.length === 1 ? `${extents.counts[0]} elements` : `${extents.offsets.length} segments`;
|
|
262
|
+
throw new Error(`beast2 v5: recut: a segment's reference says ${ref.count} elements, and its blob holds ${holds}`);
|
|
263
|
+
}
|
|
264
|
+
const head = new BufferReader(blob, MAGIC_BYTES_V5.length);
|
|
265
|
+
readTypeSection(head);
|
|
266
|
+
const sourceMap = readSourceMapSectionV5(head);
|
|
267
|
+
const reader = new FrameReader(blob, extents.offsets[0]).next();
|
|
268
|
+
const count = reader.readVarint();
|
|
269
|
+
if (count !== ref.count) {
|
|
270
|
+
throw new Error(`beast2 v5: recut: a segment's frame declares ${count} elements, its index ${ref.count}`);
|
|
271
|
+
}
|
|
272
|
+
const platform = this.platform;
|
|
273
|
+
const content = new SegmentContent(reader.buffer, reader.offset, count, this.parse, () => ({ containers: [], sourceMap, frozen: false, ...platform }));
|
|
274
|
+
this.contents.set(ref, content);
|
|
275
|
+
return content;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Builds a re-cut: `recut(pieces, sink)` writes the canonical whole of a
|
|
280
|
+
* collection given as pieces in order, segment by segment.
|
|
281
|
+
*
|
|
282
|
+
* A piece is either a run of consecutive segments of a collection the Writer
|
|
283
|
+
* wrote — given by reference, and read only where the re-cut must look inside
|
|
284
|
+
* one — or elements. A segment of a piece that is a segment of the whole is
|
|
285
|
+
* carried over as it came; everything else is written, cut by the rule. The
|
|
286
|
+
* segments are exactly those the Writer writes for the whole value, so an edit
|
|
287
|
+
* is a re-cut of the edited elements between the segments around them, and
|
|
288
|
+
* costs what the edit touched plus at most a segment or two each side.
|
|
289
|
+
*
|
|
290
|
+
* @param type - the collection type (Array/Set/Dict)
|
|
291
|
+
* @param options - the Writer's options for the written segments; its
|
|
292
|
+
* `headerPrefix` is the header the carried segments are under
|
|
293
|
+
* @returns a function taking the pieces, in order, and the sink the whole's
|
|
294
|
+
* segments go to, and resolving to what the re-cut came to
|
|
295
|
+
* @throws {TypeError} When `type` is not an Array, Set or Dict type.
|
|
296
|
+
*
|
|
297
|
+
* @remarks
|
|
298
|
+
* The pieces' segments must be the Writer's own — cut by the current rule,
|
|
299
|
+
* with aliasing scoped per element and the same codec — or they are carried
|
|
300
|
+
* over as something the Writer would not write; a collection stored some other
|
|
301
|
+
* way is given as its elements. Elements in one piece must ascend in East
|
|
302
|
+
* order (a Set's elements, a Dict's keys), which the re-cut checks; the order
|
|
303
|
+
* across pieces is the caller's. The returned function rejects, having written
|
|
304
|
+
* part of the whole, when a piece is out of order, a segment's blob is not the
|
|
305
|
+
* one segment its reference describes, or the sink rejects.
|
|
306
|
+
*
|
|
307
|
+
* @example
|
|
308
|
+
* ```ts
|
|
309
|
+
* const type = ArrayType(IntegerType);
|
|
310
|
+
* const blob = encodeBeast2PagedFor(type)(values);
|
|
311
|
+
* const extents = readBeast2Extents(blob);
|
|
312
|
+
* const segments = extents.counts.map((count, i) => ({
|
|
313
|
+
* count,
|
|
314
|
+
* fence: new Uint8Array(0),
|
|
315
|
+
* read: () => carveBeast2(blob, i, i + 1, extents),
|
|
316
|
+
* }));
|
|
317
|
+
* const parts: Uint8Array[] = [];
|
|
318
|
+
* await recutBeast2For(type)([{ segments }, { elements: [42n] }], {
|
|
319
|
+
* written: (segment) => { parts.push(segment.blob); },
|
|
320
|
+
* carried: (segment) => { parts.push(segment.read() as Uint8Array); },
|
|
321
|
+
* });
|
|
322
|
+
* spliceBeast2(parts); // encodeBeast2PagedFor(type)([...values, 42n])
|
|
323
|
+
* ```
|
|
324
|
+
*/
|
|
325
|
+
export function recutBeast2For(type, options) {
|
|
326
|
+
const typeValue = asTypeValue(type);
|
|
327
|
+
if (!isSegmentedRoot(typeValue)) {
|
|
328
|
+
throw new TypeError(`beast2 v5: recut addresses Array, Set or Dict values, not ${typeValue.type}`);
|
|
329
|
+
}
|
|
330
|
+
const parse = elementParserFor(typeValue);
|
|
331
|
+
const dict = typeValue.type === "Dict";
|
|
332
|
+
const keyType = typeValue.type === "Array" ? null
|
|
333
|
+
: dict ? typeValue.value.key : typeValue.value;
|
|
334
|
+
const cmp = keyType === null ? null : compareFor(keyType);
|
|
335
|
+
const printKey = keyType === null ? null : printFor(keyType);
|
|
336
|
+
return async (pieces, sink) => {
|
|
337
|
+
const recut = new Recutter(typeValue, parse, sink, options);
|
|
338
|
+
for await (const piece of pieces) {
|
|
339
|
+
if ("segments" in piece) {
|
|
340
|
+
let continues = false;
|
|
341
|
+
for await (const segment of piece.segments) {
|
|
342
|
+
await recut.segment(segment, continues);
|
|
343
|
+
continues = true;
|
|
344
|
+
}
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
let prev;
|
|
348
|
+
let has = false;
|
|
349
|
+
for await (const element of piece.elements) {
|
|
350
|
+
if (cmp !== null) {
|
|
351
|
+
const key = dict ? element[0] : element;
|
|
352
|
+
if (has && cmp(prev, key) >= 0) {
|
|
353
|
+
const noun = dict ? "Dict keys" : "Set elements";
|
|
354
|
+
throw new Error(`beast2 v5: recut: a piece's ${noun} must ascend strictly in East order, and ${printKey(key)} follows ${printKey(prev)}`);
|
|
355
|
+
}
|
|
356
|
+
prev = key;
|
|
357
|
+
has = true;
|
|
358
|
+
}
|
|
359
|
+
await recut.element(element);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return recut.finish();
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
//# sourceMappingURL=recut.js.map
|