@opendatalabs/personal-server-ts-stream-sync 0.0.0-stage → 0.0.1-canary.14eb2b2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +73 -2
  2. package/dist/bytes.d.ts +43 -0
  3. package/dist/bytes.d.ts.map +1 -0
  4. package/dist/bytes.js +45 -0
  5. package/dist/bytes.js.map +1 -0
  6. package/dist/control.d.ts +38 -0
  7. package/dist/control.d.ts.map +1 -0
  8. package/dist/control.js +45 -0
  9. package/dist/control.js.map +1 -0
  10. package/dist/decide.d.ts +108 -0
  11. package/dist/decide.d.ts.map +1 -0
  12. package/dist/decide.js +144 -0
  13. package/dist/decide.js.map +1 -0
  14. package/dist/eligible.d.ts +67 -0
  15. package/dist/eligible.d.ts.map +1 -0
  16. package/dist/eligible.js +60 -0
  17. package/dist/eligible.js.map +1 -0
  18. package/dist/errors.d.ts +41 -0
  19. package/dist/errors.d.ts.map +1 -0
  20. package/dist/errors.js +59 -0
  21. package/dist/errors.js.map +1 -0
  22. package/dist/head.d.ts +160 -0
  23. package/dist/head.d.ts.map +1 -0
  24. package/dist/head.js +315 -0
  25. package/dist/head.js.map +1 -0
  26. package/dist/index.d.ts +18 -0
  27. package/dist/index.d.ts.map +1 -0
  28. package/dist/index.js +18 -0
  29. package/dist/index.js.map +1 -0
  30. package/dist/json.d.ts +20 -0
  31. package/dist/json.d.ts.map +1 -0
  32. package/dist/json.js +67 -0
  33. package/dist/json.js.map +1 -0
  34. package/dist/master-key.d.ts +21 -0
  35. package/dist/master-key.d.ts.map +1 -0
  36. package/dist/master-key.js +76 -0
  37. package/dist/master-key.js.map +1 -0
  38. package/dist/merge.d.ts +70 -0
  39. package/dist/merge.d.ts.map +1 -0
  40. package/dist/merge.js +80 -0
  41. package/dist/merge.js.map +1 -0
  42. package/dist/names.d.ts +44 -0
  43. package/dist/names.d.ts.map +1 -0
  44. package/dist/names.js +105 -0
  45. package/dist/names.js.map +1 -0
  46. package/dist/namespace.d.ts +32 -0
  47. package/dist/namespace.d.ts.map +1 -0
  48. package/dist/namespace.js +51 -0
  49. package/dist/namespace.js.map +1 -0
  50. package/dist/needs-full.d.ts +34 -0
  51. package/dist/needs-full.d.ts.map +1 -0
  52. package/dist/needs-full.js +52 -0
  53. package/dist/needs-full.js.map +1 -0
  54. package/dist/outbox.d.ts +67 -0
  55. package/dist/outbox.d.ts.map +1 -0
  56. package/dist/outbox.js +57 -0
  57. package/dist/outbox.js.map +1 -0
  58. package/dist/parts.d.ts +37 -0
  59. package/dist/parts.d.ts.map +1 -0
  60. package/dist/parts.js +93 -0
  61. package/dist/parts.js.map +1 -0
  62. package/dist/planner.d.ts +56 -0
  63. package/dist/planner.d.ts.map +1 -0
  64. package/dist/planner.js +127 -0
  65. package/dist/planner.js.map +1 -0
  66. package/dist/records.d.ts +51 -0
  67. package/dist/records.d.ts.map +1 -0
  68. package/dist/records.js +113 -0
  69. package/dist/records.js.map +1 -0
  70. package/dist/vectors.d.ts +24 -0
  71. package/dist/vectors.d.ts.map +1 -0
  72. package/dist/vectors.js +289 -0
  73. package/dist/vectors.js.map +1 -0
  74. package/package.json +36 -4
  75. package/vectors/v1.json +4563 -0
@@ -0,0 +1,51 @@
1
+ /**
2
+ * §2.1 namespace and key check, §2.3 control-row hashes.
3
+ *
4
+ * K_ns = deriveScopeKey(masterKey, NAMESPACE_KEY_LABEL) (host)
5
+ * ns = hex(first 8 bytes of HMAC-SHA256(K_ns, utf8(canonicalJson([c, accountId]))))
6
+ * keyCheck = hex(first 8 bytes of HMAC-SHA256(K_ns, utf8("key-check")))
7
+ *
8
+ * The host derives K_ns and passes an HMAC port bound to it.
9
+ */
10
+ import { HEX16, keccakHex, toHex, utf8, } from "./bytes.js";
11
+ import { InvalidArgumentError } from "./errors.js";
12
+ import { canonicalJson } from "./json.js";
13
+ /** The `deriveScopeKey` label the host uses to derive `K_ns`. */
14
+ export const NAMESPACE_KEY_LABEL = "pdpp-stream-sync/namespace/v1";
15
+ /** The namespace of connectors that declare no account identity. */
16
+ export const IDENTITYLESS_NS = "";
17
+ export const KEY_CHECK_MESSAGE = "key-check";
18
+ export const ACTIVE_DATA_PREFIX = "pdpp.stream-sync.connection.active.v1:";
19
+ export const ACTIVE_METADATA_LABEL = "pdpp.stream-sync.connection.active.metadata.v1";
20
+ /** Same labels as the Personal Server tombstone constants. */
21
+ export const TOMBSTONE_DATA_LABEL = "vana.data-point.tombstone.v1";
22
+ export const TOMBSTONE_METADATA_LABEL = "vana.data-point.tombstone.metadata.v1";
23
+ async function first8Hex(hmac, message) {
24
+ const digest = await hmac(message);
25
+ if (digest.length < 8) {
26
+ throw new InvalidArgumentError("namespace HMAC returned fewer than 8 bytes");
27
+ }
28
+ return toHex(digest.subarray(0, 8));
29
+ }
30
+ export async function deriveNs(hmac, connector, accountId) {
31
+ if (typeof accountId !== "string" || accountId === "") {
32
+ throw new InvalidArgumentError("accountId must be a non-empty string; identity-less connectors use IDENTITYLESS_NS");
33
+ }
34
+ return first8Hex(hmac, utf8(canonicalJson([connector, accountId])));
35
+ }
36
+ export async function deriveKeyCheck(hmac) {
37
+ return first8Hex(hmac, utf8(KEY_CHECK_MESSAGE));
38
+ }
39
+ /** The registry hashes a device writes and recognises for its own key. */
40
+ export async function controlHashes(keccak256, keyCheck) {
41
+ if (!HEX16.test(keyCheck)) {
42
+ throw new InvalidArgumentError("keyCheck must be 16 lowercase hex");
43
+ }
44
+ return {
45
+ activeData: await keccakHex(keccak256, utf8(ACTIVE_DATA_PREFIX + keyCheck)),
46
+ activeMetadata: await keccakHex(keccak256, utf8(ACTIVE_METADATA_LABEL)),
47
+ tombstoneData: await keccakHex(keccak256, utf8(TOMBSTONE_DATA_LABEL)),
48
+ tombstoneMetadata: await keccakHex(keccak256, utf8(TOMBSTONE_METADATA_LABEL)),
49
+ };
50
+ }
51
+ //# sourceMappingURL=namespace.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"namespace.js","sourceRoot":"","sources":["../src/namespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,KAAK,EACL,SAAS,EACT,KAAK,EACL,IAAI,GAGL,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,CAAC,MAAM,mBAAmB,GAAG,+BAA+B,CAAC;AAEnE,oEAAoE;AACpE,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAElC,MAAM,CAAC,MAAM,iBAAiB,GAAG,WAAW,CAAC;AAC7C,MAAM,CAAC,MAAM,kBAAkB,GAAG,wCAAwC,CAAC;AAC3E,MAAM,CAAC,MAAM,qBAAqB,GAChC,gDAAgD,CAAC;AACnD,8DAA8D;AAC9D,MAAM,CAAC,MAAM,oBAAoB,GAAG,8BAA8B,CAAC;AACnE,MAAM,CAAC,MAAM,wBAAwB,GAAG,uCAAuC,CAAC;AAEhF,KAAK,UAAU,SAAS,CACtB,IAAmB,EACnB,OAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,oBAAoB,CAC5B,4CAA4C,CAC7C,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAmB,EACnB,SAAiB,EACjB,SAAiB;IAEjB,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;QACtD,MAAM,IAAI,oBAAoB,CAC5B,oFAAoF,CACrF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAmB;IACtD,OAAO,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;AAClD,CAAC;AAUD,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAoB,EACpB,QAAgB;IAEhB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,oBAAoB,CAAC,mCAAmC,CAAC,CAAC;IACtE,CAAC;IACD,OAAO;QACL,UAAU,EAAE,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC,CAAC;QAC3E,cAAc,EAAE,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACvE,aAAa,EAAE,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACrE,iBAAiB,EAAE,MAAM,SAAS,CAChC,SAAS,EACT,IAAI,CAAC,wBAAwB,CAAC,CAC/B;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,34 @@
1
+ /**
2
+ * §2.4, §2.7 durable `needsFull[c, ns, s]`.
3
+ *
4
+ * - Publishing `complete:false` sets `needsFull[c, ns, s]` and clears the
5
+ * checkpoint for c, so the next run is full. A no-op onto a
6
+ * `complete:false` base does the same (ruling Q9): hosts call
7
+ * `afterCommit` for a committed head and for an unchanged (no-op) head.
8
+ * - A full run that commits `complete:true` clears it.
9
+ * - Any `needsFull` for c forces the next run of c to be full.
10
+ *
11
+ * Every device that publishes onto an incomplete base sets its own entry, so
12
+ * incomplete heads converge. The set is a sorted array so hosts can store it
13
+ * as JSON and compare it byte for byte.
14
+ */
15
+ export interface NeedsFullKey {
16
+ readonly connector: string;
17
+ readonly ns: string;
18
+ readonly stream: string;
19
+ }
20
+ export type NeedsFull = readonly NeedsFullKey[];
21
+ export declare function runMustBeFull(needsFull: NeedsFull, connector: string): boolean;
22
+ export interface CommitEffect {
23
+ readonly needsFull: NeedsFull;
24
+ /** Clear the checkpoint for the connector. */
25
+ readonly clearCheckpoint: boolean;
26
+ }
27
+ /** Apply one committed or unchanged (no-op) head to the `needsFull` set. */
28
+ export declare function afterCommit(needsFull: NeedsFull, commit: NeedsFullKey & {
29
+ /** `complete` of the committed head. */
30
+ readonly complete: boolean;
31
+ /** The run used no checkpoint. */
32
+ readonly runFull: boolean;
33
+ }): CommitEffect;
34
+ //# sourceMappingURL=needs-full.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"needs-full.d.ts","sourceRoot":"","sources":["../src/needs-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,SAAS,GAAG,SAAS,YAAY,EAAE,CAAC;AAahD,wBAAgB,aAAa,CAC3B,SAAS,EAAE,SAAS,EACpB,SAAS,EAAE,MAAM,GAChB,OAAO,CAET;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,8CAA8C;IAC9C,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,4EAA4E;AAC5E,wBAAgB,WAAW,CACzB,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,YAAY,GAAG;IACrB,wCAAwC;IACxC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B,GACA,YAAY,CAsBd"}
@@ -0,0 +1,52 @@
1
+ /**
2
+ * §2.4, §2.7 durable `needsFull[c, ns, s]`.
3
+ *
4
+ * - Publishing `complete:false` sets `needsFull[c, ns, s]` and clears the
5
+ * checkpoint for c, so the next run is full. A no-op onto a
6
+ * `complete:false` base does the same (ruling Q9): hosts call
7
+ * `afterCommit` for a committed head and for an unchanged (no-op) head.
8
+ * - A full run that commits `complete:true` clears it.
9
+ * - Any `needsFull` for c forces the next run of c to be full.
10
+ *
11
+ * Every device that publishes onto an incomplete base sets its own entry, so
12
+ * incomplete heads converge. The set is a sorted array so hosts can store it
13
+ * as JSON and compare it byte for byte.
14
+ */
15
+ function compareKeys(a, b) {
16
+ for (const field of ["connector", "ns", "stream"]) {
17
+ if (a[field] !== b[field])
18
+ return a[field] < b[field] ? -1 : 1;
19
+ }
20
+ return 0;
21
+ }
22
+ function same(a, b) {
23
+ return compareKeys(a, b) === 0;
24
+ }
25
+ export function runMustBeFull(needsFull, connector) {
26
+ return needsFull.some((key) => key.connector === connector);
27
+ }
28
+ /** Apply one committed or unchanged (no-op) head to the `needsFull` set. */
29
+ export function afterCommit(needsFull, commit) {
30
+ const key = {
31
+ connector: commit.connector,
32
+ ns: commit.ns,
33
+ stream: commit.stream,
34
+ };
35
+ if (!commit.complete) {
36
+ const next = needsFull.some((k) => same(k, key))
37
+ ? [...needsFull]
38
+ : [...needsFull, key];
39
+ return { needsFull: next.sort(compareKeys), clearCheckpoint: true };
40
+ }
41
+ if (commit.runFull) {
42
+ return {
43
+ needsFull: needsFull.filter((k) => !same(k, key)).sort(compareKeys),
44
+ clearCheckpoint: false,
45
+ };
46
+ }
47
+ return {
48
+ needsFull: [...needsFull].sort(compareKeys),
49
+ clearCheckpoint: false,
50
+ };
51
+ }
52
+ //# sourceMappingURL=needs-full.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"needs-full.js","sourceRoot":"","sources":["../src/needs-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAUH,SAAS,WAAW,CAAC,CAAe,EAAE,CAAe;IACnD,KAAK,MAAM,KAAK,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,CAAU,EAAE,CAAC;QAC3D,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,IAAI,CAAC,CAAe,EAAE,CAAe;IAC5C,OAAO,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,SAAoB,EACpB,SAAiB;IAEjB,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;AAC9D,CAAC;AAQD,4EAA4E;AAC5E,MAAM,UAAU,WAAW,CACzB,SAAoB,EACpB,MAKC;IAED,MAAM,GAAG,GAAiB;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9C,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;YAChB,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE,GAAG,CAAC,CAAC;QACxB,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IACtE,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO;YACL,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;YACnE,eAAe,EAAE,KAAK;SACvB,CAAC;IACJ,CAAC;IACD,OAAO;QACL,SAAS,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;QAC3C,eAAe,EAAE,KAAK;KACvB,CAAC;AACJ,CAAC"}
@@ -0,0 +1,67 @@
1
+ /**
2
+ * §2.8 outbox entry. One entry per run, stored by the host on its durable
3
+ * spool (filesystem + SQLite, native files, OPFS or IndexedDB). The host
4
+ * resumes each stream from its `state`. Failures are visible and retried;
5
+ * no attempt cap drops data.
6
+ *
7
+ * Stream states:
8
+ * planned nothing durable yet; redo from "read control"
9
+ * request_persisted head ciphertext and signed request persisted; on an
10
+ * ambiguous reply, resend the request byte-for-byte
11
+ * committed the head POST succeeded at `generation`
12
+ * noop every part and `complete` unchanged; nothing posted
13
+ * discarded `decide` discarded this run
14
+ */
15
+ import type { StreamRunResult } from "./decide.js";
16
+ import type { StartSnapshot } from "./decide.js";
17
+ import type { JsonValue } from "./json.js";
18
+ import { type NeedsFull } from "./needs-full.js";
19
+ export type OutboxStreamState = "planned" | "request_persisted" | "committed" | "noop" | "discarded";
20
+ /** The visible reason a stream has not committed yet. */
21
+ export type OutboxProblem = "key_mismatch" | "registry_inconsistent" | "bucket_too_large" | "connector_update_required";
22
+ export interface OutboxStream {
23
+ readonly state: OutboxStreamState;
24
+ readonly requestRef?: string;
25
+ readonly headCipherRef?: string;
26
+ /** Set with `committed` and `noop`. */
27
+ readonly generation?: number;
28
+ /** `complete` of the committed or unchanged head. */
29
+ readonly complete?: boolean;
30
+ readonly problem?: OutboxProblem;
31
+ /** §2.2: records not published because their line is over 8 MiB. */
32
+ readonly recordTooLarge?: number;
33
+ }
34
+ export interface OutboxEntry {
35
+ readonly runId: string;
36
+ readonly connector: string;
37
+ /** `null` until the run's account identity is known. */
38
+ readonly ns: string | null;
39
+ /** Control states for the connector at run start, by ns; `null` = no listing yet. */
40
+ readonly start: StartSnapshot;
41
+ readonly userConnect: boolean;
42
+ /** The run used no checkpoint. */
43
+ readonly full: boolean;
44
+ /** Run start, in milliseconds since the epoch; the `o` of every run line. */
45
+ readonly observedAt: number;
46
+ readonly spoolRef: string;
47
+ readonly checkpointCandidate: JsonValue | null;
48
+ readonly streams: Readonly<Record<string, OutboxStream>>;
49
+ }
50
+ /**
51
+ * Build the entry at run start. Refuses a non-full run while the connector
52
+ * has any `needsFull` (§2.4: any `needsFull` for c forces `full`).
53
+ */
54
+ export declare function createOutboxEntry(input: {
55
+ readonly runId: string;
56
+ readonly connector: string;
57
+ readonly start: StartSnapshot;
58
+ readonly userConnect: boolean;
59
+ readonly full: boolean;
60
+ readonly observedAt: number;
61
+ readonly spoolRef: string;
62
+ readonly checkpointCandidate: JsonValue | null;
63
+ readonly needsFull: NeedsFull;
64
+ }): OutboxEntry;
65
+ /** The checkpoint view of one outbox stream (`checkpointDecision` input). */
66
+ export declare function streamRunResult(stream: OutboxStream): StreamRunResult;
67
+ //# sourceMappingURL=outbox.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../src/outbox.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAiB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAC3B,SAAS,GAAG,mBAAmB,GAAG,WAAW,GAAG,MAAM,GAAG,WAAW,CAAC;AAEvE,yDAAyD;AACzD,MAAM,MAAM,aAAa,GACrB,cAAc,GACd,uBAAuB,GACvB,kBAAkB,GAClB,2BAA2B,CAAC;AAEhC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,uCAAuC;IACvC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,qDAAqD;IACrD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC;IACjC,oEAAoE;IACpE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,qFAAqF;IACrF,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,mBAAmB,EAAE,SAAS,GAAG,IAAI,CAAC;IAC/C,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;CAC1D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,mBAAmB,EAAE,SAAS,GAAG,IAAI,CAAC;IAC/C,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;CAC/B,GAAG,WAAW,CAsBd;AAED,6EAA6E;AAC7E,wBAAgB,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,eAAe,CAcrE"}
package/dist/outbox.js ADDED
@@ -0,0 +1,57 @@
1
+ /**
2
+ * §2.8 outbox entry. One entry per run, stored by the host on its durable
3
+ * spool (filesystem + SQLite, native files, OPFS or IndexedDB). The host
4
+ * resumes each stream from its `state`. Failures are visible and retried;
5
+ * no attempt cap drops data.
6
+ *
7
+ * Stream states:
8
+ * planned nothing durable yet; redo from "read control"
9
+ * request_persisted head ciphertext and signed request persisted; on an
10
+ * ambiguous reply, resend the request byte-for-byte
11
+ * committed the head POST succeeded at `generation`
12
+ * noop every part and `complete` unchanged; nothing posted
13
+ * discarded `decide` discarded this run
14
+ */
15
+ import { InvalidArgumentError } from "./errors.js";
16
+ import { runMustBeFull } from "./needs-full.js";
17
+ /**
18
+ * Build the entry at run start. Refuses a non-full run while the connector
19
+ * has any `needsFull` (§2.4: any `needsFull` for c forces `full`).
20
+ */
21
+ export function createOutboxEntry(input) {
22
+ if (input.runId === "")
23
+ throw new InvalidArgumentError("runId is empty");
24
+ if (!Number.isSafeInteger(input.observedAt) || input.observedAt < 0) {
25
+ throw new InvalidArgumentError("observedAt must be a non-negative integer");
26
+ }
27
+ if (!input.full && runMustBeFull(input.needsFull, input.connector)) {
28
+ throw new InvalidArgumentError(`connector ${input.connector} has needsFull: the run must be full`);
29
+ }
30
+ return {
31
+ runId: input.runId,
32
+ connector: input.connector,
33
+ ns: null,
34
+ start: input.start === null ? null : { ...input.start },
35
+ userConnect: input.userConnect,
36
+ full: input.full,
37
+ observedAt: input.observedAt,
38
+ spoolRef: input.spoolRef,
39
+ checkpointCandidate: input.checkpointCandidate,
40
+ streams: {},
41
+ };
42
+ }
43
+ /** The checkpoint view of one outbox stream (`checkpointDecision` input). */
44
+ export function streamRunResult(stream) {
45
+ if (stream.state === "committed" || stream.state === "noop") {
46
+ if (stream.generation === undefined || stream.complete === undefined) {
47
+ throw new InvalidArgumentError(`${stream.state} stream needs generation and complete`);
48
+ }
49
+ return {
50
+ state: stream.state,
51
+ generation: stream.generation,
52
+ complete: stream.complete,
53
+ };
54
+ }
55
+ return { state: stream.state === "discarded" ? "discarded" : "pending" };
56
+ }
57
+ //# sourceMappingURL=outbox.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"outbox.js","sourceRoot":"","sources":["../src/outbox.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAkB,MAAM,iBAAiB,CAAC;AA0ChE;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAUjC;IACC,IAAI,KAAK,CAAC,KAAK,KAAK,EAAE;QAAE,MAAM,IAAI,oBAAoB,CAAC,gBAAgB,CAAC,CAAC;IACzE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,oBAAoB,CAAC,2CAA2C,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QACnE,MAAM,IAAI,oBAAoB,CAC5B,aAAa,KAAK,CAAC,SAAS,sCAAsC,CACnE,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,EAAE,EAAE,IAAI;QACR,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE;QACvD,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,OAAO,EAAE,EAAE;KACZ,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe,CAAC,MAAoB;IAClD,IAAI,MAAM,CAAC,KAAK,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC5D,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACrE,MAAM,IAAI,oBAAoB,CAC5B,GAAG,MAAM,CAAC,KAAK,uCAAuC,CACvD,CAAC;QACJ,CAAC;QACD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AAC3E,CAAC"}
@@ -0,0 +1,37 @@
1
+ /**
2
+ * §2.2 part codec. A part is NDJSON: one canonical line per winning record,
3
+ * sorted by pkHash, each line terminated by `\n`. A prefix with no records
4
+ * has no part.
5
+ */
6
+ import { type Sha256 } from "./bytes.js";
7
+ import type { BucketResult, Winner } from "./merge.js";
8
+ import { type Line } from "./records.js";
9
+ /** One entry of a head's `parts` list. */
10
+ export interface HeadPart {
11
+ readonly bytes: number;
12
+ readonly count: number;
13
+ /** SHA-256 of the clear part bytes, 64 lowercase hex. */
14
+ readonly hash: string;
15
+ /** The bucket's bit prefix, e.g. `""`, `"0"`, `"1011"`. */
16
+ readonly prefix: string;
17
+ }
18
+ export interface EncodedPart {
19
+ readonly entry: HeadPart;
20
+ readonly data: Uint8Array;
21
+ }
22
+ /** Encode a reduced bucket. Returns `null` for an empty bucket. */
23
+ export declare function encodePart(sha256: Sha256, bucket: Extract<BucketResult, {
24
+ kind: "ok";
25
+ }>): Promise<EncodedPart | null>;
26
+ export declare function encodeWinners(sha256: Sha256, prefix: string, winners: readonly Winner[]): Promise<EncodedPart | null>;
27
+ export interface DecodedLine {
28
+ readonly text: string;
29
+ readonly line: Line;
30
+ }
31
+ /**
32
+ * Verify part bytes against their head entry and decode the lines. When
33
+ * `primaryKey` is given, also check every line is inside the part's prefix
34
+ * and lines are strictly sorted by pkHash.
35
+ */
36
+ export declare function decodePart(sha256: Sha256, data: Uint8Array, entry: HeadPart, primaryKey?: readonly string[]): Promise<DecodedLine[]>;
37
+ //# sourceMappingURL=parts.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parts.d.ts","sourceRoot":"","sources":["../src/parts.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqC,KAAK,MAAM,EAAE,MAAM,YAAY,CAAC;AAE5E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAgC,KAAK,IAAI,EAAE,MAAM,cAAc,CAAC;AAEvE,0CAA0C;AAC1C,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2DAA2D;IAC3D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;CAC3B;AAED,mEAAmE;AACnE,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE;IAAE,IAAI,EAAE,IAAI,CAAA;CAAE,CAAC,GAC5C,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAE7B;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CA0B7B;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;CACrB;AAID;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,UAAU,EAChB,KAAK,EAAE,QAAQ,EACf,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,CAiDxB"}
package/dist/parts.js ADDED
@@ -0,0 +1,93 @@
1
+ /**
2
+ * §2.2 part codec. A part is NDJSON: one canonical line per winning record,
3
+ * sorted by pkHash, each line terminated by `\n`. A prefix with no records
4
+ * has no part.
5
+ */
6
+ import { PART_LIMIT_BYTES, sha256Hex, utf8 } from "./bytes.js";
7
+ import { InvalidArgumentError, InvalidPartError } from "./errors.js";
8
+ import { decodeLine, inPrefix, pkHash } from "./records.js";
9
+ /** Encode a reduced bucket. Returns `null` for an empty bucket. */
10
+ export async function encodePart(sha256, bucket) {
11
+ return encodeWinners(sha256, bucket.prefix, bucket.winners);
12
+ }
13
+ export async function encodeWinners(sha256, prefix, winners) {
14
+ if (winners.length === 0)
15
+ return null;
16
+ let text = "";
17
+ let previous = null;
18
+ for (const winner of winners) {
19
+ if (previous !== null && winner.pkHash <= previous) {
20
+ throw new InvalidArgumentError("winners must be strictly sorted by pkHash");
21
+ }
22
+ previous = winner.pkHash;
23
+ text += `${winner.text}\n`;
24
+ }
25
+ const data = utf8(text);
26
+ if (data.length > PART_LIMIT_BYTES) {
27
+ throw new InvalidArgumentError("part exceeds the 16 MiB part limit");
28
+ }
29
+ return {
30
+ data,
31
+ entry: {
32
+ bytes: data.length,
33
+ count: winners.length,
34
+ hash: await sha256Hex(sha256, data),
35
+ prefix,
36
+ },
37
+ };
38
+ }
39
+ const decoder = new TextDecoder("utf-8", { fatal: true });
40
+ /**
41
+ * Verify part bytes against their head entry and decode the lines. When
42
+ * `primaryKey` is given, also check every line is inside the part's prefix
43
+ * and lines are strictly sorted by pkHash.
44
+ */
45
+ export async function decodePart(sha256, data, entry, primaryKey) {
46
+ if (data.length > PART_LIMIT_BYTES) {
47
+ throw new InvalidPartError("part exceeds the 16 MiB decoder limit");
48
+ }
49
+ if (data.length !== entry.bytes) {
50
+ throw new InvalidPartError(`part has ${data.length} bytes, head says ${entry.bytes}`);
51
+ }
52
+ if ((await sha256Hex(sha256, data)) !== entry.hash) {
53
+ throw new InvalidPartError("part SHA-256 does not match the head");
54
+ }
55
+ let text;
56
+ try {
57
+ text = decoder.decode(data);
58
+ }
59
+ catch {
60
+ throw new InvalidPartError("part is not valid UTF-8");
61
+ }
62
+ if (!text.endsWith("\n")) {
63
+ throw new InvalidPartError("part does not end with a line terminator");
64
+ }
65
+ const lines = text.slice(0, -1).split("\n");
66
+ if (lines.length !== entry.count) {
67
+ throw new InvalidPartError(`part has ${lines.length} lines, head says ${entry.count}`);
68
+ }
69
+ const out = [];
70
+ let previous = null;
71
+ for (const lineText of lines) {
72
+ let line;
73
+ try {
74
+ line = decodeLine(lineText);
75
+ }
76
+ catch (cause) {
77
+ throw new InvalidPartError(`bad line: ${cause.message}`);
78
+ }
79
+ if (primaryKey) {
80
+ const hash = await pkHash(sha256, line.v, primaryKey);
81
+ if (!inPrefix(hash, entry.prefix)) {
82
+ throw new InvalidPartError("line key is outside the part prefix");
83
+ }
84
+ if (previous !== null && hash <= previous) {
85
+ throw new InvalidPartError("lines are not strictly sorted by pkHash");
86
+ }
87
+ previous = hash;
88
+ }
89
+ out.push({ text: lineText, line });
90
+ }
91
+ return out;
92
+ }
93
+ //# sourceMappingURL=parts.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parts.js","sourceRoot":"","sources":["../src/parts.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,IAAI,EAAe,MAAM,YAAY,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAiBvE,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,MAA6C;IAE7C,OAAO,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,MAAc,EACd,OAA0B;IAE1B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,oBAAoB,CAC5B,2CAA2C,CAC5C,CAAC;QACJ,CAAC;QACD,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;QACzB,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC;IAC7B,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,IAAI,IAAI,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;QACnC,MAAM,IAAI,oBAAoB,CAAC,oCAAoC,CAAC,CAAC;IACvE,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE;YACL,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,IAAI,EAAE,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC;YACnC,MAAM;SACP;KACF,CAAC;AACJ,CAAC;AAOD,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE1D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,IAAgB,EAChB,KAAe,EACf,UAA8B;IAE9B,IAAI,IAAI,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;QACnC,MAAM,IAAI,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,IAAI,gBAAgB,CACxB,YAAY,IAAI,CAAC,MAAM,qBAAqB,KAAK,CAAC,KAAK,EAAE,CAC1D,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,gBAAgB,CAAC,sCAAsC,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,gBAAgB,CAAC,yBAAyB,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,gBAAgB,CAAC,0CAA0C,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,gBAAgB,CACxB,YAAY,KAAK,CAAC,MAAM,qBAAqB,KAAK,CAAC,KAAK,EAAE,CAC3D,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,IAAU,CAAC;QACf,IAAI,CAAC;YACH,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,gBAAgB,CAAC,aAAc,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClC,MAAM,IAAI,gBAAgB,CAAC,qCAAqC,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,QAAQ,KAAK,IAAI,IAAI,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC1C,MAAM,IAAI,gBAAgB,CAAC,yCAAyC,CAAC,CAAC;YACxE,CAAC;YACD,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,56 @@
1
+ /**
2
+ * §2.6 bucket planner.
3
+ *
4
+ * The run spool has 256 shard files keyed by the first pkHash byte, with a
5
+ * byte count per shard. The planner starts with one bucket (prefix `""`) and
6
+ * splits any bucket whose estimate is over the target by the next bit, never
7
+ * deeper than 16 bits. A bucket's estimate is its base part bytes plus its
8
+ * run shard bytes.
9
+ *
10
+ * Run bytes are exact down to 8 bits (sums of shard counts). Deeper than 8
11
+ * bits they need bytes per 16-bit prefix, which the host gets with at most
12
+ * one counting pass per shard: the planner asks for those shards with
13
+ * `needs_counts`, and the host calls it again with the counts.
14
+ *
15
+ * Base bytes: a base part inside the bucket counts in full. A base part that
16
+ * covers a larger range than the bucket counts in proportion to the share of
17
+ * the key space the bucket covers (see DESIGN-QUESTIONS.md, Q3).
18
+ *
19
+ * Buckets with an estimate of 0 hold no records and are left out. The output
20
+ * is prefix-free and sorted by prefix.
21
+ */
22
+ export interface PlannerInput {
23
+ /** Writer target per part, in bytes. */
24
+ readonly targetBytes: number;
25
+ /** The base head's parts (prefix and bytes). Empty for an empty base. */
26
+ readonly baseParts: readonly {
27
+ readonly prefix: string;
28
+ readonly bytes: number;
29
+ }[];
30
+ /** Run spool bytes per shard (first pkHash byte), 256 entries. */
31
+ readonly shardBytes: readonly number[];
32
+ /**
33
+ * Counting-pass results: for a shard, run bytes per second pkHash byte
34
+ * (256 entries, summing to that shard's bytes). Keyed by shard index.
35
+ */
36
+ readonly prefix16Bytes?: Readonly<Record<number, readonly number[]>>;
37
+ }
38
+ export interface PlannedBucket {
39
+ readonly prefix: string;
40
+ readonly estimate: number;
41
+ /** At the 16-bit cap and still over target. */
42
+ readonly capped: boolean;
43
+ }
44
+ export type PlannerResult = {
45
+ readonly kind: "needs_counts";
46
+ readonly shards: readonly number[];
47
+ } | {
48
+ readonly kind: "plan";
49
+ readonly buckets: readonly PlannedBucket[];
50
+ };
51
+ export declare function planBuckets(input: PlannerInput): PlannerResult;
52
+ /** The base parts a bucket's reduce must read (their ranges overlap). */
53
+ export declare function basePartsFor<P extends {
54
+ readonly prefix: string;
55
+ }>(bucketPrefix: string, baseParts: readonly P[]): P[];
56
+ //# sourceMappingURL=planner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../src/planner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAKH,MAAM,WAAW,YAAY;IAC3B,wCAAwC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,yEAAyE;IACzE,QAAQ,CAAC,SAAS,EAAE,SAAS;QAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;KACxB,EAAE,CAAC;IACJ,kEAAkE;IAClE,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC;;;OAGG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,+CAA+C;IAC/C,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GACrB;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACrE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAA;CAAE,CAAC;AA+D1E,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,aAAa,CAoD9D;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,CAAC,SAAS;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAChE,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,SAAS,CAAC,EAAE,GACtB,CAAC,EAAE,CAML"}
@@ -0,0 +1,127 @@
1
+ /**
2
+ * §2.6 bucket planner.
3
+ *
4
+ * The run spool has 256 shard files keyed by the first pkHash byte, with a
5
+ * byte count per shard. The planner starts with one bucket (prefix `""`) and
6
+ * splits any bucket whose estimate is over the target by the next bit, never
7
+ * deeper than 16 bits. A bucket's estimate is its base part bytes plus its
8
+ * run shard bytes.
9
+ *
10
+ * Run bytes are exact down to 8 bits (sums of shard counts). Deeper than 8
11
+ * bits they need bytes per 16-bit prefix, which the host gets with at most
12
+ * one counting pass per shard: the planner asks for those shards with
13
+ * `needs_counts`, and the host calls it again with the counts.
14
+ *
15
+ * Base bytes: a base part inside the bucket counts in full. A base part that
16
+ * covers a larger range than the bucket counts in proportion to the share of
17
+ * the key space the bucket covers (see DESIGN-QUESTIONS.md, Q3).
18
+ *
19
+ * Buckets with an estimate of 0 hold no records and are left out. The output
20
+ * is prefix-free and sorted by prefix.
21
+ */
22
+ import { PLANNER_DEPTH_CAP, SHARD_COUNT } from "./bytes.js";
23
+ import { InvalidArgumentError } from "./errors.js";
24
+ function checkCount(value, label) {
25
+ if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) {
26
+ throw new InvalidArgumentError(`${label} must be a non-negative safe integer`);
27
+ }
28
+ return value;
29
+ }
30
+ function checkInput(input) {
31
+ if (!Number.isSafeInteger(input.targetBytes) || input.targetBytes < 1) {
32
+ throw new InvalidArgumentError("targetBytes must be a positive safe integer");
33
+ }
34
+ if (input.shardBytes.length !== SHARD_COUNT) {
35
+ throw new InvalidArgumentError("shardBytes must have 256 entries");
36
+ }
37
+ input.shardBytes.forEach((v, i) => checkCount(v, `shardBytes[${i}]`));
38
+ for (const part of input.baseParts) {
39
+ if (!/^[01]*$/.test(part.prefix) ||
40
+ part.prefix.length > PLANNER_DEPTH_CAP) {
41
+ throw new InvalidArgumentError(`bad base part prefix ${JSON.stringify(part.prefix)}`);
42
+ }
43
+ checkCount(part.bytes, "base part bytes");
44
+ }
45
+ for (const [key, counts] of Object.entries(input.prefix16Bytes ?? {})) {
46
+ const shard = Number(key);
47
+ if (!Number.isInteger(shard) || shard < 0 || shard >= SHARD_COUNT) {
48
+ throw new InvalidArgumentError(`bad counting-pass shard ${key}`);
49
+ }
50
+ if (counts.length !== SHARD_COUNT) {
51
+ throw new InvalidArgumentError("counting-pass result must have 256 entries");
52
+ }
53
+ let sum = 0;
54
+ counts.forEach((v, i) => (sum += checkCount(v, `prefix16Bytes[${key}][${i}]`)));
55
+ if (sum !== input.shardBytes[shard]) {
56
+ throw new InvalidArgumentError(`counting pass for shard ${shard} sums to ${sum}, shard has ${input.shardBytes[shard]}`);
57
+ }
58
+ }
59
+ }
60
+ /** Sum `values[from .. from + 2^(8 - bits.length))` for a prefix of ≤ 8 bits. */
61
+ function sumRange(values, bits) {
62
+ const span = 1 << (8 - bits.length);
63
+ const from = bits === "" ? 0 : parseInt(bits, 2) * span;
64
+ let sum = 0;
65
+ for (let i = from; i < from + span; i += 1)
66
+ sum += values[i];
67
+ return sum;
68
+ }
69
+ export function planBuckets(input) {
70
+ checkInput(input);
71
+ const counts = input.prefix16Bytes ?? {};
72
+ const needs = new Set();
73
+ const buckets = [];
74
+ /** Run bytes, or `null` when a counting pass is missing. */
75
+ const runBytes = (prefix) => {
76
+ if (prefix.length <= 8)
77
+ return sumRange(input.shardBytes, prefix);
78
+ const shard = parseInt(prefix.slice(0, 8), 2);
79
+ const shardCounts = counts[shard];
80
+ if (!shardCounts)
81
+ return null;
82
+ return sumRange(shardCounts, prefix.slice(8));
83
+ };
84
+ const baseBytes = (prefix) => {
85
+ let sum = 0;
86
+ for (const part of input.baseParts) {
87
+ if (part.prefix.startsWith(prefix))
88
+ sum += part.bytes;
89
+ else if (prefix.startsWith(part.prefix)) {
90
+ sum += part.bytes / 2 ** (prefix.length - part.prefix.length);
91
+ }
92
+ }
93
+ return sum;
94
+ };
95
+ const visit = (prefix, run) => {
96
+ const estimate = baseBytes(prefix) + run;
97
+ if (estimate === 0)
98
+ return;
99
+ if (estimate <= input.targetBytes || prefix.length >= PLANNER_DEPTH_CAP) {
100
+ buckets.push({
101
+ prefix,
102
+ estimate,
103
+ capped: estimate > input.targetBytes,
104
+ });
105
+ return;
106
+ }
107
+ const left = runBytes(`${prefix}0`);
108
+ const right = runBytes(`${prefix}1`);
109
+ if (left === null || right === null) {
110
+ needs.add(parseInt(prefix.slice(0, 8), 2));
111
+ return;
112
+ }
113
+ visit(`${prefix}0`, left);
114
+ visit(`${prefix}1`, right);
115
+ };
116
+ visit("", runBytes(""));
117
+ if (needs.size > 0) {
118
+ return { kind: "needs_counts", shards: [...needs].sort((a, b) => a - b) };
119
+ }
120
+ return { kind: "plan", buckets };
121
+ }
122
+ /** The base parts a bucket's reduce must read (their ranges overlap). */
123
+ export function basePartsFor(bucketPrefix, baseParts) {
124
+ return baseParts.filter((part) => part.prefix.startsWith(bucketPrefix) ||
125
+ bucketPrefix.startsWith(part.prefix));
126
+ }
127
+ //# sourceMappingURL=planner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"planner.js","sourceRoot":"","sources":["../src/planner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AA8BnD,SAAS,UAAU,CAAC,KAAc,EAAE,KAAa;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,oBAAoB,CAC5B,GAAG,KAAK,sCAAsC,CAC/C,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,KAAmB;IACrC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,oBAAoB,CAC5B,6CAA6C,CAC9C,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,IAAI,oBAAoB,CAAC,kCAAkC,CAAC,CAAC;IACrE,CAAC;IACD,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACnC,IACE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,iBAAiB,EACtC,CAAC;YACD,MAAM,IAAI,oBAAoB,CAC5B,wBAAwB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACtD,CAAC;QACJ,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IAC5C,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;YAClE,MAAM,IAAI,oBAAoB,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAC5B,4CAA4C,CAC7C,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,MAAM,CAAC,OAAO,CACZ,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,UAAU,CAAC,CAAC,EAAE,iBAAiB,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAChE,CAAC;QACF,IAAI,GAAG,KAAK,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,oBAAoB,CAC5B,2BAA2B,KAAK,YAAY,GAAG,eAAe,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CACxF,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,SAAS,QAAQ,CAAC,MAAyB,EAAE,IAAY;IACvD,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;IACxD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC;QAAE,GAAG,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAmB;IAC7C,UAAU,CAAC,KAAK,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,IAAI,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,OAAO,GAAoB,EAAE,CAAC;IAEpC,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAiB,EAAE;QACjD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAC9B,OAAO,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,MAAc,EAAU,EAAE;QAC3C,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC;iBACjD,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxC,GAAG,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,GAAW,EAAQ,EAAE;QAClD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QACzC,IAAI,QAAQ,KAAK,CAAC;YAAE,OAAO;QAC3B,IAAI,QAAQ,IAAI,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM;gBACN,QAAQ;gBACR,MAAM,EAAE,QAAQ,GAAG,KAAK,CAAC,WAAW;aACrC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACpC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,OAAO;QACT,CAAC;QACD,KAAK,CAAC,GAAG,MAAM,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,KAAK,CAAC,GAAG,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAE,CAAC,CAAC;IACzB,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;IAC5E,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACnC,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,YAAY,CAC1B,YAAoB,EACpB,SAAuB;IAEvB,OAAO,SAAS,CAAC,MAAM,CACrB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC;QACpC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CACvC,CAAC;AACJ,CAAC"}