@indigoai-us/hq-cloud 6.15.78 → 6.15.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/area-ledger-migration.d.ts +13 -0
- package/dist/bin/area-ledger-migration.d.ts.map +1 -0
- package/dist/bin/area-ledger-migration.js +64 -0
- package/dist/bin/area-ledger-migration.js.map +1 -0
- package/dist/bin/area-ledger-migration.test.d.ts +2 -0
- package/dist/bin/area-ledger-migration.test.d.ts.map +1 -0
- package/dist/bin/area-ledger-migration.test.js +55 -0
- package/dist/bin/area-ledger-migration.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +6 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +6 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +47 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/cli/share.d.ts +8 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +184 -10
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +330 -7
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +7 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +11 -1
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +25 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/index.test.js +11 -1
- package/dist/index.test.js.map +1 -1
- package/dist/journal-store.test.js +42 -1
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +13 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +104 -3
- package/dist/journal.js.map +1 -1
- package/dist/sync/area-ledger-migration.d.ts +116 -0
- package/dist/sync/area-ledger-migration.d.ts.map +1 -0
- package/dist/sync/area-ledger-migration.js +505 -0
- package/dist/sync/area-ledger-migration.js.map +1 -0
- package/dist/sync/area-ledger-migration.test.d.ts +2 -0
- package/dist/sync/area-ledger-migration.test.d.ts.map +1 -0
- package/dist/sync/area-ledger-migration.test.js +274 -0
- package/dist/sync/area-ledger-migration.test.js.map +1 -0
- package/dist/sync/area-ledger.d.ts +93 -0
- package/dist/sync/area-ledger.d.ts.map +1 -0
- package/dist/sync/area-ledger.js +460 -0
- package/dist/sync/area-ledger.js.map +1 -0
- package/dist/sync/area-ledger.test.d.ts +2 -0
- package/dist/sync/area-ledger.test.d.ts.map +1 -0
- package/dist/sync/area-ledger.test.js +199 -0
- package/dist/sync/area-ledger.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts +14 -4
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +177 -61
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +140 -10
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/index.d.ts +2 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +22 -4
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +45 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/reconcile-cursor.d.ts +97 -0
- package/dist/sync/reconcile-cursor.d.ts.map +1 -0
- package/dist/sync/reconcile-cursor.js +215 -0
- package/dist/sync/reconcile-cursor.js.map +1 -0
- package/dist/sync/reconcile-cursor.test.d.ts +2 -0
- package/dist/sync/reconcile-cursor.test.d.ts.map +1 -0
- package/dist/sync/reconcile-cursor.test.js +139 -0
- package/dist/sync/reconcile-cursor.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +76 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +317 -3
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +39 -1
- package/dist/sync/state-store.test.js.map +1 -1
- package/dist/sync/sync-work.d.ts +2 -1
- package/dist/sync/sync-work.d.ts.map +1 -1
- package/dist/sync/sync-work.js.map +1 -1
- package/dist/types.d.ts +6 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts +9 -2
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +3 -2
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +26 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +3 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"area-ledger-migration.test.d.ts","sourceRoot":"","sources":["../../src/sync/area-ledger-migration.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import * as crypto from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
6
|
+
import { AreaLedgerMigration, areaLayoutManifestPath, assertV3OnlyPinAllowed, resolveAreaLayoutAuthority, setAreaLedgerMigrationTestHooksForTest } from "./area-ledger-migration.js";
|
|
7
|
+
import { AreaResolver } from "./area-resolver.js";
|
|
8
|
+
import { StateStore } from "./state-store.js";
|
|
9
|
+
const roots = [];
|
|
10
|
+
afterEach(() => {
|
|
11
|
+
setAreaLedgerMigrationTestHooksForTest(undefined);
|
|
12
|
+
for (const root of roots.splice(0))
|
|
13
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
14
|
+
});
|
|
15
|
+
function root() {
|
|
16
|
+
const value = fs.mkdtempSync(path.join(os.tmpdir(), "hq-area-migration-test-"));
|
|
17
|
+
roots.push(value);
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
20
|
+
function entry(size, tombstone = false) {
|
|
21
|
+
return {
|
|
22
|
+
hash: `${size}`.padStart(64, "0"),
|
|
23
|
+
size,
|
|
24
|
+
syncedAt: "2026-08-27T00:00:00.000Z",
|
|
25
|
+
direction: "up",
|
|
26
|
+
...(tombstone ? { removedAt: "2026-08-27T00:00:00.000Z", removedReason: "manual" } : {}),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function journal(files = {}) {
|
|
30
|
+
return { version: "2", lastSync: "before", files, pulls: [] };
|
|
31
|
+
}
|
|
32
|
+
const layout = {
|
|
33
|
+
format: 1,
|
|
34
|
+
routerVersion: 1,
|
|
35
|
+
areas: [
|
|
36
|
+
{ id: "root:alpha", prefixes: ["alpha"], paths: [] },
|
|
37
|
+
{ id: "root:beta", prefixes: ["beta"], paths: [] },
|
|
38
|
+
{ id: "root:@files", prefixes: [], paths: [], rootFiles: true },
|
|
39
|
+
],
|
|
40
|
+
};
|
|
41
|
+
function openLegacy(stateRoot, initial) {
|
|
42
|
+
return StateStore.open({
|
|
43
|
+
rootDir: stateRoot,
|
|
44
|
+
scopeId: "journal:personal-vault",
|
|
45
|
+
initialState: { journal: initial },
|
|
46
|
+
validateRecord: (record) => {
|
|
47
|
+
if (record.type !== 2)
|
|
48
|
+
throw new Error("unexpected test journal record");
|
|
49
|
+
},
|
|
50
|
+
reduce: (state, record) => {
|
|
51
|
+
const delta = record.payload;
|
|
52
|
+
for (const key of delta.deletes)
|
|
53
|
+
delete state.journal.files[key];
|
|
54
|
+
Object.assign(state.journal.files, delta.upserts);
|
|
55
|
+
state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
|
|
56
|
+
state.journal.pulls = delta.pulls ?? state.journal.pulls;
|
|
57
|
+
return state;
|
|
58
|
+
},
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
function migration(stateRoot, store) {
|
|
62
|
+
return new AreaLedgerMigration({
|
|
63
|
+
rootDir: stateRoot,
|
|
64
|
+
journalSlug: "personal-vault",
|
|
65
|
+
resolver: new AreaResolver(layout),
|
|
66
|
+
legacyStore: store,
|
|
67
|
+
maxRowsPerRecord: 2,
|
|
68
|
+
maxRecordBytes: 8 * 1024,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
function migrationWithLimits(stateRoot, store, maxRecordBytes) {
|
|
72
|
+
return new AreaLedgerMigration({
|
|
73
|
+
rootDir: stateRoot,
|
|
74
|
+
journalSlug: "personal-vault",
|
|
75
|
+
resolver: new AreaResolver(layout),
|
|
76
|
+
legacyStore: store,
|
|
77
|
+
maxRowsPerRecord: 100,
|
|
78
|
+
maxRecordBytes,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
function append(store, delta) {
|
|
82
|
+
store.append(2, delta);
|
|
83
|
+
}
|
|
84
|
+
/** Mutate a tiny fixture's authenticated generation number without generating 11,694 files. */
|
|
85
|
+
function retagGeneration(store, generation) {
|
|
86
|
+
const snapshot = path.join(store.stateDir, "snapshot-1.bin");
|
|
87
|
+
const wal = path.join(store.stateDir, "wal-1.bin");
|
|
88
|
+
const retag = (file, checksumOffset) => {
|
|
89
|
+
const bytes = fs.readFileSync(file);
|
|
90
|
+
bytes.writeBigUInt64BE(BigInt(generation), 8);
|
|
91
|
+
crypto.createHash("sha256").update(bytes.subarray(0, checksumOffset)).digest().copy(bytes, checksumOffset);
|
|
92
|
+
fs.writeFileSync(file, bytes);
|
|
93
|
+
};
|
|
94
|
+
// Snapshot checksum covers header + payload, while WAL checksum covers header.
|
|
95
|
+
retag(snapshot, fs.statSync(snapshot).size - 32);
|
|
96
|
+
retag(wal, 48);
|
|
97
|
+
fs.renameSync(snapshot, path.join(store.stateDir, `snapshot-${generation}.bin`));
|
|
98
|
+
fs.renameSync(wal, path.join(store.stateDir, `wal-${generation}.bin`));
|
|
99
|
+
fs.writeFileSync(path.join(store.stateDir, "current"), `${generation}\n`);
|
|
100
|
+
}
|
|
101
|
+
describe("AreaLedgerMigration", () => {
|
|
102
|
+
it("pins and projects a generation-11694 source into bounded area records", () => {
|
|
103
|
+
const stateRoot = root();
|
|
104
|
+
const initial = journal({ "alpha/a": entry(1), "alpha/b": entry(2), "beta/c": entry(3, true) });
|
|
105
|
+
const first = openLegacy(stateRoot, initial);
|
|
106
|
+
retagGeneration(first, 11_694);
|
|
107
|
+
const source = openLegacy(stateRoot, journal());
|
|
108
|
+
const result = migration(stateRoot, source).copyAndCatchUp();
|
|
109
|
+
expect(result.state.sourceGeneration).toBe(11_694);
|
|
110
|
+
expect(result.projectedRows).toBe(3);
|
|
111
|
+
expect(result.maxProjectedRecordBytes).toBeLessThanOrEqual(8 * 1024);
|
|
112
|
+
expect(resolveAreaLayoutAuthority(stateRoot, "personal-vault")).toBe("legacy");
|
|
113
|
+
// The fixture has one snapshot/WAL pair, rather than 11,694 large files.
|
|
114
|
+
expect(fs.readdirSync(source.stateDir).filter((name) => name.startsWith("snapshot-")).length).toBe(1);
|
|
115
|
+
});
|
|
116
|
+
it("replays a warm WAL tail idempotently after source rotation", () => {
|
|
117
|
+
const stateRoot = root();
|
|
118
|
+
const reader = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
119
|
+
const bridge = migration(stateRoot, reader);
|
|
120
|
+
bridge.copyAndCatchUp();
|
|
121
|
+
const writer = openLegacy(stateRoot, journal());
|
|
122
|
+
append(writer, { version: "2", upserts: { "beta/b": entry(2) }, deletes: [], lastSync: "tail" });
|
|
123
|
+
writer.replaceState(structuredClone(writer.getState()));
|
|
124
|
+
bridge.copyAndCatchUp();
|
|
125
|
+
bridge.copyAndCatchUp();
|
|
126
|
+
const cutover = bridge.cutOver();
|
|
127
|
+
expect(cutover.phase).toBe("area-authoritative");
|
|
128
|
+
});
|
|
129
|
+
it("reconciles a persisted source cursor when a fresh process already materialized the tail", () => {
|
|
130
|
+
const stateRoot = root();
|
|
131
|
+
const initial = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
132
|
+
migration(stateRoot, initial).copyAndCatchUp();
|
|
133
|
+
const writer = openLegacy(stateRoot, journal());
|
|
134
|
+
append(writer, { version: "2", upserts: { "beta/from-new-process": entry(2) }, deletes: [] });
|
|
135
|
+
// A new StateStore.open has no refresh records because it already replayed
|
|
136
|
+
// the tail. The persisted migration cursor must still drive reconciliation.
|
|
137
|
+
const reopened = openLegacy(stateRoot, journal());
|
|
138
|
+
const resumed = migration(stateRoot, reopened);
|
|
139
|
+
expect(resumed.cutOver().phase).toBe("area-authoritative");
|
|
140
|
+
expect(resumed.readAreaJournal().files["beta/from-new-process"]).toMatchObject({ size: 2 });
|
|
141
|
+
});
|
|
142
|
+
it("flushes before a multi-row projection crosses its byte limit", () => {
|
|
143
|
+
const stateRoot = root();
|
|
144
|
+
const long = "x".repeat(420);
|
|
145
|
+
const source = openLegacy(stateRoot, journal({
|
|
146
|
+
[`alpha/${long}-one`]: entry(1),
|
|
147
|
+
[`alpha/${long}-two`]: entry(2),
|
|
148
|
+
}));
|
|
149
|
+
const bridge = migrationWithLimits(stateRoot, source, 1024);
|
|
150
|
+
const result = bridge.copyAndCatchUp();
|
|
151
|
+
expect(result.maxProjectedRecordBytes).toBeLessThanOrEqual(1024);
|
|
152
|
+
expect(bridge.cutOver().phase).toBe("area-authoritative");
|
|
153
|
+
});
|
|
154
|
+
it("rejects only a single projection row that cannot fit the limit", () => {
|
|
155
|
+
const stateRoot = root();
|
|
156
|
+
const source = openLegacy(stateRoot, journal({ [`alpha/${"x".repeat(2_000)}`]: entry(1) }));
|
|
157
|
+
expect(() => migrationWithLimits(stateRoot, source, 1024).copyAndCatchUp())
|
|
158
|
+
.toThrow("projected record exceeded bounded size");
|
|
159
|
+
});
|
|
160
|
+
it("persists and retires the source pin across separate migration processes", () => {
|
|
161
|
+
const stateRoot = root();
|
|
162
|
+
const source = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
163
|
+
const copied = migration(stateRoot, source).copyAndCatchUp();
|
|
164
|
+
expect(copied.state.sourcePinPath).toBeDefined();
|
|
165
|
+
expect(fs.readdirSync(source.stateDir).filter((name) => name.startsWith("generation-pin-")).length).toBe(1);
|
|
166
|
+
const reopened = openLegacy(stateRoot, journal());
|
|
167
|
+
expect(migration(stateRoot, reopened).cutOver().sourcePinPath).toBeUndefined();
|
|
168
|
+
expect(fs.readdirSync(source.stateDir).filter((name) => name.startsWith("generation-pin-")).length).toBe(0);
|
|
169
|
+
});
|
|
170
|
+
it("retires a persisted source pin when an uncommitted migration is abandoned", () => {
|
|
171
|
+
const stateRoot = root();
|
|
172
|
+
const source = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
173
|
+
migration(stateRoot, source).copyAndCatchUp();
|
|
174
|
+
expect(fs.readdirSync(source.stateDir).some((name) => name.startsWith("generation-pin-"))).toBe(true);
|
|
175
|
+
expect(migration(stateRoot, openLegacy(stateRoot, journal())).abandon().phase).toBe("legacy-authoritative");
|
|
176
|
+
expect(fs.readdirSync(source.stateDir).some((name) => name.startsWith("generation-pin-"))).toBe(false);
|
|
177
|
+
});
|
|
178
|
+
it("chooses legacy before the manifest rename and areas after it, including a torn primary manifest", () => {
|
|
179
|
+
const stateRoot = root();
|
|
180
|
+
const source = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
181
|
+
const bridge = migration(stateRoot, source);
|
|
182
|
+
setAreaLedgerMigrationTestHooksForTest({ beforeManifestCommit: () => { throw new Error("crash before commit"); } });
|
|
183
|
+
expect(() => bridge.cutOver()).toThrow("crash before commit");
|
|
184
|
+
expect(resolveAreaLayoutAuthority(stateRoot, "personal-vault")).toBe("legacy");
|
|
185
|
+
setAreaLedgerMigrationTestHooksForTest(undefined);
|
|
186
|
+
expect(bridge.cutOver().phase).toBe("area-authoritative");
|
|
187
|
+
fs.writeFileSync(areaLayoutManifestPath(stateRoot, "personal-vault"), "{torn");
|
|
188
|
+
expect(resolveAreaLayoutAuthority(stateRoot, "personal-vault")).toBe("areas");
|
|
189
|
+
});
|
|
190
|
+
it("keeps legacy authoritative across every interrupted pre-commit durable phase", () => {
|
|
191
|
+
for (const phase of ["source-pinned", "area-cursor-chunk", "area-cursor", "snapshot-projected", "tail-replayed"]) {
|
|
192
|
+
const stateRoot = root();
|
|
193
|
+
const source = openLegacy(stateRoot, journal({ "alpha/a": entry(1), "alpha/b": entry(2), "alpha/c": entry(3) }));
|
|
194
|
+
setAreaLedgerMigrationTestHooksForTest({ afterDurablePhase: (actual) => {
|
|
195
|
+
if (actual === phase)
|
|
196
|
+
throw new Error(`crash at ${phase}`);
|
|
197
|
+
} });
|
|
198
|
+
expect(() => migration(stateRoot, source).copyAndCatchUp()).toThrow(`crash at ${phase}`);
|
|
199
|
+
expect(resolveAreaLayoutAuthority(stateRoot, "personal-vault")).toBe("legacy");
|
|
200
|
+
setAreaLedgerMigrationTestHooksForTest(undefined);
|
|
201
|
+
}
|
|
202
|
+
const stateRoot = root();
|
|
203
|
+
const source = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
204
|
+
setAreaLedgerMigrationTestHooksForTest({ afterDurablePhase: (phase) => {
|
|
205
|
+
if (phase === "manifest-commit")
|
|
206
|
+
throw new Error("crash after commit");
|
|
207
|
+
} });
|
|
208
|
+
expect(() => migration(stateRoot, source).cutOver()).toThrow("crash after commit");
|
|
209
|
+
expect(resolveAreaLayoutAuthority(stateRoot, "personal-vault")).toBe("areas");
|
|
210
|
+
});
|
|
211
|
+
it("keeps rollback unsafe during mirror lag, then composes back before allowing a v3-only pin", () => {
|
|
212
|
+
const stateRoot = root();
|
|
213
|
+
const source = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
214
|
+
const bridge = migration(stateRoot, source);
|
|
215
|
+
bridge.cutOver();
|
|
216
|
+
setAreaLedgerMigrationTestHooksForTest({ afterDurablePhase: (phase) => {
|
|
217
|
+
if (phase === "area-commit")
|
|
218
|
+
throw new Error("crash before mirror");
|
|
219
|
+
} });
|
|
220
|
+
expect(() => bridge.reverseShadow({ upserts: { "beta/b": entry(2) } })).toThrow("crash before mirror");
|
|
221
|
+
expect(() => assertV3OnlyPinAllowed(stateRoot, "personal-vault")).toThrow("refusing v3-only pin");
|
|
222
|
+
setAreaLedgerMigrationTestHooksForTest(undefined);
|
|
223
|
+
expect(migration(stateRoot, source).composeBackForV3Rollback().rollbackSafe).toBe(true);
|
|
224
|
+
expect(() => assertV3OnlyPinAllowed(stateRoot, "personal-vault")).not.toThrow();
|
|
225
|
+
});
|
|
226
|
+
it("publishes rollback-unsafety before the area append can become durable", () => {
|
|
227
|
+
const stateRoot = root();
|
|
228
|
+
const source = openLegacy(stateRoot, journal({ "alpha/a": entry(1) }));
|
|
229
|
+
const bridge = migration(stateRoot, source);
|
|
230
|
+
bridge.cutOver();
|
|
231
|
+
setAreaLedgerMigrationTestHooksForTest({ afterDurablePhase: (phase) => {
|
|
232
|
+
if (phase === "area-commit-intent")
|
|
233
|
+
throw new Error("crash before area append");
|
|
234
|
+
} });
|
|
235
|
+
expect(() => bridge.reverseShadow({ upserts: { "beta/not-appended": entry(2) } }))
|
|
236
|
+
.toThrow("crash before area append");
|
|
237
|
+
expect(() => assertV3OnlyPinAllowed(stateRoot, "personal-vault")).toThrow("refusing v3-only pin");
|
|
238
|
+
expect(migration(stateRoot, source).readAreaJournal().files["beta/not-appended"]).toBeUndefined();
|
|
239
|
+
});
|
|
240
|
+
it("computes canonical parity digests incrementally with the existing digest contract", () => {
|
|
241
|
+
const stateRoot = root();
|
|
242
|
+
const sourceJournal = journal({ "alpha/a": entry(1), "beta/b": entry(2, true) });
|
|
243
|
+
const source = openLegacy(stateRoot, sourceJournal);
|
|
244
|
+
const bridge = migration(stateRoot, source);
|
|
245
|
+
bridge.copyAndCatchUp();
|
|
246
|
+
const expectedRows = Object.keys(sourceJournal.files).sort()
|
|
247
|
+
.map((key) => `${key}\n${canonical(sourceJournal.files[key])}`).join("\n");
|
|
248
|
+
expect(bridge.verifyParity(sourceJournal)).toMatchObject({
|
|
249
|
+
rowDigest: crypto.createHash("sha256").update(expectedRows).digest("hex"),
|
|
250
|
+
unionDigest: crypto.createHash("sha256").update("alpha/a\nbeta/b").digest("hex"),
|
|
251
|
+
tombstoneDigest: crypto.createHash("sha256").update("beta/b").digest("hex"),
|
|
252
|
+
});
|
|
253
|
+
});
|
|
254
|
+
it("models the 300k-row / 232MB production shape without materializing 232MB payloads", () => {
|
|
255
|
+
const logicalRows = 300_000;
|
|
256
|
+
const logicalBytes = 232 * 1024 * 1024;
|
|
257
|
+
const rowBytes = Math.floor(logicalBytes / logicalRows);
|
|
258
|
+
const maxRecordBytes = DEFAULT_SHAPE_RECORD_BYTES;
|
|
259
|
+
const records = Math.ceil(logicalRows / Math.floor(maxRecordBytes / rowBytes));
|
|
260
|
+
expect(records).toBeGreaterThan(1);
|
|
261
|
+
expect(maxRecordBytes).toBeLessThan(256 * 1024 * 1024);
|
|
262
|
+
});
|
|
263
|
+
});
|
|
264
|
+
const DEFAULT_SHAPE_RECORD_BYTES = 256 * 1024;
|
|
265
|
+
function canonical(value) {
|
|
266
|
+
if (value === null || typeof value === "boolean" || typeof value === "string" || typeof value === "number") {
|
|
267
|
+
return JSON.stringify(value);
|
|
268
|
+
}
|
|
269
|
+
if (Array.isArray(value))
|
|
270
|
+
return `[${value.map(canonical).join(",")}]`;
|
|
271
|
+
const object = value;
|
|
272
|
+
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`).join(",")}}`;
|
|
273
|
+
}
|
|
274
|
+
//# sourceMappingURL=area-ledger-migration.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"area-ledger-migration.test.js","sourceRoot":"","sources":["../../src/sync/area-ledger-migration.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAEzD,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,sCAAsC,EAAE,MAAM,4BAA4B,CAAC;AACrL,OAAO,EAAE,YAAY,EAAmB,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,KAAK,GAAa,EAAE,CAAC;AAE3B,SAAS,CAAC,GAAG,EAAE;IACb,sCAAsC,CAAC,SAAS,CAAC,CAAC;IAClD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACxF,CAAC,CAAC,CAAC;AAEH,SAAS,IAAI;IACX,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC,CAAC;IAChF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,KAAK,CAAC,IAAY,EAAE,SAAS,GAAG,KAAK;IAC5C,OAAO;QACL,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC;QACjC,IAAI;QACJ,QAAQ,EAAE,0BAA0B;QACpC,SAAS,EAAE,IAAI;QACf,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,aAAa,EAAE,QAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClG,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,QAAsC,EAAE;IACvD,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,MAAM,GAAe;IACzB,MAAM,EAAE,CAAC;IACT,aAAa,EAAE,CAAC;IAChB,KAAK,EAAE;QACL,EAAE,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QACpD,EAAE,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QAClD,EAAE,EAAE,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;KAChE;CACF,CAAC;AAEF,SAAS,UAAU,CAAC,SAAiB,EAAE,OAAoB;IACzD,OAAO,UAAU,CAAC,IAAI,CAAC;QACrB,OAAO,EAAE,SAAS;QAClB,OAAO,EAAE,wBAAwB;QACjC,YAAY,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE;QAClC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE;YACzB,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,MAAM,KAAK,GAAG,MAAM,CAAC,OAAgC,CAAC;YACtD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO;gBAAE,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAClD,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAClE,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACzD,OAAO,KAAK,CAAC;QACf,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,SAAiB,EAAE,KAAiC;IACrE,OAAO,IAAI,mBAAmB,CAAC;QAC7B,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,gBAAgB;QAC7B,QAAQ,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC;QAClC,WAAW,EAAE,KAAK;QAClB,gBAAgB,EAAE,CAAC;QACnB,cAAc,EAAE,CAAC,GAAG,IAAI;KACzB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAC1B,SAAiB,EACjB,KAAiC,EACjC,cAAsB;IAEtB,OAAO,IAAI,mBAAmB,CAAC;QAC7B,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,gBAAgB;QAC7B,QAAQ,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC;QAClC,WAAW,EAAE,KAAK;QAClB,gBAAgB,EAAE,GAAG;QACrB,cAAc;KACf,CAAC,CAAC;AACL,CAAC;AAED,SAAS,MAAM,CAAC,KAAiC,EAAE,KAA4B;IAC7E,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACzB,CAAC;AAED,+FAA+F;AAC/F,SAAS,eAAe,CAAC,KAA0B,EAAE,UAAkB;IACrE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,cAAsB,EAAE,EAAE;QACrD,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACpC,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QAC3G,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IACjD,KAAK,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACf,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,YAAY,UAAU,MAAM,CAAC,CAAC,CAAC;IACjF,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,UAAU,MAAM,CAAC,CAAC,CAAC;IACvE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,GAAG,UAAU,IAAI,CAAC,CAAC;AAC5E,CAAC;AAED,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,uEAAuE,EAAE,GAAG,EAAE;QAC/E,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAChG,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7C,eAAe,CAAC,KAAuC,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,cAAc,EAAE,CAAC;QAE7D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,mBAAmB,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACrE,MAAM,CAAC,0BAA0B,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/E,yEAAyE;QACzE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QACjG,MAAM,CAAC,YAAY,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yFAAyF,EAAE,GAAG,EAAE;QACjG,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,cAAc,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,uBAAuB,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAE9F,2EAA2E;QAC3E,4EAA4E;QAC5E,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC3D,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC;YAC3C,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;SAChC,CAAC,CAAC,CAAC;QACJ,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5F,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;aACxE,OAAO,CAAC,wCAAwC,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACjF,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,cAAc,EAAE,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE5G,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAClD,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,CAAC,aAAa,EAAE,CAAC;QAC/E,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9G,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,cAAc,EAAE,CAAC;QAC9C,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtG,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAC5G,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iGAAiG,EAAE,GAAG,EAAE;QACzG,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,sCAAsC,CAAC,EAAE,oBAAoB,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpH,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAC9D,MAAM,CAAC,0BAA0B,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/E,sCAAsC,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC1D,EAAE,CAAC,aAAa,CAAC,sBAAsB,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAC;QAC/E,MAAM,CAAC,0BAA0B,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,GAAG,EAAE;QACtF,KAAK,MAAM,KAAK,IAAI,CAAC,eAAe,EAAE,mBAAmB,EAAE,aAAa,EAAE,oBAAoB,EAAE,eAAe,CAAC,EAAE,CAAC;YACjH,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjH,sCAAsC,CAAC,EAAE,iBAAiB,EAAE,CAAC,MAAM,EAAE,EAAE;oBACrE,IAAI,MAAM,KAAK,KAAK;wBAAE,MAAM,IAAI,KAAK,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC;gBAC7D,CAAC,EAAE,CAAC,CAAC;YACL,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC;YACzF,MAAM,CAAC,0BAA0B,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/E,sCAAsC,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,sCAAsC,CAAC,EAAE,iBAAiB,EAAE,CAAC,KAAK,EAAE,EAAE;gBACpE,IAAI,KAAK,KAAK,iBAAiB;oBAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACzE,CAAC,EAAE,CAAC,CAAC;QACL,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACnF,MAAM,CAAC,0BAA0B,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2FAA2F,EAAE,GAAG,EAAE;QACnG,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,sCAAsC,CAAC,EAAE,iBAAiB,EAAE,CAAC,KAAK,EAAE,EAAE;gBACpE,IAAI,KAAK,KAAK,aAAa;oBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACtE,CAAC,EAAE,CAAC,CAAC;QACL,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QACvG,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAClG,sCAAsC,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,wBAAwB,EAAE,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxF,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,GAAG,EAAE;QAC/E,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,sCAAsC,CAAC,EAAE,iBAAiB,EAAE,CAAC,KAAK,EAAE,EAAE;gBACpE,IAAI,KAAK,KAAK,oBAAoB;oBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAClF,CAAC,EAAE,CAAC,CAAC;QACL,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,EAAE,mBAAmB,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;aAC/E,OAAO,CAAC,0BAA0B,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAClG,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IACpG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,GAAG,EAAE;QAC3F,MAAM,SAAS,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,aAAa,GAAG,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE;aACzD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC;YACvD,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;YACzE,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;YAChF,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC5E,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,GAAG,EAAE;QAC3F,MAAM,WAAW,GAAG,OAAO,CAAC;QAC5B,MAAM,YAAY,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC;QACxD,MAAM,cAAc,GAAG,0BAA0B,CAAC;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC;QAC/E,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAAG,GAAG,GAAG,IAAI,CAAC;AAE9C,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC3G,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACvE,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AACtH,CAAC"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { JournalEntry, PullRecord, SyncJournal } from "../types.js";
|
|
2
|
+
type JournalMetadata = {
|
|
3
|
+
lastSync?: string;
|
|
4
|
+
pulls?: PullRecord[];
|
|
5
|
+
};
|
|
6
|
+
export interface AreaLedgerDelta {
|
|
7
|
+
upserts?: Record<string, JournalEntry>;
|
|
8
|
+
deletes?: readonly string[];
|
|
9
|
+
metadata?: JournalMetadata;
|
|
10
|
+
}
|
|
11
|
+
export interface AreaLedgerOptions {
|
|
12
|
+
rootDir: string;
|
|
13
|
+
journalSlug: string;
|
|
14
|
+
layoutEpoch: number;
|
|
15
|
+
/** Every configured data area. `@meta` is reserved for the ledger itself. */
|
|
16
|
+
areaIds: readonly string[];
|
|
17
|
+
/** Returns the one configured area responsible for a canonical journal key. */
|
|
18
|
+
resolveArea: (key: string) => string | undefined;
|
|
19
|
+
/** Used only when an area StateStore is first created; this is not migration. */
|
|
20
|
+
initialJournal?: SyncJournal;
|
|
21
|
+
maxRecordsBeforeCompaction?: number;
|
|
22
|
+
maxWalBytesBeforeCompaction?: number;
|
|
23
|
+
}
|
|
24
|
+
interface AreaLedgerTestHooks {
|
|
25
|
+
afterDataBeforeWatermark?: () => void;
|
|
26
|
+
afterWatermarkBeforeFinalize?: () => void;
|
|
27
|
+
}
|
|
28
|
+
/** Test-only crash seam: data prepares are durable before a meta watermark. */
|
|
29
|
+
export declare function setAreaLedgerTestHooksForTest(hooks: AreaLedgerTestHooks | undefined): void;
|
|
30
|
+
/**
|
|
31
|
+
* The storage half of the parallel-sync area design.
|
|
32
|
+
*
|
|
33
|
+
* Data records are prepared in area stores first. The tiny `@meta` record is
|
|
34
|
+
* the final watermark. A crash before that watermark leaves a recoverable,
|
|
35
|
+
* invisible prepare; a crash after it leaves a complete aggregate view. This
|
|
36
|
+
* makes global metadata data-first and watermark-last without putting a large
|
|
37
|
+
* aggregate snapshot behind one append lock.
|
|
38
|
+
*/
|
|
39
|
+
export declare class AreaLedger {
|
|
40
|
+
private readonly options;
|
|
41
|
+
private readonly areaIds;
|
|
42
|
+
private readonly areas;
|
|
43
|
+
private meta;
|
|
44
|
+
private readonly opened;
|
|
45
|
+
private constructor();
|
|
46
|
+
static open(options: AreaLedgerOptions): AreaLedger;
|
|
47
|
+
/** Scope identity is stable and human-auditable even though StateStore hashes it on disk. */
|
|
48
|
+
scopeId(areaId: string): string;
|
|
49
|
+
/** Opens one data area only. Useful for a one-path, metadata-free operation. */
|
|
50
|
+
readScoped(keys: readonly string[]): SyncJournal;
|
|
51
|
+
/** Aggregate read. It is the parity view used by legacy SyncJournal callers. */
|
|
52
|
+
read(): SyncJournal;
|
|
53
|
+
/**
|
|
54
|
+
* Commit a partitioned journal delta. A metadata-only update still has a
|
|
55
|
+
* `@meta` append, while all data-bearing commits use the recoverable prepare
|
|
56
|
+
* protocol so multiple areas never expose a half transaction to `read()`.
|
|
57
|
+
*/
|
|
58
|
+
append(delta: AreaLedgerDelta): void;
|
|
59
|
+
/**
|
|
60
|
+
* Finish a transaction whose process died after durable area prepares but
|
|
61
|
+
* before its meta watermark. Incomplete prepares stay invisible, so a
|
|
62
|
+
* crash can never synthesize a partial aggregate journal.
|
|
63
|
+
*/
|
|
64
|
+
recoverPendingTransactions(): void;
|
|
65
|
+
/** Test/diagnostic surface proving independent scopes have independent locks. */
|
|
66
|
+
stateDirectoriesForTest(): ReadonlyMap<string, string>;
|
|
67
|
+
openedAreasForTest(): readonly string[];
|
|
68
|
+
transactionCountsForTest(): {
|
|
69
|
+
prepared: number;
|
|
70
|
+
watermarks: number;
|
|
71
|
+
finalized: number;
|
|
72
|
+
};
|
|
73
|
+
private appendMeta;
|
|
74
|
+
private openArea;
|
|
75
|
+
private openMeta;
|
|
76
|
+
private initialAreaJournal;
|
|
77
|
+
private partition;
|
|
78
|
+
private resolveArea;
|
|
79
|
+
private assertKnownArea;
|
|
80
|
+
private combine;
|
|
81
|
+
/** A watermark publishes only when every named area has prepared or folded it. */
|
|
82
|
+
private committedTransactions;
|
|
83
|
+
/**
|
|
84
|
+
* Bound replay after publication. Area folds are individually durable; a
|
|
85
|
+
* folded marker keeps a still-present watermark atomic if the process dies
|
|
86
|
+
* between peers. The watermark is retired only once every named area has
|
|
87
|
+
* its folded base snapshot, then a later pass clears the transient markers.
|
|
88
|
+
*/
|
|
89
|
+
private finalizeCommittedTransactions;
|
|
90
|
+
private finalizePublishedTransactions;
|
|
91
|
+
}
|
|
92
|
+
export {};
|
|
93
|
+
//# sourceMappingURL=area-ledger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"area-ledger.d.ts","sourceRoot":"","sources":["../../src/sync/area-ledger.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAQzE,KAAK,eAAe,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC;AAEnE,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACvC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,QAAQ,CAAC,EAAE,eAAe,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3B,+EAA+E;IAC/E,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;IACjD,iFAAiF;IACjF,cAAc,CAAC,EAAE,WAAW,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC;AAoCD,UAAU,mBAAmB;IAC3B,wBAAwB,CAAC,EAAE,MAAM,IAAI,CAAC;IACtC,4BAA4B,CAAC,EAAE,MAAM,IAAI,CAAC;CAC3C;AAID,+EAA+E;AAC/E,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,mBAAmB,GAAG,SAAS,GAAG,IAAI,CAE1F;AAED;;;;;;;;GAQG;AACH,qBAAa,UAAU;IAMD,OAAO,CAAC,QAAQ,CAAC,OAAO;IAL5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;IAC5C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA4C;IAClE,OAAO,CAAC,IAAI,CAAoC;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAE5C,OAAO;IAWP,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,GAAG,UAAU;IAInD,6FAA6F;IAC7F,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAK/B,gFAAgF;IAChF,UAAU,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,WAAW;IA6BhD,gFAAgF;IAChF,IAAI,IAAI,WAAW;IAmBnB;;;;OAIG;IACH,MAAM,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI;IAmCpC;;;;OAIG;IACH,0BAA0B,IAAI,IAAI;IA6BlC,iFAAiF;IACjF,uBAAuB,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC;IAOtD,kBAAkB,IAAI,SAAS,MAAM,EAAE;IAIvC,wBAAwB,IAAI;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;IAYvF,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,QAAQ;IAmBhB,OAAO,CAAC,QAAQ;IAuBhB,OAAO,CAAC,kBAAkB;IAU1B,OAAO,CAAC,SAAS;IAejB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,OAAO;IAWf,kFAAkF;IAClF,OAAO,CAAC,qBAAqB;IAY7B;;;;;OAKG;IACH,OAAO,CAAC,6BAA6B;IA4BrC,OAAO,CAAC,6BAA6B;CAWtC"}
|