@indigoai-us/hq-cloud 6.16.58-alpha.3 → 6.16.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +0 -14
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +89 -57
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +313 -56
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -172
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -51
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +165 -409
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -19
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +5 -40
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +0 -47
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +13 -59
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -48
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +229 -1499
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +0 -55
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +1 -87
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +8 -12
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +152 -302
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +126 -256
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +34 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +92 -21
- package/dist/cognito-auth.js.map +1 -1
- package/dist/context.d.ts +5 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +92 -12
- package/dist/context.js.map +1 -1
- package/dist/context.test.js +77 -2
- package/dist/context.test.js.map +1 -1
- package/dist/ignore.d.ts +10 -0
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +47 -5
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +30 -3
- package/dist/ignore.test.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -40
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -240
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +0 -54
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -324
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +16 -51
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +79 -303
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/lib/net-errors.d.ts +12 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +35 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/net-errors.test.js +19 -1
- package/dist/lib/net-errors.test.js.map +1 -1
- package/dist/machine-auth.test.js +6 -4
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/machine-token-cache-isolation.test.d.ts +24 -0
- package/dist/machine-token-cache-isolation.test.d.ts.map +1 -0
- package/dist/machine-token-cache-isolation.test.js +241 -0
- package/dist/machine-token-cache-isolation.test.js.map +1 -0
- package/dist/manifest/build-manifest-resident-parity.test.js +3 -3
- package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +0 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +2 -20
- package/dist/s3.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +3 -27
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +39 -149
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +5 -192
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +0 -6
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +2 -14
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watcher.d.ts +0 -12
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +0 -131
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +0 -63
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Resident-memory gate for an all-mode pull at the list -> plan boundary.
|
|
3
|
-
*
|
|
4
|
-
* `sync()` receives every remote object through `listRemoteFilesForScope()`;
|
|
5
|
-
* all mode deliberately invokes the broad `listRemoteFiles()` branch. This
|
|
6
|
-
* fixture reaches that production seam with a paged ObjectIO transport, then
|
|
7
|
-
* stops at the emitted plan before transfer/execution can own unrelated data.
|
|
8
|
-
*
|
|
9
|
-
* The fixture is 50,000 long-path rows rather than the largest 677,732-row
|
|
10
|
-
* personal vault. It matches the steady-state phase-two retention harness's
|
|
11
|
-
* practical full-pass cardinality, makes three independent trials CI-sized,
|
|
12
|
-
* and keeps the bound per-row so the production scope remains constrained.
|
|
13
|
-
*/
|
|
14
|
-
export {};
|
|
15
|
-
//# sourceMappingURL=streaming-pull-planner.bench.test.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"streaming-pull-planner.bench.test.d.ts","sourceRoot":"","sources":["../../src/cli/streaming-pull-planner.bench.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
|
|
@@ -1,273 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Resident-memory gate for an all-mode pull at the list -> plan boundary.
|
|
3
|
-
*
|
|
4
|
-
* `sync()` receives every remote object through `listRemoteFilesForScope()`;
|
|
5
|
-
* all mode deliberately invokes the broad `listRemoteFiles()` branch. This
|
|
6
|
-
* fixture reaches that production seam with a paged ObjectIO transport, then
|
|
7
|
-
* stops at the emitted plan before transfer/execution can own unrelated data.
|
|
8
|
-
*
|
|
9
|
-
* The fixture is 50,000 long-path rows rather than the largest 677,732-row
|
|
10
|
-
* personal vault. It matches the steady-state phase-two retention harness's
|
|
11
|
-
* practical full-pass cardinality, makes three independent trials CI-sized,
|
|
12
|
-
* and keeps the bound per-row so the production scope remains constrained.
|
|
13
|
-
*/
|
|
14
|
-
import * as fs from "node:fs";
|
|
15
|
-
import * as os from "node:os";
|
|
16
|
-
import * as path from "node:path";
|
|
17
|
-
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
18
|
-
import { clearContextCache } from "../context.js";
|
|
19
|
-
import { closeCachedAreaJournalMigration, openJournalStateStoreForMigration, releaseCachedAreaJournalMigrations, writeJournal, } from "../journal.js";
|
|
20
|
-
import { acquireGc, collectGarbage, usedBytes } from "../journal-snapshot-memory-helpers.js";
|
|
21
|
-
import { setObjectIOFactory, } from "../object-io.js";
|
|
22
|
-
import { AreaLedgerMigration } from "../sync/area-ledger-migration.js";
|
|
23
|
-
import { AreaResolver } from "../sync/area-resolver.js";
|
|
24
|
-
import { sync } from "./sync.js";
|
|
25
|
-
const ROW_COUNT = 50_000;
|
|
26
|
-
const PAGE_SIZE = 1_000;
|
|
27
|
-
const TRIALS = 3;
|
|
28
|
-
const CONTROL_BYTES = 16 * 1024 * 1024;
|
|
29
|
-
// The current released phase-two store retains 117.73 B/row. A streaming
|
|
30
|
-
// planner should need no more than two similarly compact per-key structures:
|
|
31
|
-
// a canonical presence marker for delete detection and a decision marker for
|
|
32
|
-
// an actionable item. 2 * 117.73 rounds up to 236 B/row. The fixed 3 MiB
|
|
33
|
-
// covers one 1,000-object page (~0.37 MiB at the measured old shape) plus
|
|
34
|
-
// table capacity and one pass's non-row bookkeeping. The journal itself is
|
|
35
|
-
// deliberately in the pre-measurement baseline.
|
|
36
|
-
const MAX_LIVE_BOUNDARY_BYTES_PER_ROW = 236;
|
|
37
|
-
const MAX_FIXED_LIVE_BOUNDARY_BYTES = 3 * 1024 * 1024;
|
|
38
|
-
const MAX_LIVE_BOUNDARY_BYTES = MAX_FIXED_LIVE_BOUNDARY_BYTES + ROW_COUNT * MAX_LIVE_BOUNDARY_BYTES_PER_ROW;
|
|
39
|
-
const EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
40
|
-
const STOP_AFTER_PLAN = Symbol("streaming-pull-planner-boundary");
|
|
41
|
-
const ORIGINAL_STATE_DIR = process.env.HQ_STATE_DIR;
|
|
42
|
-
const layout = {
|
|
43
|
-
format: 1,
|
|
44
|
-
routerVersion: 1,
|
|
45
|
-
areas: [
|
|
46
|
-
{ id: "root:docs", prefixes: ["docs"], paths: [] },
|
|
47
|
-
{ id: "root:@files", prefixes: [], paths: [], rootFiles: true },
|
|
48
|
-
],
|
|
49
|
-
};
|
|
50
|
-
const vaultConfig = {
|
|
51
|
-
apiUrl: "https://vault-api.test",
|
|
52
|
-
authToken: "streaming-pull-planner-test-token",
|
|
53
|
-
region: "us-east-1",
|
|
54
|
-
companyVaultUsesPresign: true,
|
|
55
|
-
};
|
|
56
|
-
let baseDir = "";
|
|
57
|
-
let hqRoot = "";
|
|
58
|
-
let stateDir = "";
|
|
59
|
-
let collect;
|
|
60
|
-
function fixtureKey(index) {
|
|
61
|
-
const token = index.toString(36).padStart(8, "0");
|
|
62
|
-
return `docs/directory-${index % 400}/nested-${index % 37}/entry-${token.repeat(15)}.md`;
|
|
63
|
-
}
|
|
64
|
-
function remoteEtag(index) {
|
|
65
|
-
return `streaming-boundary-${index.toString(16).padStart(8, "0")}`;
|
|
66
|
-
}
|
|
67
|
-
function pagedObjectIO() {
|
|
68
|
-
return {
|
|
69
|
-
putObject: async () => ({ etag: "unused" }),
|
|
70
|
-
getObject: async () => {
|
|
71
|
-
throw new Error("all-mode boundary fixture must stop before download");
|
|
72
|
-
},
|
|
73
|
-
deleteObject: async () => {
|
|
74
|
-
throw new Error("all-mode boundary fixture must stop before delete");
|
|
75
|
-
},
|
|
76
|
-
headObject: async () => {
|
|
77
|
-
throw new Error("all-mode boundary fixture must stop before HEAD");
|
|
78
|
-
},
|
|
79
|
-
listObjects: async (input) => {
|
|
80
|
-
expect(input.prefix).toBeUndefined();
|
|
81
|
-
const start = input.continuationToken === undefined ? 0 : Number(input.continuationToken);
|
|
82
|
-
const end = Math.min(start + PAGE_SIZE, ROW_COUNT);
|
|
83
|
-
const objects = [];
|
|
84
|
-
for (let index = start; index < end; index += 1) {
|
|
85
|
-
objects.push({
|
|
86
|
-
key: fixtureKey(index),
|
|
87
|
-
size: 0,
|
|
88
|
-
lastModified: new Date(1_700_000_000_000 + index),
|
|
89
|
-
etag: remoteEtag(index),
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
return {
|
|
93
|
-
objects,
|
|
94
|
-
nextContinuationToken: end < ROW_COUNT ? String(end) : undefined,
|
|
95
|
-
};
|
|
96
|
-
},
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
function seedAreaAuthoritativeJournal() {
|
|
100
|
-
const slug = "streaming-boundary";
|
|
101
|
-
const companyRoot = path.join(hqRoot, "companies", slug);
|
|
102
|
-
const files = {};
|
|
103
|
-
for (let index = 0; index < ROW_COUNT; index += 1) {
|
|
104
|
-
const key = fixtureKey(index);
|
|
105
|
-
const localPath = path.join(companyRoot, key);
|
|
106
|
-
fs.mkdirSync(path.dirname(localPath), { recursive: true });
|
|
107
|
-
fs.writeFileSync(localPath, "");
|
|
108
|
-
const stats = fs.lstatSync(localPath);
|
|
109
|
-
files[key] = {
|
|
110
|
-
hash: EMPTY_FILE_SHA256,
|
|
111
|
-
size: 0,
|
|
112
|
-
syncedAt: "2026-09-15T00:00:00.000Z",
|
|
113
|
-
direction: "down",
|
|
114
|
-
kind: "file",
|
|
115
|
-
remoteEtag: remoteEtag(index),
|
|
116
|
-
mtimeMs: Math.floor(stats.mtimeMs),
|
|
117
|
-
ctimeMs: Math.floor(stats.ctimeMs),
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
writeJournal(slug, { version: "2", lastSync: "seed", files, pulls: [] });
|
|
121
|
-
const migration = new AreaLedgerMigration({
|
|
122
|
-
rootDir: stateDir,
|
|
123
|
-
journalSlug: slug,
|
|
124
|
-
resolver: new AreaResolver(layout),
|
|
125
|
-
legacyStore: openJournalStateStoreForMigration(slug),
|
|
126
|
-
maxRecordBytes: 32 * 1024 * 1024,
|
|
127
|
-
maxRowsPerRecord: ROW_COUNT,
|
|
128
|
-
});
|
|
129
|
-
migration.copyAndCatchUp();
|
|
130
|
-
expect(migration.cutOver(Number.POSITIVE_INFINITY).phase).toBe("area-authoritative");
|
|
131
|
-
migration.dispose();
|
|
132
|
-
}
|
|
133
|
-
async function runThroughAllModePlan(onPlan) {
|
|
134
|
-
try {
|
|
135
|
-
await sync({
|
|
136
|
-
company: "streaming-boundary",
|
|
137
|
-
vaultConfig,
|
|
138
|
-
hqRoot,
|
|
139
|
-
fileTombstones: new Map(),
|
|
140
|
-
skipReindex: true,
|
|
141
|
-
operationLockAlreadyHeld: true,
|
|
142
|
-
onEvent: (event) => {
|
|
143
|
-
if (event.type !== "plan")
|
|
144
|
-
return;
|
|
145
|
-
onPlan(event);
|
|
146
|
-
throw STOP_AFTER_PLAN;
|
|
147
|
-
},
|
|
148
|
-
});
|
|
149
|
-
}
|
|
150
|
-
catch (error) {
|
|
151
|
-
if (error === STOP_AFTER_PLAN)
|
|
152
|
-
return;
|
|
153
|
-
throw error;
|
|
154
|
-
}
|
|
155
|
-
throw new Error("all-mode boundary fixture reached an executor instead of stopping at plan");
|
|
156
|
-
}
|
|
157
|
-
async function warmAllModePlan() {
|
|
158
|
-
let filesToSkip;
|
|
159
|
-
await runThroughAllModePlan((event) => {
|
|
160
|
-
filesToSkip = event.filesToSkip;
|
|
161
|
-
});
|
|
162
|
-
expect(filesToSkip).toBe(ROW_COUNT);
|
|
163
|
-
}
|
|
164
|
-
async function measureLiveBoundary() {
|
|
165
|
-
collectGarbage(collect);
|
|
166
|
-
const baselineBytes = usedBytes();
|
|
167
|
-
let boundaryBytes;
|
|
168
|
-
let planFilesToSkip;
|
|
169
|
-
await runThroughAllModePlan((event) => {
|
|
170
|
-
planFilesToSkip = event.filesToSkip;
|
|
171
|
-
// This event is emitted while both the all-mode RemoteFile[] and PullPlan
|
|
172
|
-
// are still live in syncWithOperationLockHeld's stack frame.
|
|
173
|
-
collectGarbage(collect);
|
|
174
|
-
boundaryBytes = usedBytes();
|
|
175
|
-
});
|
|
176
|
-
if (boundaryBytes === undefined || planFilesToSkip === undefined) {
|
|
177
|
-
throw new Error("all-mode boundary fixture did not observe the pull plan");
|
|
178
|
-
}
|
|
179
|
-
return {
|
|
180
|
-
baselineBytes,
|
|
181
|
-
liveBytes: boundaryBytes - baselineBytes,
|
|
182
|
-
planFilesToSkip,
|
|
183
|
-
};
|
|
184
|
-
}
|
|
185
|
-
function measureAllocationControl() {
|
|
186
|
-
collectGarbage(collect);
|
|
187
|
-
const baselineBytes = usedBytes();
|
|
188
|
-
let control = Buffer.alloc(CONTROL_BYTES, 0x5a);
|
|
189
|
-
expect(control[0]).toBe(0x5a);
|
|
190
|
-
const detectedBytes = usedBytes() - baselineBytes;
|
|
191
|
-
control = undefined;
|
|
192
|
-
collectGarbage(collect);
|
|
193
|
-
return detectedBytes;
|
|
194
|
-
}
|
|
195
|
-
describe("all-mode list to plan resident boundary", () => {
|
|
196
|
-
beforeAll(async () => {
|
|
197
|
-
baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-streaming-pull-planner-"));
|
|
198
|
-
hqRoot = path.join(baseDir, "hq");
|
|
199
|
-
stateDir = path.join(baseDir, "state");
|
|
200
|
-
fs.mkdirSync(stateDir, { recursive: true });
|
|
201
|
-
process.env.HQ_STATE_DIR = stateDir;
|
|
202
|
-
clearContextCache();
|
|
203
|
-
releaseCachedAreaJournalMigrations();
|
|
204
|
-
vi.stubGlobal("fetch", async (url) => {
|
|
205
|
-
const target = String(url);
|
|
206
|
-
if (target.includes("/entity/check-slug/me")) {
|
|
207
|
-
return {
|
|
208
|
-
ok: true,
|
|
209
|
-
status: 200,
|
|
210
|
-
json: async () => ({ available: false, conflictingCompanyUid: "cmp_streaming_boundary" }),
|
|
211
|
-
text: async () => "",
|
|
212
|
-
};
|
|
213
|
-
}
|
|
214
|
-
if (target.includes("/entity/by-slug/") || target.includes("/entity/cmp_")) {
|
|
215
|
-
return {
|
|
216
|
-
ok: true,
|
|
217
|
-
status: 200,
|
|
218
|
-
json: async () => ({
|
|
219
|
-
entity: {
|
|
220
|
-
uid: "cmp_streaming_boundary",
|
|
221
|
-
slug: "streaming-boundary",
|
|
222
|
-
bucketName: "hq-vault-streaming-boundary",
|
|
223
|
-
status: "active",
|
|
224
|
-
},
|
|
225
|
-
}),
|
|
226
|
-
text: async () => "",
|
|
227
|
-
};
|
|
228
|
-
}
|
|
229
|
-
return { ok: false, status: 404, json: async () => ({}), text: async () => "not found" };
|
|
230
|
-
});
|
|
231
|
-
setObjectIOFactory(() => pagedObjectIO());
|
|
232
|
-
seedAreaAuthoritativeJournal();
|
|
233
|
-
collect = acquireGc();
|
|
234
|
-
// The first pass opens the cached area migration. The measured passes are
|
|
235
|
-
// the steady state after that one-time work has settled.
|
|
236
|
-
await warmAllModePlan();
|
|
237
|
-
collectGarbage(collect);
|
|
238
|
-
}, 300_000);
|
|
239
|
-
afterAll(() => {
|
|
240
|
-
setObjectIOFactory(null);
|
|
241
|
-
clearContextCache();
|
|
242
|
-
releaseCachedAreaJournalMigrations();
|
|
243
|
-
closeCachedAreaJournalMigration("streaming-boundary");
|
|
244
|
-
vi.unstubAllGlobals();
|
|
245
|
-
if (ORIGINAL_STATE_DIR === undefined)
|
|
246
|
-
delete process.env.HQ_STATE_DIR;
|
|
247
|
-
else
|
|
248
|
-
process.env.HQ_STATE_DIR = ORIGINAL_STATE_DIR;
|
|
249
|
-
fs.rmSync(baseDir, { recursive: true, force: true });
|
|
250
|
-
});
|
|
251
|
-
it(`retains at most ${MAX_LIVE_BOUNDARY_BYTES_PER_ROW} B/row after all-mode list and plan`, async () => {
|
|
252
|
-
const controlBytes = measureAllocationControl();
|
|
253
|
-
expect(controlBytes).toBeGreaterThan(CONTROL_BYTES * 0.9);
|
|
254
|
-
const measurements = [];
|
|
255
|
-
for (let trial = 0; trial < TRIALS; trial += 1) {
|
|
256
|
-
const measurement = await measureLiveBoundary();
|
|
257
|
-
expect(measurement.planFilesToSkip).toBe(ROW_COUNT);
|
|
258
|
-
measurements.push(measurement);
|
|
259
|
-
}
|
|
260
|
-
const liveBytes = measurements.map((measurement) => measurement.liveBytes);
|
|
261
|
-
const minBytes = Math.min(...liveBytes);
|
|
262
|
-
const maxBytes = Math.max(...liveBytes);
|
|
263
|
-
const spreadBytes = maxBytes - minBytes;
|
|
264
|
-
process.stdout.write(`all-mode list->plan resident: rows=${ROW_COUNT}; trials=${liveBytes.join(",")}; ` +
|
|
265
|
-
`min=${minBytes}; max=${maxBytes}; spread=${spreadBytes}; ` +
|
|
266
|
-
`ceiling=${MAX_LIVE_BOUNDARY_BYTES}; control=${controlBytes}\n`);
|
|
267
|
-
// Use the worst of three steady-state trials: a streaming redesign must
|
|
268
|
-
// keep this boundary below the budget on every isolated pass, not merely
|
|
269
|
-
// win one favourable GC sample.
|
|
270
|
-
expect(maxBytes).toBeLessThanOrEqual(MAX_LIVE_BOUNDARY_BYTES);
|
|
271
|
-
}, 300_000);
|
|
272
|
-
});
|
|
273
|
-
//# sourceMappingURL=streaming-pull-planner.bench.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"streaming-pull-planner.bench.test.js","sourceRoot":"","sources":["../../src/cli/streaming-pull-planner.bench.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAEvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EACL,+BAA+B,EAC/B,iCAAiC,EACjC,kCAAkC,EAClC,YAAY,GACb,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,EACL,kBAAkB,GAInB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAmB,MAAM,0BAA0B,CAAC;AAEzE,OAAO,EAAE,IAAI,EAA0B,MAAM,WAAW,CAAC;AAEzD,MAAM,SAAS,GAAG,MAAM,CAAC;AACzB,MAAM,SAAS,GAAG,KAAK,CAAC;AACxB,MAAM,MAAM,GAAG,CAAC,CAAC;AACjB,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,yEAAyE;AACzE,6EAA6E;AAC7E,6EAA6E;AAC7E,yEAAyE;AACzE,0EAA0E;AAC1E,2EAA2E;AAC3E,gDAAgD;AAChD,MAAM,+BAA+B,GAAG,GAAG,CAAC;AAC5C,MAAM,6BAA6B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACtD,MAAM,uBAAuB,GAC3B,6BAA6B,GAAG,SAAS,GAAG,+BAA+B,CAAC;AAE9E,MAAM,iBAAiB,GAAG,kEAAkE,CAAC;AAC7F,MAAM,eAAe,GAAG,MAAM,CAAC,iCAAiC,CAAC,CAAC;AAClE,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;AAEpD,MAAM,MAAM,GAAe;IACzB,MAAM,EAAE,CAAC;IACT,aAAa,EAAE,CAAC;IAChB,KAAK,EAAE;QACL,EAAE,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QAClD,EAAE,EAAE,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;KAChE;CACF,CAAC;AAEF,MAAM,WAAW,GAAuB;IACtC,MAAM,EAAE,wBAAwB;IAChC,SAAS,EAAE,mCAAmC;IAC9C,MAAM,EAAE,WAAW;IACnB,uBAAuB,EAAE,IAAI;CAC9B,CAAC;AAQF,IAAI,OAAO,GAAG,EAAE,CAAC;AACjB,IAAI,MAAM,GAAG,EAAE,CAAC;AAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;AAClB,IAAI,OAAmB,CAAC;AAExB,SAAS,UAAU,CAAC,KAAa;IAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAClD,OAAO,kBAAkB,KAAK,GAAG,GAAG,WAAW,KAAK,GAAG,EAAE,UAAU,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC;AAC3F,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,sBAAsB,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,aAAa;IACpB,OAAO;QACL,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC3C,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,YAAY,EAAE,KAAK,IAAI,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QACD,UAAU,EAAE,KAAK,IAAI,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QACD,WAAW,EAAE,KAAK,EAAE,KAAuB,EAA8B,EAAE;YACzE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,KAAK,CAAC,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC1F,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,SAAS,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,EAAE,CAAC;YACnB,KAAK,IAAI,KAAK,GAAG,KAAK,EAAE,KAAK,GAAG,GAAG,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;gBAChD,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC;oBACtB,IAAI,EAAE,CAAC;oBACP,YAAY,EAAE,IAAI,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;oBACjD,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC;iBACxB,CAAC,CAAC;YACL,CAAC;YACD,OAAO;gBACL,OAAO;gBACP,qBAAqB,EAAE,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS;aACjE,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,4BAA4B;IACnC,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IACzD,MAAM,KAAK,GAAiC,EAAE,CAAC;IAE/C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAC9C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG;YACX,IAAI,EAAE,iBAAiB;YACvB,IAAI,EAAE,CAAC;YACP,QAAQ,EAAE,0BAA0B;YACpC,SAAS,EAAE,MAAM;YACjB,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC;YAC7B,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;YAClC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;SACnC,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACzE,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC;QACxC,OAAO,EAAE,QAAQ;QACjB,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC;QAClC,WAAW,EAAE,iCAAiC,CAAC,IAAI,CAAC;QACpD,cAAc,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAChC,gBAAgB,EAAE,SAAS;KAC5B,CAAC,CAAC;IACH,SAAS,CAAC,cAAc,EAAE,CAAC;IAC3B,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACrF,SAAS,CAAC,OAAO,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,MAAqE;IACxG,IAAI,CAAC;QACH,MAAM,IAAI,CAAC;YACT,OAAO,EAAE,oBAAoB;YAC7B,WAAW;YACX,MAAM;YACN,cAAc,EAAE,IAAI,GAAG,EAAE;YACzB,WAAW,EAAE,IAAI;YACjB,wBAAwB,EAAE,IAAI;YAC9B,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;oBAAE,OAAO;gBAClC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACd,MAAM,eAAe,CAAC;YACxB,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,KAAK,eAAe;YAAE,OAAO;QACtC,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;AAC/F,CAAC;AAED,KAAK,UAAU,eAAe;IAC5B,IAAI,WAA+B,CAAC;IACpC,MAAM,qBAAqB,CAAC,CAAC,KAAK,EAAE,EAAE;QACpC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACtC,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,cAAc,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,aAAa,GAAG,SAAS,EAAE,CAAC;IAClC,IAAI,aAAiC,CAAC;IACtC,IAAI,eAAmC,CAAC;IAExC,MAAM,qBAAqB,CAAC,CAAC,KAAK,EAAE,EAAE;QACpC,eAAe,GAAG,KAAK,CAAC,WAAW,CAAC;QACpC,0EAA0E;QAC1E,6DAA6D;QAC7D,cAAc,CAAC,OAAO,CAAC,CAAC;QACxB,aAAa,GAAG,SAAS,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,IAAI,aAAa,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO;QACL,aAAa;QACb,SAAS,EAAE,aAAa,GAAG,aAAa;QACxC,eAAe;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB;IAC/B,cAAc,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,aAAa,GAAG,SAAS,EAAE,CAAC;IAClC,IAAI,OAAO,GAAuB,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IACpE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,aAAa,GAAG,SAAS,EAAE,GAAG,aAAa,CAAC;IAClD,OAAO,GAAG,SAAS,CAAC;IACpB,cAAc,CAAC,OAAO,CAAC,CAAC;IACxB,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,QAAQ,CAAC,yCAAyC,EAAE,GAAG,EAAE;IACvD,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC,CAAC;QAC/E,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAClC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACvC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,QAAQ,CAAC;QACpC,iBAAiB,EAAE,CAAC;QACpB,kCAAkC,EAAE,CAAC;QACrC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,GAAW,EAAE,EAAE;YAC3C,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBAC7C,OAAO;oBACL,EAAE,EAAE,IAAI;oBACR,MAAM,EAAE,GAAG;oBACX,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,CAAC;oBACzF,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE;iBACrB,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC3E,OAAO;oBACL,EAAE,EAAE,IAAI;oBACR,MAAM,EAAE,GAAG;oBACX,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;wBACjB,MAAM,EAAE;4BACN,GAAG,EAAE,wBAAwB;4BAC7B,IAAI,EAAE,oBAAoB;4BAC1B,UAAU,EAAE,6BAA6B;4BACzC,MAAM,EAAE,QAAQ;yBACjB;qBACF,CAAC;oBACF,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE;iBACrB,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3F,CAAC,CAAC,CAAC;QACH,kBAAkB,CAAC,GAAG,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC;QAC1C,4BAA4B,EAAE,CAAC;QAC/B,OAAO,GAAG,SAAS,EAAE,CAAC;QAEtB,0EAA0E;QAC1E,yDAAyD;QACzD,MAAM,eAAe,EAAE,CAAC;QACxB,cAAc,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC,EAAE,OAAO,CAAC,CAAC;IAEZ,QAAQ,CAAC,GAAG,EAAE;QACZ,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACzB,iBAAiB,EAAE,CAAC;QACpB,kCAAkC,EAAE,CAAC;QACrC,+BAA+B,CAAC,oBAAoB,CAAC,CAAC;QACtD,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACtB,IAAI,kBAAkB,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;;YACjE,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,kBAAkB,CAAC;QACnD,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mBAAmB,+BAA+B,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACrG,MAAM,YAAY,GAAG,wBAAwB,EAAE,CAAC;QAChD,MAAM,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC;QAE1D,MAAM,YAAY,GAA0B,EAAE,CAAC;QAC/C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,WAAW,GAAG,MAAM,mBAAmB,EAAE,CAAC;YAChD,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACpD,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;QACxC,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sCAAsC,SAAS,YAAY,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI;YAChF,OAAO,QAAQ,SAAS,QAAQ,YAAY,WAAW,IAAI;YAC3D,WAAW,uBAAuB,aAAa,YAAY,IAAI,CAClE,CAAC;QAEF,wEAAwE;QACxE,yEAAyE;QACzE,gCAAgC;QAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,mBAAmB,CAAC,uBAAuB,CAAC,CAAC;IAChE,CAAC,EAAE,OAAO,CAAC,CAAC;AACd,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3.list-cost.test.d.ts","sourceRoot":"","sources":["../src/s3.list-cost.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { createHook } from "node:async_hooks";
|
|
2
|
-
import { afterEach, expect, it } from "vitest";
|
|
3
|
-
import { listRemoteFiles, streamRemoteFiles } from "./s3.js";
|
|
4
|
-
import { setObjectIOFactory } from "./object-io.js";
|
|
5
|
-
const ctx = {
|
|
6
|
-
uid: "cmp_list_cost", slug: "list-cost", bucketName: "list-cost", region: "us-east-1",
|
|
7
|
-
credentials: { accessKeyId: "fixture", secretAccessKey: "fixture", sessionToken: "fixture" },
|
|
8
|
-
expiresAt: "2099-01-01T00:00:00Z",
|
|
9
|
-
};
|
|
10
|
-
afterEach(() => setObjectIOFactory(null));
|
|
11
|
-
async function measure(rows, perObjectControl) {
|
|
12
|
-
const forbidden = async () => { throw new Error("unexpected transfer"); };
|
|
13
|
-
const io = {
|
|
14
|
-
putObject: forbidden, getObject: forbidden, headObject: forbidden, deleteObject: forbidden,
|
|
15
|
-
listObjects: async () => ({ objects: Array.from({ length: rows }, (_, i) => ({
|
|
16
|
-
key: `docs/row-${i}.md`, size: i, etag: `etag-${i}`, lastModified: new Date(1700000000000),
|
|
17
|
-
})) }),
|
|
18
|
-
};
|
|
19
|
-
setObjectIOFactory(() => io);
|
|
20
|
-
let promises = 0;
|
|
21
|
-
const hook = createHook({ init(_id, type) { if (type === "PROMISE")
|
|
22
|
-
promises++; } });
|
|
23
|
-
hook.enable();
|
|
24
|
-
let files;
|
|
25
|
-
try {
|
|
26
|
-
if (perObjectControl) {
|
|
27
|
-
files = [];
|
|
28
|
-
// This is the old collector shape, observed by the SAME async hook.
|
|
29
|
-
for await (const file of streamRemoteFiles(ctx, "docs/"))
|
|
30
|
-
files.push(file);
|
|
31
|
-
}
|
|
32
|
-
else
|
|
33
|
-
files = await listRemoteFiles(ctx, "docs/");
|
|
34
|
-
}
|
|
35
|
-
finally {
|
|
36
|
-
hook.disable();
|
|
37
|
-
}
|
|
38
|
-
expect(files).toHaveLength(rows);
|
|
39
|
-
expect(files.at(-1)?.key).toBe(`docs/row-${rows - 1}.md`);
|
|
40
|
-
return promises;
|
|
41
|
-
}
|
|
42
|
-
it("array LIST promise allocations scale with pages rather than rows", async () => {
|
|
43
|
-
const smallControl = await measure(1, true);
|
|
44
|
-
const largeControl = await measure(200, true);
|
|
45
|
-
expect(largeControl - smallControl).toBeGreaterThan(199);
|
|
46
|
-
const small = await measure(1, false);
|
|
47
|
-
const large = await measure(200, false);
|
|
48
|
-
expect(large).toBe(small);
|
|
49
|
-
console.log(`LIST promise control: per-object 1/200 rows=${smallControl}/${largeControl}; array collector 1/200 rows=${small}/${large}; each is one complete page`);
|
|
50
|
-
});
|
|
51
|
-
//# sourceMappingURL=s3.list-cost.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3.list-cost.test.js","sourceRoot":"","sources":["../src/s3.list-cost.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAmB,MAAM,SAAS,CAAC;AAC9E,OAAO,EAAE,kBAAkB,EAAiB,MAAM,gBAAgB,CAAC;AAGnE,MAAM,GAAG,GAAkB;IACzB,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW;IACrF,WAAW,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE;IAC5F,SAAS,EAAE,sBAAsB;CAClC,CAAC;AACF,SAAS,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1C,KAAK,UAAU,OAAO,CAAC,IAAY,EAAE,gBAAyB;IAC5D,MAAM,SAAS,GAAG,KAAK,IAAoB,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,EAAE,GAAa;QACnB,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS;QAC1F,WAAW,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3E,GAAG,EAAE,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,aAAa,CAAC;aAC3F,CAAC,CAAC,EAAE,CAAC;KACP,CAAC;IACF,kBAAkB,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,IAAI,IAAI,KAAK,SAAS;YAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrF,IAAI,CAAC,MAAM,EAAE,CAAC;IACd,IAAI,KAAmB,CAAC;IACxB,IAAI,CAAC;QACH,IAAI,gBAAgB,EAAE,CAAC;YACrB,KAAK,GAAG,EAAE,CAAC;YACX,oEAAoE;YACpE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC;;YAAM,KAAK,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;YAAS,CAAC;QAAC,IAAI,CAAC,OAAO,EAAE,CAAC;IAAC,CAAC;IAC7B,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;IAChF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC9C,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAC,+CAA+C,YAAY,IAAI,YAAY,gCAAgC,KAAK,IAAI,KAAK,6BAA6B,CAAC,CAAC;AACtK,CAAC,CAAC,CAAC"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { type RemoteFile } from "../s3.js";
|
|
2
|
-
export declare function collectRemoteAncestors(files: Iterable<RemoteFile>): Set<string>;
|
|
3
|
-
export interface SpooledRemoteFiles {
|
|
4
|
-
files: Iterable<RemoteFile>;
|
|
5
|
-
remoteKeysWithChildren: ReadonlySet<string>;
|
|
6
|
-
dispose(): void;
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* Preserve one LIST snapshot without retaining every row in memory. Deferring
|
|
10
|
-
* only directory collisions cannot preserve the journal freshness contract:
|
|
11
|
-
* an intent for ANY earlier row can arrive on the last page. Replay after the
|
|
12
|
-
* caller refreshes its journal, with complete overlay evidence and no remote
|
|
13
|
-
* second pass. Disk cost is one JSONL write and read per row (metadata only).
|
|
14
|
-
*/
|
|
15
|
-
export declare function spoolRemoteFiles(source: AsyncIterable<RemoteFile>): Promise<SpooledRemoteFiles>;
|
|
16
|
-
//# sourceMappingURL=remote-file-spool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"remote-file-spool.d.ts","sourceRoot":"","sources":["../../src/sync/remote-file-spool.ts"],"names":[],"mappings":"AAIA,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAWvD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAI/E;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC5B,sBAAsB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC5C,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAkCrG"}
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import * as fs from "node:fs";
|
|
2
|
-
import * as os from "node:os";
|
|
3
|
-
import * as path from "node:path";
|
|
4
|
-
import { StringDecoder } from "node:string_decoder";
|
|
5
|
-
import { toPosixKey } from "../s3.js";
|
|
6
|
-
const CHUNK_BYTES = 64 * 1024;
|
|
7
|
-
function addAncestors(keys, remoteKey) {
|
|
8
|
-
const key = toPosixKey(remoteKey);
|
|
9
|
-
for (let slash = key.indexOf("/"); slash !== -1; slash = key.indexOf("/", slash + 1)) {
|
|
10
|
-
keys.add(key.slice(0, slash));
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
export function collectRemoteAncestors(files) {
|
|
14
|
-
const keys = new Set();
|
|
15
|
-
for (const file of files)
|
|
16
|
-
addAncestors(keys, file.key);
|
|
17
|
-
return keys;
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* Preserve one LIST snapshot without retaining every row in memory. Deferring
|
|
21
|
-
* only directory collisions cannot preserve the journal freshness contract:
|
|
22
|
-
* an intent for ANY earlier row can arrive on the last page. Replay after the
|
|
23
|
-
* caller refreshes its journal, with complete overlay evidence and no remote
|
|
24
|
-
* second pass. Disk cost is one JSONL write and read per row (metadata only).
|
|
25
|
-
*/
|
|
26
|
-
export async function spoolRemoteFiles(source) {
|
|
27
|
-
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "hq-pull-list-"));
|
|
28
|
-
const filename = path.join(directory, "rows.jsonl");
|
|
29
|
-
const remoteKeysWithChildren = new Set();
|
|
30
|
-
let writer;
|
|
31
|
-
const dispose = () => fs.rmSync(directory, { recursive: true, force: true });
|
|
32
|
-
try {
|
|
33
|
-
writer = fs.openSync(filename, "wx", 0o600);
|
|
34
|
-
let pending = "";
|
|
35
|
-
const flush = () => {
|
|
36
|
-
// writeFileSync handles partial writes to this already-open descriptor.
|
|
37
|
-
fs.writeFileSync(writer, pending, "utf8");
|
|
38
|
-
pending = "";
|
|
39
|
-
};
|
|
40
|
-
for await (const file of source) {
|
|
41
|
-
addAncestors(remoteKeysWithChildren, file.key);
|
|
42
|
-
// Encode the timestamp numerically; replay restores the Date used by
|
|
43
|
-
// tombstone/recreate classification. JSON escapes newlines in keys.
|
|
44
|
-
pending += JSON.stringify({ ...file, lastModified: file.lastModified.getTime() }) + "\n";
|
|
45
|
-
if (pending.length >= CHUNK_BYTES)
|
|
46
|
-
flush();
|
|
47
|
-
}
|
|
48
|
-
if (pending.length > 0)
|
|
49
|
-
flush();
|
|
50
|
-
fs.closeSync(writer);
|
|
51
|
-
writer = undefined;
|
|
52
|
-
return {
|
|
53
|
-
remoteKeysWithChildren,
|
|
54
|
-
files: { [Symbol.iterator]: () => readRows(filename) },
|
|
55
|
-
dispose,
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
catch (error) {
|
|
59
|
-
if (writer !== undefined)
|
|
60
|
-
fs.closeSync(writer);
|
|
61
|
-
dispose();
|
|
62
|
-
throw error;
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
function* readRows(filename) {
|
|
66
|
-
const reader = fs.openSync(filename, "r");
|
|
67
|
-
try {
|
|
68
|
-
const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
|
|
69
|
-
const decoder = new StringDecoder("utf8");
|
|
70
|
-
let pending = "";
|
|
71
|
-
let bytes;
|
|
72
|
-
while ((bytes = fs.readSync(reader, buffer, 0, buffer.length, null)) > 0) {
|
|
73
|
-
pending += decoder.write(buffer.subarray(0, bytes));
|
|
74
|
-
let start = 0;
|
|
75
|
-
let newline;
|
|
76
|
-
while ((newline = pending.indexOf("\n", start)) !== -1) {
|
|
77
|
-
const row = JSON.parse(pending.slice(start, newline));
|
|
78
|
-
yield { ...row, lastModified: new Date(row.lastModified) };
|
|
79
|
-
start = newline + 1;
|
|
80
|
-
}
|
|
81
|
-
pending = pending.slice(start);
|
|
82
|
-
}
|
|
83
|
-
pending += decoder.end();
|
|
84
|
-
if (pending.length !== 0)
|
|
85
|
-
throw new Error("Incomplete pull LIST spool");
|
|
86
|
-
}
|
|
87
|
-
finally {
|
|
88
|
-
fs.closeSync(reader);
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
//# sourceMappingURL=remote-file-spool.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"remote-file-spool.js","sourceRoot":"","sources":["../../src/sync/remote-file-spool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAmB,MAAM,UAAU,CAAC;AAEvD,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC;AAE9B,SAAS,YAAY,CAAC,IAAiB,EAAE,SAAiB;IACxD,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAClC,KAAK,IAAI,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QACrF,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA2B;IAChE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC;AACd,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAiC;IACtE,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACpD,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;IACjD,IAAI,MAA0B,CAAC;IAC/B,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7E,IAAI,CAAC;QACH,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC5C,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,GAAG,EAAE;YACjB,wEAAwE;YACxE,EAAE,CAAC,aAAa,CAAC,MAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3C,OAAO,GAAG,EAAE,CAAC;QACf,CAAC,CAAC;QACF,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAChC,YAAY,CAAC,sBAAsB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/C,qEAAqE;YACrE,oEAAoE;YACpE,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC;YACzF,IAAI,OAAO,CAAC,MAAM,IAAI,WAAW;gBAAE,KAAK,EAAE,CAAC;QAC7C,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,EAAE,CAAC;QAChC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,GAAG,SAAS,CAAC;QACnB,OAAO;YACL,sBAAsB;YACtB,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACtD,OAAO;SACR,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,KAAK,SAAS;YAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC/C,OAAO,EAAE,CAAC;QACV,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAgB;IACjC,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC1C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,KAAa,CAAC;QAClB,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YACzE,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACpD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,OAAe,CAAC;YACpB,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACvD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAgE,CAAC;gBACrH,MAAM,EAAE,GAAG,GAAG,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC3D,KAAK,GAAG,OAAO,GAAG,CAAC,CAAC;YACtB,CAAC;YACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"remote-file-spool.test.d.ts","sourceRoot":"","sources":["../../src/sync/remote-file-spool.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
import * as fs from "node:fs";
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
4
|
-
import { spoolRemoteFiles } from "./remote-file-spool.js";
|
|
5
|
-
vi.mock("node:fs", async (importOriginal) => ({ ...await importOriginal() }));
|
|
6
|
-
const row = (key) => ({ key, size: 17, lastModified: new Date(1_700_000_000_123), etag: '"abc"', storageClass: "DEEP_ARCHIVE" });
|
|
7
|
-
async function* stream(files) { yield* files; }
|
|
8
|
-
afterEach(() => vi.restoreAllMocks());
|
|
9
|
-
function observeDirectory() {
|
|
10
|
-
const mkdir = vi.spyOn(fs, "mkdtempSync");
|
|
11
|
-
return () => {
|
|
12
|
-
const result = mkdir.mock.results.find((entry) => entry.type === "return");
|
|
13
|
-
expect(result).toBeDefined();
|
|
14
|
-
return result.value;
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
|
-
describe("remote LIST spool", () => {
|
|
18
|
-
it("round-trips metadata and UTF-8 across chunks, with complete unordered ancestors", async () => {
|
|
19
|
-
const directory = observeDirectory();
|
|
20
|
-
const rows = [row("docs/overlay"), ...Array.from({ length: 1000 }, (_, i) => row(`docs/overlay/é漢😀-${i}-line\nback\\slash.md`)), row("parent/child/grandchild.md"), row("parent")];
|
|
21
|
-
const source = await spoolRemoteFiles(stream(rows));
|
|
22
|
-
try {
|
|
23
|
-
expect(fs.existsSync(directory())).toBe(true);
|
|
24
|
-
expect(fs.statSync(directory()).mode & 0o777).toBe(0o700);
|
|
25
|
-
expect(fs.statSync(path.join(directory(), "rows.jsonl")).mode & 0o777).toBe(0o600);
|
|
26
|
-
expect([...source.files]).toEqual(rows);
|
|
27
|
-
expect(source.remoteKeysWithChildren.has("docs/overlay")).toBe(true);
|
|
28
|
-
expect(source.remoteKeysWithChildren.has("parent/child")).toBe(true);
|
|
29
|
-
expect(source.remoteKeysWithChildren.has("parent")).toBe(true);
|
|
30
|
-
console.log(`spool round-trip control: rows=${rows.length}; bytes=${fs.statSync(path.join(directory(), "rows.jsonl")).size}; Date/storageClass/UTF-8 preserved=true; private permissions=700/600`);
|
|
31
|
-
}
|
|
32
|
-
finally {
|
|
33
|
-
source.dispose();
|
|
34
|
-
}
|
|
35
|
-
expect(fs.existsSync(directory())).toBe(false);
|
|
36
|
-
console.log("spool cleanup control: directory existed=true; after dispose exists=false");
|
|
37
|
-
});
|
|
38
|
-
it("removes partial spool on a late LIST failure", async () => {
|
|
39
|
-
const directory = observeDirectory();
|
|
40
|
-
async function* failingList() {
|
|
41
|
-
yield row("docs/first.md");
|
|
42
|
-
expect(fs.existsSync(directory())).toBe(true);
|
|
43
|
-
throw new Error("LIST failed");
|
|
44
|
-
}
|
|
45
|
-
await expect(spoolRemoteFiles(failingList())).rejects.toThrow("LIST failed");
|
|
46
|
-
expect(fs.existsSync(directory())).toBe(false);
|
|
47
|
-
console.log("LIST failure cleanup control: directory existed=true; after rejected LIST exists=false");
|
|
48
|
-
});
|
|
49
|
-
it("removes partial spool on disk write failure", async () => {
|
|
50
|
-
const directory = observeDirectory();
|
|
51
|
-
vi.spyOn(fs, "writeFileSync").mockImplementation(() => {
|
|
52
|
-
expect(fs.existsSync(directory())).toBe(true);
|
|
53
|
-
throw new Error("ENOSPC control");
|
|
54
|
-
});
|
|
55
|
-
await expect(spoolRemoteFiles(stream([row("docs/first.md")]))).rejects.toThrow("ENOSPC control");
|
|
56
|
-
expect(fs.existsSync(directory())).toBe(false);
|
|
57
|
-
console.log("disk failure cleanup control: directory existed=true; after ENOSPC exists=false");
|
|
58
|
-
});
|
|
59
|
-
it("closes the replay descriptor if classification throws", async () => {
|
|
60
|
-
const source = await spoolRemoteFiles(stream([row("docs/first.md"), row("docs/second.md")]));
|
|
61
|
-
const close = vi.spyOn(fs, "closeSync");
|
|
62
|
-
const open = vi.spyOn(fs, "openSync");
|
|
63
|
-
try {
|
|
64
|
-
expect(() => {
|
|
65
|
-
for (const _file of source.files)
|
|
66
|
-
throw new Error("classification failed");
|
|
67
|
-
}).toThrow("classification failed");
|
|
68
|
-
expect(open).toHaveBeenCalledTimes(1);
|
|
69
|
-
expect(close).toHaveBeenCalledWith(open.mock.results[0].value);
|
|
70
|
-
console.log("replay failure descriptor control: opened=1; matching descriptor closed=true");
|
|
71
|
-
}
|
|
72
|
-
finally {
|
|
73
|
-
source.dispose();
|
|
74
|
-
}
|
|
75
|
-
});
|
|
76
|
-
});
|
|
77
|
-
//# sourceMappingURL=remote-file-spool.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"remote-file-spool.test.js","sourceRoot":"","sources":["../../src/sync/remote-file-spool.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,cAAc,EAA4B,EAAE,CAAC,CAAC,CAAC;AAExG,MAAM,GAAG,GAAG,CAAC,GAAW,EAAc,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC;AACrJ,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,KAAmB,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7D,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC;AAEtC,SAAS,gBAAgB;IACvB,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;IAC1C,OAAO,GAAG,EAAE;QACV,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;QAC7B,OAAO,MAAO,CAAC,KAAe,CAAC;IACjC,CAAC,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;QAC/F,MAAM,SAAS,GAAG,gBAAgB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,qBAAqB,CAAC,uBAAuB,CAAC,CAAC,EAAE,GAAG,CAAC,4BAA4B,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpL,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnF,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,OAAO,CAAC,GAAG,CAAC,kCAAkC,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,uEAAuE,CAAC,CAAC;QACrM,CAAC;gBAAS,CAAC;YAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAAC,CAAC;QAC/B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,2EAA2E,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,SAAS,GAAG,gBAAgB,EAAE,CAAC;QACrC,KAAK,SAAS,CAAC,CAAC,WAAW;YACzB,MAAM,GAAG,CAAC,eAAe,CAAC,CAAC;YAC3B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;QACjC,CAAC;QACD,MAAM,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,SAAS,GAAG,gBAAgB,EAAE,CAAC;QACrC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE;YACpD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACjG,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,iFAAiF,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,EAAE;gBACV,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK;oBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/D,OAAO,CAAC,GAAG,CAAC,8EAA8E,CAAC,CAAC;QAC9F,CAAC;gBAAS,CAAC;YAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|