@indigoai-us/hq-cloud 5.34.0 → 5.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +196 -27
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +532 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +182 -49
- package/dist/cli/sync.js.map +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +13 -0
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +28 -0
- package/dist/ignore.test.js.map +1 -1
- package/dist/journal.d.ts +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +4 -1
- package/dist/journal.js.map +1 -1
- package/dist/types.d.ts +16 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/cli/share.test.ts +594 -0
- package/src/cli/share.ts +201 -27
- package/src/cli/sync.ts +200 -48
- package/src/ignore.test.ts +37 -0
- package/src/ignore.ts +14 -0
- package/src/journal.ts +4 -0
- package/src/types.ts +16 -0
package/dist/cli/share.test.js
CHANGED
|
@@ -19,6 +19,17 @@ vi.mock("../s3.js", () => ({
|
|
|
19
19
|
deleteRemoteFile: vi.fn().mockResolvedValue(undefined),
|
|
20
20
|
headRemoteFile: vi.fn().mockResolvedValue(null),
|
|
21
21
|
}));
|
|
22
|
+
// Mock readline so the interactive-conflict serialization test can observe
|
|
23
|
+
// prompt lifecycle (createInterface + close) without ever touching stdin.
|
|
24
|
+
// Default impl never resolves; conflict-prompt tests below override per-test
|
|
25
|
+
// to script answers.
|
|
26
|
+
vi.mock("readline", () => ({
|
|
27
|
+
createInterface: vi.fn(() => ({
|
|
28
|
+
question: vi.fn(),
|
|
29
|
+
close: vi.fn(),
|
|
30
|
+
})),
|
|
31
|
+
}));
|
|
32
|
+
import * as readline from "readline";
|
|
22
33
|
import { share, _testing as shareTesting } from "./share.js";
|
|
23
34
|
import { deleteRemoteFile, headRemoteFile, uploadFile, uploadSymlink } from "../s3.js";
|
|
24
35
|
const mockConfig = {
|
|
@@ -2213,5 +2224,526 @@ describe("currency-gated: journal version 2 fixtures", () => {
|
|
|
2213
2224
|
expect(result.filesDeleted).toBe(1);
|
|
2214
2225
|
expect(deleteRemoteFile).toHaveBeenCalledWith(expect.anything(), "core/policies/old.md");
|
|
2215
2226
|
});
|
|
2227
|
+
// ── Fix #1 (5.36.0): lstat fast-path ─────────────────────────────────────
|
|
2228
|
+
//
|
|
2229
|
+
// When skipUnchanged is on AND the journal carries an mtimeMs for the
|
|
2230
|
+
// file AND (lstat.size, lstat.mtimeMs) match the recorded values, the
|
|
2231
|
+
// push planner classifies the file `unchanged` WITHOUT reading its
|
|
2232
|
+
// bytes. Same trade-off rsync/gitignore use; turns no-op syncs from
|
|
2233
|
+
// O(file bytes) into O(file count).
|
|
2234
|
+
//
|
|
2235
|
+
// We prove "the SHA256 path didn't run" by seeding the journal with an
|
|
2236
|
+
// intentionally WRONG hash but matching mtimeMs+size. If the fast-path
|
|
2237
|
+
// fires → file is classified `unchanged` purely on lstat (no hash
|
|
2238
|
+
// compare). If the fast-path is skipped → hashFile runs, hashes the
|
|
2239
|
+
// real file, fails the !== journal-hash check, and uploads. The
|
|
2240
|
+
// upload-vs-skip outcome is the diagnostic signal.
|
|
2241
|
+
describe("lstat fast-path (5.36.0)", () => {
|
|
2242
|
+
it("skips upload when mtimeMs + size match journal (even with wrong stored hash)", async () => {
|
|
2243
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2244
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2245
|
+
const testFile = path.join(companyRoot, "stable.md");
|
|
2246
|
+
fs.writeFileSync(testFile, "stable bytes");
|
|
2247
|
+
const lstat = fs.lstatSync(testFile);
|
|
2248
|
+
// Intentionally-wrong hash; matching mtimeMs+size. If the fast-path
|
|
2249
|
+
// doesn't fire, hashFile runs, the real hash != "sentinel-wrong-hash",
|
|
2250
|
+
// and the file gets uploaded — which would fail this test.
|
|
2251
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2252
|
+
fs.writeFileSync(journalPath, JSON.stringify({
|
|
2253
|
+
version: "2",
|
|
2254
|
+
lastSync: new Date().toISOString(),
|
|
2255
|
+
files: {
|
|
2256
|
+
"stable.md": {
|
|
2257
|
+
hash: "sentinel-wrong-hash-fast-path-should-skip-without-checking",
|
|
2258
|
+
size: lstat.size,
|
|
2259
|
+
mtimeMs: lstat.mtimeMs,
|
|
2260
|
+
syncedAt: new Date().toISOString(),
|
|
2261
|
+
direction: "up",
|
|
2262
|
+
},
|
|
2263
|
+
},
|
|
2264
|
+
pulls: [],
|
|
2265
|
+
}));
|
|
2266
|
+
const result = await share({
|
|
2267
|
+
paths: [testFile],
|
|
2268
|
+
company: "acme",
|
|
2269
|
+
vaultConfig: mockConfig,
|
|
2270
|
+
hqRoot: tmpDir,
|
|
2271
|
+
skipUnchanged: true,
|
|
2272
|
+
});
|
|
2273
|
+
// Fast-path fired → no hash read → no upload despite wrong stored hash.
|
|
2274
|
+
expect(result.filesUploaded).toBe(0);
|
|
2275
|
+
expect(result.filesSkipped).toBe(1);
|
|
2276
|
+
expect(uploadFile).not.toHaveBeenCalled();
|
|
2277
|
+
});
|
|
2278
|
+
it("uploads when mtime changes (fast-path misses → hash runs → mismatch)", async () => {
|
|
2279
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2280
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2281
|
+
const testFile = path.join(companyRoot, "touched.md");
|
|
2282
|
+
fs.writeFileSync(testFile, "original bytes!");
|
|
2283
|
+
const originalLstat = fs.lstatSync(testFile);
|
|
2284
|
+
// Wrong hash + matching mtimeMs+size — would skip via fast-path if
|
|
2285
|
+
// mtime stayed the same. We're about to bump mtime, so fast-path
|
|
2286
|
+
// MUST miss, hashFile MUST run, hash MUST mismatch the wrong stored
|
|
2287
|
+
// value, and the file MUST upload.
|
|
2288
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2289
|
+
fs.writeFileSync(journalPath, JSON.stringify({
|
|
2290
|
+
version: "2",
|
|
2291
|
+
lastSync: new Date().toISOString(),
|
|
2292
|
+
files: {
|
|
2293
|
+
"touched.md": {
|
|
2294
|
+
hash: "sentinel-wrong-hash-mtime-bump-forces-hashFile-run",
|
|
2295
|
+
size: originalLstat.size,
|
|
2296
|
+
mtimeMs: originalLstat.mtimeMs,
|
|
2297
|
+
syncedAt: new Date().toISOString(),
|
|
2298
|
+
direction: "up",
|
|
2299
|
+
},
|
|
2300
|
+
},
|
|
2301
|
+
pulls: [],
|
|
2302
|
+
}));
|
|
2303
|
+
// Bump mtime forward — fast-path comparison mtime fails.
|
|
2304
|
+
const futureTime = new Date(Date.now() + 60_000);
|
|
2305
|
+
fs.utimesSync(testFile, futureTime, futureTime);
|
|
2306
|
+
const result = await share({
|
|
2307
|
+
paths: [testFile],
|
|
2308
|
+
company: "acme",
|
|
2309
|
+
vaultConfig: mockConfig,
|
|
2310
|
+
hqRoot: tmpDir,
|
|
2311
|
+
skipUnchanged: true,
|
|
2312
|
+
});
|
|
2313
|
+
// Fast-path missed → second-stage hash compare ran → wrong stored hash
|
|
2314
|
+
// forced the upload.
|
|
2315
|
+
expect(result.filesUploaded).toBe(1);
|
|
2316
|
+
expect(uploadFile).toHaveBeenCalled();
|
|
2317
|
+
});
|
|
2318
|
+
it("stamps mtimeMs on the journal entry after a successful upload", async () => {
|
|
2319
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2320
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2321
|
+
const testFile = path.join(companyRoot, "fresh.md");
|
|
2322
|
+
fs.writeFileSync(testFile, "fresh content");
|
|
2323
|
+
const lstat = fs.lstatSync(testFile);
|
|
2324
|
+
const result = await share({
|
|
2325
|
+
paths: [testFile],
|
|
2326
|
+
company: "acme",
|
|
2327
|
+
vaultConfig: mockConfig,
|
|
2328
|
+
hqRoot: tmpDir,
|
|
2329
|
+
});
|
|
2330
|
+
expect(result.filesUploaded).toBe(1);
|
|
2331
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2332
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2333
|
+
expect(journal.files["fresh.md"]).toBeDefined();
|
|
2334
|
+
expect(journal.files["fresh.md"].mtimeMs).toBe(lstat.mtimeMs);
|
|
2335
|
+
expect(journal.files["fresh.md"].size).toBe(lstat.size);
|
|
2336
|
+
});
|
|
2337
|
+
it("falls through to hashFile when journal has no mtimeMs (pre-5.36 entry)", async () => {
|
|
2338
|
+
// Back-compat: pre-5.36 journals don't carry mtimeMs. The first sync
|
|
2339
|
+
// after upgrade must fall through to the hash path so legacy entries
|
|
2340
|
+
// still work; the post-upload stamp adds the field so subsequent
|
|
2341
|
+
// syncs use the fast-path.
|
|
2342
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2343
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2344
|
+
const testFile = path.join(companyRoot, "legacy.md");
|
|
2345
|
+
fs.writeFileSync(testFile, "legacy bytes");
|
|
2346
|
+
const { hashFile: realHashFile } = await import("../journal.js");
|
|
2347
|
+
const realHash = realHashFile(testFile);
|
|
2348
|
+
const realSize = fs.statSync(testFile).size;
|
|
2349
|
+
// Real hash but NO mtimeMs — must take hash path. Hash matches →
|
|
2350
|
+
// second-stage compare skips upload.
|
|
2351
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2352
|
+
fs.writeFileSync(journalPath, JSON.stringify({
|
|
2353
|
+
version: "1",
|
|
2354
|
+
lastSync: new Date().toISOString(),
|
|
2355
|
+
files: {
|
|
2356
|
+
"legacy.md": {
|
|
2357
|
+
hash: realHash,
|
|
2358
|
+
size: realSize,
|
|
2359
|
+
syncedAt: new Date().toISOString(),
|
|
2360
|
+
direction: "up",
|
|
2361
|
+
// No mtimeMs — pre-5.36 shape.
|
|
2362
|
+
},
|
|
2363
|
+
},
|
|
2364
|
+
}));
|
|
2365
|
+
const result = await share({
|
|
2366
|
+
paths: [testFile],
|
|
2367
|
+
company: "acme",
|
|
2368
|
+
vaultConfig: mockConfig,
|
|
2369
|
+
hqRoot: tmpDir,
|
|
2370
|
+
skipUnchanged: true,
|
|
2371
|
+
});
|
|
2372
|
+
// Pre-5.36 entry → fast-path skipped → hash path ran → real hash
|
|
2373
|
+
// matched → upload skipped at second stage. Same outcome as 5.35.0.
|
|
2374
|
+
expect(result.filesSkipped).toBe(1);
|
|
2375
|
+
expect(uploadFile).not.toHaveBeenCalled();
|
|
2376
|
+
// And the upgrade path: a follow-up upload (forced by changing the
|
|
2377
|
+
// file) must stamp mtimeMs so the NEXT sync gets the fast-path.
|
|
2378
|
+
// Verified separately by the "stamps mtimeMs" test above.
|
|
2379
|
+
});
|
|
2380
|
+
});
|
|
2381
|
+
// ── Fix #2 (5.36.0): bounded-parallel transfer pool ──────────────────────
|
|
2382
|
+
//
|
|
2383
|
+
// Replaces the serial PUT loop with a bounded-concurrent pool (default 16,
|
|
2384
|
+
// env-tunable via HQ_SYNC_TRANSFER_CONCURRENCY). Per-file progress events
|
|
2385
|
+
// still fire (now at file-settle time, not plan-walk time) so the menubar
|
|
2386
|
+
// stream parser sees the same shape. Aborts mid-batch let in-flight PUTs
|
|
2387
|
+
// finish but stop queueing new ones.
|
|
2388
|
+
describe("parallel upload pool (5.36.0)", () => {
|
|
2389
|
+
it("runs uploads with measurable parallelism (8+ in flight before first settles)", async () => {
|
|
2390
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2391
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2392
|
+
// Plan 32 files so the pool of 16 has clear evidence of parallelism.
|
|
2393
|
+
const FILE_COUNT = 32;
|
|
2394
|
+
for (let i = 0; i < FILE_COUNT; i++) {
|
|
2395
|
+
fs.writeFileSync(path.join(companyRoot, `file-${i.toString().padStart(2, "0")}.bin`), `content-${i}`);
|
|
2396
|
+
}
|
|
2397
|
+
// Instrument uploadFile to record start + end timestamps and
|
|
2398
|
+
// artificially delay each PUT by 50ms. Serial would be ~1600ms;
|
|
2399
|
+
// pool of 16 should be ~100ms (two waves of 16).
|
|
2400
|
+
const UPLOAD_DELAY_MS = 50;
|
|
2401
|
+
const startTimes = [];
|
|
2402
|
+
const endTimes = [];
|
|
2403
|
+
vi.mocked(uploadFile).mockImplementation(async (_ctx, _abs, key) => {
|
|
2404
|
+
startTimes.push({ key, t: Date.now() });
|
|
2405
|
+
await new Promise((r) => setTimeout(r, UPLOAD_DELAY_MS));
|
|
2406
|
+
endTimes.push({ key, t: Date.now() });
|
|
2407
|
+
return { etag: '"upload-etag"' };
|
|
2408
|
+
});
|
|
2409
|
+
const events = [];
|
|
2410
|
+
const result = await share({
|
|
2411
|
+
paths: [companyRoot],
|
|
2412
|
+
company: "acme",
|
|
2413
|
+
vaultConfig: mockConfig,
|
|
2414
|
+
hqRoot: tmpDir,
|
|
2415
|
+
onEvent: (e) => events.push(e),
|
|
2416
|
+
// onConflict must be set so the pool uses its env-tunable
|
|
2417
|
+
// concurrency (Codex P1 fix forces concurrency=1 in interactive
|
|
2418
|
+
// mode to serialize readline prompts).
|
|
2419
|
+
onConflict: "keep",
|
|
2420
|
+
});
|
|
2421
|
+
expect(result.filesUploaded).toBe(FILE_COUNT);
|
|
2422
|
+
expect(uploadFile).toHaveBeenCalledTimes(FILE_COUNT);
|
|
2423
|
+
// Parallelism check #1: by the time the 8th PUT *started*, the 1st
|
|
2424
|
+
// PUT had NOT yet ended. (Stronger than just "wall-clock fast" —
|
|
2425
|
+
// proves true overlap.)
|
|
2426
|
+
expect(startTimes.length).toBeGreaterThanOrEqual(8);
|
|
2427
|
+
const eighthStart = startTimes[7].t;
|
|
2428
|
+
const firstEnd = endTimes[0]?.t ?? Number.POSITIVE_INFINITY;
|
|
2429
|
+
expect(eighthStart).toBeLessThan(firstEnd);
|
|
2430
|
+
// Parallelism check #2: at least 8 PUTs were in flight simultaneously
|
|
2431
|
+
// at some point.
|
|
2432
|
+
let maxConcurrent = 0;
|
|
2433
|
+
const sortedEvents = [];
|
|
2434
|
+
for (const s of startTimes)
|
|
2435
|
+
sortedEvents.push({ t: s.t, kind: "start" });
|
|
2436
|
+
for (const e of endTimes)
|
|
2437
|
+
sortedEvents.push({ t: e.t, kind: "end" });
|
|
2438
|
+
// Equal-t ordering: starts before ends gives an upper bound. For the
|
|
2439
|
+
// lower-bound assertion below (>= 8) this is conservative-safe.
|
|
2440
|
+
sortedEvents.sort((a, b) => a.t - b.t || (a.kind === "start" ? -1 : 1));
|
|
2441
|
+
let cur = 0;
|
|
2442
|
+
for (const ev of sortedEvents) {
|
|
2443
|
+
if (ev.kind === "start") {
|
|
2444
|
+
cur++;
|
|
2445
|
+
if (cur > maxConcurrent)
|
|
2446
|
+
maxConcurrent = cur;
|
|
2447
|
+
}
|
|
2448
|
+
else {
|
|
2449
|
+
cur--;
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
expect(maxConcurrent).toBeGreaterThanOrEqual(8);
|
|
2453
|
+
// Event-count correctness: one progress event per file.
|
|
2454
|
+
const progressEvents = events.filter((e) => !!e && typeof e === "object" && e.type === "progress");
|
|
2455
|
+
expect(progressEvents.length).toBe(FILE_COUNT);
|
|
2456
|
+
const progressKeys = new Set(progressEvents.map((e) => e.path));
|
|
2457
|
+
expect(progressKeys.size).toBe(FILE_COUNT);
|
|
2458
|
+
});
|
|
2459
|
+
it("HQ_SYNC_TRANSFER_CONCURRENCY env var caps the pool size", async () => {
|
|
2460
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2461
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2462
|
+
for (let i = 0; i < 10; i++) {
|
|
2463
|
+
fs.writeFileSync(path.join(companyRoot, `f-${i}.bin`), `c-${i}`);
|
|
2464
|
+
}
|
|
2465
|
+
process.env.HQ_SYNC_TRANSFER_CONCURRENCY = "2";
|
|
2466
|
+
const startTimes = [];
|
|
2467
|
+
const endTimes = [];
|
|
2468
|
+
vi.mocked(uploadFile).mockImplementation(async () => {
|
|
2469
|
+
startTimes.push(Date.now());
|
|
2470
|
+
await new Promise((r) => setTimeout(r, 30));
|
|
2471
|
+
endTimes.push(Date.now());
|
|
2472
|
+
return { etag: '"upload-etag"' };
|
|
2473
|
+
});
|
|
2474
|
+
try {
|
|
2475
|
+
await share({
|
|
2476
|
+
paths: [companyRoot],
|
|
2477
|
+
company: "acme",
|
|
2478
|
+
vaultConfig: mockConfig,
|
|
2479
|
+
hqRoot: tmpDir,
|
|
2480
|
+
// Force non-interactive mode so the pool honors the env cap
|
|
2481
|
+
// (interactive mode forces concurrency=1 — see Codex P1 fix).
|
|
2482
|
+
onConflict: "keep",
|
|
2483
|
+
});
|
|
2484
|
+
// Compute max concurrency from start/end timeline. With cap=2,
|
|
2485
|
+
// we should never see more than 2 in flight. Date.now() has only
|
|
2486
|
+
// ms granularity so simultaneous start/end timestamps are common;
|
|
2487
|
+
// process ENDs before STARTs at the same ms (the actual ordering
|
|
2488
|
+
// — a task can't start until the prior one's `finally` removed it
|
|
2489
|
+
// from inFlight).
|
|
2490
|
+
const sortedEvents = [];
|
|
2491
|
+
for (const t of startTimes)
|
|
2492
|
+
sortedEvents.push({ t, kind: "start" });
|
|
2493
|
+
for (const t of endTimes)
|
|
2494
|
+
sortedEvents.push({ t, kind: "end" });
|
|
2495
|
+
sortedEvents.sort((a, b) => a.t - b.t || (a.kind === "end" ? -1 : 1));
|
|
2496
|
+
let cur = 0;
|
|
2497
|
+
let maxConcurrent = 0;
|
|
2498
|
+
for (const ev of sortedEvents) {
|
|
2499
|
+
if (ev.kind === "start") {
|
|
2500
|
+
cur++;
|
|
2501
|
+
if (cur > maxConcurrent)
|
|
2502
|
+
maxConcurrent = cur;
|
|
2503
|
+
}
|
|
2504
|
+
else {
|
|
2505
|
+
cur--;
|
|
2506
|
+
}
|
|
2507
|
+
}
|
|
2508
|
+
expect(maxConcurrent).toBeLessThanOrEqual(2);
|
|
2509
|
+
}
|
|
2510
|
+
finally {
|
|
2511
|
+
delete process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
2512
|
+
}
|
|
2513
|
+
});
|
|
2514
|
+
it("one failed upload does not abort sibling uploads in the pool", async () => {
|
|
2515
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2516
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2517
|
+
for (let i = 0; i < 6; i++) {
|
|
2518
|
+
fs.writeFileSync(path.join(companyRoot, `f-${i}.bin`), `c-${i}`);
|
|
2519
|
+
}
|
|
2520
|
+
// One file fails; others succeed.
|
|
2521
|
+
vi.mocked(uploadFile).mockImplementation(async (_ctx, _abs, key) => {
|
|
2522
|
+
if (key === "f-2.bin")
|
|
2523
|
+
throw new Error("simulated upload failure");
|
|
2524
|
+
return { etag: '"upload-etag"' };
|
|
2525
|
+
});
|
|
2526
|
+
const events = [];
|
|
2527
|
+
const result = await share({
|
|
2528
|
+
paths: [companyRoot],
|
|
2529
|
+
company: "acme",
|
|
2530
|
+
vaultConfig: mockConfig,
|
|
2531
|
+
hqRoot: tmpDir,
|
|
2532
|
+
onEvent: (e) => events.push(e),
|
|
2533
|
+
// Non-interactive so the pool runs in parallel (Codex P1 fix).
|
|
2534
|
+
onConflict: "keep",
|
|
2535
|
+
});
|
|
2536
|
+
// 5 succeeded + 1 errored.
|
|
2537
|
+
expect(result.filesUploaded).toBe(5);
|
|
2538
|
+
const errorEvents = events.filter((e) => !!e && typeof e === "object" && e.type === "error");
|
|
2539
|
+
expect(errorEvents.length).toBe(1);
|
|
2540
|
+
expect(errorEvents[0].path).toBe("f-2.bin");
|
|
2541
|
+
});
|
|
2542
|
+
// ── Codex P1 (5.36.x): interactive conflict prompt serialization ────
|
|
2543
|
+
//
|
|
2544
|
+
// The parallel pool can run multiple processUploadItem workers
|
|
2545
|
+
// concurrently; each worker calls resolveConflict() on a conflict, and
|
|
2546
|
+
// in interactive mode (onConflict === undefined) that opens a readline
|
|
2547
|
+
// prompt on process.stdin. Two prompts open at once would race for the
|
|
2548
|
+
// same terminal input — answers would interleave nondeterministically.
|
|
2549
|
+
//
|
|
2550
|
+
// Trade-off (Option A): when onConflict is undefined we force pool
|
|
2551
|
+
// concurrency to 1 for the whole run. Interactive sessions are rare
|
|
2552
|
+
// (operator is already at the terminal clicking through prompts), so
|
|
2553
|
+
// serializing the entire pool is acceptable and has the smallest
|
|
2554
|
+
// blast radius vs. a per-prompt mutex. Non-interactive (onConflict set)
|
|
2555
|
+
// keeps full parallelism.
|
|
2556
|
+
it("serializes interactive conflict prompts (no two readline prompts open at once)", async () => {
|
|
2557
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2558
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2559
|
+
// Two files that will both trigger a fresh-collision conflict —
|
|
2560
|
+
// remote already has them (different bytes) and there's no journal
|
|
2561
|
+
// entry yet, so the push planner classifies as a conflict.
|
|
2562
|
+
fs.writeFileSync(path.join(companyRoot, "a.bin"), "local-a");
|
|
2563
|
+
fs.writeFileSync(path.join(companyRoot, "b.bin"), "local-b");
|
|
2564
|
+
// headRemoteFile returns a remote with a non-matching MD5 (the local
|
|
2565
|
+
// body hashes differently), forcing the fresh-collision branch.
|
|
2566
|
+
vi.mocked(headRemoteFile).mockResolvedValue({
|
|
2567
|
+
etag: '"deadbeefdeadbeefdeadbeefdeadbeef"',
|
|
2568
|
+
lastModified: new Date(Date.now() - 60_000),
|
|
2569
|
+
size: 999,
|
|
2570
|
+
metadata: {},
|
|
2571
|
+
});
|
|
2572
|
+
// Track prompt lifecycle: each createInterface increments openCount;
|
|
2573
|
+
// each close decrements. If concurrency leaks, openCount will exceed
|
|
2574
|
+
// 1 between question() and close().
|
|
2575
|
+
let openCount = 0;
|
|
2576
|
+
let maxOpenAtOnce = 0;
|
|
2577
|
+
const promptOrder = [];
|
|
2578
|
+
vi.mocked(readline.createInterface).mockImplementation(() => {
|
|
2579
|
+
openCount++;
|
|
2580
|
+
if (openCount > maxOpenAtOnce)
|
|
2581
|
+
maxOpenAtOnce = openCount;
|
|
2582
|
+
promptOrder.push("open");
|
|
2583
|
+
return {
|
|
2584
|
+
question: (_q, cb) => {
|
|
2585
|
+
// Yield a tick so any other concurrent worker that *was* going
|
|
2586
|
+
// to open a prompt would do so before we answer + close.
|
|
2587
|
+
setTimeout(() => cb("k"), 25);
|
|
2588
|
+
},
|
|
2589
|
+
close: () => {
|
|
2590
|
+
openCount--;
|
|
2591
|
+
promptOrder.push("close");
|
|
2592
|
+
},
|
|
2593
|
+
};
|
|
2594
|
+
});
|
|
2595
|
+
const result = await share({
|
|
2596
|
+
paths: [companyRoot],
|
|
2597
|
+
company: "acme",
|
|
2598
|
+
vaultConfig: mockConfig,
|
|
2599
|
+
hqRoot: tmpDir,
|
|
2600
|
+
// onConflict intentionally omitted → interactive mode
|
|
2601
|
+
});
|
|
2602
|
+
// Both conflicts surfaced.
|
|
2603
|
+
expect(result.conflictPaths.length).toBe(2);
|
|
2604
|
+
// Both prompts opened (one per conflicting file).
|
|
2605
|
+
expect(readline.createInterface).toHaveBeenCalledTimes(2);
|
|
2606
|
+
// CRITICAL: no two prompts open simultaneously.
|
|
2607
|
+
expect(maxOpenAtOnce).toBe(1);
|
|
2608
|
+
// CRITICAL: strict open → close → open → close ordering.
|
|
2609
|
+
expect(promptOrder).toEqual(["open", "close", "open", "close"]);
|
|
2610
|
+
});
|
|
2611
|
+
it("parallel pool stays at full concurrency when onConflict is set", async () => {
|
|
2612
|
+
// Sanity check: the interactive-mode fallback to concurrency=1 must
|
|
2613
|
+
// NOT regress non-interactive throughput. With onConflict provided
|
|
2614
|
+
// (no readline involved) the pool keeps the 16-wide default.
|
|
2615
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2616
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2617
|
+
const FILE_COUNT = 16;
|
|
2618
|
+
for (let i = 0; i < FILE_COUNT; i++) {
|
|
2619
|
+
fs.writeFileSync(path.join(companyRoot, `f-${i.toString().padStart(2, "0")}.bin`), `c-${i}`);
|
|
2620
|
+
}
|
|
2621
|
+
const startTimes = [];
|
|
2622
|
+
const endTimes = [];
|
|
2623
|
+
vi.mocked(uploadFile).mockImplementation(async () => {
|
|
2624
|
+
startTimes.push(Date.now());
|
|
2625
|
+
await new Promise((r) => setTimeout(r, 30));
|
|
2626
|
+
endTimes.push(Date.now());
|
|
2627
|
+
return { etag: '"upload-etag"' };
|
|
2628
|
+
});
|
|
2629
|
+
await share({
|
|
2630
|
+
paths: [companyRoot],
|
|
2631
|
+
company: "acme",
|
|
2632
|
+
vaultConfig: mockConfig,
|
|
2633
|
+
hqRoot: tmpDir,
|
|
2634
|
+
onConflict: "keep",
|
|
2635
|
+
});
|
|
2636
|
+
// Compute max concurrency from start/end timeline.
|
|
2637
|
+
const sortedEvents = [];
|
|
2638
|
+
for (const t of startTimes)
|
|
2639
|
+
sortedEvents.push({ t, kind: "start" });
|
|
2640
|
+
for (const t of endTimes)
|
|
2641
|
+
sortedEvents.push({ t, kind: "end" });
|
|
2642
|
+
sortedEvents.sort((a, b) => a.t - b.t || (a.kind === "start" ? -1 : 1));
|
|
2643
|
+
let cur = 0;
|
|
2644
|
+
let maxConcurrent = 0;
|
|
2645
|
+
for (const ev of sortedEvents) {
|
|
2646
|
+
if (ev.kind === "start") {
|
|
2647
|
+
cur++;
|
|
2648
|
+
if (cur > maxConcurrent)
|
|
2649
|
+
maxConcurrent = cur;
|
|
2650
|
+
}
|
|
2651
|
+
else {
|
|
2652
|
+
cur--;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
// Should be significantly > 1 (full pool); use 4 as a conservative
|
|
2656
|
+
// lower bound so the test isn't flaky on slow CI.
|
|
2657
|
+
expect(maxConcurrent).toBeGreaterThanOrEqual(4);
|
|
2658
|
+
});
|
|
2659
|
+
// ── Codex P1 (5.36.x): pool drains in-flight on worker rejection ────
|
|
2660
|
+
//
|
|
2661
|
+
// Pre-fix: the scheduler waited on `Promise.race(inFlight)`. If any
|
|
2662
|
+
// worker rejected (e.g. headRemoteFile threw before its caller's
|
|
2663
|
+
// try/catch was reached), the race rejected immediately, the await
|
|
2664
|
+
// unwound out of the pool, and remaining in-flight workers kept
|
|
2665
|
+
// running — their PUTs still mutated S3 while share() had already
|
|
2666
|
+
// failed and skipped writeJournal. Remote and journal drifted out
|
|
2667
|
+
// of sync, breaking the next run's planner.
|
|
2668
|
+
//
|
|
2669
|
+
// Fix: wrap each worker so it never rejects from the pool's
|
|
2670
|
+
// perspective; collect errors in a side array; after the pool drains
|
|
2671
|
+
// (queue empty + inFlight empty), if errors exist, throw an
|
|
2672
|
+
// aggregated error so the caller still sees the failure.
|
|
2673
|
+
it("drains all in-flight uploads before throwing when a worker rejects (headRemoteFile)", async () => {
|
|
2674
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2675
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2676
|
+
const FILE_COUNT = 10;
|
|
2677
|
+
for (let i = 0; i < FILE_COUNT; i++) {
|
|
2678
|
+
fs.writeFileSync(path.join(companyRoot, `f-${i.toString().padStart(2, "0")}.bin`), `content-${i}`);
|
|
2679
|
+
}
|
|
2680
|
+
// headRemoteFile throws on file index 3 (an unhandled-by-pool
|
|
2681
|
+
// rejection — the surrounding try/catch in processUploadItem
|
|
2682
|
+
// covers uploadFile but NOT headRemoteFile). All other files
|
|
2683
|
+
// resolve to null (no remote → straight upload). Each call takes
|
|
2684
|
+
// ~50ms so multiple workers are in flight when the rejection
|
|
2685
|
+
// surfaces.
|
|
2686
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => {
|
|
2687
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
2688
|
+
if (key === "f-03.bin") {
|
|
2689
|
+
throw new Error("simulated headRemoteFile failure");
|
|
2690
|
+
}
|
|
2691
|
+
return null;
|
|
2692
|
+
});
|
|
2693
|
+
// Per-file uploadFile delay so there's measurable overlap with
|
|
2694
|
+
// the head-rejecting worker.
|
|
2695
|
+
vi.mocked(uploadFile).mockImplementation(async (_ctx, _abs, _key) => {
|
|
2696
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
2697
|
+
return { etag: '"upload-etag"' };
|
|
2698
|
+
});
|
|
2699
|
+
let thrown = null;
|
|
2700
|
+
try {
|
|
2701
|
+
await share({
|
|
2702
|
+
paths: [companyRoot],
|
|
2703
|
+
company: "acme",
|
|
2704
|
+
vaultConfig: mockConfig,
|
|
2705
|
+
hqRoot: tmpDir,
|
|
2706
|
+
// onConflict set so the pool runs in parallel (interactive
|
|
2707
|
+
// mode forces concurrency=1 and the test wants overlap).
|
|
2708
|
+
onConflict: "keep",
|
|
2709
|
+
});
|
|
2710
|
+
}
|
|
2711
|
+
catch (err) {
|
|
2712
|
+
thrown = err;
|
|
2713
|
+
}
|
|
2714
|
+
// share() rejects with the worker's error surfaced.
|
|
2715
|
+
expect(thrown).toBeInstanceOf(Error);
|
|
2716
|
+
expect(thrown.message).toMatch(/headRemoteFile failure/);
|
|
2717
|
+
// CRITICAL: headRemoteFile was called for every file — proves the
|
|
2718
|
+
// pool kept scheduling work after the rejecting worker started,
|
|
2719
|
+
// and (combined with the writeJournal assertion below) that the
|
|
2720
|
+
// pool drained the in-flight set instead of bailing on first
|
|
2721
|
+
// rejection. Pre-fix this would be < FILE_COUNT.
|
|
2722
|
+
expect(headRemoteFile).toHaveBeenCalledTimes(FILE_COUNT);
|
|
2723
|
+
// CRITICAL: uploadFile was called for the 9 non-rejecting files.
|
|
2724
|
+
// (f-03 rejected at headRemoteFile, so its uploadFile never
|
|
2725
|
+
// fires.) Pre-fix this could be < 9 because the pool would
|
|
2726
|
+
// abandon in-flight workers before they reached their PUT.
|
|
2727
|
+
expect(uploadFile).toHaveBeenCalledTimes(FILE_COUNT - 1);
|
|
2728
|
+
// CRITICAL: writeJournal still ran — the journal on disk has
|
|
2729
|
+
// entries for the 9 successful uploads. Pre-fix the journal was
|
|
2730
|
+
// never written (the throw escaped before writeJournal()), so
|
|
2731
|
+
// the next sync re-uploaded all 9 successful files unnecessarily
|
|
2732
|
+
// AND lost the etag/mtime stamps needed by the lstat fast-path.
|
|
2733
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2734
|
+
expect(fs.existsSync(journalPath)).toBe(true);
|
|
2735
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2736
|
+
const stampedKeys = Object.keys(journal.files);
|
|
2737
|
+
expect(stampedKeys.length).toBe(FILE_COUNT - 1);
|
|
2738
|
+
// The failed file is NOT in the journal (no successful upload).
|
|
2739
|
+
expect(stampedKeys).not.toContain("f-03.bin");
|
|
2740
|
+
// The 9 successful files ARE in the journal.
|
|
2741
|
+
for (let i = 0; i < FILE_COUNT; i++) {
|
|
2742
|
+
if (i === 3)
|
|
2743
|
+
continue;
|
|
2744
|
+
expect(stampedKeys).toContain(`f-${i.toString().padStart(2, "0")}.bin`);
|
|
2745
|
+
}
|
|
2746
|
+
});
|
|
2747
|
+
});
|
|
2216
2748
|
});
|
|
2217
2749
|
//# sourceMappingURL=share.test.js.map
|