@indigoai-us/hq-cloud 6.11.6 → 6.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +1 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +23 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +83 -15
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +198 -7
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +16 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +1 -62
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/tombstones.d.ts +43 -0
- package/dist/cli/tombstones.d.ts.map +1 -0
- package/dist/cli/tombstones.js +78 -0
- package/dist/cli/tombstones.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/personal-vault.d.ts +36 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +89 -1
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +143 -1
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +22 -1
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +29 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +1 -0
- package/src/bin/sync-runner.ts +1 -0
- package/src/cli/share.test.ts +228 -7
- package/src/cli/share.ts +120 -24
- package/src/cli/sync.ts +21 -88
- package/src/cli/tombstones.ts +106 -0
- package/src/index.ts +2 -0
- package/src/personal-vault.test.ts +175 -0
- package/src/personal-vault.ts +86 -1
- package/src/watcher.test.ts +41 -0
- package/src/watcher.ts +24 -1
package/src/bin/sync-runner.ts
CHANGED
package/src/cli/share.test.ts
CHANGED
|
@@ -336,6 +336,175 @@ describe("share", () => {
|
|
|
336
336
|
expect(uploadFile).toHaveBeenCalledWith(expect.anything(), testFile, "changed.md", undefined, expect.anything());
|
|
337
337
|
});
|
|
338
338
|
|
|
339
|
+
// ── Push-side FILE_TOMBSTONE consult (delete-resync, 3B) ────────────────────
|
|
340
|
+
// An authoritative delete (`hq files delete`) writes a FILE_TOMBSTONE and
|
|
341
|
+
// removes the S3 object. Without a push-side consult, a behind peer that still
|
|
342
|
+
// holds the deleted file RE-UPLOADS it on a skipUnchanged=false push (e.g.
|
|
343
|
+
// `hq cloud share <path>`); the re-upload post-dates the tombstone, so the
|
|
344
|
+
// pull planner's timestamp-only re-create heuristic treats it as a genuine
|
|
345
|
+
// re-create and resurrects the key for everyone. These tests pin the fix: a
|
|
346
|
+
// stale-baseline copy is suppressed, while genuine content (locally-changed or
|
|
347
|
+
// no-journal) still uploads. Differential proof: reverting the consult in
|
|
348
|
+
// share.ts makes the first test fail (uploadFile IS called → resurrection).
|
|
349
|
+
describe("push-side tombstone consult", () => {
|
|
350
|
+
function seedJournal(key: string, hash: string, size: number) {
|
|
351
|
+
fs.writeFileSync(
|
|
352
|
+
path.join(stateDir, "sync-journal.acme.json"),
|
|
353
|
+
JSON.stringify({
|
|
354
|
+
version: "1",
|
|
355
|
+
lastSync: new Date().toISOString(),
|
|
356
|
+
files: {
|
|
357
|
+
[key]: {
|
|
358
|
+
hash,
|
|
359
|
+
size,
|
|
360
|
+
syncedAt: new Date().toISOString(),
|
|
361
|
+
direction: "down",
|
|
362
|
+
},
|
|
363
|
+
},
|
|
364
|
+
}),
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
it("suppresses re-upload of a stale-baseline copy of a tombstoned key", async () => {
|
|
369
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
370
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
371
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
372
|
+
fs.writeFileSync(f, "shared content");
|
|
373
|
+
const { hashFile } = await import("../journal.js");
|
|
374
|
+
// Journal hash === current file hash → this machine holds the exact
|
|
375
|
+
// deleted baseline (a behind peer that never pulled the delete).
|
|
376
|
+
seedJournal("docs/shared.md", hashFile(f), 14);
|
|
377
|
+
|
|
378
|
+
const events: Array<{ type: string; path?: string }> = [];
|
|
379
|
+
const result = await share({
|
|
380
|
+
paths: [path.join(companyRoot, "docs")],
|
|
381
|
+
company: "acme",
|
|
382
|
+
vaultConfig: mockConfig,
|
|
383
|
+
hqRoot: tmpDir,
|
|
384
|
+
// skipUnchanged omitted (=false): models `hq cloud share <path>`, the
|
|
385
|
+
// path where a behind peer would otherwise re-upload the stale copy.
|
|
386
|
+
fileTombstones: new Map([
|
|
387
|
+
["docs/shared.md", { deletedAt: new Date().toISOString() }],
|
|
388
|
+
]),
|
|
389
|
+
onEvent: (e) => events.push(e as { type: string; path?: string }),
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
// The resurrection is blocked at the source: no upload for the tombstoned key.
|
|
393
|
+
expect(uploadFile).not.toHaveBeenCalled();
|
|
394
|
+
expect(result.filesUploaded).toBe(0);
|
|
395
|
+
expect(result.filesSuppressedByTombstone).toBe(1);
|
|
396
|
+
expect(
|
|
397
|
+
events.some(
|
|
398
|
+
(e) => e.type === "upload-suppressed-tombstone" && e.path === "docs/shared.md",
|
|
399
|
+
),
|
|
400
|
+
).toBe(true);
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
it("still uploads a LOCALLY-CHANGED file even when tombstoned (genuine edit/re-create)", async () => {
|
|
404
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
405
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
406
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
407
|
+
fs.writeFileSync(f, "edited-since-delete");
|
|
408
|
+
// Journal hash is stale (the file was edited after the delete) → genuine
|
|
409
|
+
// new content, must NOT be suppressed.
|
|
410
|
+
seedJournal("docs/shared.md", "stale-baseline-hash", 14);
|
|
411
|
+
|
|
412
|
+
const result = await share({
|
|
413
|
+
paths: [path.join(companyRoot, "docs")],
|
|
414
|
+
company: "acme",
|
|
415
|
+
vaultConfig: mockConfig,
|
|
416
|
+
hqRoot: tmpDir,
|
|
417
|
+
fileTombstones: new Map([
|
|
418
|
+
["docs/shared.md", { deletedAt: new Date().toISOString() }],
|
|
419
|
+
]),
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
expect(result.filesSuppressedByTombstone).toBe(0);
|
|
423
|
+
expect(result.filesUploaded).toBe(1);
|
|
424
|
+
expect(uploadFile).toHaveBeenCalledWith(
|
|
425
|
+
expect.anything(),
|
|
426
|
+
f,
|
|
427
|
+
"docs/shared.md",
|
|
428
|
+
undefined,
|
|
429
|
+
expect.anything(),
|
|
430
|
+
);
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
it("still uploads a tombstoned key with NO journal entry (genuine re-create)", async () => {
|
|
434
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
435
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
436
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
437
|
+
fs.writeFileSync(f, "freshly created");
|
|
438
|
+
// No journal entry seeded → indistinguishable from genuine new content;
|
|
439
|
+
// fail-open and upload (mirrors the pull side's `!journalEntry` branch).
|
|
440
|
+
|
|
441
|
+
const result = await share({
|
|
442
|
+
paths: [path.join(companyRoot, "docs")],
|
|
443
|
+
company: "acme",
|
|
444
|
+
vaultConfig: mockConfig,
|
|
445
|
+
hqRoot: tmpDir,
|
|
446
|
+
fileTombstones: new Map([
|
|
447
|
+
["docs/shared.md", { deletedAt: new Date().toISOString() }],
|
|
448
|
+
]),
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
expect(result.filesSuppressedByTombstone).toBe(0);
|
|
452
|
+
expect(result.filesUploaded).toBe(1);
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it("auto-fetches tombstones via vaultConfig (no injection) and suppresses", async () => {
|
|
456
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
457
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
458
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
459
|
+
fs.writeFileSync(f, "shared content");
|
|
460
|
+
const { hashFile } = await import("../journal.js");
|
|
461
|
+
seedJournal("docs/shared.md", hashFile(f), 14);
|
|
462
|
+
|
|
463
|
+
// Re-stub fetch to also answer GET /v1/files/tombstones, proving share()
|
|
464
|
+
// fetches and consults tombstones on its own (the production wiring) when
|
|
465
|
+
// no map is injected.
|
|
466
|
+
const fetchMock = vi.fn().mockImplementation(async (url: string) => {
|
|
467
|
+
const u = String(url);
|
|
468
|
+
if (u.includes("/entity/check-slug/me")) {
|
|
469
|
+
return {
|
|
470
|
+
ok: true,
|
|
471
|
+
status: 200,
|
|
472
|
+
json: async () => ({ available: false, conflictingCompanyUid: mockEntity.uid }),
|
|
473
|
+
text: async () => "",
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
if (u.includes("/entity/by-slug/") || /\/entity\/cmp_/.test(u)) {
|
|
477
|
+
return { ok: true, status: 200, json: async () => ({ entity: mockEntity }), text: async () => "" };
|
|
478
|
+
}
|
|
479
|
+
if (u.includes("/sts/vend")) {
|
|
480
|
+
return { ok: true, status: 200, json: async () => mockVendResponse, text: async () => "" };
|
|
481
|
+
}
|
|
482
|
+
if (u.includes("/v1/files/tombstones")) {
|
|
483
|
+
return {
|
|
484
|
+
ok: true,
|
|
485
|
+
status: 200,
|
|
486
|
+
json: async () => ({
|
|
487
|
+
tombstones: [{ key: "docs/shared.md", deletedAt: new Date().toISOString() }],
|
|
488
|
+
}),
|
|
489
|
+
text: async () => "",
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
return { ok: false, status: 404, text: async () => "Not found" };
|
|
493
|
+
});
|
|
494
|
+
vi.stubGlobal("fetch", fetchMock);
|
|
495
|
+
|
|
496
|
+
const result = await share({
|
|
497
|
+
paths: [path.join(companyRoot, "docs")],
|
|
498
|
+
company: "acme",
|
|
499
|
+
vaultConfig: mockConfig,
|
|
500
|
+
hqRoot: tmpDir,
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
expect(uploadFile).not.toHaveBeenCalled();
|
|
504
|
+
expect(result.filesSuppressedByTombstone).toBe(1);
|
|
505
|
+
});
|
|
506
|
+
});
|
|
507
|
+
|
|
339
508
|
it("populates conflictPaths and emits a conflict event when both local and remote drifted from journal", async () => {
|
|
340
509
|
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
341
510
|
fs.mkdirSync(companyRoot, { recursive: true });
|
|
@@ -4157,7 +4326,7 @@ describe("currency-gated: journal version 2 fixtures", () => {
|
|
|
4157
4326
|
expect(errorEvents[0]!.path).toBe("f-2.bin");
|
|
4158
4327
|
});
|
|
4159
4328
|
|
|
4160
|
-
// ──
|
|
4329
|
+
// ── Interactive conflict prompt serialization ───────────────────────
|
|
4161
4330
|
//
|
|
4162
4331
|
// The parallel pool can run multiple processUploadItem workers
|
|
4163
4332
|
// concurrently; each worker calls resolveConflict() on a conflict, and
|
|
@@ -4165,12 +4334,11 @@ describe("currency-gated: journal version 2 fixtures", () => {
|
|
|
4165
4334
|
// prompt on process.stdin. Two prompts open at once would race for the
|
|
4166
4335
|
// same terminal input — answers would interleave nondeterministically.
|
|
4167
4336
|
//
|
|
4168
|
-
//
|
|
4169
|
-
//
|
|
4170
|
-
//
|
|
4171
|
-
//
|
|
4172
|
-
//
|
|
4173
|
-
// keeps full parallelism.
|
|
4337
|
+
// Fix: rather than force the WHOLE pool to concurrency=1 (which made an
|
|
4338
|
+
// interactive `hq sync now` crawl even with zero conflicts), the pool
|
|
4339
|
+
// keeps full concurrency and serializes ONLY the prompt via a chained
|
|
4340
|
+
// lock (`resolveConflictSerialized`). At most one prompt awaits input at
|
|
4341
|
+
// a time; this test pins that invariant.
|
|
4174
4342
|
it("serializes interactive conflict prompts (no two readline prompts open at once)", async () => {
|
|
4175
4343
|
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
4176
4344
|
fs.mkdirSync(companyRoot, { recursive: true });
|
|
@@ -4282,6 +4450,59 @@ describe("currency-gated: journal version 2 fixtures", () => {
|
|
|
4282
4450
|
expect(maxConcurrent).toBeGreaterThanOrEqual(4);
|
|
4283
4451
|
});
|
|
4284
4452
|
|
|
4453
|
+
it("interactive mode (onConflict unset) still runs transfers concurrently", async () => {
|
|
4454
|
+
// REGRESSION: the 5.36.x interactive guard forced the WHOLE pool to
|
|
4455
|
+
// concurrency=1 whenever onConflict was unset — so `hq sync now` (which
|
|
4456
|
+
// omits --on-conflict) crawled through transfers one at a time even when
|
|
4457
|
+
// no conflict ever occurred. The prompt is now serialized on its own
|
|
4458
|
+
// chained lock, leaving the transfer pool at full concurrency.
|
|
4459
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
4460
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
4461
|
+
const FILE_COUNT = 16;
|
|
4462
|
+
for (let i = 0; i < FILE_COUNT; i++) {
|
|
4463
|
+
fs.writeFileSync(
|
|
4464
|
+
path.join(companyRoot, `f-${i.toString().padStart(2, "0")}.bin`),
|
|
4465
|
+
`c-${i}`,
|
|
4466
|
+
);
|
|
4467
|
+
}
|
|
4468
|
+
|
|
4469
|
+
const startTimes: number[] = [];
|
|
4470
|
+
const endTimes: number[] = [];
|
|
4471
|
+
vi.mocked(uploadFile).mockImplementation(async () => {
|
|
4472
|
+
startTimes.push(Date.now());
|
|
4473
|
+
await new Promise((r) => setTimeout(r, 30));
|
|
4474
|
+
endTimes.push(Date.now());
|
|
4475
|
+
return { etag: '"upload-etag"' };
|
|
4476
|
+
});
|
|
4477
|
+
|
|
4478
|
+
await share({
|
|
4479
|
+
paths: [companyRoot],
|
|
4480
|
+
company: "acme",
|
|
4481
|
+
vaultConfig: mockConfig,
|
|
4482
|
+
hqRoot: tmpDir,
|
|
4483
|
+
// onConflict intentionally omitted → interactive mode. No conflicts
|
|
4484
|
+
// occur, so the transfer pool must NOT serialize.
|
|
4485
|
+
});
|
|
4486
|
+
|
|
4487
|
+
const sortedEvents: Array<{ t: number; kind: "start" | "end" }> = [];
|
|
4488
|
+
for (const t of startTimes) sortedEvents.push({ t, kind: "start" });
|
|
4489
|
+
for (const t of endTimes) sortedEvents.push({ t, kind: "end" });
|
|
4490
|
+
sortedEvents.sort((a, b) => a.t - b.t || (a.kind === "start" ? -1 : 1));
|
|
4491
|
+
let cur = 0;
|
|
4492
|
+
let maxConcurrent = 0;
|
|
4493
|
+
for (const ev of sortedEvents) {
|
|
4494
|
+
if (ev.kind === "start") {
|
|
4495
|
+
cur++;
|
|
4496
|
+
if (cur > maxConcurrent) maxConcurrent = cur;
|
|
4497
|
+
} else {
|
|
4498
|
+
cur--;
|
|
4499
|
+
}
|
|
4500
|
+
}
|
|
4501
|
+
// Pre-fix this was exactly 1 (forced serial). Now it tracks the full
|
|
4502
|
+
// pool; 4 is a conservative lower bound to avoid CI flakiness.
|
|
4503
|
+
expect(maxConcurrent).toBeGreaterThanOrEqual(4);
|
|
4504
|
+
});
|
|
4505
|
+
|
|
4285
4506
|
// ── Codex P1 (5.36.x): pool drains in-flight on worker rejection ────
|
|
4286
4507
|
//
|
|
4287
4508
|
// Pre-fix: the scheduler waited on `Promise.race(inFlight)`. If any
|
package/src/cli/share.ts
CHANGED
|
@@ -42,6 +42,10 @@ import {
|
|
|
42
42
|
import { resolveConflict } from "./conflict.js";
|
|
43
43
|
import type { ConflictStrategy } from "./conflict.js";
|
|
44
44
|
import type { SyncProgressEvent } from "./sync.js";
|
|
45
|
+
import {
|
|
46
|
+
fetchCompanyTombstones,
|
|
47
|
+
type CompanyTombstone,
|
|
48
|
+
} from "./tombstones.js";
|
|
45
49
|
import { isCoveredByAny, isDirInScope } from "../prefix-coalesce.js";
|
|
46
50
|
import {
|
|
47
51
|
buildConflictId,
|
|
@@ -634,6 +638,19 @@ export interface ShareOptions {
|
|
|
634
638
|
* path is out of scope (mirrors the pull side's `isCoveredByAny([])`).
|
|
635
639
|
*/
|
|
636
640
|
prefixSet?: string[];
|
|
641
|
+
/**
|
|
642
|
+
* Pre-fetched FILE_TOMBSTONE map (POSIX key → tombstone) for the push-side
|
|
643
|
+
* delete-resync consult. When omitted, share() fetches it itself via
|
|
644
|
+
* `fetchCompanyTombstones` for COMPANY vaults that have a `vaultConfig`
|
|
645
|
+
* (personal vaults and pre-vended `entityContext`-only callers without a
|
|
646
|
+
* `vaultConfig` degrade to no-suppression — the safe, legacy direction).
|
|
647
|
+
*
|
|
648
|
+
* Injection is an optimization + test seam: a sync run that already fetched
|
|
649
|
+
* tombstones for the pull leg can hand the same map to the push leg to avoid a
|
|
650
|
+
* second round-trip, and tests can supply a controlled map without stubbing
|
|
651
|
+
* the network. See the consult in the Stage-2 classification pass.
|
|
652
|
+
*/
|
|
653
|
+
fileTombstones?: Map<string, CompanyTombstone>;
|
|
637
654
|
}
|
|
638
655
|
|
|
639
656
|
export interface ShareResult {
|
|
@@ -678,6 +695,15 @@ export interface ShareResult {
|
|
|
678
695
|
* once the runner has folded them into the totals.
|
|
679
696
|
*/
|
|
680
697
|
filesRefusedStalePaths: string[];
|
|
698
|
+
/**
|
|
699
|
+
* Number of uploads suppressed by the push-side FILE_TOMBSTONE consult — keys
|
|
700
|
+
* an authoritative delete (`hq files delete`) tombstoned that this machine
|
|
701
|
+
* still held as the unchanged synced baseline. Skipping the upload is what
|
|
702
|
+
* stops a behind peer from resurrecting an authoritatively-deleted key. Always
|
|
703
|
+
* 0 when there are no tombstones for in-scope keys (the common case) or when
|
|
704
|
+
* the run can't load tombstones (personal vault / no `vaultConfig`).
|
|
705
|
+
*/
|
|
706
|
+
filesSuppressedByTombstone: number;
|
|
681
707
|
/**
|
|
682
708
|
* Number of paths blocked by `PERSONAL_VAULT_DEFAULT_EXCLUSIONS` during this
|
|
683
709
|
* run (push leg, personalMode=true). Includes both files that would have
|
|
@@ -919,6 +945,10 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
919
945
|
// propagateDeletes=false path.
|
|
920
946
|
let filesTombstoned = 0;
|
|
921
947
|
let filesRefusedStale = 0;
|
|
948
|
+
// Count of uploads suppressed by the push-side FILE_TOMBSTONE consult (a
|
|
949
|
+
// behind peer's stale copy of an authoritatively-deleted key). Always 0 when
|
|
950
|
+
// there are no tombstones for in-scope keys.
|
|
951
|
+
let filesSuppressedByTombstone = 0;
|
|
922
952
|
// Capped at 50 to bound event payload size — `newFiles` uses the same cap.
|
|
923
953
|
const REFUSED_STALE_PATH_CAP = 50;
|
|
924
954
|
const filesRefusedStalePaths: string[] = [];
|
|
@@ -1040,24 +1070,64 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1040
1070
|
// PUTs that already issued will complete (S3 doesn't have client-side
|
|
1041
1071
|
// cancellation in this code path); their results are still recorded on
|
|
1042
1072
|
// the journal so the next sync's planner doesn't re-fire them.
|
|
1043
|
-
// Interactive-mode
|
|
1044
|
-
//
|
|
1045
|
-
//
|
|
1046
|
-
//
|
|
1047
|
-
//
|
|
1048
|
-
//
|
|
1049
|
-
//
|
|
1050
|
-
//
|
|
1051
|
-
//
|
|
1052
|
-
const isInteractiveConflictMode = onConflict === undefined;
|
|
1073
|
+
// Interactive-mode prompts: when `onConflict` is unset the per-item conflict
|
|
1074
|
+
// path calls resolveConflict()'s readline prompt on process.stdin, and two
|
|
1075
|
+
// pool workers prompting at once would race for the terminal and interleave
|
|
1076
|
+
// answers. The 5.36.x guard solved this by forcing the WHOLE pool to
|
|
1077
|
+
// concurrency=1 — which made an interactive `hq sync now` crawl even when
|
|
1078
|
+
// zero conflicts existed (every transfer serialized just in case one might
|
|
1079
|
+
// prompt). Instead, keep full env-tunable concurrency and serialize ONLY the
|
|
1080
|
+
// prompt (see `resolveConflictSerialized` below): at most one prompt awaits
|
|
1081
|
+
// input at a time while transfers stay parallel.
|
|
1053
1082
|
const TRANSFER_CONCURRENCY = (() => {
|
|
1054
|
-
if (isInteractiveConflictMode) return 1;
|
|
1055
1083
|
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
1056
1084
|
if (raw === undefined || raw === "") return 16;
|
|
1057
1085
|
const parsed = Number.parseInt(raw, 10);
|
|
1058
1086
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
1059
1087
|
})();
|
|
1060
1088
|
|
|
1089
|
+
// Chained lock around the (possibly interactive) conflict prompt. Each
|
|
1090
|
+
// resolveConflict() runs only after the previous one settles, so concurrent
|
|
1091
|
+
// pool workers never prompt over each other on stdin — without dropping the
|
|
1092
|
+
// transfer pool's parallelism. A rejected prompt must not wedge the chain,
|
|
1093
|
+
// so the link swallows errors (the original promise still rejects to its
|
|
1094
|
+
// awaiter). In non-interactive mode resolveConflict applies the configured
|
|
1095
|
+
// strategy without reading stdin, so the lock adds no real serialization.
|
|
1096
|
+
let conflictPromptChain: Promise<unknown> = Promise.resolve();
|
|
1097
|
+
const resolveConflictSerialized = (
|
|
1098
|
+
info: Parameters<typeof resolveConflict>[0],
|
|
1099
|
+
): ReturnType<typeof resolveConflict> => {
|
|
1100
|
+
const run = conflictPromptChain.then(() => resolveConflict(info, onConflict));
|
|
1101
|
+
conflictPromptChain = run.then(
|
|
1102
|
+
() => undefined,
|
|
1103
|
+
() => undefined,
|
|
1104
|
+
);
|
|
1105
|
+
return run;
|
|
1106
|
+
};
|
|
1107
|
+
|
|
1108
|
+
// Push-side FILE_TOMBSTONE consult (delete-resync) — symmetric to the pull
|
|
1109
|
+
// planner's suppression in sync.ts. An authoritative delete
|
|
1110
|
+
// (`hq files delete <prefix>`) writes a FILE_TOMBSTONE and removes the S3
|
|
1111
|
+
// object; the pull side already honors it. But the PUSH side did not: a behind
|
|
1112
|
+
// peer who still holds the deleted file locally would re-upload it here, and
|
|
1113
|
+
// because the re-uploaded object post-dates the tombstone, the pull planner's
|
|
1114
|
+
// timestamp-only re-create heuristic (`isRemoteRecreateAfterTombstone`) treats
|
|
1115
|
+
// it as a genuine re-create and resurrects the key for EVERYONE — defeating the
|
|
1116
|
+
// authoritative delete. Consult the tombstones so a stale-baseline upload is
|
|
1117
|
+
// skipped at the source.
|
|
1118
|
+
//
|
|
1119
|
+
// Source: an injected `fileTombstones` (a sync run can hand the push leg the
|
|
1120
|
+
// map it already fetched for the pull leg), else a self-fetch for COMPANY
|
|
1121
|
+
// vaults that have a `vaultConfig`. Personal vaults have no company tombstones
|
|
1122
|
+
// (the pull side skips them too), and `entityContext`-only callers have no
|
|
1123
|
+
// auth to fetch with — both degrade to an empty map (no suppression), the
|
|
1124
|
+
// safe/legacy direction.
|
|
1125
|
+
const fileTombstones: Map<string, CompanyTombstone> =
|
|
1126
|
+
options.fileTombstones ??
|
|
1127
|
+
(!ctx.uid.startsWith("prs_") && vaultConfig
|
|
1128
|
+
? await fetchCompanyTombstones(vaultConfig, ctx.uid)
|
|
1129
|
+
: new Map<string, CompanyTombstone>());
|
|
1130
|
+
|
|
1061
1131
|
// Phase A: serial classification pass — handle skips inline, collect
|
|
1062
1132
|
// upload candidates for the parallel pool.
|
|
1063
1133
|
const uploadItems: Array<typeof plan.items[number] & { action: "upload" }> = [];
|
|
@@ -1071,6 +1141,30 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1071
1141
|
filesSkipped++;
|
|
1072
1142
|
continue;
|
|
1073
1143
|
}
|
|
1144
|
+
// Suppress the re-upload of an authoritatively-deleted key when the local
|
|
1145
|
+
// copy is still the deleted baseline. Discrimination mirrors the pull side's
|
|
1146
|
+
// `localChanged || !journalEntry` test: a journal entry whose hash matches
|
|
1147
|
+
// the upload's localHash means the file is byte-identical to what was last
|
|
1148
|
+
// synced — i.e. the deleted version a behind peer never pulled away — so
|
|
1149
|
+
// SKIP it. A locally-changed file (hash differs) or one with no journal
|
|
1150
|
+
// entry is genuine new content (a real re-create or a post-delete edit) and
|
|
1151
|
+
// uploads normally. The deleter's own stale local copy is removed by the
|
|
1152
|
+
// pull leg's `tombstone-delete`; this guard only stops the resurrection.
|
|
1153
|
+
if (item.action === "upload" && fileTombstones.size > 0) {
|
|
1154
|
+
const ts = fileTombstones.get(toPosixKey(item.relativePath));
|
|
1155
|
+
if (ts !== undefined) {
|
|
1156
|
+
const entry = journal.files[item.relativePath];
|
|
1157
|
+
if (entry && entry.hash === item.localHash) {
|
|
1158
|
+
filesSuppressedByTombstone++;
|
|
1159
|
+
emit({
|
|
1160
|
+
type: "upload-suppressed-tombstone",
|
|
1161
|
+
path: item.relativePath,
|
|
1162
|
+
deletedAt: ts.deletedAt,
|
|
1163
|
+
});
|
|
1164
|
+
continue;
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1074
1168
|
if (item.action === "skip-unchanged") {
|
|
1075
1169
|
// Refresh the journal's (mtimeMs, size) for a touched-but-identical file
|
|
1076
1170
|
// so the next sync's lstat fast-path matches and skips without re-hashing.
|
|
@@ -1256,15 +1350,12 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1256
1350
|
if ((localChanged && remoteChanged) || isFreshCollision) {
|
|
1257
1351
|
conflictPaths.push(relativePath);
|
|
1258
1352
|
|
|
1259
|
-
const resolution = await
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
},
|
|
1266
|
-
onConflict,
|
|
1267
|
-
);
|
|
1353
|
+
const resolution = await resolveConflictSerialized({
|
|
1354
|
+
path: relativePath,
|
|
1355
|
+
localHash,
|
|
1356
|
+
remoteModified: remoteMeta.lastModified,
|
|
1357
|
+
direction: "push",
|
|
1358
|
+
});
|
|
1268
1359
|
|
|
1269
1360
|
emit({
|
|
1270
1361
|
type: "conflict",
|
|
@@ -1404,10 +1495,11 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1404
1495
|
// the race the fence exists to catch. Surface as a push conflict;
|
|
1405
1496
|
// never silently overwrite.
|
|
1406
1497
|
conflictPaths.push(relativePath);
|
|
1407
|
-
const resolution = await
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1498
|
+
const resolution = await resolveConflictSerialized({
|
|
1499
|
+
path: relativePath,
|
|
1500
|
+
localHash,
|
|
1501
|
+
direction: "push",
|
|
1502
|
+
});
|
|
1411
1503
|
emit({
|
|
1412
1504
|
type: "conflict",
|
|
1413
1505
|
path: relativePath,
|
|
@@ -1546,6 +1638,9 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1546
1638
|
// defaulting fallback. Empty on the abort path because the
|
|
1547
1639
|
// delete-plan execution loop is short-circuited.
|
|
1548
1640
|
filesRefusedStalePaths,
|
|
1641
|
+
// Tombstone-suppressed uploads are classified in Phase A, which runs
|
|
1642
|
+
// before the upload pool can abort, so the count is meaningful here.
|
|
1643
|
+
filesSuppressedByTombstone,
|
|
1549
1644
|
// Exclusions are computed during the upload walk which has
|
|
1550
1645
|
// already completed by the time we hit a per-file conflict-
|
|
1551
1646
|
// abort, so the count is meaningful here. No event emit on
|
|
@@ -1717,6 +1812,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1717
1812
|
filesTombstoned,
|
|
1718
1813
|
filesRefusedStale,
|
|
1719
1814
|
filesRefusedStalePaths,
|
|
1815
|
+
filesSuppressedByTombstone,
|
|
1720
1816
|
filesExcludedByPolicy: excludedSet.size,
|
|
1721
1817
|
filesExcludedByScope: scopeExcludedSet.size,
|
|
1722
1818
|
conflictPaths,
|
package/src/cli/sync.ts
CHANGED
|
@@ -54,6 +54,10 @@ import {
|
|
|
54
54
|
} from "../lib/conflict-file.js";
|
|
55
55
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
56
56
|
import { reindex } from "./reindex.js";
|
|
57
|
+
import {
|
|
58
|
+
fetchCompanyTombstones,
|
|
59
|
+
type CompanyTombstone,
|
|
60
|
+
} from "./tombstones.js";
|
|
57
61
|
|
|
58
62
|
/**
|
|
59
63
|
* Per-file events emitted by `sync()` as it progresses.
|
|
@@ -265,6 +269,23 @@ export type SyncProgressEvent =
|
|
|
265
269
|
type: "scope-excluded";
|
|
266
270
|
count: number;
|
|
267
271
|
samplePaths: string[];
|
|
272
|
+
}
|
|
273
|
+
| {
|
|
274
|
+
/**
|
|
275
|
+
* Emitted by the PUSH leg (`share()`) once per key whose upload was
|
|
276
|
+
* suppressed because an authoritative FILE_TOMBSTONE marks it deleted and
|
|
277
|
+
* the local copy is still the deleted baseline (journal hash unchanged).
|
|
278
|
+
* Without this, a behind peer that still holds the file would re-upload it,
|
|
279
|
+
* and the pull planner's timestamp-only re-create heuristic
|
|
280
|
+
* (`isRemoteRecreateAfterTombstone`) would treat the re-upload as a genuine
|
|
281
|
+
* re-create and resurrect the key for everyone. The deleter's own stale
|
|
282
|
+
* local copy is cleaned by the pull leg's `tombstone-delete`; this event
|
|
283
|
+
* records that the push refused to resurrect it. `deletedAt` is the
|
|
284
|
+
* tombstone's delete time.
|
|
285
|
+
*/
|
|
286
|
+
type: "upload-suppressed-tombstone";
|
|
287
|
+
path: string;
|
|
288
|
+
deletedAt: string;
|
|
268
289
|
};
|
|
269
290
|
|
|
270
291
|
export interface SyncOptions {
|
|
@@ -539,94 +560,6 @@ async function reportNewFilesToNotify(
|
|
|
539
560
|
}
|
|
540
561
|
}
|
|
541
562
|
|
|
542
|
-
/** Timeout for the best-effort FILE_TOMBSTONE fetch (GET /v1/files/tombstones). */
|
|
543
|
-
const FETCH_TOMBSTONES_TIMEOUT_MS = 5000;
|
|
544
|
-
|
|
545
|
-
/**
|
|
546
|
-
* A FILE_TOMBSTONE as the pull planner needs it: the deleted key + when it was
|
|
547
|
-
* deleted. The `deletedAt` timestamp is the decisive precedence signal — a
|
|
548
|
-
* remote object newer than it is a genuine re-create (sync it), an object at or
|
|
549
|
-
* older than it is a stale resurrection of a deleted key (suppress it).
|
|
550
|
-
*/
|
|
551
|
-
interface CompanyTombstone {
|
|
552
|
-
deletedAt: string;
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
/**
|
|
556
|
-
* Fetch the company's FILE_TOMBSTONE rows from hq-pro (GET /v1/files/tombstones)
|
|
557
|
-
* and return them as a POSIX-keyed map the pull planner consults to avoid
|
|
558
|
-
* resurrecting an intentionally-deleted object (delete-resync). The endpoint is
|
|
559
|
-
* ACL-filtered server-side, so the map only ever contains keys this caller can
|
|
560
|
-
* read — exactly the keys that can appear in the (STS-scoped) remote LIST.
|
|
561
|
-
*
|
|
562
|
-
* Best-effort and bounded by a 5s timeout: a tombstone read that fails, times
|
|
563
|
-
* out, or returns non-2xx degrades to an EMPTY map — i.e. to the pre-fix
|
|
564
|
-
* behavior (no suppression). That is the safe failure direction: a missed
|
|
565
|
-
* tombstone re-pulls a deleted file (a known, recoverable annoyance), whereas a
|
|
566
|
-
* spurious tombstone would HIDE a file the user wants. The failure is logged
|
|
567
|
-
* (never silently swallowed) so a persistently-degraded read is visible.
|
|
568
|
-
*/
|
|
569
|
-
async function fetchCompanyTombstones(
|
|
570
|
-
vaultConfig: VaultServiceConfig,
|
|
571
|
-
companyUid: string,
|
|
572
|
-
): Promise<Map<string, CompanyTombstone>> {
|
|
573
|
-
const out = new Map<string, CompanyTombstone>();
|
|
574
|
-
try {
|
|
575
|
-
const token =
|
|
576
|
-
typeof vaultConfig.authToken === "function"
|
|
577
|
-
? await vaultConfig.authToken()
|
|
578
|
-
: vaultConfig.authToken;
|
|
579
|
-
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
580
|
-
const url = `${base}/v1/files/tombstones?company=${encodeURIComponent(
|
|
581
|
-
companyUid,
|
|
582
|
-
)}`;
|
|
583
|
-
const controller = new AbortController();
|
|
584
|
-
const timer = setTimeout(
|
|
585
|
-
() => controller.abort(),
|
|
586
|
-
FETCH_TOMBSTONES_TIMEOUT_MS,
|
|
587
|
-
);
|
|
588
|
-
try {
|
|
589
|
-
const res = await fetch(url, {
|
|
590
|
-
method: "GET",
|
|
591
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
592
|
-
signal: controller.signal,
|
|
593
|
-
});
|
|
594
|
-
if (!res.ok) {
|
|
595
|
-
// Non-2xx is non-fatal: log and degrade to no-suppression. A 404 means
|
|
596
|
-
// the endpoint is not deployed yet (hq-pro release lag) — the pull
|
|
597
|
-
// proceeds with the legacy behavior until it lands.
|
|
598
|
-
console.error(
|
|
599
|
-
`[hq-sync] tombstone fetch returned ${res.status} (degrading to no-suppression)`,
|
|
600
|
-
);
|
|
601
|
-
return out;
|
|
602
|
-
}
|
|
603
|
-
const body = (await res.json()) as {
|
|
604
|
-
tombstones?: Array<{ key?: string; deletedAt?: string }>;
|
|
605
|
-
};
|
|
606
|
-
for (const t of body.tombstones ?? []) {
|
|
607
|
-
if (typeof t.key === "string" && typeof t.deletedAt === "string") {
|
|
608
|
-
out.set(toPosixKey(t.key), { deletedAt: t.deletedAt });
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
} finally {
|
|
612
|
-
clearTimeout(timer);
|
|
613
|
-
}
|
|
614
|
-
} catch (err) {
|
|
615
|
-
// Best-effort: a failed tombstone read must never break the pull. Log
|
|
616
|
-
// (policy: never silently swallow) and degrade to no-suppression.
|
|
617
|
-
try {
|
|
618
|
-
console.error(
|
|
619
|
-
`[hq-sync] tombstone fetch failed (non-fatal, degrading to no-suppression): ${
|
|
620
|
-
err instanceof Error ? err.message : String(err)
|
|
621
|
-
}`,
|
|
622
|
-
);
|
|
623
|
-
} catch {
|
|
624
|
-
// swallow — logging must never break sync
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
|
-
return out;
|
|
628
|
-
}
|
|
629
|
-
|
|
630
563
|
/**
|
|
631
564
|
* Sync (pull) all allowed files from the entity vault.
|
|
632
565
|
*/
|