@indigoai-us/hq-cloud 5.36.0 → 5.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +128 -1
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +256 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +35 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +10 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/s3.d.ts +54 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +150 -1
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +261 -1
- package/dist/s3.test.js.map +1 -1
- package/package.json +1 -1
- package/src/cli/share.test.ts +311 -1
- package/src/cli/share.ts +146 -2
- package/src/cli/sync.ts +46 -1
- package/src/s3.test.ts +296 -0
- package/src/s3.ts +155 -1
package/src/cli/share.test.ts
CHANGED
|
@@ -702,7 +702,8 @@ describe("share", () => {
|
|
|
702
702
|
if (
|
|
703
703
|
e.type === "plan" ||
|
|
704
704
|
e.type === "new-files" ||
|
|
705
|
-
e.type === "personal-vault-out-of-policy"
|
|
705
|
+
e.type === "personal-vault-out-of-policy" ||
|
|
706
|
+
e.type === "delete-refused-bulk-asymmetry"
|
|
706
707
|
) return;
|
|
707
708
|
events.push({
|
|
708
709
|
type: e.type,
|
|
@@ -2533,6 +2534,315 @@ describe("share", () => {
|
|
|
2533
2534
|
expect(calls.find((c) => c[2].includes("secret.md"))).toBeUndefined();
|
|
2534
2535
|
});
|
|
2535
2536
|
});
|
|
2537
|
+
|
|
2538
|
+
// ── Bulk-asymmetry circuit-breaker ─────────────────────────────────────
|
|
2539
|
+
//
|
|
2540
|
+
// Defends against the "local mirror lost, journal still full" failure
|
|
2541
|
+
// mode that caused the 2026-05-25 indigo vault mass-delete (269 signals/
|
|
2542
|
+
// + 290 sources/ delete-markers in one afternoon).
|
|
2543
|
+
//
|
|
2544
|
+
// Trip rule: candidates >= 10 absolute AND candidates / inScope >= 0.10.
|
|
2545
|
+
// Bypass: HQ_SYNC_DELETE_BULK_OVERRIDE=1 OR propagateDeletePolicy: "all".
|
|
2546
|
+
//
|
|
2547
|
+
// See `workspace/reports/indigo-vault-mass-delete-debug.md` for the full
|
|
2548
|
+
// investigation and `BULK_ASYMMETRY_*` constants in share.ts for the
|
|
2549
|
+
// rationale behind these specific thresholds.
|
|
2550
|
+
describe("bulk-asymmetry circuit-breaker", () => {
|
|
2551
|
+
// Helper — write a journal of N entries, marking the first `missing`
|
|
2552
|
+
// entries as absent locally (the rest get a real file on disk).
|
|
2553
|
+
// `direction` default "up" so the owned-only branch accepts every
|
|
2554
|
+
// candidate (direction:"down" silently skips and never trips the guard).
|
|
2555
|
+
function setupJournal({
|
|
2556
|
+
total,
|
|
2557
|
+
missing,
|
|
2558
|
+
direction = "up" as "up" | "down",
|
|
2559
|
+
}: {
|
|
2560
|
+
total: number;
|
|
2561
|
+
missing: number;
|
|
2562
|
+
direction?: "up" | "down";
|
|
2563
|
+
}): { companyRoot: string; journalPath: string } {
|
|
2564
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2565
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2566
|
+
const files: Record<
|
|
2567
|
+
string,
|
|
2568
|
+
{
|
|
2569
|
+
hash: string;
|
|
2570
|
+
size: number;
|
|
2571
|
+
syncedAt: string;
|
|
2572
|
+
direction: "up" | "down";
|
|
2573
|
+
remoteEtag: string;
|
|
2574
|
+
}
|
|
2575
|
+
> = {};
|
|
2576
|
+
for (let i = 0; i < total; i++) {
|
|
2577
|
+
const key = `f-${i.toString().padStart(4, "0")}.md`;
|
|
2578
|
+
files[key] = {
|
|
2579
|
+
hash: "h",
|
|
2580
|
+
size: 5,
|
|
2581
|
+
syncedAt: new Date().toISOString(),
|
|
2582
|
+
direction,
|
|
2583
|
+
remoteEtag: `etag-${i}`,
|
|
2584
|
+
};
|
|
2585
|
+
if (i >= missing) {
|
|
2586
|
+
fs.writeFileSync(path.join(companyRoot, key), "hi");
|
|
2587
|
+
}
|
|
2588
|
+
}
|
|
2589
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2590
|
+
fs.writeFileSync(
|
|
2591
|
+
journalPath,
|
|
2592
|
+
JSON.stringify({
|
|
2593
|
+
version: "1",
|
|
2594
|
+
lastSync: new Date().toISOString(),
|
|
2595
|
+
files,
|
|
2596
|
+
}),
|
|
2597
|
+
);
|
|
2598
|
+
return { companyRoot, journalPath };
|
|
2599
|
+
}
|
|
2600
|
+
|
|
2601
|
+
afterEach(() => {
|
|
2602
|
+
delete process.env.HQ_SYNC_DELETE_BULK_OVERRIDE;
|
|
2603
|
+
});
|
|
2604
|
+
|
|
2605
|
+
it("trips at the 11/100 boundary (11% > 10%, abs >= 10): refuses every delete and emits bulk-asymmetry event", async () => {
|
|
2606
|
+
const { companyRoot, journalPath } = setupJournal({ total: 100, missing: 11 });
|
|
2607
|
+
// currency-gated: remote HEAD would match every journal etag — guard
|
|
2608
|
+
// must trip BEFORE any HEAD round-trip occurs.
|
|
2609
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
2610
|
+
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
2611
|
+
lastModified: new Date(),
|
|
2612
|
+
size: 5,
|
|
2613
|
+
}));
|
|
2614
|
+
|
|
2615
|
+
const events: Array<{ type: string; [k: string]: unknown }> = [];
|
|
2616
|
+
const result = await share({
|
|
2617
|
+
paths: [companyRoot],
|
|
2618
|
+
company: "acme",
|
|
2619
|
+
vaultConfig: mockConfig,
|
|
2620
|
+
hqRoot: tmpDir,
|
|
2621
|
+
skipUnchanged: true,
|
|
2622
|
+
propagateDeletes: true,
|
|
2623
|
+
propagateDeletePolicy: "currency-gated",
|
|
2624
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
2625
|
+
});
|
|
2626
|
+
|
|
2627
|
+
expect(result.filesDeleted).toBe(0);
|
|
2628
|
+
expect(deleteRemoteFile).not.toHaveBeenCalled();
|
|
2629
|
+
// No HEAD was issued for any of the 11 MISSING keys — the guard
|
|
2630
|
+
// short-circuits before Stage 2 HEAD pass. `headRemoteFile` may still
|
|
2631
|
+
// be called for the 89 present-locally upload candidates (3-way
|
|
2632
|
+
// conflict check), so the assertion targets only the would-be
|
|
2633
|
+
// delete-candidates.
|
|
2634
|
+
const headedKeys = vi
|
|
2635
|
+
.mocked(headRemoteFile)
|
|
2636
|
+
.mock.calls.map((c) => c[1] as string);
|
|
2637
|
+
for (let i = 0; i < 11; i++) {
|
|
2638
|
+
const missingKey = `f-${i.toString().padStart(4, "0")}.md`;
|
|
2639
|
+
expect(headedKeys).not.toContain(missingKey);
|
|
2640
|
+
}
|
|
2641
|
+
// All 11 missing entries refused with reason "bulk-asymmetry".
|
|
2642
|
+
expect(result.filesRefusedStale).toBe(11);
|
|
2643
|
+
const summary = events.find((e) => e.type === "delete-refused-bulk-asymmetry") as
|
|
2644
|
+
| { candidates: number; inScope: number; ratio: number; samplePaths: string[] }
|
|
2645
|
+
| undefined;
|
|
2646
|
+
expect(summary).toBeDefined();
|
|
2647
|
+
expect(summary!.candidates).toBe(11);
|
|
2648
|
+
expect(summary!.inScope).toBe(100);
|
|
2649
|
+
expect(summary!.ratio).toBeCloseTo(0.11, 5);
|
|
2650
|
+
expect(summary!.samplePaths.length).toBeGreaterThan(0);
|
|
2651
|
+
expect(summary!.samplePaths.length).toBeLessThanOrEqual(10);
|
|
2652
|
+
// Journal entries for the missing files SURVIVE — guard never mutates.
|
|
2653
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2654
|
+
expect(Object.keys(journal.files).length).toBe(100);
|
|
2655
|
+
// Per-key delete-refused-stale-etag events for each candidate.
|
|
2656
|
+
const perKeyRefusals = events.filter(
|
|
2657
|
+
(e) => e.type === "delete-refused-stale-etag" && e.reason === "bulk-asymmetry",
|
|
2658
|
+
);
|
|
2659
|
+
expect(perKeyRefusals.length).toBe(11);
|
|
2660
|
+
});
|
|
2661
|
+
|
|
2662
|
+
it("does NOT trip at 9/100 (below ratio): all candidates delete normally", async () => {
|
|
2663
|
+
const { companyRoot } = setupJournal({ total: 100, missing: 9 });
|
|
2664
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
2665
|
+
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
2666
|
+
lastModified: new Date(),
|
|
2667
|
+
size: 5,
|
|
2668
|
+
}));
|
|
2669
|
+
|
|
2670
|
+
const events: Array<{ type: string }> = [];
|
|
2671
|
+
const result = await share({
|
|
2672
|
+
paths: [companyRoot],
|
|
2673
|
+
company: "acme",
|
|
2674
|
+
vaultConfig: mockConfig,
|
|
2675
|
+
hqRoot: tmpDir,
|
|
2676
|
+
skipUnchanged: true,
|
|
2677
|
+
propagateDeletes: true,
|
|
2678
|
+
propagateDeletePolicy: "currency-gated",
|
|
2679
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
2680
|
+
});
|
|
2681
|
+
|
|
2682
|
+
expect(result.filesDeleted).toBe(9);
|
|
2683
|
+
expect(deleteRemoteFile).toHaveBeenCalledTimes(9);
|
|
2684
|
+
expect(events.find((e) => e.type === "delete-refused-bulk-asymmetry")).toBeUndefined();
|
|
2685
|
+
});
|
|
2686
|
+
|
|
2687
|
+
it("HQ_SYNC_DELETE_BULK_OVERRIDE=1 bypasses the guard: 11/100 deletes proceed", async () => {
|
|
2688
|
+
process.env.HQ_SYNC_DELETE_BULK_OVERRIDE = "1";
|
|
2689
|
+
const { companyRoot } = setupJournal({ total: 100, missing: 11 });
|
|
2690
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
2691
|
+
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
2692
|
+
lastModified: new Date(),
|
|
2693
|
+
size: 5,
|
|
2694
|
+
}));
|
|
2695
|
+
|
|
2696
|
+
const result = await share({
|
|
2697
|
+
paths: [companyRoot],
|
|
2698
|
+
company: "acme",
|
|
2699
|
+
vaultConfig: mockConfig,
|
|
2700
|
+
hqRoot: tmpDir,
|
|
2701
|
+
skipUnchanged: true,
|
|
2702
|
+
propagateDeletes: true,
|
|
2703
|
+
propagateDeletePolicy: "currency-gated",
|
|
2704
|
+
});
|
|
2705
|
+
|
|
2706
|
+
expect(result.filesDeleted).toBe(11);
|
|
2707
|
+
expect(deleteRemoteFile).toHaveBeenCalledTimes(11);
|
|
2708
|
+
});
|
|
2709
|
+
|
|
2710
|
+
it("propagateDeletePolicy: 'all' bypasses the guard: 11/100 deletes proceed", async () => {
|
|
2711
|
+
const { companyRoot } = setupJournal({ total: 100, missing: 11 });
|
|
2712
|
+
|
|
2713
|
+
const result = await share({
|
|
2714
|
+
paths: [companyRoot],
|
|
2715
|
+
company: "acme",
|
|
2716
|
+
vaultConfig: mockConfig,
|
|
2717
|
+
hqRoot: tmpDir,
|
|
2718
|
+
skipUnchanged: true,
|
|
2719
|
+
propagateDeletes: true,
|
|
2720
|
+
propagateDeletePolicy: "all",
|
|
2721
|
+
});
|
|
2722
|
+
|
|
2723
|
+
expect(result.filesDeleted).toBe(11);
|
|
2724
|
+
expect(deleteRemoteFile).toHaveBeenCalledTimes(11);
|
|
2725
|
+
});
|
|
2726
|
+
|
|
2727
|
+
it("fully-empty mirror (100/100) trips the guard — the canonical mass-delete failure mode", async () => {
|
|
2728
|
+
const { companyRoot, journalPath } = setupJournal({ total: 50, missing: 50 });
|
|
2729
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
2730
|
+
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
2731
|
+
lastModified: new Date(),
|
|
2732
|
+
size: 5,
|
|
2733
|
+
}));
|
|
2734
|
+
|
|
2735
|
+
const events: Array<{ type: string; [k: string]: unknown }> = [];
|
|
2736
|
+
const result = await share({
|
|
2737
|
+
paths: [companyRoot],
|
|
2738
|
+
company: "acme",
|
|
2739
|
+
vaultConfig: mockConfig,
|
|
2740
|
+
hqRoot: tmpDir,
|
|
2741
|
+
skipUnchanged: true,
|
|
2742
|
+
propagateDeletes: true,
|
|
2743
|
+
propagateDeletePolicy: "currency-gated",
|
|
2744
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
2745
|
+
});
|
|
2746
|
+
|
|
2747
|
+
expect(result.filesDeleted).toBe(0);
|
|
2748
|
+
expect(deleteRemoteFile).not.toHaveBeenCalled();
|
|
2749
|
+
expect(result.filesRefusedStale).toBe(50);
|
|
2750
|
+
const summary = events.find((e) => e.type === "delete-refused-bulk-asymmetry") as
|
|
2751
|
+
| { candidates: number; inScope: number; ratio: number }
|
|
2752
|
+
| undefined;
|
|
2753
|
+
expect(summary).toBeDefined();
|
|
2754
|
+
expect(summary!.ratio).toBeCloseTo(1.0, 5);
|
|
2755
|
+
// Journal untouched.
|
|
2756
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2757
|
+
expect(Object.keys(journal.files).length).toBe(50);
|
|
2758
|
+
});
|
|
2759
|
+
|
|
2760
|
+
it("small absolute count (9/9 = 100% ratio but < MIN_ABS=10) does NOT trip: all 9 delete normally", async () => {
|
|
2761
|
+
const { companyRoot } = setupJournal({ total: 9, missing: 9 });
|
|
2762
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
2763
|
+
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
2764
|
+
lastModified: new Date(),
|
|
2765
|
+
size: 5,
|
|
2766
|
+
}));
|
|
2767
|
+
|
|
2768
|
+
const result = await share({
|
|
2769
|
+
paths: [companyRoot],
|
|
2770
|
+
company: "acme",
|
|
2771
|
+
vaultConfig: mockConfig,
|
|
2772
|
+
hqRoot: tmpDir,
|
|
2773
|
+
skipUnchanged: true,
|
|
2774
|
+
propagateDeletes: true,
|
|
2775
|
+
propagateDeletePolicy: "currency-gated",
|
|
2776
|
+
});
|
|
2777
|
+
|
|
2778
|
+
// 9 candidates, ratio 1.0, but abs < 10 → guard does not trip.
|
|
2779
|
+
expect(result.filesDeleted).toBe(9);
|
|
2780
|
+
expect(deleteRemoteFile).toHaveBeenCalledTimes(9);
|
|
2781
|
+
});
|
|
2782
|
+
|
|
2783
|
+
it("owned-only policy also goes through the guard: 11/100 missing direction:'up' entries refused", async () => {
|
|
2784
|
+
const { companyRoot, journalPath } = setupJournal({ total: 100, missing: 11 });
|
|
2785
|
+
|
|
2786
|
+
const events: Array<{ type: string }> = [];
|
|
2787
|
+
const result = await share({
|
|
2788
|
+
paths: [companyRoot],
|
|
2789
|
+
company: "acme",
|
|
2790
|
+
vaultConfig: mockConfig,
|
|
2791
|
+
hqRoot: tmpDir,
|
|
2792
|
+
skipUnchanged: true,
|
|
2793
|
+
propagateDeletes: true,
|
|
2794
|
+
propagateDeletePolicy: "owned-only",
|
|
2795
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
2796
|
+
});
|
|
2797
|
+
|
|
2798
|
+
expect(result.filesDeleted).toBe(0);
|
|
2799
|
+
expect(deleteRemoteFile).not.toHaveBeenCalled();
|
|
2800
|
+
expect(events.find((e) => e.type === "delete-refused-bulk-asymmetry")).toBeDefined();
|
|
2801
|
+
// Journal entries survive.
|
|
2802
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2803
|
+
expect(Object.keys(journal.files).length).toBe(100);
|
|
2804
|
+
});
|
|
2805
|
+
|
|
2806
|
+
it("direction:'down' entries don't count toward the bulk numerator (owned-only silently skips them)", async () => {
|
|
2807
|
+
// 50 "down" entries all missing locally + 1 "up" entry missing.
|
|
2808
|
+
// Under owned-only: only the 1 "up" candidate goes to toDelete; the
|
|
2809
|
+
// 50 "down" entries are silently dropped from the plan (existing
|
|
2810
|
+
// behavior). The bulk numerator should be 1, not 51 — so the guard
|
|
2811
|
+
// does NOT trip and the 1 legitimate delete proceeds.
|
|
2812
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
2813
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
2814
|
+
const files: Record<string, {
|
|
2815
|
+
hash: string; size: number; syncedAt: string;
|
|
2816
|
+
direction: "up" | "down"; remoteEtag: string;
|
|
2817
|
+
}> = {};
|
|
2818
|
+
const now = new Date().toISOString();
|
|
2819
|
+
for (let i = 0; i < 50; i++) {
|
|
2820
|
+
files[`d-${i.toString().padStart(4, "0")}.md`] = {
|
|
2821
|
+
hash: "h", size: 5, syncedAt: now,
|
|
2822
|
+
direction: "down", remoteEtag: `down-etag-${i}`,
|
|
2823
|
+
};
|
|
2824
|
+
}
|
|
2825
|
+
files["only-up.md"] = {
|
|
2826
|
+
hash: "h", size: 5, syncedAt: now,
|
|
2827
|
+
direction: "up", remoteEtag: "up-etag",
|
|
2828
|
+
};
|
|
2829
|
+
const journalPath = path.join(stateDir, "sync-journal.acme.json");
|
|
2830
|
+
fs.writeFileSync(journalPath, JSON.stringify({ version: "1", lastSync: now, files }));
|
|
2831
|
+
|
|
2832
|
+
const result = await share({
|
|
2833
|
+
paths: [companyRoot],
|
|
2834
|
+
company: "acme",
|
|
2835
|
+
vaultConfig: mockConfig,
|
|
2836
|
+
hqRoot: tmpDir,
|
|
2837
|
+
skipUnchanged: true,
|
|
2838
|
+
propagateDeletes: true,
|
|
2839
|
+
propagateDeletePolicy: "owned-only",
|
|
2840
|
+
});
|
|
2841
|
+
|
|
2842
|
+
expect(result.filesDeleted).toBe(1);
|
|
2843
|
+
expect(deleteRemoteFile).toHaveBeenCalledWith(expect.anything(), "only-up.md");
|
|
2844
|
+
});
|
|
2845
|
+
});
|
|
2536
2846
|
});
|
|
2537
2847
|
|
|
2538
2848
|
// ── Pure-function unit coverage: isEphemeralPath ─────────────────────────────
|
package/src/cli/share.ts
CHANGED
|
@@ -716,6 +716,20 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
716
716
|
filesToDelete: deletePlan.toDelete.length + decommissionPlan.length,
|
|
717
717
|
});
|
|
718
718
|
|
|
719
|
+
// Bulk-asymmetry summary event. Emitted once if the circuit-breaker
|
|
720
|
+
// tripped inside `computeDeletePlan` — see DeletePlan.bulkAsymmetry doc.
|
|
721
|
+
// Per-key refusal events fire later in the refusedStale loop with
|
|
722
|
+
// reason: "bulk-asymmetry" so the UI can also show the affected paths.
|
|
723
|
+
if (deletePlan.bulkAsymmetry) {
|
|
724
|
+
emit({
|
|
725
|
+
type: "delete-refused-bulk-asymmetry",
|
|
726
|
+
candidates: deletePlan.bulkAsymmetry.candidates,
|
|
727
|
+
inScope: deletePlan.bulkAsymmetry.inScope,
|
|
728
|
+
ratio: deletePlan.bulkAsymmetry.ratio,
|
|
729
|
+
samplePaths: deletePlan.bulkAsymmetry.samplePaths,
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
|
|
719
733
|
// Stage 2: execute. Skip items pre-classified as no-ops, then for each
|
|
720
734
|
// upload candidate run the HEAD + 3-way conflict check + actual PUT.
|
|
721
735
|
//
|
|
@@ -1235,11 +1249,24 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
|
|
|
1235
1249
|
console.error(
|
|
1236
1250
|
` ⚠ no-etag-on-record, kept on remote: ${event.path} (journal entry predates etag tracking)`,
|
|
1237
1251
|
);
|
|
1252
|
+
} else if (event.reason === "bulk-asymmetry") {
|
|
1253
|
+
// The one-shot summary below carries the count + bypass instructions.
|
|
1254
|
+
// Per-key lines stay compact so a 100-candidate refusal doesn't bury
|
|
1255
|
+
// the summary banner.
|
|
1256
|
+
console.error(` ⚠ bulk-asymmetry refusal, kept on remote: ${event.path}`);
|
|
1238
1257
|
} else {
|
|
1239
1258
|
console.error(
|
|
1240
1259
|
` ⚠ stale-etag, kept on remote: ${event.path} (journal=${event.journalEtag}, remote=${event.remoteEtag})`,
|
|
1241
1260
|
);
|
|
1242
1261
|
}
|
|
1262
|
+
} else if (event.type === "delete-refused-bulk-asymmetry") {
|
|
1263
|
+
const pct = (event.ratio * 100).toFixed(1);
|
|
1264
|
+
console.error(
|
|
1265
|
+
` ⚠ bulk-asymmetry circuit-breaker tripped: ${event.candidates}/${event.inScope} in-scope journal entries (${pct}%) are missing locally.\n` +
|
|
1266
|
+
` Refusing to propagate deletes — this looks like a local-mirror loss (moved hqRoot, partial restore, fresh clone, unmounted volume, accidental rm) rather than an intentional cleanup.\n` +
|
|
1267
|
+
` Sample refused paths: ${event.samplePaths.slice(0, 5).join(", ")}${event.samplePaths.length > 5 ? ", …" : ""}\n` +
|
|
1268
|
+
` To proceed anyway: re-run with HQ_SYNC_DELETE_BULK_OVERRIDE=1 (or propagateDeletePolicy:"all").`,
|
|
1269
|
+
);
|
|
1243
1270
|
}
|
|
1244
1271
|
}
|
|
1245
1272
|
|
|
@@ -1536,7 +1563,55 @@ function resolveDeleteScopeRoots(
|
|
|
1536
1563
|
* tracked. `journalEtag` and `remoteEtag` are
|
|
1537
1564
|
* placeholder sentinels — do not display as ETags.
|
|
1538
1565
|
*/
|
|
1539
|
-
type RefusedStaleReason = "stale-etag" | "legacy-no-etag";
|
|
1566
|
+
type RefusedStaleReason = "stale-etag" | "legacy-no-etag" | "bulk-asymmetry";
|
|
1567
|
+
|
|
1568
|
+
/**
|
|
1569
|
+
* Bulk-asymmetry circuit-breaker — refuses to convert a suspiciously-large
|
|
1570
|
+
* fraction of the in-scope journal entries into remote `DeleteObject` calls.
|
|
1571
|
+
*
|
|
1572
|
+
* Failure mode this defends against: the local mirror is corrupt
|
|
1573
|
+
* (moved hqRoot, partial restore from backup, fresh clone over an inherited
|
|
1574
|
+
* `~/.hq/sync-journal.*.json`, unmounted external volume, accidental
|
|
1575
|
+
* `rm -rf` of a populated subtree). `computeDeletePlan` walks every journal
|
|
1576
|
+
* entry, lstats locally, and any ENOENT is a delete candidate; under
|
|
1577
|
+
* `currency-gated` the per-file HEAD passes in the steady state (nobody
|
|
1578
|
+
* else rewrote the object) so the delete proceeds. Per-file currency is
|
|
1579
|
+
* orthogonal to whole-set health — when a large slice of the mirror is
|
|
1580
|
+
* absent at once, the engine should refuse rather than amplify the
|
|
1581
|
+
* accident into a mass-DELETE.
|
|
1582
|
+
*
|
|
1583
|
+
* The guard fires when **both** conditions hold:
|
|
1584
|
+
* - `candidates / inScope >= BULK_ASYMMETRY_RATIO` (default 10%).
|
|
1585
|
+
* - `candidates >= BULK_ASYMMETRY_MIN_ABS` (default 10).
|
|
1586
|
+
*
|
|
1587
|
+
* The MIN_ABS floor is what lets small intentional deletes through —
|
|
1588
|
+
* `rm 1 file` from a 5-entry mirror is 20% but 1 absolute candidate; never
|
|
1589
|
+
* tripped. The RATIO is what lets large intentional uploads-then-deletes
|
|
1590
|
+
* through — a 10000-entry journal where 50 files were intentionally
|
|
1591
|
+
* removed is 0.5%; never tripped.
|
|
1592
|
+
*
|
|
1593
|
+
* Bypass paths (both intentional opt-outs of safety):
|
|
1594
|
+
* - Env `HQ_SYNC_DELETE_BULK_OVERRIDE=1|true|yes` (case-insensitive). The
|
|
1595
|
+
* rollback knob for the rare legitimate mass-delete; symmetric to
|
|
1596
|
+
* `HQ_SYNC_DELETE_POLICY` as a per-invocation policy override.
|
|
1597
|
+
* - `propagateDeletePolicy: "all"`. The emergency-reconcile policy
|
|
1598
|
+
* already opts out of safety gates (currency check, owned-only).
|
|
1599
|
+
* This guard honors that explicit opt-out.
|
|
1600
|
+
*
|
|
1601
|
+
* `"owned-only"` and `"currency-gated"` both go through the guard. The
|
|
1602
|
+
* `direction === "up"` filter in `owned-only` is no defense once a user
|
|
1603
|
+
* has ever run full bidirectional sync (every uploaded file gets
|
|
1604
|
+
* direction:'up') — see investigation report
|
|
1605
|
+
* `workspace/reports/indigo-vault-mass-delete-debug.md`.
|
|
1606
|
+
*/
|
|
1607
|
+
const BULK_ASYMMETRY_RATIO = 0.10;
|
|
1608
|
+
const BULK_ASYMMETRY_MIN_ABS = 10;
|
|
1609
|
+
const BULK_ASYMMETRY_SAMPLE_CAP = 10;
|
|
1610
|
+
|
|
1611
|
+
function isBulkAsymmetryOverride(): boolean {
|
|
1612
|
+
const v = (process.env.HQ_SYNC_DELETE_BULK_OVERRIDE ?? "").toLowerCase();
|
|
1613
|
+
return v === "1" || v === "true" || v === "yes";
|
|
1614
|
+
}
|
|
1540
1615
|
|
|
1541
1616
|
/**
|
|
1542
1617
|
* Three buckets returned by computeDeletePlan, exposed so the execution
|
|
@@ -1560,6 +1635,18 @@ interface DeletePlan {
|
|
|
1560
1635
|
remoteEtag: string;
|
|
1561
1636
|
reason: RefusedStaleReason;
|
|
1562
1637
|
}>;
|
|
1638
|
+
/**
|
|
1639
|
+
* Populated only when the bulk-asymmetry circuit-breaker tripped. The
|
|
1640
|
+
* caller emits `delete-refused-bulk-asymmetry` as a one-shot summary so
|
|
1641
|
+
* the UI gets one banner-grade signal in addition to the per-key
|
|
1642
|
+
* `delete-refused-stale-etag` events under `refusedStale`.
|
|
1643
|
+
*/
|
|
1644
|
+
bulkAsymmetry?: {
|
|
1645
|
+
candidates: number;
|
|
1646
|
+
inScope: number;
|
|
1647
|
+
ratio: number;
|
|
1648
|
+
samplePaths: string[];
|
|
1649
|
+
};
|
|
1563
1650
|
}
|
|
1564
1651
|
|
|
1565
1652
|
/**
|
|
@@ -1662,6 +1749,14 @@ async function computeDeletePlan(
|
|
|
1662
1749
|
// and the journal-mutation buckets are already settled before any I/O.
|
|
1663
1750
|
type HeadCandidate = { key: string; journalEtag: string };
|
|
1664
1751
|
const headCandidates: HeadCandidate[] = [];
|
|
1752
|
+
// Bulk-asymmetry tracking: count every in-scope journal entry (denominator)
|
|
1753
|
+
// and every entry that would have been a delete-candidate before the guard
|
|
1754
|
+
// (numerator). Numerator = headCandidates + owned-only/all toDelete picks +
|
|
1755
|
+
// legacy-no-etag refusals. We do NOT count "ENOENT but ignore-filtered" or
|
|
1756
|
+
// "ENOENT but ephemeral" — those drop out of the plan entirely on their own
|
|
1757
|
+
// and don't reflect mirror-loss intent.
|
|
1758
|
+
let inScopeJournalEntries = 0;
|
|
1759
|
+
let bulkCandidatePicks = 0;
|
|
1665
1760
|
|
|
1666
1761
|
for (const [relativeKey, entry] of Object.entries(journal.files)) {
|
|
1667
1762
|
const inScope = scopeRoots.some(
|
|
@@ -1671,6 +1766,7 @@ async function computeDeletePlan(
|
|
|
1671
1766
|
relativeKey.startsWith(`${root}/`),
|
|
1672
1767
|
);
|
|
1673
1768
|
if (!inScope) continue;
|
|
1769
|
+
inScopeJournalEntries++;
|
|
1674
1770
|
const localPath = path.join(syncRoot, relativeKey);
|
|
1675
1771
|
let presentLocally = true;
|
|
1676
1772
|
try {
|
|
@@ -1694,11 +1790,21 @@ async function computeDeletePlan(
|
|
|
1694
1790
|
if (isEphemeralPath(relativeKey)) continue;
|
|
1695
1791
|
|
|
1696
1792
|
if (policy === "all") {
|
|
1793
|
+
// policy:"all" is the explicit-opt-out emergency-reconcile mode; the
|
|
1794
|
+
// bulk-asymmetry guard skips this branch (caller asserted intent).
|
|
1697
1795
|
plan.toDelete.push(relativeKey);
|
|
1698
1796
|
continue;
|
|
1699
1797
|
}
|
|
1798
|
+
bulkCandidatePicks++;
|
|
1700
1799
|
if (policy === "owned-only") {
|
|
1701
|
-
if (entry.direction !== "up")
|
|
1800
|
+
if (entry.direction !== "up") {
|
|
1801
|
+
// Not a delete candidate under owned-only, but it WAS missing
|
|
1802
|
+
// locally. Don't count it for the bulk guard — direction:'down'
|
|
1803
|
+
// entries that vanish locally are silently ignored by this policy
|
|
1804
|
+
// anyway, so they don't represent intent to mass-delete.
|
|
1805
|
+
bulkCandidatePicks--;
|
|
1806
|
+
continue;
|
|
1807
|
+
}
|
|
1702
1808
|
plan.toDelete.push(relativeKey);
|
|
1703
1809
|
continue;
|
|
1704
1810
|
}
|
|
@@ -1716,6 +1822,44 @@ async function computeDeletePlan(
|
|
|
1716
1822
|
headCandidates.push({ key: relativeKey, journalEtag });
|
|
1717
1823
|
}
|
|
1718
1824
|
|
|
1825
|
+
// Bulk-asymmetry circuit-breaker. See `BULK_ASYMMETRY_*` constants for
|
|
1826
|
+
// rationale + bypass paths. Skipped under policy:"all" (caller asserted
|
|
1827
|
+
// intent) and under `HQ_SYNC_DELETE_BULK_OVERRIDE` (operator rollback knob).
|
|
1828
|
+
if (
|
|
1829
|
+
policy !== "all" &&
|
|
1830
|
+
!isBulkAsymmetryOverride() &&
|
|
1831
|
+
bulkCandidatePicks >= BULK_ASYMMETRY_MIN_ABS &&
|
|
1832
|
+
inScopeJournalEntries > 0 &&
|
|
1833
|
+
bulkCandidatePicks / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO
|
|
1834
|
+
) {
|
|
1835
|
+
// Move every staged candidate (both already-bucketed toDelete from
|
|
1836
|
+
// owned-only and queued headCandidates from currency-gated) into
|
|
1837
|
+
// refusedStale with reason "bulk-asymmetry". Journal is not mutated;
|
|
1838
|
+
// no DeleteObject is issued.
|
|
1839
|
+
const samplePaths: string[] = [];
|
|
1840
|
+
const pushRefused = (key: string): void => {
|
|
1841
|
+
plan.refusedStale.push({
|
|
1842
|
+
key,
|
|
1843
|
+
journalEtag: "<bulk-asymmetry>",
|
|
1844
|
+
remoteEtag: "<not-checked>",
|
|
1845
|
+
reason: "bulk-asymmetry",
|
|
1846
|
+
});
|
|
1847
|
+
if (samplePaths.length < BULK_ASYMMETRY_SAMPLE_CAP) {
|
|
1848
|
+
samplePaths.push(key);
|
|
1849
|
+
}
|
|
1850
|
+
};
|
|
1851
|
+
for (const key of plan.toDelete) pushRefused(key);
|
|
1852
|
+
for (const c of headCandidates) pushRefused(c.key);
|
|
1853
|
+
plan.toDelete = [];
|
|
1854
|
+
plan.bulkAsymmetry = {
|
|
1855
|
+
candidates: bulkCandidatePicks,
|
|
1856
|
+
inScope: inScopeJournalEntries,
|
|
1857
|
+
ratio: bulkCandidatePicks / inScopeJournalEntries,
|
|
1858
|
+
samplePaths,
|
|
1859
|
+
};
|
|
1860
|
+
return plan;
|
|
1861
|
+
}
|
|
1862
|
+
|
|
1719
1863
|
// Stage 2: bounded-parallel HEAD pass. Promise.all over chunks of size
|
|
1720
1864
|
// `DELETE_PLAN_HEAD_CONCURRENCY` so a large candidate set doesn't
|
|
1721
1865
|
// serialize into N round-trips, and so we don't burst past the AWS-SDK
|
package/src/cli/sync.ts
CHANGED
|
@@ -135,12 +135,47 @@ export type SyncProgressEvent =
|
|
|
135
135
|
* `journalEtag` and `remoteEtag` are sentinel strings
|
|
136
136
|
* (`<legacy-no-etag>` / `<unknown>`) — do not render as ETags.
|
|
137
137
|
* Consumers should branch on `reason`, not on the etag values.
|
|
138
|
+
*
|
|
139
|
+
* - `"bulk-asymmetry"`: the bulk-asymmetry circuit-breaker tripped
|
|
140
|
+
* (see `delete-refused-bulk-asymmetry`). Each candidate that would
|
|
141
|
+
* have been deleted also surfaces here for path-level visibility.
|
|
142
|
+
* `journalEtag` and `remoteEtag` are sentinel strings
|
|
143
|
+
* (`<bulk-asymmetry>` / `<not-checked>`) since no HEAD was issued.
|
|
138
144
|
*/
|
|
139
145
|
type: "delete-refused-stale-etag";
|
|
140
146
|
path: string;
|
|
141
147
|
journalEtag: string;
|
|
142
148
|
remoteEtag: string;
|
|
143
|
-
reason: "stale-etag" | "legacy-no-etag";
|
|
149
|
+
reason: "stale-etag" | "legacy-no-etag" | "bulk-asymmetry";
|
|
150
|
+
}
|
|
151
|
+
| {
|
|
152
|
+
/**
|
|
153
|
+
* Emitted at most ONCE per `share()` call when the bulk-asymmetry
|
|
154
|
+
* circuit-breaker tripped: a large fraction of the in-scope journal
|
|
155
|
+
* entries are missing locally, suggesting the local mirror is corrupt
|
|
156
|
+
* (moved hqRoot, partial restore, fresh clone over inherited state,
|
|
157
|
+
* unmounted volume, accidental `rm -rf`) rather than a deliberate
|
|
158
|
+
* delete. The engine refuses to convert the missing entries into
|
|
159
|
+
* remote `DeleteObject` calls; every refused candidate also surfaces
|
|
160
|
+
* via `delete-refused-stale-etag` with `reason: "bulk-asymmetry"`.
|
|
161
|
+
*
|
|
162
|
+
* Trip condition: `candidates >= 10 AND candidates / inScope >= 0.10`.
|
|
163
|
+
* Bypass: set `HQ_SYNC_DELETE_BULK_OVERRIDE=1` (truthy: `1|true|yes`,
|
|
164
|
+
* case-insensitive) OR pass `propagateDeletePolicy: "all"` (the
|
|
165
|
+
* existing emergency-reconcile policy which already opts out of
|
|
166
|
+
* safety gates).
|
|
167
|
+
*
|
|
168
|
+
* `candidates` is the number of journal entries whose local file is
|
|
169
|
+
* missing AND would have been delete-candidates absent the guard;
|
|
170
|
+
* `inScope` is the total journal entries under the share's scope
|
|
171
|
+
* roots; `ratio = candidates / inScope`; `samplePaths` carries up to
|
|
172
|
+
* 10 refused keys for diagnostic display.
|
|
173
|
+
*/
|
|
174
|
+
type: "delete-refused-bulk-asymmetry";
|
|
175
|
+
candidates: number;
|
|
176
|
+
inScope: number;
|
|
177
|
+
ratio: number;
|
|
178
|
+
samplePaths: string[];
|
|
144
179
|
}
|
|
145
180
|
| {
|
|
146
181
|
/**
|
|
@@ -576,6 +611,16 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
576
611
|
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
577
612
|
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
578
613
|
// SHA256 for this file without reading its bytes.
|
|
614
|
+
//
|
|
615
|
+
// 5.37.0 ordering invariant: downloadFile applies hq-mtime via
|
|
616
|
+
// utimesSync AFTER its byte write but BEFORE returning, and this
|
|
617
|
+
// lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
|
|
618
|
+
// already reflects the source-stamped mtime, not the wall-clock
|
|
619
|
+
// write-time. The journal therefore matches what the next push's
|
|
620
|
+
// lstat fast-path will see, and the file is correctly skipped on
|
|
621
|
+
// re-sync instead of being hashed every tick. Do not move this
|
|
622
|
+
// lstat earlier; do not stamp the journal from any pre-download
|
|
623
|
+
// mtime.
|
|
579
624
|
updateEntry(
|
|
580
625
|
journal,
|
|
581
626
|
remoteFile.key,
|