@indigoai-us/hq-cloud 6.14.32 → 6.14.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +14 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +53 -1
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +80 -5
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.js +76 -14
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +139 -7
- package/dist/cli/share.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-watch-loop.ts +70 -2
- package/src/bin/sync-runner.test.ts +103 -4
- package/src/cli/share.test.ts +161 -7
- package/src/cli/share.ts +73 -14
package/package.json
CHANGED
|
@@ -136,6 +136,46 @@ export function journalEntriesForWatcherDelete(
|
|
|
136
136
|
);
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
/**
|
|
140
|
+
* Bulk-intent quarantine thresholds. Deliberately the same shape and values as
|
|
141
|
+
* the engine's `BULK_ASYMMETRY_*` constants (`src/cli/share.ts`) so operators
|
|
142
|
+
* learn one rule and one knob: a disappearance covering >= 10 journal entries
|
|
143
|
+
* AND >= 10% of the shard is treated as suspect rather than deliberate.
|
|
144
|
+
*/
|
|
145
|
+
const WATCHER_BULK_INTENT_MIN_ABS = 10;
|
|
146
|
+
const WATCHER_BULK_INTENT_RATIO = 0.1;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Should a single `unlinkDir` expansion be quarantined instead of stamping a
|
|
150
|
+
* version-bound delete intent on every descendant journal entry?
|
|
151
|
+
*
|
|
152
|
+
* One filesystem event can cover an arbitrary subtree, so the producer is
|
|
153
|
+
* capable of minting thousands of individually-valid intents from a checkout
|
|
154
|
+
* swap, an unmount, a partial restore or an `rm -rf` — the 2026-07-30 mass
|
|
155
|
+
* deletion did exactly that. Quarantined entries are simply left intent-less,
|
|
156
|
+
* which the engine already refuses (`missing-delete-intent`); nothing is
|
|
157
|
+
* deleted locally or remotely and the next pull restores the tree.
|
|
158
|
+
*
|
|
159
|
+
* Pure so the threshold is unit-testable without a watcher or a journal.
|
|
160
|
+
*/
|
|
161
|
+
export function shouldQuarantineWatcherDeleteIntents(
|
|
162
|
+
expandedEntries: number,
|
|
163
|
+
journalEntries: number,
|
|
164
|
+
): boolean {
|
|
165
|
+
if (expandedEntries < WATCHER_BULK_INTENT_MIN_ABS) return false;
|
|
166
|
+
if (journalEntries <= 0) return false;
|
|
167
|
+
return expandedEntries / journalEntries >= WATCHER_BULK_INTENT_RATIO;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Same operator escape hatch as the engine breaker — one knob for both layers.
|
|
172
|
+
* Set when a genuinely intentional large folder removal must propagate.
|
|
173
|
+
*/
|
|
174
|
+
function isBulkIntentOverride(): boolean {
|
|
175
|
+
const v = (process.env.HQ_SYNC_DELETE_BULK_OVERRIDE ?? "").toLowerCase();
|
|
176
|
+
return v === "1" || v === "true" || v === "yes";
|
|
177
|
+
}
|
|
178
|
+
|
|
139
179
|
function passExitCode(outcome: number | RunnerPassOutcome): number {
|
|
140
180
|
return typeof outcome === "number" ? outcome : outcome.exitCode;
|
|
141
181
|
}
|
|
@@ -490,11 +530,39 @@ export async function runWatchLoop(
|
|
|
490
530
|
// Chokidar emits one unlink per file during rm -rf. Avoid walking the
|
|
491
531
|
// whole shard for every exact-key event; directory unlinks alone need
|
|
492
532
|
// descendant expansion.
|
|
493
|
-
|
|
533
|
+
const covered = journalEntriesForWatcherDelete(
|
|
494
534
|
journal.files,
|
|
495
535
|
coordinates.key,
|
|
496
536
|
kind,
|
|
497
|
-
)
|
|
537
|
+
);
|
|
538
|
+
|
|
539
|
+
// Bulk-intent quarantine: one directory event must not be able to mint
|
|
540
|
+
// intents across a large fraction of the shard. Leaving them intent-less
|
|
541
|
+
// is the safe state — the engine refuses intent-less disappearances and
|
|
542
|
+
// the next pull restores the subtree.
|
|
543
|
+
const stampable = covered.filter(
|
|
544
|
+
([, entry]) =>
|
|
545
|
+
entry?.remoteEtag && entry.kind && entry.removedAt === undefined,
|
|
546
|
+
);
|
|
547
|
+
if (
|
|
548
|
+
kind === "unlinkDir" &&
|
|
549
|
+
!isBulkIntentOverride() &&
|
|
550
|
+
shouldQuarantineWatcherDeleteIntents(
|
|
551
|
+
stampable.length,
|
|
552
|
+
Object.keys(journal.files).length,
|
|
553
|
+
)
|
|
554
|
+
) {
|
|
555
|
+
process.stderr.write(
|
|
556
|
+
`hq-sync-runner: refusing to record delete intent for ${stampable.length} ` +
|
|
557
|
+
`of ${Object.keys(journal.files).length} journal entries under ` +
|
|
558
|
+
`"${coordinates.key || "<root>"}" — a single directory removal that large ` +
|
|
559
|
+
`is treated as a possible mirror loss. Nothing was deleted. ` +
|
|
560
|
+
`Set HQ_SYNC_DELETE_BULK_OVERRIDE=1 to propagate it deliberately.\n`,
|
|
561
|
+
);
|
|
562
|
+
return [];
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
for (const [key, entry] of stampable) {
|
|
498
566
|
captureEntry(key, entry);
|
|
499
567
|
}
|
|
500
568
|
|
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
import {
|
|
43
43
|
adaptivePollMs,
|
|
44
44
|
journalEntriesForWatcherDelete,
|
|
45
|
+
shouldQuarantineWatcherDeleteIntents,
|
|
45
46
|
} from "./sync-runner-watch-loop.js";
|
|
46
47
|
import {
|
|
47
48
|
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
@@ -7800,6 +7801,32 @@ describe("watcher delete intents without pre-seeded authorization", () => {
|
|
|
7800
7801
|
).toEqual([["knowledge/exact.md", exactEntry]]);
|
|
7801
7802
|
});
|
|
7802
7803
|
|
|
7804
|
+
// Bulk-intent quarantine. A single `unlinkDir` expands to every descendant
|
|
7805
|
+
// journal entry, so without a ceiling one filesystem event — a checkout
|
|
7806
|
+
// swap, an unmount, a partial restore — mints thousands of individually
|
|
7807
|
+
// valid delete intents. Same thresholds as the engine's bulk-asymmetry
|
|
7808
|
+
// breaker so there is one rule and one override knob across both layers.
|
|
7809
|
+
it.each([
|
|
7810
|
+
// [expanded, journalEntries, quarantined, why]
|
|
7811
|
+
[11, 100, true, "11% of the shard, above both thresholds"],
|
|
7812
|
+
[10, 100, true, "exactly at MIN_ABS and RATIO"],
|
|
7813
|
+
[9, 100, false, "below MIN_ABS — ordinary cleanup must propagate"],
|
|
7814
|
+
[9, 9, false, "100% ratio but below MIN_ABS floor"],
|
|
7815
|
+
[10, 1000, false, "at MIN_ABS but only 1% of the shard"],
|
|
7816
|
+
[50, 50, true, "the whole shard vanished at once"],
|
|
7817
|
+
[0, 0, false, "empty journal never quarantines"],
|
|
7818
|
+
])(
|
|
7819
|
+
"bulk-intent quarantine for %i of %i journal entries → %s (%s)",
|
|
7820
|
+
(expanded, journalEntries, quarantined) => {
|
|
7821
|
+
expect(
|
|
7822
|
+
shouldQuarantineWatcherDeleteIntents(
|
|
7823
|
+
expanded as number,
|
|
7824
|
+
journalEntries as number,
|
|
7825
|
+
),
|
|
7826
|
+
).toBe(quarantined);
|
|
7827
|
+
},
|
|
7828
|
+
);
|
|
7829
|
+
|
|
7803
7830
|
it("keeps a deleted top-level personal file in the scoped push delete roots", async () => {
|
|
7804
7831
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-personal-delete-"));
|
|
7805
7832
|
const key = "personal-note.md";
|
|
@@ -7974,7 +8001,14 @@ describe("watcher delete intents without pre-seeded authorization", () => {
|
|
|
7974
8001
|
await stopHarness(h);
|
|
7975
8002
|
});
|
|
7976
8003
|
|
|
7977
|
-
|
|
8004
|
+
// Replaces #227's "unlinkDir records every tracked descendant across a
|
|
8005
|
+
// 12-file breaker-sized subtree", which asserted that one directory event
|
|
8006
|
+
// may mint an intent for the entire shard and wipe it remotely. That is the
|
|
8007
|
+
// shape of the 2026-07-30 incident (1,108 files in one push), and the test
|
|
8008
|
+
// name conceded it was breaker-sized. Quarantining is now the contract; the
|
|
8009
|
+
// three tests below cover the refusal, the operator escape hatch, and the
|
|
8010
|
+
// ordinary folder removal that must keep working.
|
|
8011
|
+
it("unlinkDir covering the whole shard is quarantined: no intents, nothing deleted", async () => {
|
|
7978
8012
|
const keys = Array.from(
|
|
7979
8013
|
{ length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
|
|
7980
8014
|
);
|
|
@@ -7984,7 +8018,72 @@ describe("watcher delete intents without pre-seeded authorization", () => {
|
|
|
7984
8018
|
const snapshots = h.capture()(
|
|
7985
8019
|
"companies/indigo/knowledge/moved", "unlinkDir",
|
|
7986
8020
|
);
|
|
7987
|
-
expect(snapshots).toHaveLength(
|
|
8021
|
+
expect(snapshots).toHaveLength(0);
|
|
8022
|
+
expect(
|
|
8023
|
+
Object.values(readJournal("indigo").files).filter(
|
|
8024
|
+
(entry) => entry.localDeleteIntent,
|
|
8025
|
+
),
|
|
8026
|
+
).toHaveLength(0);
|
|
8027
|
+
fs.rmSync(subtree, { recursive: true, force: true });
|
|
8028
|
+
h.watcher.emit("companies/indigo/knowledge/moved", {
|
|
8029
|
+
paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
|
|
8030
|
+
changes: new Map([
|
|
8031
|
+
[subtree, { kind: "unlinkDir", deleteSnapshots: snapshots }],
|
|
8032
|
+
]),
|
|
8033
|
+
} as TreeChangeBatch);
|
|
8034
|
+
h.tick();
|
|
8035
|
+
await settle();
|
|
8036
|
+
expect(h.remote.size).toBe(12);
|
|
8037
|
+
expect(Object.keys(readJournal("indigo").files)).toHaveLength(12);
|
|
8038
|
+
await stopHarness(h);
|
|
8039
|
+
});
|
|
8040
|
+
|
|
8041
|
+
it("HQ_SYNC_DELETE_BULK_OVERRIDE=1 lets a whole-shard unlinkDir propagate", async () => {
|
|
8042
|
+
process.env.HQ_SYNC_DELETE_BULK_OVERRIDE = "1";
|
|
8043
|
+
try {
|
|
8044
|
+
const keys = Array.from(
|
|
8045
|
+
{ length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
|
|
8046
|
+
);
|
|
8047
|
+
const h = startDeleteHarness(keys);
|
|
8048
|
+
await settle();
|
|
8049
|
+
const subtree = path.join(h.companyRoot, "knowledge/moved");
|
|
8050
|
+
const snapshots = h.capture()(
|
|
8051
|
+
"companies/indigo/knowledge/moved", "unlinkDir",
|
|
8052
|
+
);
|
|
8053
|
+
expect(snapshots).toHaveLength(12);
|
|
8054
|
+
fs.rmSync(subtree, { recursive: true, force: true });
|
|
8055
|
+
h.watcher.emit("companies/indigo/knowledge/moved", {
|
|
8056
|
+
paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
|
|
8057
|
+
changes: new Map([
|
|
8058
|
+
[subtree, { kind: "unlinkDir", deleteSnapshots: snapshots }],
|
|
8059
|
+
]),
|
|
8060
|
+
} as TreeChangeBatch);
|
|
8061
|
+
h.tick();
|
|
8062
|
+
await settle();
|
|
8063
|
+
expect(h.remote.size).toBe(0);
|
|
8064
|
+
expect(Object.keys(readJournal("indigo").files)).toHaveLength(0);
|
|
8065
|
+
await stopHarness(h);
|
|
8066
|
+
} finally {
|
|
8067
|
+
delete process.env.HQ_SYNC_DELETE_BULK_OVERRIDE;
|
|
8068
|
+
}
|
|
8069
|
+
});
|
|
8070
|
+
|
|
8071
|
+
// The quarantine must not touch ordinary folder cleanup — that is the
|
|
8072
|
+
// delete-resurrection bug #227 was written to fix, and it stays fixed.
|
|
8073
|
+
it("an ordinary folder removal (10 of 120 entries) still records intents and propagates", async () => {
|
|
8074
|
+
const removed = Array.from(
|
|
8075
|
+
{ length: 10 }, (_, i) => `knowledge/moved/f-${i}.md`,
|
|
8076
|
+
);
|
|
8077
|
+
const kept = Array.from(
|
|
8078
|
+
{ length: 110 }, (_, i) => `knowledge/keep/k-${i}.md`,
|
|
8079
|
+
);
|
|
8080
|
+
const h = startDeleteHarness([...removed, ...kept]);
|
|
8081
|
+
await settle();
|
|
8082
|
+
const subtree = path.join(h.companyRoot, "knowledge/moved");
|
|
8083
|
+
const snapshots = h.capture()(
|
|
8084
|
+
"companies/indigo/knowledge/moved", "unlinkDir",
|
|
8085
|
+
);
|
|
8086
|
+
expect(snapshots).toHaveLength(10);
|
|
7988
8087
|
fs.rmSync(subtree, { recursive: true, force: true });
|
|
7989
8088
|
h.watcher.emit("companies/indigo/knowledge/moved", {
|
|
7990
8089
|
paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
|
|
@@ -7994,8 +8093,8 @@ describe("watcher delete intents without pre-seeded authorization", () => {
|
|
|
7994
8093
|
} as TreeChangeBatch);
|
|
7995
8094
|
h.tick();
|
|
7996
8095
|
await settle();
|
|
7997
|
-
expect(h.remote.
|
|
7998
|
-
expect(
|
|
8096
|
+
for (const key of removed) expect(h.remote.has(key)).toBe(false);
|
|
8097
|
+
expect(h.remote.size).toBe(110);
|
|
7999
8098
|
await stopHarness(h);
|
|
8000
8099
|
});
|
|
8001
8100
|
|
package/src/cli/share.test.ts
CHANGED
|
@@ -4222,12 +4222,16 @@ describe("share", () => {
|
|
|
4222
4222
|
missing,
|
|
4223
4223
|
direction = "up" as "up" | "down",
|
|
4224
4224
|
withIntents = true,
|
|
4225
|
+
intentFor,
|
|
4225
4226
|
}: {
|
|
4226
4227
|
total: number;
|
|
4227
4228
|
missing: number;
|
|
4228
4229
|
direction?: "up" | "down";
|
|
4229
4230
|
withIntents?: boolean;
|
|
4231
|
+
/** Per-entry override of `withIntents`, for mixed-bucket cases. */
|
|
4232
|
+
intentFor?: (index: number) => boolean;
|
|
4230
4233
|
}): { companyRoot: string; journalPath: string } {
|
|
4234
|
+
const hasIntent = intentFor ?? ((): boolean => withIntents);
|
|
4231
4235
|
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
4232
4236
|
fs.mkdirSync(companyRoot, { recursive: true });
|
|
4233
4237
|
const files: Record<
|
|
@@ -4251,7 +4255,7 @@ describe("share", () => {
|
|
|
4251
4255
|
direction,
|
|
4252
4256
|
remoteEtag: `etag-${i}`,
|
|
4253
4257
|
kind: "file",
|
|
4254
|
-
...(
|
|
4258
|
+
...(hasIntent(i)
|
|
4255
4259
|
? { localDeleteIntent: deleteIntent(`etag-${i}`, "h") }
|
|
4256
4260
|
: {}),
|
|
4257
4261
|
};
|
|
@@ -4336,14 +4340,25 @@ describe("share", () => {
|
|
|
4336
4340
|
expect(perKeyRefusals.length).toBe(11);
|
|
4337
4341
|
});
|
|
4338
4342
|
|
|
4339
|
-
|
|
4340
|
-
|
|
4343
|
+
// Regression: hq-cloud #227 (v6.14.19) narrowed the breaker numerator to
|
|
4344
|
+
// intent-less disappearances and, in the same change, shipped the watcher
|
|
4345
|
+
// producer that stamps one intent per descendant of a vanished directory.
|
|
4346
|
+
// The assertion this replaces ("intent-backed 11/100 bypasses the breaker
|
|
4347
|
+
// and deletes every covered path") is that PR's own seven-day-old
|
|
4348
|
+
// assertion, not established contract; it is what the 2026-07-30
|
|
4349
|
+
// 1,108-file mass deletion looked like in test form. Intent currency is a
|
|
4350
|
+
// per-file property and cannot stand in for whole-set health.
|
|
4351
|
+
it("intent-backed 11/100 subtree disappearance ALSO trips the breaker: no delete is issued", async () => {
|
|
4352
|
+
const { companyRoot, journalPath } = setupJournal({
|
|
4353
|
+
total: 100,
|
|
4354
|
+
missing: 11,
|
|
4355
|
+
});
|
|
4341
4356
|
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
4342
4357
|
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
4343
4358
|
lastModified: new Date(),
|
|
4344
4359
|
size: 5,
|
|
4345
4360
|
}));
|
|
4346
|
-
const events: Array<{ type: string }> = [];
|
|
4361
|
+
const events: Array<{ type: string; [k: string]: unknown }> = [];
|
|
4347
4362
|
|
|
4348
4363
|
const result = await share({
|
|
4349
4364
|
paths: [companyRoot],
|
|
@@ -4356,11 +4371,147 @@ describe("share", () => {
|
|
|
4356
4371
|
onEvent: (e) => events.push(e as { type: string }),
|
|
4357
4372
|
});
|
|
4358
4373
|
|
|
4359
|
-
expect(result.filesDeleted).toBe(
|
|
4360
|
-
expect(deleteRemoteFile).
|
|
4374
|
+
expect(result.filesDeleted).toBe(0);
|
|
4375
|
+
expect(deleteRemoteFile).not.toHaveBeenCalled();
|
|
4376
|
+
const summary = events.find(
|
|
4377
|
+
(e) => e.type === "delete-refused-bulk-asymmetry",
|
|
4378
|
+
) as { candidates: number; inScope: number; ratio: number } | undefined;
|
|
4379
|
+
expect(summary).toBeDefined();
|
|
4380
|
+
expect(summary!.candidates).toBe(11);
|
|
4381
|
+
expect(summary!.inScope).toBe(100);
|
|
4382
|
+
// No HEAD for the would-be delete candidates — the trip short-circuits
|
|
4383
|
+
// Stage 2 exactly as it does for intent-less disappearances.
|
|
4384
|
+
const headedKeys = vi
|
|
4385
|
+
.mocked(headRemoteFile)
|
|
4386
|
+
.mock.calls.map((c) => c[1] as string);
|
|
4387
|
+
for (let i = 0; i < 11; i++) {
|
|
4388
|
+
expect(headedKeys).not.toContain(`f-${i.toString().padStart(4, "0")}.md`);
|
|
4389
|
+
}
|
|
4390
|
+
// Journal untouched — the entries stay recoverable.
|
|
4391
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
4392
|
+
expect(Object.keys(journal.files).length).toBe(100);
|
|
4393
|
+
});
|
|
4394
|
+
|
|
4395
|
+
// The core of the #227 regression, isolated: pre-#227 a trip emptied
|
|
4396
|
+
// `plan.toDelete`; #227 replaced that with an in-place re-label of entries
|
|
4397
|
+
// already in `refusedStale`, so the breaker stopped stopping anything it
|
|
4398
|
+
// had not already refused for another reason. owned-only stages its picks
|
|
4399
|
+
// directly into `toDelete` (no HEAD pass), so it is the sharpest probe of
|
|
4400
|
+
// that one line.
|
|
4401
|
+
it("a trip empties plan.toDelete: owned-only intent-backed 11/100 issues no DeleteObject", async () => {
|
|
4402
|
+
const { companyRoot, journalPath } = setupJournal({
|
|
4403
|
+
total: 100,
|
|
4404
|
+
missing: 11,
|
|
4405
|
+
});
|
|
4406
|
+
|
|
4407
|
+
const events: Array<{ type: string; [k: string]: unknown }> = [];
|
|
4408
|
+
const result = await share({
|
|
4409
|
+
paths: [companyRoot],
|
|
4410
|
+
company: "acme",
|
|
4411
|
+
vaultConfig: mockConfig,
|
|
4412
|
+
hqRoot: tmpDir,
|
|
4413
|
+
skipUnchanged: true,
|
|
4414
|
+
propagateDeletes: true,
|
|
4415
|
+
propagateDeletePolicy: "owned-only",
|
|
4416
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
4417
|
+
});
|
|
4418
|
+
|
|
4419
|
+
expect(result.filesDeleted).toBe(0);
|
|
4420
|
+
expect(deleteRemoteFile).not.toHaveBeenCalled();
|
|
4421
|
+
expect(result.filesRefusedStale).toBe(11);
|
|
4422
|
+
const perKeyRefusals = events.filter(
|
|
4423
|
+
(e) =>
|
|
4424
|
+
e.type === "delete-refused-stale-etag" &&
|
|
4425
|
+
e.reason === "bulk-asymmetry",
|
|
4426
|
+
);
|
|
4427
|
+
expect(perKeyRefusals.length).toBe(11);
|
|
4428
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
4429
|
+
expect(Object.keys(journal.files).length).toBe(100);
|
|
4430
|
+
});
|
|
4431
|
+
|
|
4432
|
+
it("vault litter still drains when the breaker trips", async () => {
|
|
4433
|
+
// The trip short-circuits the Stage-2 HEAD pass, so the litter merge has
|
|
4434
|
+
// to happen on that path too. Litter is never a user content loss, and
|
|
4435
|
+
// refusing the ratchet-drain would leave conflict mirrors stuck in the
|
|
4436
|
+
// vault for as long as the mirror looks unhealthy.
|
|
4437
|
+
const { companyRoot, journalPath } = setupJournal({
|
|
4438
|
+
total: 100,
|
|
4439
|
+
missing: 11,
|
|
4440
|
+
withIntents: false,
|
|
4441
|
+
});
|
|
4442
|
+
const litterKey = "CLAUDE.md.conflict-2026-05-13T19-40-40Z-e5797a.md";
|
|
4443
|
+
const seeded = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
4444
|
+
seeded.files[litterKey] = {
|
|
4445
|
+
hash: "h",
|
|
4446
|
+
size: 5,
|
|
4447
|
+
syncedAt: new Date().toISOString(),
|
|
4448
|
+
direction: "up",
|
|
4449
|
+
remoteEtag: "litter-etag",
|
|
4450
|
+
kind: "file",
|
|
4451
|
+
};
|
|
4452
|
+
fs.writeFileSync(journalPath, JSON.stringify(seeded));
|
|
4453
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
4454
|
+
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
4455
|
+
lastModified: new Date(),
|
|
4456
|
+
size: 5,
|
|
4457
|
+
}));
|
|
4458
|
+
|
|
4459
|
+
const events: Array<{ type: string }> = [];
|
|
4460
|
+
const result = await share({
|
|
4461
|
+
paths: [companyRoot],
|
|
4462
|
+
company: "acme",
|
|
4463
|
+
vaultConfig: mockConfig,
|
|
4464
|
+
hqRoot: tmpDir,
|
|
4465
|
+
skipUnchanged: true,
|
|
4466
|
+
propagateDeletes: true,
|
|
4467
|
+
propagateDeletePolicy: "currency-gated",
|
|
4468
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
4469
|
+
});
|
|
4470
|
+
|
|
4361
4471
|
expect(
|
|
4362
4472
|
events.find((e) => e.type === "delete-refused-bulk-asymmetry"),
|
|
4363
|
-
).
|
|
4473
|
+
).toBeDefined();
|
|
4474
|
+
expect(result.filesDeleted).toBe(1);
|
|
4475
|
+
expect(deleteRemoteFile).toHaveBeenCalledTimes(1);
|
|
4476
|
+
expect(deleteRemoteFile).toHaveBeenCalledWith(
|
|
4477
|
+
expect.anything(),
|
|
4478
|
+
litterKey,
|
|
4479
|
+
);
|
|
4480
|
+
});
|
|
4481
|
+
|
|
4482
|
+
it("intent-backed and intent-less disappearances share one numerator: 6 + 5 of 100 trips", async () => {
|
|
4483
|
+
const { companyRoot } = setupJournal({
|
|
4484
|
+
total: 100,
|
|
4485
|
+
missing: 11,
|
|
4486
|
+
intentFor: (i) => i < 6,
|
|
4487
|
+
});
|
|
4488
|
+
vi.mocked(headRemoteFile).mockImplementation(async (_ctx, key) => ({
|
|
4489
|
+
etag: `etag-${parseInt(key.replace(/[^0-9]/g, ""), 10)}`,
|
|
4490
|
+
lastModified: new Date(),
|
|
4491
|
+
size: 5,
|
|
4492
|
+
}));
|
|
4493
|
+
|
|
4494
|
+
const events: Array<{ type: string; [k: string]: unknown }> = [];
|
|
4495
|
+
const result = await share({
|
|
4496
|
+
paths: [companyRoot],
|
|
4497
|
+
company: "acme",
|
|
4498
|
+
vaultConfig: mockConfig,
|
|
4499
|
+
hqRoot: tmpDir,
|
|
4500
|
+
skipUnchanged: true,
|
|
4501
|
+
propagateDeletes: true,
|
|
4502
|
+
propagateDeletePolicy: "currency-gated",
|
|
4503
|
+
onEvent: (e) => events.push(e as { type: string }),
|
|
4504
|
+
});
|
|
4505
|
+
|
|
4506
|
+
expect(result.filesDeleted).toBe(0);
|
|
4507
|
+
expect(deleteRemoteFile).not.toHaveBeenCalled();
|
|
4508
|
+
const summary = events.find(
|
|
4509
|
+
(e) => e.type === "delete-refused-bulk-asymmetry",
|
|
4510
|
+
) as { candidates: number } | undefined;
|
|
4511
|
+
expect(summary).toBeDefined();
|
|
4512
|
+
// Neither bucket reaches MIN_ABS alone; together they are the whole
|
|
4513
|
+
// prospective destructive set and must be counted as one.
|
|
4514
|
+
expect(summary!.candidates).toBe(11);
|
|
4364
4515
|
});
|
|
4365
4516
|
|
|
4366
4517
|
it("does NOT trip at 9/100 (below ratio): all candidates delete normally", async () => {
|
|
@@ -4388,6 +4539,9 @@ describe("share", () => {
|
|
|
4388
4539
|
expect(events.find((e) => e.type === "delete-refused-bulk-asymmetry")).toBeUndefined();
|
|
4389
4540
|
});
|
|
4390
4541
|
|
|
4542
|
+
// The two bypasses below are the sanctioned route for a genuinely
|
|
4543
|
+
// intentional mass delete now that intent-backed candidates count toward
|
|
4544
|
+
// the numerator. Both journals here carry current delete intents.
|
|
4391
4545
|
it("HQ_SYNC_DELETE_BULK_OVERRIDE=1 bypasses the guard: 11/100 deletes proceed", async () => {
|
|
4392
4546
|
process.env.HQ_SYNC_DELETE_BULK_OVERRIDE = "1";
|
|
4393
4547
|
const { companyRoot } = setupJournal({ total: 100, missing: 11 });
|
package/src/cli/share.ts
CHANGED
|
@@ -2233,6 +2233,12 @@ type RefusedStaleReason =
|
|
|
2233
2233
|
* - `candidates / inScope >= BULK_ASYMMETRY_RATIO` (default 10%).
|
|
2234
2234
|
* - `candidates >= BULK_ASYMMETRY_MIN_ABS` (default 10).
|
|
2235
2235
|
*
|
|
2236
|
+
* `candidates` is the whole prospective destructive set, whether or not each
|
|
2237
|
+
* entry carries a current watcher delete-intent. A live watcher stamps one
|
|
2238
|
+
* intent per descendant when a directory disappears, so intent presence says
|
|
2239
|
+
* nothing about whether the disappearance was deliberate — see the numerator
|
|
2240
|
+
* comment in `computeDeletePlan` and the 2026-07-30 incident.
|
|
2241
|
+
*
|
|
2236
2242
|
* The MIN_ABS floor is what lets small intentional deletes through —
|
|
2237
2243
|
* `rm 1 file` from a 5-entry mirror is 20% but 1 absolute candidate; never
|
|
2238
2244
|
* tripped. The RATIO is what lets large intentional uploads-then-deletes
|
|
@@ -2431,18 +2437,37 @@ async function computeDeletePlan(
|
|
|
2431
2437
|
// and the journal-mutation buckets are already settled before any I/O.
|
|
2432
2438
|
type HeadCandidate = { key: string; journalEtag: string; intentVersion: 1 };
|
|
2433
2439
|
const headCandidates: HeadCandidate[] = [];
|
|
2434
|
-
// Litter drain bucket — kept separate from
|
|
2435
|
-
//
|
|
2440
|
+
// Litter drain bucket — kept separate from `plan.toDelete` so the
|
|
2441
|
+
// bulk-asymmetry breaker (which moves toDelete + headCandidates into
|
|
2442
|
+
// `refusedStale` when it trips) can't sweep these out. See
|
|
2436
2443
|
// `isVaultLitterArtifact` for the patterns and rationale: by construction
|
|
2437
2444
|
// these aren't user content losses, so a high ratio of litter must not
|
|
2438
2445
|
// refuse the drain — that's the whole point of the bypass. Merged into
|
|
2439
2446
|
// `plan.toDelete` after the breaker check.
|
|
2440
2447
|
const litterToDelete: string[] = [];
|
|
2441
|
-
//
|
|
2442
|
-
//
|
|
2443
|
-
//
|
|
2448
|
+
// Bulk-asymmetry tracking: count every in-scope journal entry (denominator)
|
|
2449
|
+
// and every entry that would have been a delete-candidate before the guard
|
|
2450
|
+
// (numerator). The numerator is the WHOLE prospective destructive set:
|
|
2451
|
+
// intent-backed picks (owned-only/all toDelete, currency-gated
|
|
2452
|
+
// headCandidates, legacy-no-etag refusals) AND intent-less disappearances.
|
|
2453
|
+
//
|
|
2454
|
+
// 6.14.19 (#227) narrowed it to intent-less disappearances only, on the
|
|
2455
|
+
// premise that a current watcher intent is affirmative evidence of a
|
|
2456
|
+
// deliberate delete. A wiped, swapped, unmounted or partially-restored tree
|
|
2457
|
+
// observed by a live watcher mints an intent per descendant, so that premise
|
|
2458
|
+
// does not hold and the breaker stopped covering the exact failure mode it
|
|
2459
|
+
// exists for (incident 2026-07-30: 1,108 intent-backed deletions in one
|
|
2460
|
+
// push). Intent currency is a per-file property; the breaker is a whole-set
|
|
2461
|
+
// health check, and the two must not be conflated.
|
|
2462
|
+
//
|
|
2463
|
+
// We do NOT count "ENOENT but ignore-filtered" or "ENOENT but ephemeral" —
|
|
2464
|
+
// those drop out of the plan entirely on their own and don't reflect
|
|
2465
|
+
// mirror-loss intent — nor litter drains (see above), which are intentional
|
|
2466
|
+
// ratchet-cleanup, not mass-delete intent.
|
|
2444
2467
|
let inScopeJournalEntries = 0;
|
|
2445
|
-
let
|
|
2468
|
+
let bulkCandidatePicks = 0;
|
|
2469
|
+
// Intent-less candidates already parked in `refusedStale`. On a trip they are
|
|
2470
|
+
// re-labelled `bulk-asymmetry` in place rather than refused twice.
|
|
2446
2471
|
const intentlessKeys = new Set<string>();
|
|
2447
2472
|
|
|
2448
2473
|
for (const [relativeKey, entry] of Object.entries(journal.files)) {
|
|
@@ -2534,7 +2559,7 @@ async function computeDeletePlan(
|
|
|
2534
2559
|
policy === "currency-gated" ||
|
|
2535
2560
|
(policy === "owned-only" && entry.direction === "up")
|
|
2536
2561
|
) {
|
|
2537
|
-
|
|
2562
|
+
bulkCandidatePicks++;
|
|
2538
2563
|
intentlessKeys.add(relativeKey);
|
|
2539
2564
|
}
|
|
2540
2565
|
plan.refusedStale.push({
|
|
@@ -2546,6 +2571,7 @@ async function computeDeletePlan(
|
|
|
2546
2571
|
continue;
|
|
2547
2572
|
}
|
|
2548
2573
|
|
|
2574
|
+
bulkCandidatePicks++;
|
|
2549
2575
|
if (policy === "all") {
|
|
2550
2576
|
// policy:"all" is the explicit-opt-out emergency-reconcile mode; the
|
|
2551
2577
|
// bulk-asymmetry guard skips this branch (caller asserted intent).
|
|
@@ -2557,6 +2583,11 @@ async function computeDeletePlan(
|
|
|
2557
2583
|
}
|
|
2558
2584
|
if (policy === "owned-only") {
|
|
2559
2585
|
if (entry.direction !== "up") {
|
|
2586
|
+
// Not a delete candidate under owned-only, but it WAS missing
|
|
2587
|
+
// locally. Don't count it for the bulk guard — direction:'down'
|
|
2588
|
+
// entries that vanish locally are silently ignored by this policy
|
|
2589
|
+
// anyway, so they don't represent intent to mass-delete.
|
|
2590
|
+
bulkCandidatePicks--;
|
|
2560
2591
|
continue;
|
|
2561
2592
|
}
|
|
2562
2593
|
plan.toDelete.push({
|
|
@@ -2589,26 +2620,54 @@ async function computeDeletePlan(
|
|
|
2589
2620
|
if (
|
|
2590
2621
|
policy !== "all" &&
|
|
2591
2622
|
!isBulkAsymmetryOverride() &&
|
|
2592
|
-
|
|
2623
|
+
bulkCandidatePicks >= BULK_ASYMMETRY_MIN_ABS &&
|
|
2593
2624
|
inScopeJournalEntries > 0 &&
|
|
2594
|
-
|
|
2625
|
+
bulkCandidatePicks / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO
|
|
2595
2626
|
) {
|
|
2627
|
+
// Move every staged candidate (both already-bucketed toDelete from
|
|
2628
|
+
// owned-only and queued headCandidates from currency-gated) into
|
|
2629
|
+
// refusedStale with reason "bulk-asymmetry", and re-label the intent-less
|
|
2630
|
+
// entries already parked there. Journal is not mutated; no DeleteObject is
|
|
2631
|
+
// issued; the Stage-2 HEAD pass never runs.
|
|
2596
2632
|
const samplePaths: string[] = [];
|
|
2633
|
+
const noteSample = (key: string): void => {
|
|
2634
|
+
if (samplePaths.length < BULK_ASYMMETRY_SAMPLE_CAP) samplePaths.push(key);
|
|
2635
|
+
};
|
|
2597
2636
|
for (const refused of plan.refusedStale) {
|
|
2598
2637
|
if (!intentlessKeys.has(refused.key)) continue;
|
|
2599
2638
|
refused.journalEtag = "<bulk-asymmetry>";
|
|
2600
2639
|
refused.remoteEtag = "<not-checked>";
|
|
2601
2640
|
refused.reason = "bulk-asymmetry";
|
|
2602
|
-
|
|
2603
|
-
samplePaths.push(refused.key);
|
|
2604
|
-
}
|
|
2641
|
+
noteSample(refused.key);
|
|
2605
2642
|
}
|
|
2643
|
+
const pushRefused = (key: string): void => {
|
|
2644
|
+
plan.refusedStale.push({
|
|
2645
|
+
key,
|
|
2646
|
+
journalEtag: "<bulk-asymmetry>",
|
|
2647
|
+
remoteEtag: "<not-checked>",
|
|
2648
|
+
reason: "bulk-asymmetry",
|
|
2649
|
+
});
|
|
2650
|
+
noteSample(key);
|
|
2651
|
+
};
|
|
2652
|
+
for (const item of plan.toDelete) pushRefused(item.key);
|
|
2653
|
+
for (const c of headCandidates) pushRefused(c.key);
|
|
2654
|
+
// Emptying the delete list is what makes the trip a refusal rather than a
|
|
2655
|
+
// label. #227 dropped this line, so a tripped breaker stopped nothing it
|
|
2656
|
+
// had not already refused for another reason.
|
|
2657
|
+
plan.toDelete = [];
|
|
2606
2658
|
plan.bulkAsymmetry = {
|
|
2607
|
-
candidates:
|
|
2659
|
+
candidates: bulkCandidatePicks,
|
|
2608
2660
|
inScope: inScopeJournalEntries,
|
|
2609
|
-
ratio:
|
|
2661
|
+
ratio: bulkCandidatePicks / inScopeJournalEntries,
|
|
2610
2662
|
samplePaths,
|
|
2611
2663
|
};
|
|
2664
|
+
// Litter still drains even when the breaker trips — see `litterToDelete`
|
|
2665
|
+
// declaration. The breaker protects against corrupt-mirror mass-deletes
|
|
2666
|
+
// of user content; litter cleanup is orthogonal and should never be
|
|
2667
|
+
// refused for the same reason new-litter producer-side exclusions
|
|
2668
|
+
// shouldn't block it.
|
|
2669
|
+
plan.toDelete.push(...litterToDelete.map((key) => ({ key, intentVersion: null })));
|
|
2670
|
+
return plan;
|
|
2612
2671
|
}
|
|
2613
2672
|
|
|
2614
2673
|
// Stage 2: bounded-parallel HEAD pass. Promise.all over chunks of size
|