@indigoai-us/hq-cloud 6.14.18 → 6.14.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +35 -0
- package/dist/bin/sync-runner-company.d.ts +7 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +21 -1
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +8 -16
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +138 -0
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +10 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +201 -25
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +14 -2
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +17 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +36 -6
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +550 -3
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-core.js +27 -1
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-drop-dir-symlink.test.d.ts +2 -0
- package/dist/cli/rescue-drop-dir-symlink.test.d.ts.map +1 -0
- package/dist/cli/rescue-drop-dir-symlink.test.js +206 -0
- package/dist/cli/rescue-drop-dir-symlink.test.js.map +1 -0
- package/dist/cli/share.d.ts +15 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -55
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +119 -6
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +19 -25
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +121 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal.d.ts +3 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +13 -2
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +26 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/personal-vault.d.ts +12 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +20 -0
- package/dist/personal-vault.js.map +1 -1
- package/dist/qmd-reindex.d.ts +88 -11
- package/dist/qmd-reindex.d.ts.map +1 -1
- package/dist/qmd-reindex.js +354 -65
- package/dist/qmd-reindex.js.map +1 -1
- package/dist/qmd-reindex.test.d.ts +4 -0
- package/dist/qmd-reindex.test.d.ts.map +1 -1
- package/dist/qmd-reindex.test.js +285 -3
- package/dist/qmd-reindex.test.js.map +1 -1
- package/dist/s3.d.ts +13 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +34 -1
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +60 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/telemetry.d.ts +22 -0
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +79 -0
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +117 -1
- package/dist/telemetry.test.js.map +1 -1
- package/dist/watcher.d.ts +26 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +75 -12
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +29 -1
- package/src/bin/sync-runner-planning.test.ts +153 -0
- package/src/bin/sync-runner-planning.ts +10 -17
- package/src/bin/sync-runner-watch-loop.ts +322 -32
- package/src/bin/sync-runner-watch-routes.ts +14 -1
- package/src/bin/sync-runner.test.ts +662 -6
- package/src/bin/sync-runner.ts +65 -7
- package/src/cli/rescue-core.ts +26 -1
- package/src/cli/rescue-drop-dir-symlink.test.ts +224 -0
- package/src/cli/share.test.ts +150 -6
- package/src/cli/share.ts +143 -48
- package/src/cli/sync.test.ts +150 -1
- package/src/cli/sync.ts +25 -29
- package/src/journal.test.ts +45 -0
- package/src/journal.ts +18 -1
- package/src/personal-vault.ts +23 -0
- package/src/qmd-reindex.test.ts +324 -4
- package/src/qmd-reindex.ts +414 -76
- package/src/s3.test.ts +79 -0
- package/src/s3.ts +48 -1
- package/src/telemetry.test.ts +128 -0
- package/src/telemetry.ts +80 -0
- package/src/watcher.ts +111 -14
- package/test/e2e/watcher-real-chokidar.test.ts +62 -2
- package/test/e2e/watcher-recursive-backend.test.ts +67 -1
package/src/cli/share.ts
CHANGED
|
@@ -41,7 +41,10 @@ import {
|
|
|
41
41
|
wrapFilterWithPersonalVaultDefaults,
|
|
42
42
|
type PersonalVaultExclusion,
|
|
43
43
|
} from "../personal-vault-exclusions.js";
|
|
44
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
isGeneratedCoreMirrorKey,
|
|
46
|
+
isPersonalVaultDeletionExcluded,
|
|
47
|
+
} from "../personal-vault.js";
|
|
45
48
|
import { resolveConflict } from "./conflict.js";
|
|
46
49
|
import type { ConflictStrategy } from "./conflict.js";
|
|
47
50
|
import type { SyncProgressEvent } from "./sync.js";
|
|
@@ -590,6 +593,12 @@ export interface ShareOptions {
|
|
|
590
593
|
* next push to erase it.
|
|
591
594
|
*/
|
|
592
595
|
propagateDeletePolicy?: "currency-gated" | "owned-only" | "all";
|
|
596
|
+
/**
|
|
597
|
+
* Explicit vault-relative delete roots captured by a live watcher before a
|
|
598
|
+
* scoped push. These roots do not need to exist locally; delete execution
|
|
599
|
+
* still requires a current version-bound intent (or server tombstone).
|
|
600
|
+
*/
|
|
601
|
+
deleteScopeRoots?: string[];
|
|
593
602
|
/**
|
|
594
603
|
* Hq-root-relative key prefixes whose journal entries should be
|
|
595
604
|
* unconditionally decommissioned from the remote bucket and journal,
|
|
@@ -753,9 +762,27 @@ export interface ShareResult {
|
|
|
753
762
|
* way to runner/UI consumers.
|
|
754
763
|
*/
|
|
755
764
|
conflictPaths: string[];
|
|
765
|
+
/** Uncapped per-path delete outcomes used by scoped watcher publication. */
|
|
766
|
+
pathResults?: SharePathResult[];
|
|
756
767
|
aborted: boolean;
|
|
757
768
|
}
|
|
758
769
|
|
|
770
|
+
export type ShareDeleteRefusalReason =
|
|
771
|
+
| "stale-etag"
|
|
772
|
+
| "legacy-no-etag"
|
|
773
|
+
| "bulk-asymmetry"
|
|
774
|
+
| "divergent-local"
|
|
775
|
+
| "missing-delete-intent"
|
|
776
|
+
| "intent-changed"
|
|
777
|
+
| "transfer-error";
|
|
778
|
+
|
|
779
|
+
export interface SharePathResult {
|
|
780
|
+
path: string;
|
|
781
|
+
status: "accepted" | "refused";
|
|
782
|
+
operation: "delete" | "tombstone";
|
|
783
|
+
reason?: ShareDeleteRefusalReason;
|
|
784
|
+
}
|
|
785
|
+
|
|
759
786
|
/**
|
|
760
787
|
* A conditional-write fence rejection — the SDK's 412 (`name:
|
|
761
788
|
* "PreconditionFailed"`) or the presigned transport's mirror of it. Means
|
|
@@ -838,6 +865,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
838
865
|
const counters = createShareCounters();
|
|
839
866
|
const filesRefusedStalePaths: string[] = [];
|
|
840
867
|
const conflictPaths: string[] = [];
|
|
868
|
+
const pathResults: SharePathResult[] = [];
|
|
841
869
|
|
|
842
870
|
const plans = await buildSharePlans(run);
|
|
843
871
|
const uploadResult = await executeUploads(
|
|
@@ -852,6 +880,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
852
880
|
counters,
|
|
853
881
|
filesRefusedStalePaths,
|
|
854
882
|
uploadResult.abortFlightConflictPaths,
|
|
883
|
+
pathResults,
|
|
855
884
|
true,
|
|
856
885
|
);
|
|
857
886
|
}
|
|
@@ -862,11 +891,19 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
862
891
|
plans.decommissionPlan,
|
|
863
892
|
counters,
|
|
864
893
|
filesRefusedStalePaths,
|
|
894
|
+
pathResults,
|
|
865
895
|
);
|
|
866
896
|
finalizeShareJournal(run);
|
|
867
897
|
throwUploadWorkerErrors(uploadResult.workerErrors);
|
|
868
898
|
|
|
869
|
-
return buildShareResult(
|
|
899
|
+
return buildShareResult(
|
|
900
|
+
run,
|
|
901
|
+
counters,
|
|
902
|
+
filesRefusedStalePaths,
|
|
903
|
+
conflictPaths,
|
|
904
|
+
pathResults,
|
|
905
|
+
false,
|
|
906
|
+
);
|
|
870
907
|
}
|
|
871
908
|
|
|
872
909
|
type DeletePolicy = NonNullable<ShareOptions["propagateDeletePolicy"]>;
|
|
@@ -1070,7 +1107,27 @@ async function buildSharePlans(run: PushRunContext): Promise<SharePlans> {
|
|
|
1070
1107
|
// `companies/{slug}/…` keys, so this only applies in company mode.
|
|
1071
1108
|
const filesToShare =
|
|
1072
1109
|
run.options.personalMode === true
|
|
1073
|
-
? collected
|
|
1110
|
+
? collected.filter((entry) => {
|
|
1111
|
+
// `core/<type>` symlinks are generated projections of canonical
|
|
1112
|
+
// personal/package content. Reindex or package wiring owns their
|
|
1113
|
+
// lifecycle, so uploading them creates remote objects that a later
|
|
1114
|
+
// pull can only materialize temporarily before reindex removes them.
|
|
1115
|
+
// Keep ordinary core files eligible: only a symlink is derived.
|
|
1116
|
+
const generatedCoreMirror =
|
|
1117
|
+
entry.kind === "symlink" && isGeneratedCoreMirrorKey(entry.relativePath)
|
|
1118
|
+
if (!generatedCoreMirror) return true;
|
|
1119
|
+
|
|
1120
|
+
// This is a type-aware companion to the path-only personal-vault
|
|
1121
|
+
// default filters. `collectFiles` is the first seam that knows the
|
|
1122
|
+
// entry is a symlink, so record it here for the same summary/result
|
|
1123
|
+
// accounting used by the static exclusions.
|
|
1124
|
+
if (!run.excludedSet.has(entry.relativePath)) {
|
|
1125
|
+
run.excludedSet.add(entry.relativePath);
|
|
1126
|
+
run.excludedById["generated-core-mirror"] =
|
|
1127
|
+
(run.excludedById["generated-core-mirror"] ?? 0) + 1;
|
|
1128
|
+
}
|
|
1129
|
+
return false;
|
|
1130
|
+
})
|
|
1074
1131
|
: collected.filter((entry) => {
|
|
1075
1132
|
if (!entry.relativePath.startsWith("companies/")) return true;
|
|
1076
1133
|
run.emit({
|
|
@@ -1085,7 +1142,12 @@ async function buildSharePlans(run: PushRunContext): Promise<SharePlans> {
|
|
|
1085
1142
|
run.skipUnchanged === true,
|
|
1086
1143
|
);
|
|
1087
1144
|
const deleteScopeRoots = run.propagateDeletes === true
|
|
1088
|
-
? resolveDeleteScopeRoots(
|
|
1145
|
+
? resolveDeleteScopeRoots(
|
|
1146
|
+
run.paths,
|
|
1147
|
+
run.hqRoot,
|
|
1148
|
+
run.syncRoot,
|
|
1149
|
+
run.options.deleteScopeRoots,
|
|
1150
|
+
)
|
|
1089
1151
|
: [];
|
|
1090
1152
|
const deletePlan: DeletePlan = run.propagateDeletes === true
|
|
1091
1153
|
? await computeDeletePlan(
|
|
@@ -1522,6 +1584,7 @@ async function executeDeletes(
|
|
|
1522
1584
|
decommissionPlan: string[],
|
|
1523
1585
|
counters: ShareCounters,
|
|
1524
1586
|
filesRefusedStalePaths: string[],
|
|
1587
|
+
pathResults: SharePathResult[],
|
|
1525
1588
|
): Promise<void> {
|
|
1526
1589
|
const deleteItems = [
|
|
1527
1590
|
...deletePlan.toDelete.map((item) => ({ ...item, decommission: false })),
|
|
@@ -1543,6 +1606,9 @@ async function executeDeletes(
|
|
|
1543
1606
|
entry!.localDeleteIntent!.version !== item.intentVersion ||
|
|
1544
1607
|
!isLocallyAbsent(path.join(run.syncRoot, relativePath)))
|
|
1545
1608
|
) {
|
|
1609
|
+
if (entry?.localDeleteIntent) {
|
|
1610
|
+
delete entry.localDeleteIntent;
|
|
1611
|
+
}
|
|
1546
1612
|
counters.filesRefusedStale++;
|
|
1547
1613
|
if (filesRefusedStalePaths.length < REFUSED_STALE_PATH_CAP) {
|
|
1548
1614
|
filesRefusedStalePaths.push(relativePath);
|
|
@@ -1554,6 +1620,12 @@ async function executeDeletes(
|
|
|
1554
1620
|
remoteEtag: "<not-checked>",
|
|
1555
1621
|
reason: "intent-changed",
|
|
1556
1622
|
});
|
|
1623
|
+
pathResults.push({
|
|
1624
|
+
path: relativePath,
|
|
1625
|
+
status: "refused",
|
|
1626
|
+
operation: "delete",
|
|
1627
|
+
reason: "intent-changed",
|
|
1628
|
+
});
|
|
1557
1629
|
continue;
|
|
1558
1630
|
}
|
|
1559
1631
|
const size = entry?.size ?? 0;
|
|
@@ -1566,6 +1638,11 @@ async function executeDeletes(
|
|
|
1566
1638
|
bytes: size,
|
|
1567
1639
|
deleted: true,
|
|
1568
1640
|
});
|
|
1641
|
+
pathResults.push({
|
|
1642
|
+
path: relativePath,
|
|
1643
|
+
status: "accepted",
|
|
1644
|
+
operation: "delete",
|
|
1645
|
+
});
|
|
1569
1646
|
} catch (err) {
|
|
1570
1647
|
if (err instanceof VaultAuthError) throw err;
|
|
1571
1648
|
run.emit({
|
|
@@ -1573,6 +1650,12 @@ async function executeDeletes(
|
|
|
1573
1650
|
path: relativePath,
|
|
1574
1651
|
message: err instanceof Error ? err.message : String(err),
|
|
1575
1652
|
});
|
|
1653
|
+
pathResults.push({
|
|
1654
|
+
path: relativePath,
|
|
1655
|
+
status: "refused",
|
|
1656
|
+
operation: "delete",
|
|
1657
|
+
reason: "transfer-error",
|
|
1658
|
+
});
|
|
1576
1659
|
}
|
|
1577
1660
|
}
|
|
1578
1661
|
for (const relativePath of deletePlan.toTombstone) {
|
|
@@ -1585,6 +1668,11 @@ async function executeDeletes(
|
|
|
1585
1668
|
deleted: true,
|
|
1586
1669
|
message: "tombstone (remote already 404)",
|
|
1587
1670
|
});
|
|
1671
|
+
pathResults.push({
|
|
1672
|
+
path: relativePath,
|
|
1673
|
+
status: "accepted",
|
|
1674
|
+
operation: "tombstone",
|
|
1675
|
+
});
|
|
1588
1676
|
}
|
|
1589
1677
|
const decommissionedSet =
|
|
1590
1678
|
decommissionPlan.length > 0 ? new Set(decommissionPlan) : null;
|
|
@@ -1601,6 +1689,12 @@ async function executeDeletes(
|
|
|
1601
1689
|
remoteEtag: refused.remoteEtag,
|
|
1602
1690
|
reason: refused.reason,
|
|
1603
1691
|
});
|
|
1692
|
+
pathResults.push({
|
|
1693
|
+
path: refused.key,
|
|
1694
|
+
status: "refused",
|
|
1695
|
+
operation: "delete",
|
|
1696
|
+
reason: refused.reason,
|
|
1697
|
+
});
|
|
1604
1698
|
}
|
|
1605
1699
|
}
|
|
1606
1700
|
|
|
@@ -1669,6 +1763,7 @@ function buildShareResult(
|
|
|
1669
1763
|
counters: ShareCounters,
|
|
1670
1764
|
filesRefusedStalePaths: string[],
|
|
1671
1765
|
conflictPaths: string[],
|
|
1766
|
+
pathResults: SharePathResult[],
|
|
1672
1767
|
aborted: boolean,
|
|
1673
1768
|
): ShareResult {
|
|
1674
1769
|
return {
|
|
@@ -1684,6 +1779,7 @@ function buildShareResult(
|
|
|
1684
1779
|
filesExcludedByScope: run.scopeExcludedSet.size,
|
|
1685
1780
|
filesExcludedByIgnore: run.ignoreExcludedSet.size,
|
|
1686
1781
|
conflictPaths,
|
|
1782
|
+
pathResults,
|
|
1687
1783
|
aborted,
|
|
1688
1784
|
};
|
|
1689
1785
|
}
|
|
@@ -2043,8 +2139,26 @@ function resolveDeleteScopeRoots(
|
|
|
2043
2139
|
paths: string[],
|
|
2044
2140
|
hqRoot: string,
|
|
2045
2141
|
syncRoot: string,
|
|
2142
|
+
explicitRoots: readonly string[] = [],
|
|
2046
2143
|
): string[] {
|
|
2047
2144
|
const prefixes = new Set<string>();
|
|
2145
|
+
for (const root of explicitRoots) {
|
|
2146
|
+
const normalized = root
|
|
2147
|
+
.split("\\")
|
|
2148
|
+
.join("/")
|
|
2149
|
+
.replace(/^\.\/+/, "")
|
|
2150
|
+
.replace(/\/+$/, "");
|
|
2151
|
+
if (
|
|
2152
|
+
normalized === "" ||
|
|
2153
|
+
normalized === "." ||
|
|
2154
|
+
path.posix.isAbsolute(normalized) ||
|
|
2155
|
+
normalized === ".." ||
|
|
2156
|
+
normalized.startsWith("../")
|
|
2157
|
+
) {
|
|
2158
|
+
continue;
|
|
2159
|
+
}
|
|
2160
|
+
prefixes.add(normalized);
|
|
2161
|
+
}
|
|
2048
2162
|
for (const p of paths) {
|
|
2049
2163
|
const absolutePath = path.isAbsolute(p) ? p : path.resolve(hqRoot, p);
|
|
2050
2164
|
if (!fs.existsSync(absolutePath)) continue;
|
|
@@ -2296,23 +2410,19 @@ async function computeDeletePlan(
|
|
|
2296
2410
|
// and the journal-mutation buckets are already settled before any I/O.
|
|
2297
2411
|
type HeadCandidate = { key: string; journalEtag: string; intentVersion: 1 };
|
|
2298
2412
|
const headCandidates: HeadCandidate[] = [];
|
|
2299
|
-
// Litter drain bucket — kept separate from
|
|
2300
|
-
// bulk-asymmetry
|
|
2301
|
-
// `refusedStale` when it trips) can't sweep these out. See
|
|
2413
|
+
// Litter drain bucket — kept separate from normal delete candidates so
|
|
2414
|
+
// the bulk-asymmetry accounting cannot sweep these in. See
|
|
2302
2415
|
// `isVaultLitterArtifact` for the patterns and rationale: by construction
|
|
2303
2416
|
// these aren't user content losses, so a high ratio of litter must not
|
|
2304
2417
|
// refuse the drain — that's the whole point of the bypass. Merged into
|
|
2305
2418
|
// `plan.toDelete` after the breaker check.
|
|
2306
2419
|
const litterToDelete: string[] = [];
|
|
2307
|
-
//
|
|
2308
|
-
//
|
|
2309
|
-
//
|
|
2310
|
-
// legacy-no-etag refusals. We do NOT count "ENOENT but ignore-filtered" or
|
|
2311
|
-
// "ENOENT but ephemeral" — those drop out of the plan entirely on their own
|
|
2312
|
-
// and don't reflect mirror-loss intent — and litter drains (see above),
|
|
2313
|
-
// which are intentional ratchet-cleanup, not mass-delete intent.
|
|
2420
|
+
// Current watcher intents are affirmative evidence of an observed delete,
|
|
2421
|
+
// so only intent-less disappearances contribute to the mirror-loss breaker.
|
|
2422
|
+
// An unmounted/wiped tree produces no intents and still trips it.
|
|
2314
2423
|
let inScopeJournalEntries = 0;
|
|
2315
|
-
let
|
|
2424
|
+
let intentlessDisappearances = 0;
|
|
2425
|
+
const intentlessKeys = new Set<string>();
|
|
2316
2426
|
|
|
2317
2427
|
for (const [relativeKey, entry] of Object.entries(journal.files)) {
|
|
2318
2428
|
const inScope = scopeRoots.some(
|
|
@@ -2399,6 +2509,13 @@ async function computeDeletePlan(
|
|
|
2399
2509
|
}
|
|
2400
2510
|
|
|
2401
2511
|
if (!hasCurrentLocalDeleteIntent(entry)) {
|
|
2512
|
+
if (
|
|
2513
|
+
policy === "currency-gated" ||
|
|
2514
|
+
(policy === "owned-only" && entry.direction === "up")
|
|
2515
|
+
) {
|
|
2516
|
+
intentlessDisappearances++;
|
|
2517
|
+
intentlessKeys.add(relativeKey);
|
|
2518
|
+
}
|
|
2402
2519
|
plan.refusedStale.push({
|
|
2403
2520
|
key: relativeKey,
|
|
2404
2521
|
journalEtag: entry.remoteEtag ?? "<missing-delete-intent>",
|
|
@@ -2417,14 +2534,8 @@ async function computeDeletePlan(
|
|
|
2417
2534
|
});
|
|
2418
2535
|
continue;
|
|
2419
2536
|
}
|
|
2420
|
-
bulkCandidatePicks++;
|
|
2421
2537
|
if (policy === "owned-only") {
|
|
2422
2538
|
if (entry.direction !== "up") {
|
|
2423
|
-
// Not a delete candidate under owned-only, but it WAS missing
|
|
2424
|
-
// locally. Don't count it for the bulk guard — direction:'down'
|
|
2425
|
-
// entries that vanish locally are silently ignored by this policy
|
|
2426
|
-
// anyway, so they don't represent intent to mass-delete.
|
|
2427
|
-
bulkCandidatePicks--;
|
|
2428
2539
|
continue;
|
|
2429
2540
|
}
|
|
2430
2541
|
plan.toDelete.push({
|
|
@@ -2457,42 +2568,26 @@ async function computeDeletePlan(
|
|
|
2457
2568
|
if (
|
|
2458
2569
|
policy !== "all" &&
|
|
2459
2570
|
!isBulkAsymmetryOverride() &&
|
|
2460
|
-
|
|
2571
|
+
intentlessDisappearances >= BULK_ASYMMETRY_MIN_ABS &&
|
|
2461
2572
|
inScopeJournalEntries > 0 &&
|
|
2462
|
-
|
|
2573
|
+
intentlessDisappearances / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO
|
|
2463
2574
|
) {
|
|
2464
|
-
// Move every staged candidate (both already-bucketed toDelete from
|
|
2465
|
-
// owned-only and queued headCandidates from currency-gated) into
|
|
2466
|
-
// refusedStale with reason "bulk-asymmetry". Journal is not mutated;
|
|
2467
|
-
// no DeleteObject is issued.
|
|
2468
2575
|
const samplePaths: string[] = [];
|
|
2469
|
-
const
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
reason: "bulk-asymmetry",
|
|
2475
|
-
});
|
|
2576
|
+
for (const refused of plan.refusedStale) {
|
|
2577
|
+
if (!intentlessKeys.has(refused.key)) continue;
|
|
2578
|
+
refused.journalEtag = "<bulk-asymmetry>";
|
|
2579
|
+
refused.remoteEtag = "<not-checked>";
|
|
2580
|
+
refused.reason = "bulk-asymmetry";
|
|
2476
2581
|
if (samplePaths.length < BULK_ASYMMETRY_SAMPLE_CAP) {
|
|
2477
|
-
samplePaths.push(key);
|
|
2582
|
+
samplePaths.push(refused.key);
|
|
2478
2583
|
}
|
|
2479
|
-
}
|
|
2480
|
-
for (const item of plan.toDelete) pushRefused(item.key);
|
|
2481
|
-
for (const c of headCandidates) pushRefused(c.key);
|
|
2482
|
-
plan.toDelete = [];
|
|
2584
|
+
}
|
|
2483
2585
|
plan.bulkAsymmetry = {
|
|
2484
|
-
candidates:
|
|
2586
|
+
candidates: intentlessDisappearances,
|
|
2485
2587
|
inScope: inScopeJournalEntries,
|
|
2486
|
-
ratio:
|
|
2588
|
+
ratio: intentlessDisappearances / inScopeJournalEntries,
|
|
2487
2589
|
samplePaths,
|
|
2488
2590
|
};
|
|
2489
|
-
// Litter still drains even when the breaker trips — see `litterToDelete`
|
|
2490
|
-
// declaration. The breaker protects against corrupt-mirror mass-deletes
|
|
2491
|
-
// of user content; litter cleanup is orthogonal and should never be
|
|
2492
|
-
// refused for the same reason new-litter producer-side exclusions
|
|
2493
|
-
// shouldn't block it.
|
|
2494
|
-
plan.toDelete.push(...litterToDelete.map((key) => ({ key, intentVersion: null })));
|
|
2495
|
-
return plan;
|
|
2496
2591
|
}
|
|
2497
2592
|
|
|
2498
2593
|
// Stage 2: bounded-parallel HEAD pass. Promise.all over chunks of size
|
package/src/cli/sync.test.ts
CHANGED
|
@@ -84,7 +84,10 @@ const mockVendResponse = {
|
|
|
84
84
|
};
|
|
85
85
|
|
|
86
86
|
function setupFetchMock(
|
|
87
|
-
opts: {
|
|
87
|
+
opts: {
|
|
88
|
+
tombstones?: Array<{ key: string; deletedAt: string }>;
|
|
89
|
+
tombstoneFailure?: boolean;
|
|
90
|
+
} = {},
|
|
88
91
|
) {
|
|
89
92
|
const fetchMock = vi.fn().mockImplementation(async (url: string) => {
|
|
90
93
|
const urlStr = String(url);
|
|
@@ -92,6 +95,7 @@ function setupFetchMock(
|
|
|
92
95
|
// every legacy test exercises the no-suppression path; tests that need
|
|
93
96
|
// suppression pass `{ tombstones }`.
|
|
94
97
|
if (urlStr.includes("/v1/files/tombstones")) {
|
|
98
|
+
if (opts.tombstoneFailure) throw new Error("tombstone lookup unavailable");
|
|
95
99
|
return {
|
|
96
100
|
ok: true,
|
|
97
101
|
status: 200,
|
|
@@ -1085,6 +1089,53 @@ describe("sync", () => {
|
|
|
1085
1089
|
expect(journal.files[key]?.removedAt).toBeUndefined();
|
|
1086
1090
|
});
|
|
1087
1091
|
|
|
1092
|
+
it("tombstone lookup failure does not resurrect a current intent-backed delete", async () => {
|
|
1093
|
+
const key = "docs/watcher-deleted.md";
|
|
1094
|
+
const localPath = path.join(tmpDir, "companies", "acme", key);
|
|
1095
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
1096
|
+
{ key, size: 42, lastModified: new Date(), etag: '"cur123"' },
|
|
1097
|
+
]);
|
|
1098
|
+
setupFetchMock({ tombstoneFailure: true });
|
|
1099
|
+
fs.writeFileSync(
|
|
1100
|
+
journalPath,
|
|
1101
|
+
JSON.stringify({
|
|
1102
|
+
version: "2",
|
|
1103
|
+
lastSync: new Date().toISOString(),
|
|
1104
|
+
files: {
|
|
1105
|
+
[key]: {
|
|
1106
|
+
hash: "synced-hash",
|
|
1107
|
+
size: 42,
|
|
1108
|
+
syncedAt: new Date().toISOString(),
|
|
1109
|
+
direction: "down",
|
|
1110
|
+
kind: "file",
|
|
1111
|
+
remoteEtag: "cur123",
|
|
1112
|
+
localDeleteIntent: {
|
|
1113
|
+
version: 1,
|
|
1114
|
+
remoteEtag: "cur123",
|
|
1115
|
+
localHash: "synced-hash",
|
|
1116
|
+
localKind: "file",
|
|
1117
|
+
},
|
|
1118
|
+
},
|
|
1119
|
+
},
|
|
1120
|
+
pulls: [],
|
|
1121
|
+
}),
|
|
1122
|
+
);
|
|
1123
|
+
|
|
1124
|
+
await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
|
|
1125
|
+
|
|
1126
|
+
expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
|
|
1127
|
+
expect.anything(),
|
|
1128
|
+
key,
|
|
1129
|
+
expect.anything(),
|
|
1130
|
+
expect.anything(),
|
|
1131
|
+
);
|
|
1132
|
+
expect(fs.existsSync(localPath)).toBe(false);
|
|
1133
|
+
expect(
|
|
1134
|
+
JSON.parse(fs.readFileSync(journalPath, "utf8")).files[key]
|
|
1135
|
+
?.removedReason,
|
|
1136
|
+
).toBe("local-delete");
|
|
1137
|
+
});
|
|
1138
|
+
|
|
1088
1139
|
it("pull producer: a BULK disappearance is drift-restored, NOT tombstoned (ridge-incident guard in the pull leg)", async () => {
|
|
1089
1140
|
// 12 clean files vanish at once (corrupt mirror / bulk rm / checkout). The
|
|
1090
1141
|
// bulk-asymmetry guard refuses to treat this as intent — every file is
|
|
@@ -3480,6 +3531,104 @@ describe("sync", () => {
|
|
|
3480
3531
|
});
|
|
3481
3532
|
});
|
|
3482
3533
|
|
|
3534
|
+
it("converges after reindex prunes a downloaded generated core mirror (DEV-1974)", async () => {
|
|
3535
|
+
// Legacy personal-vault objects may be generated links below core/<type>.
|
|
3536
|
+
// A pull materializes the link, then reindex removes it because its
|
|
3537
|
+
// canonical source is elsewhere. The second pull must recognize the
|
|
3538
|
+
// journaled symlink as generated state rather than downloading it again.
|
|
3539
|
+
const linkKey = "core/policies/legacy-mirror.md";
|
|
3540
|
+
const linkPath = path.join(tmpDir, linkKey);
|
|
3541
|
+
const linkTarget = "../../personal/policies/legacy-mirror.md";
|
|
3542
|
+
const remote = {
|
|
3543
|
+
key: linkKey,
|
|
3544
|
+
size: 48,
|
|
3545
|
+
lastModified: new Date(),
|
|
3546
|
+
etag: '"legacy-mirror"',
|
|
3547
|
+
};
|
|
3548
|
+
|
|
3549
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValue([remote]);
|
|
3550
|
+
vi.mocked(s3Module.downloadFile).mockImplementation(
|
|
3551
|
+
async (_ctx: unknown, _key: string, localPath: string) => {
|
|
3552
|
+
fs.mkdirSync(path.dirname(localPath), { recursive: true });
|
|
3553
|
+
fs.symlinkSync(linkTarget, localPath);
|
|
3554
|
+
return { metadata: {} };
|
|
3555
|
+
},
|
|
3556
|
+
);
|
|
3557
|
+
|
|
3558
|
+
const first = await sync({
|
|
3559
|
+
company: "acme",
|
|
3560
|
+
vaultConfig: mockConfig,
|
|
3561
|
+
hqRoot: tmpDir,
|
|
3562
|
+
personalMode: true,
|
|
3563
|
+
});
|
|
3564
|
+
expect(first.filesDownloaded).toBe(1);
|
|
3565
|
+
expect(fs.lstatSync(linkPath).isSymbolicLink()).toBe(true);
|
|
3566
|
+
|
|
3567
|
+
// The real post-sync reindex removes this retired mirror. The unit test
|
|
3568
|
+
// mocks reindex globally, so make that lifecycle transition explicit.
|
|
3569
|
+
fs.unlinkSync(linkPath);
|
|
3570
|
+
|
|
3571
|
+
const second = await sync({
|
|
3572
|
+
company: "acme",
|
|
3573
|
+
vaultConfig: mockConfig,
|
|
3574
|
+
hqRoot: tmpDir,
|
|
3575
|
+
personalMode: true,
|
|
3576
|
+
});
|
|
3577
|
+
|
|
3578
|
+
expect(second.filesDownloaded).toBe(0);
|
|
3579
|
+
expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
|
|
3580
|
+
});
|
|
3581
|
+
|
|
3582
|
+
it("pulls a changed canonical object after a generated core mirror journal entry", async () => {
|
|
3583
|
+
const key = "core/policies/legacy-mirror.md";
|
|
3584
|
+
const localPath = path.join(tmpDir, key);
|
|
3585
|
+
const legacyMirror = {
|
|
3586
|
+
key,
|
|
3587
|
+
size: 48,
|
|
3588
|
+
lastModified: new Date(),
|
|
3589
|
+
etag: '"legacy-mirror"',
|
|
3590
|
+
};
|
|
3591
|
+
const canonicalReplacement = {
|
|
3592
|
+
...legacyMirror,
|
|
3593
|
+
lastModified: new Date(Date.now() + 1_000),
|
|
3594
|
+
etag: '"canonical-replacement"',
|
|
3595
|
+
};
|
|
3596
|
+
|
|
3597
|
+
vi.mocked(s3Module.listRemoteFiles)
|
|
3598
|
+
.mockResolvedValueOnce([legacyMirror])
|
|
3599
|
+
.mockResolvedValueOnce([canonicalReplacement]);
|
|
3600
|
+
vi.mocked(s3Module.downloadFile)
|
|
3601
|
+
.mockImplementationOnce(async (_ctx: unknown, _key: string, destination: string) => {
|
|
3602
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
3603
|
+
fs.symlinkSync("../../personal/policies/legacy-mirror.md", destination);
|
|
3604
|
+
return { metadata: {} };
|
|
3605
|
+
})
|
|
3606
|
+
.mockImplementationOnce(async (_ctx: unknown, _key: string, destination: string) => {
|
|
3607
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
3608
|
+
fs.writeFileSync(destination, "canonical replacement\n");
|
|
3609
|
+
return { metadata: {} };
|
|
3610
|
+
});
|
|
3611
|
+
|
|
3612
|
+
await sync({
|
|
3613
|
+
company: "acme",
|
|
3614
|
+
vaultConfig: mockConfig,
|
|
3615
|
+
hqRoot: tmpDir,
|
|
3616
|
+
personalMode: true,
|
|
3617
|
+
});
|
|
3618
|
+
fs.unlinkSync(localPath);
|
|
3619
|
+
|
|
3620
|
+
const second = await sync({
|
|
3621
|
+
company: "acme",
|
|
3622
|
+
vaultConfig: mockConfig,
|
|
3623
|
+
hqRoot: tmpDir,
|
|
3624
|
+
personalMode: true,
|
|
3625
|
+
});
|
|
3626
|
+
|
|
3627
|
+
expect(second.filesDownloaded).toBe(1);
|
|
3628
|
+
expect(fs.readFileSync(localPath, "utf8")).toBe("canonical replacement\n");
|
|
3629
|
+
expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(2);
|
|
3630
|
+
});
|
|
3631
|
+
|
|
3483
3632
|
it("does not crash when local is a real directory at a remote-key path (EISDIR guard)", async () => {
|
|
3484
3633
|
// Codex round-9 P2 follow-up: pre-fix, the pull planner's
|
|
3485
3634
|
// post-lstat branch always called hashFile (readFileSync) on a
|
package/src/cli/sync.ts
CHANGED
|
@@ -48,7 +48,10 @@ import {
|
|
|
48
48
|
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
49
49
|
migratePersonalVaultJournal,
|
|
50
50
|
} from "../journal.js";
|
|
51
|
-
import {
|
|
51
|
+
import {
|
|
52
|
+
isGeneratedCoreMirrorKey,
|
|
53
|
+
PERSONAL_VAULT_MANIFEST_KEY,
|
|
54
|
+
} from "../personal-vault.js";
|
|
52
55
|
import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
|
|
53
56
|
import {
|
|
54
57
|
buildScopeShrinkPlan,
|
|
@@ -2122,33 +2125,6 @@ interface PullPlan {
|
|
|
2122
2125
|
const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
|
|
2123
2126
|
const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
|
|
2124
2127
|
|
|
2125
|
-
// The four release-shipped `core/<type>/` namespaces the personal-overlay
|
|
2126
|
-
// mirror (reindex) materializes into. A stale mirror symlink pushed to the
|
|
2127
|
-
// vault lands as a single object at exactly `core/<type>/<...>/<entry>`, and if
|
|
2128
|
-
// a later release ships a REAL directory of the same name, pull sees a
|
|
2129
|
-
// (local real dir) vs (remote single object) collision at that key forever.
|
|
2130
|
-
const OVERLAY_MIRROR_CORE_TYPES = ["knowledge", "policies", "workers", "settings"];
|
|
2131
|
-
|
|
2132
|
-
/**
|
|
2133
|
-
* True when `key` is under a release-shipped personal-overlay-mirror core
|
|
2134
|
-
* namespace (`core/{knowledge,policies,workers,settings}/<entry>/...`).
|
|
2135
|
-
*
|
|
2136
|
-
* A remote LIST entry is always a single S3 object. A REAL release-shipped core
|
|
2137
|
-
* directory only ever syncs as child objects UNDER its key
|
|
2138
|
-
* (`core/knowledge/public/agent-browser/INDEX.md`), never as one object AT the
|
|
2139
|
-
* directory's own key — so a single object at such a key, colliding with a
|
|
2140
|
-
* local real directory, is unambiguously a stale overlay symlink marker, not
|
|
2141
|
-
* live core content. Pure path test — no I/O — safe inside the sync planner.
|
|
2142
|
-
*/
|
|
2143
|
-
function isOverlayMirrorCoreKey(key: string): boolean {
|
|
2144
|
-
const parts = key.split("/");
|
|
2145
|
-
return (
|
|
2146
|
-
parts.length >= 3 &&
|
|
2147
|
-
parts[0] === "core" &&
|
|
2148
|
-
OVERLAY_MIRROR_CORE_TYPES.includes(parts[1])
|
|
2149
|
-
);
|
|
2150
|
-
}
|
|
2151
|
-
|
|
2152
2128
|
function computePullPlan(
|
|
2153
2129
|
remoteFiles: RemoteFile[],
|
|
2154
2130
|
journal: SyncJournal,
|
|
@@ -2305,6 +2281,26 @@ function computePullPlan(
|
|
|
2305
2281
|
|
|
2306
2282
|
const journalEntry = getEntry(journal, remoteFile.key);
|
|
2307
2283
|
|
|
2284
|
+
// Reindex and package wiring own symlinks under these `core/<type>`
|
|
2285
|
+
// surfaces; their canonical sources are elsewhere. A prior pull records
|
|
2286
|
+
// the remote object as a symlink, then reindex removes the generated link.
|
|
2287
|
+
// Without this classification the next pull sees only an absent local path
|
|
2288
|
+
// and downloads the same marker forever. Retain regular core files: the
|
|
2289
|
+
// journal kind is the discriminator, because LIST exposes no object type.
|
|
2290
|
+
// An unchanged ETag is also required: a remote writer can replace a legacy
|
|
2291
|
+
// symlink with canonical file content at the same key, and that replacement
|
|
2292
|
+
// must still be pulled.
|
|
2293
|
+
if (
|
|
2294
|
+
personalMode &&
|
|
2295
|
+
journalEntry?.kind === "symlink" &&
|
|
2296
|
+
isGeneratedCoreMirrorKey(remoteFile.key) &&
|
|
2297
|
+
journalEntry.remoteEtag !== undefined &&
|
|
2298
|
+
!hasRemoteChanged(remoteFile, journalEntry)
|
|
2299
|
+
) {
|
|
2300
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2301
|
+
continue;
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2308
2304
|
// ── FILE_TOMBSTONE consult (delete-resync) ───────────────────────────────
|
|
2309
2305
|
// A remote object present in the LIST may be an intentionally-deleted key
|
|
2310
2306
|
// (a peer re-pushed it, or its delete-marker hasn't propagated to this
|
|
@@ -2414,7 +2410,7 @@ function computePullPlan(
|
|
|
2414
2410
|
// "rm -rf the local directory" advice below is actively dangerous here
|
|
2415
2411
|
// (it would destroy release-shipped core content to pull inert derived
|
|
2416
2412
|
// state), and it recurred on every sync. feedback_ce9aeede.
|
|
2417
|
-
if (
|
|
2413
|
+
if (isGeneratedCoreMirrorKey(remoteFile.key)) {
|
|
2418
2414
|
console.error(
|
|
2419
2415
|
` Note: ${remoteFile.key} is a stale personal-overlay marker ` +
|
|
2420
2416
|
`shadowing a release-shipped core directory; keeping the local ` +
|
package/src/journal.test.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
30
30
|
migratePersonalVaultJournal,
|
|
31
31
|
listJournals,
|
|
32
|
+
markLocalDeleteIntent,
|
|
32
33
|
} from "./journal.js";
|
|
33
34
|
import type { SyncJournal, PullRecord } from "./types.js";
|
|
34
35
|
|
|
@@ -745,6 +746,50 @@ describe("journal", () => {
|
|
|
745
746
|
// successful sync because it read one path keyed off the HQ-ROOT, while the
|
|
746
747
|
// engine writes slug-sharded journals (personal vault + per-company).
|
|
747
748
|
// `listJournals` is the all-shard enumeration the status surface now uses.
|
|
749
|
+
it("binds a local delete intent to the captured remote revision", () => {
|
|
750
|
+
const journal: SyncJournal = {
|
|
751
|
+
version: "2",
|
|
752
|
+
lastSync: "",
|
|
753
|
+
pulls: [],
|
|
754
|
+
files: {
|
|
755
|
+
"docs/a.md": {
|
|
756
|
+
hash: "local-hash",
|
|
757
|
+
size: 1,
|
|
758
|
+
syncedAt: new Date().toISOString(),
|
|
759
|
+
direction: "down",
|
|
760
|
+
kind: "file",
|
|
761
|
+
remoteEtag: "new-etag",
|
|
762
|
+
},
|
|
763
|
+
},
|
|
764
|
+
};
|
|
765
|
+
|
|
766
|
+
expect(
|
|
767
|
+
markLocalDeleteIntent(
|
|
768
|
+
journal,
|
|
769
|
+
"docs/a.md",
|
|
770
|
+
"local-hash",
|
|
771
|
+
"file",
|
|
772
|
+
"old-etag",
|
|
773
|
+
),
|
|
774
|
+
).toBe(false);
|
|
775
|
+
expect(journal.files["docs/a.md"]?.localDeleteIntent).toBeUndefined();
|
|
776
|
+
|
|
777
|
+
expect(
|
|
778
|
+
markLocalDeleteIntent(
|
|
779
|
+
journal,
|
|
780
|
+
"docs/a.md",
|
|
781
|
+
"local-hash",
|
|
782
|
+
"file",
|
|
783
|
+
"new-etag",
|
|
784
|
+
),
|
|
785
|
+
).toBe(true);
|
|
786
|
+
expect(journal.files["docs/a.md"]?.localDeleteIntent).toMatchObject({
|
|
787
|
+
remoteEtag: "new-etag",
|
|
788
|
+
localHash: "local-hash",
|
|
789
|
+
localKind: "file",
|
|
790
|
+
});
|
|
791
|
+
});
|
|
792
|
+
|
|
748
793
|
describe("listJournals", () => {
|
|
749
794
|
function seed(slug: string, files: SyncJournal["files"]): void {
|
|
750
795
|
writeJournal(slug, {
|