@indigoai-us/hq-cloud 6.13.2 → 6.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +15 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-rollup.d.ts +7 -0
- package/dist/bin/sync-runner-rollup.d.ts.map +1 -1
- package/dist/bin/sync-runner-rollup.js +15 -0
- package/dist/bin/sync-runner-rollup.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +7 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +27 -3
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +73 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +38 -0
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +77 -0
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/share.js +15 -2
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +39 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +22 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +60 -1
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +88 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/qmd-reindex.d.ts +14 -0
- package/dist/qmd-reindex.d.ts.map +1 -1
- package/dist/qmd-reindex.js +93 -19
- package/dist/qmd-reindex.js.map +1 -1
- package/dist/qmd-reindex.test.js +47 -0
- package/dist/qmd-reindex.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +21 -0
- package/src/bin/sync-runner-rollup.ts +23 -0
- package/src/bin/sync-runner.test.ts +90 -0
- package/src/bin/sync-runner.ts +37 -5
- package/src/cli/reindex.test.ts +92 -0
- package/src/cli/reindex.ts +35 -0
- package/src/cli/share.test.ts +46 -0
- package/src/cli/share.ts +14 -2
- package/src/cli/sync.test.ts +105 -0
- package/src/cli/sync.ts +101 -1
- package/src/qmd-reindex.test.ts +50 -0
- package/src/qmd-reindex.ts +124 -18
package/src/cli/reindex.test.ts
CHANGED
|
@@ -203,6 +203,98 @@ describe("reindex", () => {
|
|
|
203
203
|
).toBe("../../../core/skills/demo/SKILL.md");
|
|
204
204
|
});
|
|
205
205
|
|
|
206
|
+
// ── Cleanup pass C: orphan personal-overlay mirror symlinks ──────────────
|
|
207
|
+
// The mirror creates core/<type>/<entry> -> ../../personal/<type>/<entry>.
|
|
208
|
+
// When the personal source is later deleted/renamed, the next reindex must
|
|
209
|
+
// prune the now-dangling core symlink — the gap that used to leave a retired
|
|
210
|
+
// personal policy pointer surfacing in core/policies/ until hand-deleted.
|
|
211
|
+
|
|
212
|
+
it("prunes an orphan personal-overlay mirror symlink when the source is deleted", () => {
|
|
213
|
+
fs.mkdirSync(path.join(root, "personal/policies"), { recursive: true });
|
|
214
|
+
fs.writeFileSync(path.join(root, "personal/policies/myrule.md"), "rule\n");
|
|
215
|
+
reindex({ repoRoot: root });
|
|
216
|
+
const link = path.join(root, "core/policies/myrule.md");
|
|
217
|
+
expect(fs.lstatSync(link).isSymbolicLink()).toBe(true);
|
|
218
|
+
|
|
219
|
+
// Retire the personal source, then reindex again.
|
|
220
|
+
fs.rmSync(path.join(root, "personal/policies/myrule.md"));
|
|
221
|
+
expect(reindex({ repoRoot: root }).status).toBe(0);
|
|
222
|
+
|
|
223
|
+
// The dangling mirror symlink is gone — no entry at all (lstat throws).
|
|
224
|
+
expect(fs.existsSync(link)).toBe(false);
|
|
225
|
+
expect(() => fs.lstatSync(link)).toThrow();
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it("prunes mirror symlinks when the ENTIRE personal/<type>/ directory is removed", () => {
|
|
229
|
+
fs.mkdirSync(path.join(root, "personal/policies"), { recursive: true });
|
|
230
|
+
fs.writeFileSync(path.join(root, "personal/policies/a.md"), "a\n");
|
|
231
|
+
fs.writeFileSync(path.join(root, "personal/policies/b.md"), "b\n");
|
|
232
|
+
reindex({ repoRoot: root });
|
|
233
|
+
expect(fs.lstatSync(path.join(root, "core/policies/a.md")).isSymbolicLink()).toBe(true);
|
|
234
|
+
expect(fs.lstatSync(path.join(root, "core/policies/b.md")).isSymbolicLink()).toBe(true);
|
|
235
|
+
|
|
236
|
+
// Nuke the whole personal/policies dir (the creation loop's isDir guard would
|
|
237
|
+
// skip the type entirely — the prune must still run and clean both links).
|
|
238
|
+
fs.rmSync(path.join(root, "personal/policies"), { recursive: true, force: true });
|
|
239
|
+
expect(reindex({ repoRoot: root }).status).toBe(0);
|
|
240
|
+
|
|
241
|
+
expect(fs.existsSync(path.join(root, "core/policies/a.md"))).toBe(false);
|
|
242
|
+
expect(fs.existsSync(path.join(root, "core/policies/b.md"))).toBe(false);
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
it("keeps a LIVE personal-overlay mirror symlink (only orphans are pruned)", () => {
|
|
246
|
+
fs.mkdirSync(path.join(root, "personal/policies"), { recursive: true });
|
|
247
|
+
fs.writeFileSync(path.join(root, "personal/policies/keep.md"), "keep\n");
|
|
248
|
+
fs.writeFileSync(path.join(root, "personal/policies/drop.md"), "drop\n");
|
|
249
|
+
reindex({ repoRoot: root });
|
|
250
|
+
|
|
251
|
+
// Retire only one source; the other stays live.
|
|
252
|
+
fs.rmSync(path.join(root, "personal/policies/drop.md"));
|
|
253
|
+
reindex({ repoRoot: root });
|
|
254
|
+
|
|
255
|
+
const keep = path.join(root, "core/policies/keep.md");
|
|
256
|
+
expect(fs.lstatSync(keep).isSymbolicLink()).toBe(true);
|
|
257
|
+
expect(fs.readlinkSync(keep)).toBe("../../personal/policies/keep.md");
|
|
258
|
+
expect(fs.readFileSync(keep, "utf-8")).toBe("keep\n");
|
|
259
|
+
expect(fs.existsSync(path.join(root, "core/policies/drop.md"))).toBe(false);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it("prunes an orphan personal-overlay mirror DIRECTORY symlink (DEV-1767 shape)", () => {
|
|
263
|
+
// personal/knowledge/<kb>/ mirrors as a directory symlink; deleting the kb
|
|
264
|
+
// must prune the dangling directory symlink too (rmSync on a symlink, not
|
|
265
|
+
// recursive into a resolved dir).
|
|
266
|
+
fs.mkdirSync(path.join(root, "personal/knowledge/my-kb"), { recursive: true });
|
|
267
|
+
fs.writeFileSync(path.join(root, "personal/knowledge/my-kb/note.md"), "n\n");
|
|
268
|
+
reindex({ repoRoot: root });
|
|
269
|
+
const link = path.join(root, "core/knowledge/my-kb");
|
|
270
|
+
expect(fs.lstatSync(link).isSymbolicLink()).toBe(true);
|
|
271
|
+
|
|
272
|
+
fs.rmSync(path.join(root, "personal/knowledge/my-kb"), { recursive: true, force: true });
|
|
273
|
+
expect(reindex({ repoRoot: root }).status).toBe(0);
|
|
274
|
+
expect(fs.existsSync(link)).toBe(false);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
it("leaves a real core file and a foreign symlink in core/<type>/ untouched (cross-check invariant)", () => {
|
|
278
|
+
fs.mkdirSync(path.join(root, "core/policies"), { recursive: true });
|
|
279
|
+
// (a) a real release-shipped core file — not a symlink, never ours.
|
|
280
|
+
fs.writeFileSync(path.join(root, "core/policies/shipped.md"), "shipped\n");
|
|
281
|
+
// (b) a human-authored symlink in core/policies pointing SOMEWHERE ELSE
|
|
282
|
+
// (not the managed ../../personal/policies/<name> shape) — even though
|
|
283
|
+
// it is dangling, its target doesn't match, so it must survive.
|
|
284
|
+
fs.symlinkSync("../../somewhere/else.md", path.join(root, "core/policies/foreign.md"));
|
|
285
|
+
// (c) a symlink whose target's basename disagrees with its own name — the
|
|
286
|
+
// exact cross-check catches this: name 'mismatch.md' but target points
|
|
287
|
+
// at personal/policies/other.md → NOT managed, left alone.
|
|
288
|
+
fs.symlinkSync("../../personal/policies/other.md", path.join(root, "core/policies/mismatch.md"));
|
|
289
|
+
|
|
290
|
+
expect(reindex({ repoRoot: root }).status).toBe(0);
|
|
291
|
+
|
|
292
|
+
expect(fs.lstatSync(path.join(root, "core/policies/shipped.md")).isSymbolicLink()).toBe(false);
|
|
293
|
+
expect(fs.existsSync(path.join(root, "core/policies/shipped.md"))).toBe(true);
|
|
294
|
+
expect(fs.lstatSync(path.join(root, "core/policies/foreign.md")).isSymbolicLink()).toBe(true);
|
|
295
|
+
expect(fs.lstatSync(path.join(root, "core/policies/mismatch.md")).isSymbolicLink()).toBe(true);
|
|
296
|
+
});
|
|
297
|
+
|
|
206
298
|
it("skips `_`-prefixed and dotfile skill folders", () => {
|
|
207
299
|
writeSkill("core/skills/_template");
|
|
208
300
|
fs.mkdirSync(path.join(root, "core/skills/.hidden"), { recursive: true });
|
package/src/cli/reindex.ts
CHANGED
|
@@ -487,6 +487,41 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
|
|
|
487
487
|
}
|
|
488
488
|
}
|
|
489
489
|
|
|
490
|
+
// --- Cleanup pass C: drop orphan personal-overlay mirror symlinks ----------
|
|
491
|
+
// The mirror above creates core/<type>/<entry> -> ../../personal/<type>/<entry>
|
|
492
|
+
// for each LIVE personal entry, but historically never removed that symlink
|
|
493
|
+
// when its personal source was later deleted or renamed — leaving a dangling
|
|
494
|
+
// link in core/<type>/ (e.g. a retired personal policy that still surfaced via
|
|
495
|
+
// the policy-trigger hook until a human hand-deleted it). This pass prunes it.
|
|
496
|
+
//
|
|
497
|
+
// Ownership discriminator (policy wrapper-sync-orphan-cleanup-cross-check): a
|
|
498
|
+
// core/<type>/<entry> is "managed by this mirror" ONLY when it is a symlink
|
|
499
|
+
// whose target is EXACTLY `../../personal/<type>/<entry>` — i.e. the target
|
|
500
|
+
// encodes the same <type> and <entry> as the link's own location and name.
|
|
501
|
+
// That exact cross-check means a real release-shipped core file (not a
|
|
502
|
+
// symlink), or a human-authored symlink pointing anywhere else, never matches
|
|
503
|
+
// and is left untouched. A managed link is pruned IFF its target no longer
|
|
504
|
+
// resolves (existsFollow=false) — which, given the exact-target shape, is
|
|
505
|
+
// precisely "the personal source was removed". This runs independently of the
|
|
506
|
+
// creation loop above (note: no `isDir(personalDir)` guard) so that deleting
|
|
507
|
+
// the ENTIRE personal/<type>/ directory still prunes every mirror it produced.
|
|
508
|
+
for (const type of ["knowledge", "policies", "workers", "settings"]) {
|
|
509
|
+
const coreDir = path.join(root, "core", type);
|
|
510
|
+
for (const entry of globEntries(coreDir)) {
|
|
511
|
+
const linkPath = path.join(coreDir, entry);
|
|
512
|
+
const lst = lstatOrNull(linkPath);
|
|
513
|
+
if (!lst || !lst.isSymbolicLink()) continue; // real file/dir → not ours
|
|
514
|
+
if (readlinkOrNull(linkPath) !== `../../personal/${type}/${entry}`) continue; // not a managed mirror link
|
|
515
|
+
if (existsFollow(linkPath)) continue; // live: personal source still present
|
|
516
|
+
// Managed mirror symlink whose personal source is gone → prune.
|
|
517
|
+
try {
|
|
518
|
+
fs.rmSync(linkPath);
|
|
519
|
+
} catch {
|
|
520
|
+
/* best-effort */
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
490
525
|
// --- Workers registry regeneration ----------------------------------------
|
|
491
526
|
// Source of truth: each worker.yaml. The registry is a derived index — the
|
|
492
527
|
// generator (when present in the operated-on tree) keeps it in sync.
|
package/src/cli/share.test.ts
CHANGED
|
@@ -1264,6 +1264,52 @@ describe("share", () => {
|
|
|
1264
1264
|
expect(events.some((e) => e.type === "error")).toBe(false);
|
|
1265
1265
|
});
|
|
1266
1266
|
|
|
1267
|
+
it("a file over the size limit is a benign skip-size-limit event, NEVER an error (HQ-SYNC-4 exit-2 flood)", async () => {
|
|
1268
|
+
// Regression for the fleet-wide "auto-sync watcher exited unexpectedly
|
|
1269
|
+
// (code=Some(2))" flood (HQ-SYNC-4, ~93k events across 60+ machines). An
|
|
1270
|
+
// oversized file is a permanent, benign skip — it exceeds the 50 MB cap on
|
|
1271
|
+
// EVERY pass, forever. Emitting it as `type: "error"` pushed it into the
|
|
1272
|
+
// runner's `errors[]`, so the pass returned exit 2, which the menubar
|
|
1273
|
+
// supervisor reported as an unexpected watcher crash on every single tick.
|
|
1274
|
+
// A too-big file must surface for visibility but must NOT count as an error.
|
|
1275
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
1276
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
1277
|
+
const bigFile = path.join(companyRoot, "huge.mp4");
|
|
1278
|
+
// Sparse 51 MB file: statSync (isWithinSizeLimit) reports 51 MB while the
|
|
1279
|
+
// file occupies ~0 bytes on disk, so the test stays fast.
|
|
1280
|
+
const fd = fs.openSync(bigFile, "w");
|
|
1281
|
+
fs.ftruncateSync(fd, 51 * 1024 * 1024);
|
|
1282
|
+
fs.closeSync(fd);
|
|
1283
|
+
|
|
1284
|
+
const events: Array<{ type?: string; path?: string; bytes?: number }> = [];
|
|
1285
|
+
const result = await share({
|
|
1286
|
+
paths: [bigFile],
|
|
1287
|
+
company: "acme",
|
|
1288
|
+
vaultConfig: mockConfig,
|
|
1289
|
+
hqRoot: tmpDir,
|
|
1290
|
+
onEvent: (e) => events.push(e as { type?: string }),
|
|
1291
|
+
});
|
|
1292
|
+
|
|
1293
|
+
// Not uploaded, counted as a skip, company did not abort.
|
|
1294
|
+
expect(result.aborted).toBe(false);
|
|
1295
|
+
expect(result.filesUploaded).toBe(0);
|
|
1296
|
+
expect(result.filesSkipped).toBe(1);
|
|
1297
|
+
expect(uploadFile).not.toHaveBeenCalled();
|
|
1298
|
+
|
|
1299
|
+
// The oversized file surfaces as a benign skip-size-limit event for
|
|
1300
|
+
// visibility (path + byte size)…
|
|
1301
|
+
const skipEv = events.find((e) => e.type === "skip-size-limit") as
|
|
1302
|
+
| { path: string; bytes: number }
|
|
1303
|
+
| undefined;
|
|
1304
|
+
expect(skipEv).toBeDefined();
|
|
1305
|
+
expect(skipEv!.path).toBe("huge.mp4");
|
|
1306
|
+
expect(skipEv!.bytes).toBe(51 * 1024 * 1024);
|
|
1307
|
+
|
|
1308
|
+
// …and CRUCIALLY never as an error — an error event is what flipped the
|
|
1309
|
+
// pass to exit 2 and drove the watcher-crash flood.
|
|
1310
|
+
expect(events.some((e) => e.type === "error")).toBe(false);
|
|
1311
|
+
});
|
|
1312
|
+
|
|
1267
1313
|
it("uploads (no conflict) when only the local side changed since last sync", async () => {
|
|
1268
1314
|
// Regression for hq-cloud#<conflict-detection>: a local edit to a file
|
|
1269
1315
|
// that exists on S3 used to trigger a push conflict because the
|
package/src/cli/share.ts
CHANGED
|
@@ -1135,10 +1135,22 @@ async function executeUploads(
|
|
|
1135
1135
|
const uploadItems: UploadPlanItem[] = [];
|
|
1136
1136
|
for (const item of pushPlan.items) {
|
|
1137
1137
|
if (item.action === "skip-size-limit") {
|
|
1138
|
+
// A file over the size cap is a permanent, benign skip — NOT an error.
|
|
1139
|
+
// Emitting it as `type: "error"` pushed it into the runner's `errors[]`,
|
|
1140
|
+
// so every watch pass returned exit 2 and the menubar reported an
|
|
1141
|
+
// "auto-sync watcher exited unexpectedly (code=Some(2))" crash on every
|
|
1142
|
+
// tick (the fleet-wide HQ-SYNC-4 flood). Surface it for visibility, but
|
|
1143
|
+
// do not let it flip the pass to a non-zero exit.
|
|
1144
|
+
let bytes = 0;
|
|
1145
|
+
try {
|
|
1146
|
+
bytes = fs.statSync(item.absolutePath).size;
|
|
1147
|
+
} catch {
|
|
1148
|
+
// Best-effort size for display only; a stat race must not break the sync.
|
|
1149
|
+
}
|
|
1138
1150
|
run.emit({
|
|
1139
|
-
type: "
|
|
1151
|
+
type: "skip-size-limit",
|
|
1140
1152
|
path: item.relativePath,
|
|
1141
|
-
|
|
1153
|
+
bytes,
|
|
1142
1154
|
});
|
|
1143
1155
|
counters.filesSkipped++;
|
|
1144
1156
|
continue;
|
package/src/cli/sync.test.ts
CHANGED
|
@@ -3141,6 +3141,111 @@ describe("sync", () => {
|
|
|
3141
3141
|
expect(fs.existsSync(path.join(localDir, "inside.md"))).toBe(true);
|
|
3142
3142
|
});
|
|
3143
3143
|
|
|
3144
|
+
it("stale personal-overlay marker vs release-shipped core dir: skips cleanly, no 'rm -rf' advice (feedback_ce9aeede)", async () => {
|
|
3145
|
+
// Reporter feedback_ce9aeede: a stale personal-overlay symlink marker in
|
|
3146
|
+
// the vault at core/knowledge/public/agent-browser collided with the REAL,
|
|
3147
|
+
// release-shipped core directory of the same name (the overlay mirror once
|
|
3148
|
+
// symlinked it into core/, pushed the marker, then a release shipped a real
|
|
3149
|
+
// dir there). Pre-fix, the pull planner hit the generic dir-vs-object
|
|
3150
|
+
// collision branch and emitted, on EVERY sync, a "manual reconciliation
|
|
3151
|
+
// required (rm -rf the local directory to pull)" warning — advice that here
|
|
3152
|
+
// would DESTROY release-shipped core content to pull an inert marker. The
|
|
3153
|
+
// reporter had to hand-delete the vault object. Fix: recognize the
|
|
3154
|
+
// overlay-mirror core key, keep the local directory, and skip quietly.
|
|
3155
|
+
const markerKey = "core/knowledge/public/agent-browser";
|
|
3156
|
+
const localDir = path.join(
|
|
3157
|
+
tmpDir,
|
|
3158
|
+
"core",
|
|
3159
|
+
"knowledge",
|
|
3160
|
+
"public",
|
|
3161
|
+
"agent-browser",
|
|
3162
|
+
);
|
|
3163
|
+
fs.mkdirSync(localDir, { recursive: true });
|
|
3164
|
+
fs.writeFileSync(
|
|
3165
|
+
path.join(localDir, "INDEX.md"),
|
|
3166
|
+
"release-shipped core content",
|
|
3167
|
+
);
|
|
3168
|
+
|
|
3169
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
3170
|
+
{ key: markerKey, size: 40, lastModified: new Date(), etag: '"stale-marker"' },
|
|
3171
|
+
]);
|
|
3172
|
+
const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
|
3173
|
+
|
|
3174
|
+
let crashed = false;
|
|
3175
|
+
let result;
|
|
3176
|
+
try {
|
|
3177
|
+
result = await sync({
|
|
3178
|
+
company: "acme",
|
|
3179
|
+
vaultConfig: mockConfig,
|
|
3180
|
+
hqRoot: tmpDir,
|
|
3181
|
+
// personalMode → companyRoot === hqRoot, so a `core/...` key resolves
|
|
3182
|
+
// under the HQ root exactly as the reporter's personal-vault pull did.
|
|
3183
|
+
personalMode: true,
|
|
3184
|
+
});
|
|
3185
|
+
} catch {
|
|
3186
|
+
crashed = true;
|
|
3187
|
+
}
|
|
3188
|
+
const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
|
|
3189
|
+
errSpy.mockRestore();
|
|
3190
|
+
|
|
3191
|
+
expect(crashed).toBe(false);
|
|
3192
|
+
expect(result).toBeDefined();
|
|
3193
|
+
// Marker ignored, nothing downloaded, release-shipped dir intact.
|
|
3194
|
+
expect(result!.filesDownloaded).toBe(0);
|
|
3195
|
+
expect(result!.filesSkipped).toBe(1);
|
|
3196
|
+
expect(fs.readFileSync(path.join(localDir, "INDEX.md"), "utf-8")).toBe(
|
|
3197
|
+
"release-shipped core content",
|
|
3198
|
+
);
|
|
3199
|
+
// The dangerous, recurring reconciliation advice is GONE...
|
|
3200
|
+
expect(errOutput).not.toContain("rm -rf");
|
|
3201
|
+
expect(errOutput).not.toContain("manual");
|
|
3202
|
+
// ...replaced by a calm, no-action-needed note naming the stale marker.
|
|
3203
|
+
expect(errOutput).toContain("stale personal-overlay marker");
|
|
3204
|
+
expect(errOutput).toContain(markerKey);
|
|
3205
|
+
});
|
|
3206
|
+
|
|
3207
|
+
it("dir-vs-object collision OUTSIDE the overlay-mirror core namespace still warns to reconcile (no over-broadening)", async () => {
|
|
3208
|
+
// Boundary guard for the feedback_ce9aeede fix: the quiet skip is scoped to
|
|
3209
|
+
// core/{knowledge,policies,workers,settings}/ overlay-mirror keys ONLY. A
|
|
3210
|
+
// dir-vs-object collision at any OTHER key (here `core/docs/...`, a core
|
|
3211
|
+
// subtree that is NOT an overlay-mirror type) is a genuine structural
|
|
3212
|
+
// collision the operator must resolve, so the original warning path — and
|
|
3213
|
+
// its skip-local-only classification — must remain intact.
|
|
3214
|
+
const key = "core/docs/agent-browser";
|
|
3215
|
+
const localDir = path.join(tmpDir, "core", "docs", "agent-browser");
|
|
3216
|
+
fs.mkdirSync(localDir, { recursive: true });
|
|
3217
|
+
fs.writeFileSync(path.join(localDir, "inside.md"), "local content");
|
|
3218
|
+
|
|
3219
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
3220
|
+
{ key, size: 40, lastModified: new Date(), etag: '"obj"' },
|
|
3221
|
+
]);
|
|
3222
|
+
const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
|
3223
|
+
|
|
3224
|
+
let crashed = false;
|
|
3225
|
+
let result;
|
|
3226
|
+
try {
|
|
3227
|
+
result = await sync({
|
|
3228
|
+
company: "acme",
|
|
3229
|
+
vaultConfig: mockConfig,
|
|
3230
|
+
hqRoot: tmpDir,
|
|
3231
|
+
personalMode: true,
|
|
3232
|
+
});
|
|
3233
|
+
} catch {
|
|
3234
|
+
crashed = true;
|
|
3235
|
+
}
|
|
3236
|
+
const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
|
|
3237
|
+
errSpy.mockRestore();
|
|
3238
|
+
|
|
3239
|
+
expect(crashed).toBe(false);
|
|
3240
|
+
expect(result!.filesDownloaded).toBe(0);
|
|
3241
|
+
expect(result!.filesSkipped).toBe(1);
|
|
3242
|
+
expect(fs.existsSync(path.join(localDir, "inside.md"))).toBe(true);
|
|
3243
|
+
// Generic collision path preserved: reconciliation warning still emitted,
|
|
3244
|
+
// NOT the overlay-marker note.
|
|
3245
|
+
expect(errOutput).toContain("manual");
|
|
3246
|
+
expect(errOutput).not.toContain("stale personal-overlay marker");
|
|
3247
|
+
});
|
|
3248
|
+
|
|
3144
3249
|
it("pulls a remote symlink record under an .hqinclude dir-only allowlist pattern", async () => {
|
|
3145
3250
|
// Codex round-7 P1 follow-up: pre-fix, computePullPlan called
|
|
3146
3251
|
// shouldSync(localPath) with the default isDir=false. LIST gives
|
package/src/cli/sync.ts
CHANGED
|
@@ -322,6 +322,24 @@ export type SyncProgressEvent =
|
|
|
322
322
|
type: "upload-suppressed-tombstone";
|
|
323
323
|
path: string;
|
|
324
324
|
deletedAt: string;
|
|
325
|
+
}
|
|
326
|
+
| {
|
|
327
|
+
/**
|
|
328
|
+
* Emitted by the PUSH leg (`share()`) once per local file skipped because
|
|
329
|
+
* it exceeds the max sync size (`isWithinSizeLimit`, 50 MB default). This
|
|
330
|
+
* is a permanent, benign, user-policy outcome — the file is too big to
|
|
331
|
+
* sync and will be over the cap on every pass — so it is NOT an error and
|
|
332
|
+
* the run exits 0. It was previously emitted as `type: "error"`, which
|
|
333
|
+
* pushed it into the runner's `errors[]` and made every watch pass return
|
|
334
|
+
* exit 2; the menubar supervisor then reported that as
|
|
335
|
+
* "auto-sync watcher exited unexpectedly (code=Some(2))" on every tick —
|
|
336
|
+
* the fleet-wide HQ-SYNC-4 flood. Surfaced here purely for visibility
|
|
337
|
+
* (`bytes` is the offending file's size) so a UI/CLI can tell the user
|
|
338
|
+
* which file was too big without treating it as a failure.
|
|
339
|
+
*/
|
|
340
|
+
type: "skip-size-limit";
|
|
341
|
+
path: string;
|
|
342
|
+
bytes: number;
|
|
325
343
|
};
|
|
326
344
|
|
|
327
345
|
export interface SyncOptions {
|
|
@@ -503,6 +521,11 @@ export interface SyncResult {
|
|
|
503
521
|
* true they are left on disk and tombstoned, and counted here.
|
|
504
522
|
*/
|
|
505
523
|
scopeOrphansBlocked: number;
|
|
524
|
+
/**
|
|
525
|
+
* Remote keys whose local on-disk counterpart changed during this pull.
|
|
526
|
+
* Includes downloads, tombstone deletes, and clean scope-orphan removals.
|
|
527
|
+
*/
|
|
528
|
+
changedPaths?: string[];
|
|
506
529
|
}
|
|
507
530
|
|
|
508
531
|
type SyncEventEmitter = (event: SyncProgressEvent) => void;
|
|
@@ -535,6 +558,7 @@ interface PullCounters {
|
|
|
535
558
|
filesTombstoned: number;
|
|
536
559
|
filesOutOfScope: number;
|
|
537
560
|
conflictPaths: string[];
|
|
561
|
+
changedPaths: string[];
|
|
538
562
|
}
|
|
539
563
|
|
|
540
564
|
interface ScopeShrinkPlanState {
|
|
@@ -549,6 +573,7 @@ interface ScopeShrinkRun {
|
|
|
549
573
|
shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
|
|
550
574
|
shrinkResult: ReturnType<typeof applyScopeShrink>;
|
|
551
575
|
scopeOrphansRemoved: number;
|
|
576
|
+
changedPaths: string[];
|
|
552
577
|
}
|
|
553
578
|
|
|
554
579
|
/**
|
|
@@ -831,6 +856,7 @@ function createPullCounters(): PullCounters {
|
|
|
831
856
|
filesTombstoned: 0,
|
|
832
857
|
filesOutOfScope: 0,
|
|
833
858
|
conflictPaths: [],
|
|
859
|
+
changedPaths: [],
|
|
834
860
|
};
|
|
835
861
|
}
|
|
836
862
|
|
|
@@ -943,6 +969,10 @@ function executeScopeShrink(
|
|
|
943
969
|
shrinkResult,
|
|
944
970
|
scopeOrphansRemoved:
|
|
945
971
|
shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
|
|
972
|
+
changedPaths: [
|
|
973
|
+
...shrinkResult.quarantinedPaths,
|
|
974
|
+
...shrinkResult.removedPaths,
|
|
975
|
+
],
|
|
946
976
|
};
|
|
947
977
|
}
|
|
948
978
|
|
|
@@ -967,7 +997,8 @@ async function executeConflictExecutor(
|
|
|
967
997
|
item.action === "skip-ignored" ||
|
|
968
998
|
item.action === "skip-personal-mode" ||
|
|
969
999
|
item.action === "skip-unchanged" ||
|
|
970
|
-
item.action === "skip-local-only"
|
|
1000
|
+
item.action === "skip-local-only" ||
|
|
1001
|
+
item.action === "skip-stale-overlay-marker"
|
|
971
1002
|
) {
|
|
972
1003
|
counters.filesSkipped++;
|
|
973
1004
|
continue;
|
|
@@ -1038,6 +1069,7 @@ function executeFileTombstoneDelete(
|
|
|
1038
1069
|
}
|
|
1039
1070
|
removeEntry(run.journal, tombstoneKey);
|
|
1040
1071
|
counters.filesTombstoned++;
|
|
1072
|
+
counters.changedPaths.push(tombstoneKey);
|
|
1041
1073
|
run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
1042
1074
|
}
|
|
1043
1075
|
|
|
@@ -1197,6 +1229,7 @@ async function executeConflictItem(
|
|
|
1197
1229
|
filesOutOfScope: counters.filesOutOfScope,
|
|
1198
1230
|
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1199
1231
|
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1232
|
+
changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
|
|
1200
1233
|
};
|
|
1201
1234
|
}
|
|
1202
1235
|
|
|
@@ -1334,6 +1367,7 @@ async function downloadOne(
|
|
|
1334
1367
|
|
|
1335
1368
|
counters.filesDownloaded++;
|
|
1336
1369
|
counters.bytesDownloaded += size;
|
|
1370
|
+
counters.changedPaths.push(remoteFile.key);
|
|
1337
1371
|
} catch (err) {
|
|
1338
1372
|
if (isAccessDenied(err)) {
|
|
1339
1373
|
counters.filesSkipped++;
|
|
@@ -1473,6 +1507,7 @@ function executeJournalTombstoneDeletes(
|
|
|
1473
1507
|
}
|
|
1474
1508
|
removeEntry(run.journal, key);
|
|
1475
1509
|
counters.filesTombstoned++;
|
|
1510
|
+
counters.changedPaths.push(key);
|
|
1476
1511
|
run.emit({
|
|
1477
1512
|
type: "progress",
|
|
1478
1513
|
path: key,
|
|
@@ -1531,6 +1566,7 @@ function finalizePullRun(
|
|
|
1531
1566
|
filesOutOfScope: counters.filesOutOfScope,
|
|
1532
1567
|
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1533
1568
|
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1569
|
+
changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
|
|
1534
1570
|
};
|
|
1535
1571
|
}
|
|
1536
1572
|
|
|
@@ -1670,6 +1706,15 @@ type PullPlanItem =
|
|
|
1670
1706
|
| { action: "skip-personal-mode"; remoteFile: RemoteFile; localPath: string }
|
|
1671
1707
|
| { action: "skip-unchanged"; remoteFile: RemoteFile; localPath: string }
|
|
1672
1708
|
| { action: "skip-local-only"; remoteFile: RemoteFile; localPath: string }
|
|
1709
|
+
// A stale personal-overlay symlink marker in the vault sitting at a key that
|
|
1710
|
+
// is ALSO a real, release-shipped core directory (e.g. an old
|
|
1711
|
+
// `core/knowledge/public/agent-browser` overlay marker after a release
|
|
1712
|
+
// shipped a real directory of the same name). The local release-shipped
|
|
1713
|
+
// directory is authoritative; the marker is inert derived state that must
|
|
1714
|
+
// NEVER overwrite core. Distinct from skip-local-only so the benign,
|
|
1715
|
+
// no-action-needed case is never surfaced with the alarming "rm -rf the
|
|
1716
|
+
// local directory" reconciliation advice.
|
|
1717
|
+
| { action: "skip-stale-overlay-marker"; remoteFile: RemoteFile; localPath: string }
|
|
1673
1718
|
// Remote keys refused by ephemeral-mirror policy. The push walker has
|
|
1674
1719
|
// refused to upload these since 5.33.0; the pull walker now refuses to
|
|
1675
1720
|
// download them so legacy litter in cloud staging drains naturally.
|
|
@@ -1757,6 +1802,33 @@ interface PullPlan {
|
|
|
1757
1802
|
const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
|
|
1758
1803
|
const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
|
|
1759
1804
|
|
|
1805
|
+
// The four release-shipped `core/<type>/` namespaces the personal-overlay
|
|
1806
|
+
// mirror (reindex) materializes into. A stale mirror symlink pushed to the
|
|
1807
|
+
// vault lands as a single object at exactly `core/<type>/<...>/<entry>`, and if
|
|
1808
|
+
// a later release ships a REAL directory of the same name, pull sees a
|
|
1809
|
+
// (local real dir) vs (remote single object) collision at that key forever.
|
|
1810
|
+
const OVERLAY_MIRROR_CORE_TYPES = ["knowledge", "policies", "workers", "settings"];
|
|
1811
|
+
|
|
1812
|
+
/**
|
|
1813
|
+
* True when `key` is under a release-shipped personal-overlay-mirror core
|
|
1814
|
+
* namespace (`core/{knowledge,policies,workers,settings}/<entry>/...`).
|
|
1815
|
+
*
|
|
1816
|
+
* A remote LIST entry is always a single S3 object. A REAL release-shipped core
|
|
1817
|
+
* directory only ever syncs as child objects UNDER its key
|
|
1818
|
+
* (`core/knowledge/public/agent-browser/INDEX.md`), never as one object AT the
|
|
1819
|
+
* directory's own key — so a single object at such a key, colliding with a
|
|
1820
|
+
* local real directory, is unambiguously a stale overlay symlink marker, not
|
|
1821
|
+
* live core content. Pure path test — no I/O — safe inside the sync planner.
|
|
1822
|
+
*/
|
|
1823
|
+
function isOverlayMirrorCoreKey(key: string): boolean {
|
|
1824
|
+
const parts = key.split("/");
|
|
1825
|
+
return (
|
|
1826
|
+
parts.length >= 3 &&
|
|
1827
|
+
parts[0] === "core" &&
|
|
1828
|
+
OVERLAY_MIRROR_CORE_TYPES.includes(parts[1])
|
|
1829
|
+
);
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1760
1832
|
function computePullPlan(
|
|
1761
1833
|
remoteFiles: RemoteFile[],
|
|
1762
1834
|
journal: SyncJournal,
|
|
@@ -1959,6 +2031,29 @@ function computePullPlan(
|
|
|
1959
2031
|
// skip and warn so the operator can reconcile manually
|
|
1960
2032
|
// (delete the local dir, or restructure the remote).
|
|
1961
2033
|
if (localLstat!.isDirectory() && !isLocalSymlink) {
|
|
2034
|
+
// Stale personal-overlay marker vs. release-shipped core directory:
|
|
2035
|
+
// a single vault object at a `core/<overlay-type>/.../<entry>` key
|
|
2036
|
+
// whose local counterpart is a real directory is an orphaned overlay
|
|
2037
|
+
// symlink marker (reindex mirror artifact) left behind after a release
|
|
2038
|
+
// shipped a real core dir of the same name. The local release-shipped
|
|
2039
|
+
// directory is authoritative and the marker can never be materialized
|
|
2040
|
+
// over it, so keep local and ignore the marker QUIETLY — the generic
|
|
2041
|
+
// "rm -rf the local directory" advice below is actively dangerous here
|
|
2042
|
+
// (it would destroy release-shipped core content to pull inert derived
|
|
2043
|
+
// state), and it recurred on every sync. feedback_ce9aeede.
|
|
2044
|
+
if (isOverlayMirrorCoreKey(remoteFile.key)) {
|
|
2045
|
+
console.error(
|
|
2046
|
+
` Note: ${remoteFile.key} is a stale personal-overlay marker ` +
|
|
2047
|
+
`shadowing a release-shipped core directory; keeping the local ` +
|
|
2048
|
+
`directory and ignoring the marker (no action needed).`,
|
|
2049
|
+
);
|
|
2050
|
+
items.push({
|
|
2051
|
+
action: "skip-stale-overlay-marker",
|
|
2052
|
+
remoteFile,
|
|
2053
|
+
localPath,
|
|
2054
|
+
});
|
|
2055
|
+
continue;
|
|
2056
|
+
}
|
|
1962
2057
|
console.error(
|
|
1963
2058
|
` Warning: ${remoteFile.key} exists locally as a directory; ` +
|
|
1964
2059
|
`cloud has a single object at this key. Skipping; manual ` +
|
|
@@ -2402,5 +2497,10 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
|
|
|
2402
2497
|
if (event.count > event.samplePaths.length) {
|
|
2403
2498
|
console.warn(` ... and ${event.count - event.samplePaths.length} more`);
|
|
2404
2499
|
}
|
|
2500
|
+
} else if (event.type === "skip-size-limit") {
|
|
2501
|
+
const mb = (event.bytes / (1024 * 1024)).toFixed(1);
|
|
2502
|
+
console.warn(
|
|
2503
|
+
` ! ${event.path} skipped — ${mb} MB exceeds the 50 MB sync size limit`,
|
|
2504
|
+
);
|
|
2405
2505
|
}
|
|
2406
2506
|
}
|
package/src/qmd-reindex.test.ts
CHANGED
|
@@ -100,6 +100,56 @@ describe("reindexAfterSync", () => {
|
|
|
100
100
|
expect(calls).toEqual([["collection", "list"], ["update"]]);
|
|
101
101
|
});
|
|
102
102
|
|
|
103
|
+
it("skips qmd entirely when provided changed paths do not touch knowledge markdown", () => {
|
|
104
|
+
const { exec, calls } = fakeExec({ listStdout: "" });
|
|
105
|
+
const r = reindexAfterSync("/hq", {
|
|
106
|
+
exec,
|
|
107
|
+
existsSync: () => true,
|
|
108
|
+
changedPaths: ["companies/acme/docs/readme.txt"],
|
|
109
|
+
});
|
|
110
|
+
expect(r.qmdAvailable).toBe(false);
|
|
111
|
+
expect(r.updated).toBe(false);
|
|
112
|
+
expect(calls).toEqual([]);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("debounces dirty updates with a pending marker and flushes the final update later", () => {
|
|
116
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-debounce-"));
|
|
117
|
+
try {
|
|
118
|
+
const statePath = path.join(dir, "qmd-state.json");
|
|
119
|
+
const { exec, calls } = fakeExec({ listStdout: "acme (qmd://acme/)\n" });
|
|
120
|
+
|
|
121
|
+
const first = reindexAfterSync("/hq", {
|
|
122
|
+
exec,
|
|
123
|
+
existsSync: () => true,
|
|
124
|
+
changedPaths: ["companies/acme/knowledge/note.md"],
|
|
125
|
+
debounceMs: 60_000,
|
|
126
|
+
nowMs: 100_000,
|
|
127
|
+
statePath,
|
|
128
|
+
});
|
|
129
|
+
expect(first.updated).toBe(false);
|
|
130
|
+
expect(first.pendingDirty).toBe(true);
|
|
131
|
+
expect(calls).toEqual([["collection", "list"]]);
|
|
132
|
+
|
|
133
|
+
const second = reindexAfterSync("/hq", {
|
|
134
|
+
exec,
|
|
135
|
+
existsSync: () => true,
|
|
136
|
+
changedPaths: [],
|
|
137
|
+
debounceMs: 60_000,
|
|
138
|
+
nowMs: 161_000,
|
|
139
|
+
statePath,
|
|
140
|
+
});
|
|
141
|
+
expect(second.updated).toBe(true);
|
|
142
|
+
expect(second.pendingDirty).toBe(false);
|
|
143
|
+
expect(calls).toEqual([
|
|
144
|
+
["collection", "list"],
|
|
145
|
+
["collection", "list"],
|
|
146
|
+
["update"],
|
|
147
|
+
]);
|
|
148
|
+
} finally {
|
|
149
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
|
|
103
153
|
it("runs embed only when embed:true", () => {
|
|
104
154
|
const { exec, calls } = fakeExec({ listStdout: "" });
|
|
105
155
|
const r = reindexAfterSync("/hq", {
|