@indigoai-us/hq-cloud 6.5.0 → 6.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +4 -35
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -104
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +19 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +67 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +19 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +62 -19
- package/dist/cli/sync.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +15 -5
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +71 -2
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +70 -7
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +102 -23
- package/dist/scope-shrink.js.map +1 -1
- package/dist/scope-shrink.test.js +63 -0
- package/dist/scope-shrink.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts +50 -0
- package/dist/sync/pull-scope.d.ts.map +1 -0
- package/dist/sync/pull-scope.js +129 -0
- package/dist/sync/pull-scope.js.map +1 -0
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +23 -0
- package/src/bin/sync-runner.ts +19 -116
- package/src/cli/sync-scope.test.ts +84 -0
- package/src/cli/sync.ts +90 -17
- package/src/index.ts +11 -0
- package/src/s3.test.ts +91 -1
- package/src/s3.ts +15 -5
- package/src/scope-shrink.test.ts +71 -0
- package/src/scope-shrink.ts +164 -20
- package/src/sync/pull-scope.ts +161 -0
package/src/bin/sync-runner.ts
CHANGED
|
@@ -76,7 +76,11 @@ import {
|
|
|
76
76
|
type ExplicitGrant,
|
|
77
77
|
} from "../index.js";
|
|
78
78
|
import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
79
|
-
import {
|
|
79
|
+
import {
|
|
80
|
+
resolvePullScope,
|
|
81
|
+
readPinnedPrefixes,
|
|
82
|
+
type PullScope,
|
|
83
|
+
} from "../sync/pull-scope.js";
|
|
80
84
|
import {
|
|
81
85
|
PERSONAL_VAULT_EXCLUDED_TOP_LEVEL,
|
|
82
86
|
computePersonalVaultPaths,
|
|
@@ -401,121 +405,13 @@ export interface VaultClientSurface {
|
|
|
401
405
|
listMyExplicitGrants?: (companyUid: string) => Promise<ExplicitGrant[]>;
|
|
402
406
|
}
|
|
403
407
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
prefixSet?: string[];
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
/**
|
|
415
|
-
* Resolve the effective download scope for a company target.
|
|
416
|
-
*
|
|
417
|
-
* - `all` → no prefix set; full-bucket pull (legacy behavior).
|
|
418
|
-
* - `shared` → coalesced caller explicit grants (company-relative paths,
|
|
419
|
-
* same namespace as `RemoteFile.key`).
|
|
420
|
-
* - `custom` → coalesced `customPaths` from the sync-config row.
|
|
421
|
-
*
|
|
422
|
-
* DEGRADE-TO-`all` CONTRACT: any failure (missing client method, membership
|
|
423
|
-
* not found, network error, grant fetch error) returns `{ syncMode: "all" }`.
|
|
424
|
-
* A transient failure must NEVER silently narrow scope — that would prune the
|
|
425
|
-
* local tree. Mirrors the CLI's `resolvePerCompanyPullPlan` degrade behavior.
|
|
426
|
-
*/
|
|
427
|
-
export async function resolvePullScope(
|
|
428
|
-
client: VaultClientSurface,
|
|
429
|
-
companyUid: string,
|
|
430
|
-
// Company slug — required to normalize grant paths (which may be anchored
|
|
431
|
-
// at `companies/<slug>/` or `<slug>/`) into the company-relative namespace.
|
|
432
|
-
slug: string,
|
|
433
|
-
// Local HQ root — used to read the per-machine pin set (`.hq/pins.json`).
|
|
434
|
-
// When omitted, pins are simply not unioned (no behavior change).
|
|
435
|
-
hqRoot?: string,
|
|
436
|
-
): Promise<PullScope> {
|
|
437
|
-
if (!client.getMembershipSyncConfig) return { syncMode: "all" };
|
|
438
|
-
try {
|
|
439
|
-
const memberships = await client.listMyMemberships();
|
|
440
|
-
const m = memberships.find((x) => x.companyUid === companyUid);
|
|
441
|
-
if (!m) return { syncMode: "all" };
|
|
442
|
-
const cfg = await client.getMembershipSyncConfig(m.membershipKey);
|
|
443
|
-
if (cfg.syncMode === "all") return { syncMode: "all" };
|
|
444
|
-
|
|
445
|
-
// Pins are company-relative prefixes a user explicitly materialized via
|
|
446
|
-
// `hq files get`. They're unioned into the scope so a scoped pull keeps
|
|
447
|
-
// them instead of pruning them as out-of-scope orphans. Pins only WIDEN
|
|
448
|
-
// scope, never narrow — and `all` mode (handled above) ignores them since
|
|
449
|
-
// it pulls everything anyway.
|
|
450
|
-
const pinPrefixes = hqRoot ? readPinnedPrefixes(hqRoot, slug) : [];
|
|
451
|
-
|
|
452
|
-
if (cfg.syncMode === "custom") {
|
|
453
|
-
const customPrefixes = (cfg.customPaths ?? []).map((p) =>
|
|
454
|
-
grantPathToPrefix(p, slug),
|
|
455
|
-
);
|
|
456
|
-
// A bare-everything entry ("" — e.g. a `*` path) collapses under
|
|
457
|
-
// `coalescePrefixes` (which drops empties) to "nothing", which would
|
|
458
|
-
// prune the whole tree. An everything-scope is semantically `all`.
|
|
459
|
-
if (customPrefixes.some((p) => p === "")) return { syncMode: "all" };
|
|
460
|
-
return {
|
|
461
|
-
syncMode: "custom",
|
|
462
|
-
prefixSet: coalescePrefixes([...customPrefixes, ...pinPrefixes]),
|
|
463
|
-
};
|
|
464
|
-
}
|
|
465
|
-
// shared: scope to the caller's explicit grants. Real grant paths are
|
|
466
|
-
// inconsistent — full (`companies/<slug>/x/*`), slug-anchored
|
|
467
|
-
// (`<slug>/x/*`), company-relative (`x/*`), bare globs (`*`), and exact
|
|
468
|
-
// files all coexist in production — so each is normalized via
|
|
469
|
-
// `grantPathToPrefix` into a company-relative, startsWith-friendly prefix
|
|
470
|
-
// (the namespace the engine's `RemoteFile.key`s live in) before coalescing.
|
|
471
|
-
//
|
|
472
|
-
// SAFETY: if the client can't fetch grants, we must NOT fall through to an
|
|
473
|
-
// empty `shared` scope — that would tell the engine "nothing is in scope"
|
|
474
|
-
// and scope-shrink would prune every clean local file. Degrade to `all`
|
|
475
|
-
// instead. A genuinely-empty grant list (the method exists and returns
|
|
476
|
-
// []) is a real "nothing shared with me" and is allowed to narrow.
|
|
477
|
-
if (!client.listMyExplicitGrants) return { syncMode: "all" };
|
|
478
|
-
const grants = await client.listMyExplicitGrants(companyUid);
|
|
479
|
-
const sharedPrefixes = grants.map((g) => grantPathToPrefix(g.path, slug));
|
|
480
|
-
// A wildcard grant (`*`) normalizes to "" = everything. Since
|
|
481
|
-
// `coalescePrefixes` drops empties (collapsing "everything" to "nothing"),
|
|
482
|
-
// treat any such grant as full-access `all` rather than risk pruning.
|
|
483
|
-
if (sharedPrefixes.some((p) => p === "")) return { syncMode: "all" };
|
|
484
|
-
return {
|
|
485
|
-
syncMode: "shared",
|
|
486
|
-
prefixSet: coalescePrefixes([...sharedPrefixes, ...pinPrefixes]),
|
|
487
|
-
};
|
|
488
|
-
} catch {
|
|
489
|
-
// Degrade to `all` — never prune on a resolution failure.
|
|
490
|
-
return { syncMode: "all" };
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
/**
|
|
495
|
-
* Read the per-machine pin set (`<hqRoot>/.hq/pins.json`) and return the
|
|
496
|
-
* company-relative pinned prefixes for `slug`. These are prefixes the user
|
|
497
|
-
* materialized on demand via `hq files get` that must survive a scoped pull.
|
|
498
|
-
*
|
|
499
|
-
* Tolerant by construction: a missing, unreadable, or malformed file yields
|
|
500
|
-
* `[]` (no pins) — pins only ever widen scope, so "no pins" is the safe
|
|
501
|
-
* default. Empty-string entries are dropped (an everything-pin is meaningless
|
|
502
|
-
* here; `all` mode already covers that case).
|
|
503
|
-
*/
|
|
504
|
-
export function readPinnedPrefixes(hqRoot: string, slug: string): string[] {
|
|
505
|
-
try {
|
|
506
|
-
const raw = fs.readFileSync(path.join(hqRoot, ".hq", "pins.json"), "utf-8");
|
|
507
|
-
const parsed = JSON.parse(raw) as { pins?: Record<string, unknown> };
|
|
508
|
-
const list = parsed?.pins?.[slug];
|
|
509
|
-
if (Array.isArray(list)) {
|
|
510
|
-
return list.filter(
|
|
511
|
-
(p): p is string => typeof p === "string" && p.length > 0,
|
|
512
|
-
);
|
|
513
|
-
}
|
|
514
|
-
} catch {
|
|
515
|
-
/* missing / unreadable / malformed → no pins */
|
|
516
|
-
}
|
|
517
|
-
return [];
|
|
518
|
-
}
|
|
408
|
+
// `resolvePullScope`, `readPinnedPrefixes`, and the `PullScope` type now live
|
|
409
|
+
// in `../sync/pull-scope.ts` so the menubar runner and `hq sync pull|now`
|
|
410
|
+
// (hq-cli) share ONE scope resolver — the drift between them was the root
|
|
411
|
+
// cause of the all→shared scope-shrink wedge (DEV-1768). Re-exported here so
|
|
412
|
+
// existing importers (and the runner test suite) keep their import path.
|
|
413
|
+
export { resolvePullScope, readPinnedPrefixes };
|
|
414
|
+
export type { PullScope };
|
|
519
415
|
|
|
520
416
|
/**
|
|
521
417
|
* Backoff schedule (in ms) between attempts 2 and 3 of
|
|
@@ -1446,6 +1342,13 @@ export async function runRunner(
|
|
|
1446
1342
|
hqRoot: parsed.hqRoot,
|
|
1447
1343
|
onConflict: parsed.onConflict,
|
|
1448
1344
|
syncMode: pullScope.syncMode,
|
|
1345
|
+
// The menubar runner can take no interactive flag, so a scope shrink
|
|
1346
|
+
// must NEVER throw here (the old `ScopeShrinkBlockedError` → exit 2
|
|
1347
|
+
// was the permanent wedge in DEV-1768). Self-heal non-destructively:
|
|
1348
|
+
// dirty out-of-scope files stay on disk + un-tracked, clean ones are
|
|
1349
|
+
// quarantined (recoverable). This also clears an already-wedged
|
|
1350
|
+
// journal — seeded by a buggy `all`-mode CLI pull — on the next sync.
|
|
1351
|
+
scopeShrinkPolicy: "auto-recover",
|
|
1449
1352
|
// Scope-shrink authorship guard: pass the caller's own sub (the very
|
|
1450
1353
|
// sub stamped onto uploads as `created-by-sub`) so a scope shrink
|
|
1451
1354
|
// never prunes content this owner authored. Owners hold their whole
|
|
@@ -340,4 +340,88 @@ describe("sync — scope-aware download (US-005)", () => {
|
|
|
340
340
|
expect(forced.scopeOrphansBlocked).toBe(1);
|
|
341
341
|
expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(true);
|
|
342
342
|
});
|
|
343
|
+
|
|
344
|
+
// DEV-1768 fix #3: a scope shrink must QUARANTINE clean orphans (recoverable),
|
|
345
|
+
// never silently delete them. The file leaves its working-tree path but is
|
|
346
|
+
// recoverable under `.hq/scope-quarantine/<slug>/`.
|
|
347
|
+
it("scope shrink QUARANTINES a clean orphan (recoverable), never silent-deletes", async () => {
|
|
348
|
+
const quarantined = path.join(
|
|
349
|
+
tmpDir,
|
|
350
|
+
".hq",
|
|
351
|
+
"scope-quarantine",
|
|
352
|
+
"acme",
|
|
353
|
+
"docs",
|
|
354
|
+
"handoff.md",
|
|
355
|
+
);
|
|
356
|
+
await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir, syncMode: "all" });
|
|
357
|
+
const original = companyRel("docs/handoff.md");
|
|
358
|
+
const bodyBefore = fs.readFileSync(original, "utf-8");
|
|
359
|
+
|
|
360
|
+
const shared = await sync({
|
|
361
|
+
company: "acme",
|
|
362
|
+
vaultConfig: mockConfig,
|
|
363
|
+
hqRoot: tmpDir,
|
|
364
|
+
syncMode: "shared",
|
|
365
|
+
prefixSet: ["knowledge/"],
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
expect(shared.scopeOrphansRemoved).toBe(1);
|
|
369
|
+
// Gone from the working tree...
|
|
370
|
+
expect(fs.existsSync(original)).toBe(false);
|
|
371
|
+
// ...but recoverable in quarantine, byte-for-byte.
|
|
372
|
+
expect(fs.existsSync(quarantined)).toBe(true);
|
|
373
|
+
expect(fs.readFileSync(quarantined, "utf-8")).toBe(bodyBefore);
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
// DEV-1768 fix #1 (recovery) + #2: the background runner path
|
|
377
|
+
// (`scopeShrinkPolicy: "auto-recover"`) must NEVER throw on a scope shrink —
|
|
378
|
+
// it self-heals. This reproduces the EXACT wedge: a buggy `all`-mode pull
|
|
379
|
+
// seeds the journal (all-mode PullRecord + both files), one out-of-scope file
|
|
380
|
+
// is edited locally (dirty), then the real `shared` pull arrives. Under the
|
|
381
|
+
// old code this threw ScopeShrinkBlockedError(all→shared) → exit 2 forever.
|
|
382
|
+
it("auto-recover policy clears an all→shared wedge: dirty kept on disk, clean quarantined, no throw", async () => {
|
|
383
|
+
// 1. Buggy all-mode CLI seed: journals everything + stamps an all-mode record.
|
|
384
|
+
await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir, syncMode: "all" });
|
|
385
|
+
// 2. User edits an out-of-scope file → it becomes a DIRTY orphan.
|
|
386
|
+
fs.writeFileSync(companyRel("docs/handoff.md"), "LOCAL EDIT — keep me");
|
|
387
|
+
const quarantinedClean = path.join(
|
|
388
|
+
tmpDir, ".hq", "scope-quarantine", "acme", "knowledge", "readme.md",
|
|
389
|
+
);
|
|
390
|
+
|
|
391
|
+
// 3. The menubar runner's real shared pull. prefixSet covers NEITHER file
|
|
392
|
+
// (docs is dirty-orphan, knowledge becomes a clean orphan) so we exercise
|
|
393
|
+
// both dispositions. Must resolve without throwing.
|
|
394
|
+
const recovered = await sync({
|
|
395
|
+
company: "acme",
|
|
396
|
+
vaultConfig: mockConfig,
|
|
397
|
+
hqRoot: tmpDir,
|
|
398
|
+
syncMode: "shared",
|
|
399
|
+
prefixSet: ["projects/"],
|
|
400
|
+
scopeShrinkPolicy: "auto-recover",
|
|
401
|
+
});
|
|
402
|
+
expect(recovered.aborted).toBe(false);
|
|
403
|
+
// Dirty file: KEPT on disk (un-tracked), counted as blocked-but-kept.
|
|
404
|
+
expect(recovered.scopeOrphansBlocked).toBe(1);
|
|
405
|
+
expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(true);
|
|
406
|
+
expect(fs.readFileSync(companyRel("docs/handoff.md"), "utf-8")).toBe(
|
|
407
|
+
"LOCAL EDIT — keep me",
|
|
408
|
+
);
|
|
409
|
+
// Clean file: quarantined (recoverable), removed from the working tree.
|
|
410
|
+
expect(recovered.scopeOrphansRemoved).toBe(1);
|
|
411
|
+
expect(fs.existsSync(companyRel("knowledge/readme.md"))).toBe(false);
|
|
412
|
+
expect(fs.existsSync(quarantinedClean)).toBe(true);
|
|
413
|
+
|
|
414
|
+
// 4. Idempotent: the next auto-recover sync re-flags NOTHING (the orphans
|
|
415
|
+
// were tombstoned, so the wedge does not recur).
|
|
416
|
+
const second = await sync({
|
|
417
|
+
company: "acme",
|
|
418
|
+
vaultConfig: mockConfig,
|
|
419
|
+
hqRoot: tmpDir,
|
|
420
|
+
syncMode: "shared",
|
|
421
|
+
prefixSet: ["projects/"],
|
|
422
|
+
scopeShrinkPolicy: "auto-recover",
|
|
423
|
+
});
|
|
424
|
+
expect(second.scopeOrphansRemoved).toBe(0);
|
|
425
|
+
expect(second.scopeOrphansBlocked).toBe(0);
|
|
426
|
+
});
|
|
343
427
|
});
|
package/src/cli/sync.ts
CHANGED
|
@@ -40,6 +40,7 @@ import {
|
|
|
40
40
|
applyScopeShrink,
|
|
41
41
|
ScopeShrinkBlockedError,
|
|
42
42
|
ScopeShrinkLargePruneError,
|
|
43
|
+
type ScopeShrinkAdviceContext,
|
|
43
44
|
} from "../scope-shrink.js";
|
|
44
45
|
import { coalescePrefixes, isCoveredByAny } from "../prefix-coalesce.js";
|
|
45
46
|
import { createIgnoreFilter } from "../ignore.js";
|
|
@@ -355,6 +356,25 @@ export interface SyncOptions {
|
|
|
355
356
|
* tombstoned. Mirrors `hq sync narrow --force`.
|
|
356
357
|
*/
|
|
357
358
|
forceScopeShrink?: boolean;
|
|
359
|
+
/**
|
|
360
|
+
* How `sync()` handles a scope shrink (US-005 / DEV-1768):
|
|
361
|
+
*
|
|
362
|
+
* - `"block"` (default) — a human is present (foreground `hq sync`). Dirty
|
|
363
|
+
* out-of-scope orphans, or a clean prune over the safety cap, raise a
|
|
364
|
+
* structured error whose advice is followable from a terminal. Clean
|
|
365
|
+
* orphans within the cap are QUARANTINED (moved, not deleted).
|
|
366
|
+
* - `"auto-recover"` — the background menubar runner, which can take no
|
|
367
|
+
* interactive flag. NEVER throws on a shrink: dirty orphans are kept on
|
|
368
|
+
* disk + un-tracked, clean orphans are quarantined, and the bulk-prune
|
|
369
|
+
* cap is bypassed (quarantine is non-destructive). This is what clears an
|
|
370
|
+
* already-wedged journal on the next sync, idempotently and without data
|
|
371
|
+
* loss — the recovery seam for the all→shared seed bug.
|
|
372
|
+
*
|
|
373
|
+
* Both policies are non-destructive for CLEAN files (quarantine, never
|
|
374
|
+
* silent delete) — the deliberate `hq sync narrow --apply` ritual is the only
|
|
375
|
+
* path that hard-deletes, and it confirms first.
|
|
376
|
+
*/
|
|
377
|
+
scopeShrinkPolicy?: "block" | "auto-recover";
|
|
358
378
|
/**
|
|
359
379
|
* The caller's own Cognito `sub`, used by the scope-shrink authorship guard
|
|
360
380
|
* so a scope shrink never prunes content the caller authored. Injected by the
|
|
@@ -655,23 +675,38 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
655
675
|
// explicit `hq sync narrow` ritual opts out of the unknown-author shield.
|
|
656
676
|
protectUnknownAuthors: true,
|
|
657
677
|
});
|
|
658
|
-
|
|
678
|
+
// Policy: the background menubar runner ("auto-recover") can take no
|
|
679
|
+
// interactive flag, so it must never throw on a shrink — it self-heals
|
|
680
|
+
// non-destructively (dirty kept on disk + un-tracked, clean quarantined).
|
|
681
|
+
// A foreground `hq sync` ("block", the default) keeps the protective gate
|
|
682
|
+
// but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
|
|
683
|
+
// bypasses the bulk-prune cap (quarantine is non-destructive, so a large
|
|
684
|
+
// recovery move is safe). DEV-1768.
|
|
685
|
+
const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
|
|
686
|
+
const autoRecover = scopeShrinkPolicy === "auto-recover";
|
|
687
|
+
const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
|
|
688
|
+
const effectiveForce = options.forceScopeShrink === true || autoRecover;
|
|
689
|
+
|
|
690
|
+
if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
|
|
659
691
|
throw new ScopeShrinkBlockedError(
|
|
660
692
|
ctx.uid,
|
|
661
693
|
lastRecord?.syncMode ?? "unknown",
|
|
662
694
|
syncMode,
|
|
663
695
|
shrinkPlan.dirty,
|
|
664
696
|
shrinkPlan.clean,
|
|
697
|
+
adviceContext,
|
|
665
698
|
);
|
|
666
699
|
}
|
|
667
|
-
// Bulk
|
|
668
|
-
//
|
|
669
|
-
// `hq sync narrow --apply` (its own confirmation)
|
|
670
|
-
//
|
|
671
|
-
//
|
|
700
|
+
// Bulk guard: refuse to auto-move more than the safety cap of CLEAN files in
|
|
701
|
+
// a single foreground sync. A deliberate large narrow goes through
|
|
702
|
+
// `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
|
|
703
|
+
// raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
|
|
704
|
+
// under auto-recover — quarantine is non-destructive so a big recovery is
|
|
705
|
+
// safe, and the runner has no way to act on a thrown cap. The engine moves
|
|
706
|
+
// nothing when it throws here.
|
|
672
707
|
const autoPruneCap = resolveAutoPruneCap();
|
|
673
708
|
if (
|
|
674
|
-
|
|
709
|
+
!effectiveForce &&
|
|
675
710
|
autoPruneCap > 0 &&
|
|
676
711
|
shrinkPlan.clean.length > autoPruneCap
|
|
677
712
|
) {
|
|
@@ -680,27 +715,65 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
680
715
|
syncMode,
|
|
681
716
|
shrinkPlan.clean.length,
|
|
682
717
|
autoPruneCap,
|
|
718
|
+
adviceContext,
|
|
683
719
|
);
|
|
684
720
|
}
|
|
721
|
+
// Clean orphans are QUARANTINED (moved into `.hq/scope-quarantine/<slug>/`,
|
|
722
|
+
// recoverable), never silently deleted — a background sync purging local
|
|
723
|
+
// files unannounced was DEV-1768 fix #3. The quarantine root lives under the
|
|
724
|
+
// real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
|
|
725
|
+
// files don't round-trip back through S3.
|
|
726
|
+
const scopeQuarantineRoot = path.join(
|
|
727
|
+
hqRoot,
|
|
728
|
+
".hq",
|
|
729
|
+
"scope-quarantine",
|
|
730
|
+
journalSlug,
|
|
731
|
+
);
|
|
685
732
|
const shrinkResult = applyScopeShrink({
|
|
686
733
|
journal,
|
|
687
734
|
plan: shrinkPlan,
|
|
688
735
|
hqRoot: companyRoot,
|
|
689
|
-
forceScopeShrink:
|
|
736
|
+
forceScopeShrink: effectiveForce,
|
|
690
737
|
reason: "scope_shrink",
|
|
738
|
+
cleanDisposition: "quarantine",
|
|
739
|
+
quarantineRoot: scopeQuarantineRoot,
|
|
691
740
|
});
|
|
692
|
-
// Surface each
|
|
693
|
-
//
|
|
694
|
-
//
|
|
695
|
-
|
|
741
|
+
// Surface each affected orphan explicitly (named path) so the prune is never
|
|
742
|
+
// silent. Quarantined clean files render as `deleted: true` (removed from the
|
|
743
|
+
// working tree, recoverable in quarantine); dirty files KEPT on disk render
|
|
744
|
+
// as a non-deletion notice so the operator knows they were un-tracked, not
|
|
745
|
+
// removed. The Rust menubar parser already handles `deleted: true`.
|
|
746
|
+
for (const relPath of shrinkResult.quarantinedPaths) {
|
|
747
|
+
emit({
|
|
748
|
+
type: "progress",
|
|
749
|
+
path: relPath,
|
|
750
|
+
bytes: 0,
|
|
751
|
+
deleted: true,
|
|
752
|
+
message: `scope-narrowed: moved out-of-scope copy to ${scopeQuarantineRoot}`,
|
|
753
|
+
});
|
|
754
|
+
}
|
|
755
|
+
for (const relPath of shrinkResult.removedPaths) {
|
|
696
756
|
emit({
|
|
697
757
|
type: "progress",
|
|
698
|
-
path:
|
|
758
|
+
path: relPath,
|
|
699
759
|
bytes: 0,
|
|
700
760
|
deleted: true,
|
|
701
761
|
message: "scope-narrowed (removed local copy outside sync scope)",
|
|
702
762
|
});
|
|
703
763
|
}
|
|
764
|
+
for (const relPath of shrinkResult.dirtyKeptPaths) {
|
|
765
|
+
emit({
|
|
766
|
+
type: "progress",
|
|
767
|
+
path: relPath,
|
|
768
|
+
bytes: 0,
|
|
769
|
+
message:
|
|
770
|
+
"scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
// "Removed from the working tree" = deleted OR quarantined; both vacate the
|
|
774
|
+
// file's original path. Reported as `scopeOrphansRemoved` for back-compat.
|
|
775
|
+
const scopeOrphansRemoved =
|
|
776
|
+
shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined;
|
|
704
777
|
|
|
705
778
|
// Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
|
|
706
779
|
// inline loop; the only structural change is that classification has
|
|
@@ -949,7 +1022,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
949
1022
|
// a conflict abort. `filesOutOfScope` reflects how far the serial
|
|
950
1023
|
// pass got before the abort; that's acceptable for an abort result.
|
|
951
1024
|
filesOutOfScope,
|
|
952
|
-
scopeOrphansRemoved
|
|
1025
|
+
scopeOrphansRemoved,
|
|
953
1026
|
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
954
1027
|
};
|
|
955
1028
|
break;
|
|
@@ -1328,7 +1401,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1328
1401
|
syncMode,
|
|
1329
1402
|
prefixSet: currentPrefixSet,
|
|
1330
1403
|
scopeChangeDetected: shrinkPlan.scopeChangeDetected,
|
|
1331
|
-
orphansRemoved:
|
|
1404
|
+
orphansRemoved: scopeOrphansRemoved,
|
|
1332
1405
|
orphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1333
1406
|
});
|
|
1334
1407
|
|
|
@@ -1347,7 +1420,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1347
1420
|
const changedOnDisk =
|
|
1348
1421
|
filesDownloaded > 0 ||
|
|
1349
1422
|
filesTombstoned > 0 ||
|
|
1350
|
-
|
|
1423
|
+
scopeOrphansRemoved > 0;
|
|
1351
1424
|
if (!options.skipReindex && changedOnDisk) {
|
|
1352
1425
|
try {
|
|
1353
1426
|
// skipLock: the surrounding sync run already holds this root's operation
|
|
@@ -1370,7 +1443,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1370
1443
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1371
1444
|
filesTombstoned,
|
|
1372
1445
|
filesOutOfScope,
|
|
1373
|
-
scopeOrphansRemoved
|
|
1446
|
+
scopeOrphansRemoved,
|
|
1374
1447
|
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1375
1448
|
};
|
|
1376
1449
|
}
|
package/src/index.ts
CHANGED
|
@@ -69,6 +69,7 @@ export {
|
|
|
69
69
|
buildScopeShrinkPlan,
|
|
70
70
|
applyScopeShrink,
|
|
71
71
|
ScopeShrinkBlockedError,
|
|
72
|
+
ScopeShrinkLargePruneError,
|
|
72
73
|
} from "./scope-shrink.js";
|
|
73
74
|
export type {
|
|
74
75
|
OrphanClassification,
|
|
@@ -76,6 +77,8 @@ export type {
|
|
|
76
77
|
BuildScopeShrinkPlanInput,
|
|
77
78
|
ApplyScopeShrinkInput,
|
|
78
79
|
ApplyScopeShrinkResult,
|
|
80
|
+
ScopeShrinkAdviceContext,
|
|
81
|
+
CleanOrphanDisposition,
|
|
79
82
|
} from "./scope-shrink.js";
|
|
80
83
|
|
|
81
84
|
// Engine-layer ACL-aware pull orchestration (US-005)
|
|
@@ -118,6 +121,14 @@ export {
|
|
|
118
121
|
} from "./cognito-auth.js";
|
|
119
122
|
export type { CognitoAuthConfig, CognitoTokens } from "./cognito-auth.js";
|
|
120
123
|
|
|
124
|
+
// Per-company PULL scope resolver (US-005) — shared between hq-sync-runner and
|
|
125
|
+
// `hq sync pull|now` (hq-cli). Exported so hq-cli's foreground pull paths resolve
|
|
126
|
+
// the SAME effective scope the menubar runner does, instead of defaulting every
|
|
127
|
+
// CLI pull to `syncMode: "all"` (the seed of the all→shared scope-shrink wedge,
|
|
128
|
+
// DEV-1768).
|
|
129
|
+
export { resolvePullScope, readPinnedPrefixes } from "./sync/pull-scope.js";
|
|
130
|
+
export type { PullScope, PullScopeClient } from "./sync/pull-scope.js";
|
|
131
|
+
|
|
121
132
|
// Personal-vault scope helpers — shared between hq-sync-runner and `hq sync`
|
|
122
133
|
export {
|
|
123
134
|
PERSONAL_VAULT_EXCLUDED_TOP_LEVEL,
|
package/src/s3.test.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* `Metadata`, so the listing's HEAD fan-out had nothing to attribute.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { describe, it, expect, beforeEach, vi } from "vitest";
|
|
9
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
|
10
10
|
import * as fs from "fs";
|
|
11
11
|
import * as os from "os";
|
|
12
12
|
import * as path from "path";
|
|
@@ -83,6 +83,7 @@ import {
|
|
|
83
83
|
uploadFile,
|
|
84
84
|
uploadSymlink,
|
|
85
85
|
toPosixKey,
|
|
86
|
+
primeUploads,
|
|
86
87
|
downloadFile,
|
|
87
88
|
listRemoteFiles,
|
|
88
89
|
SYMLINK_BODY_PREFIX,
|
|
@@ -92,6 +93,12 @@ import {
|
|
|
92
93
|
FILE_MTIME_META_KEY,
|
|
93
94
|
FILE_BTIME_META_KEY,
|
|
94
95
|
} from "./s3.js";
|
|
96
|
+
import {
|
|
97
|
+
setObjectIOFactory,
|
|
98
|
+
presignObjectIOFactory,
|
|
99
|
+
type PresignTransportClient,
|
|
100
|
+
} from "./object-io.js";
|
|
101
|
+
import type { PresignResultRow } from "./vault-client.js";
|
|
95
102
|
import type { EntityContext } from "./types.js";
|
|
96
103
|
|
|
97
104
|
function makeCtx(): EntityContext {
|
|
@@ -158,6 +165,89 @@ describe("backslash key normalization (Windows client → POSIX S3 key)", () =>
|
|
|
158
165
|
});
|
|
159
166
|
});
|
|
160
167
|
|
|
168
|
+
describe("backslash key normalization on the PRESIGNED (GA) transport", () => {
|
|
169
|
+
// The tests above assert normalization on the S3-SDK transport (sentCommands
|
|
170
|
+
// capture SDK PutObjectCommands). As of 6.5.0 the presigned-URL transport is
|
|
171
|
+
// GA for every company vault (cmp_*) — that's the path a real Windows client
|
|
172
|
+
// like Ridge's now takes. These lock the SAME write-side invariant on that
|
|
173
|
+
// transport: a backslash key is canonicalized to POSIX before the server is
|
|
174
|
+
// ever asked to presign it, so a non-POSIX vault key can never be minted.
|
|
175
|
+
let tmpFile: string;
|
|
176
|
+
let presignCalls: Array<{ op?: string; keys: Array<{ key?: string }> }>;
|
|
177
|
+
|
|
178
|
+
function presignKeysFor(op: string): string[] {
|
|
179
|
+
return presignCalls
|
|
180
|
+
.filter((c) => c.op === op)
|
|
181
|
+
.flatMap((c) => c.keys.map((k) => k.key ?? ""));
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
beforeEach(() => {
|
|
185
|
+
tmpFile = path.join(
|
|
186
|
+
os.tmpdir(),
|
|
187
|
+
`s3-presign-backslash-${Date.now()}-${Math.random()}.md`,
|
|
188
|
+
);
|
|
189
|
+
fs.writeFileSync(tmpFile, "hello");
|
|
190
|
+
|
|
191
|
+
presignCalls = [];
|
|
192
|
+
const vault: PresignTransportClient = {
|
|
193
|
+
// Echo each requested key back as a usable URL so putObject's PUT
|
|
194
|
+
// resolves; record the op + keys so we can assert what was signed.
|
|
195
|
+
presign: async (input) => {
|
|
196
|
+
presignCalls.push({ op: input.op, keys: input.keys as Array<{ key?: string }> });
|
|
197
|
+
const results: PresignResultRow[] = input.keys.map((k) => ({
|
|
198
|
+
key: (k as { key: string }).key,
|
|
199
|
+
op: input.op ?? "put",
|
|
200
|
+
url: `https://s3.test/${(k as { key: string }).key}`,
|
|
201
|
+
}));
|
|
202
|
+
return { results, expiresAt: "2099-01-01T00:00:00.000Z" };
|
|
203
|
+
},
|
|
204
|
+
listFiles: async () => ({ objects: [], cursor: null, truncated: false }),
|
|
205
|
+
};
|
|
206
|
+
setObjectIOFactory(presignObjectIOFactory(vault));
|
|
207
|
+
// putObject moves bytes over the presigned URL via global fetch.
|
|
208
|
+
vi.stubGlobal(
|
|
209
|
+
"fetch",
|
|
210
|
+
vi.fn(async () => new Response(null, { status: 200, headers: { etag: '"e"' } })),
|
|
211
|
+
);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
afterEach(() => {
|
|
215
|
+
setObjectIOFactory(null);
|
|
216
|
+
vi.unstubAllGlobals();
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it("uploadFile presigns + PUTs a POSIX key for a Windows-style backslash path", async () => {
|
|
220
|
+
await uploadFile(makeCtx(), tmpFile, "knowledge\\books-eoi.md");
|
|
221
|
+
expect(presignKeysFor("put")).toContain("knowledge/books-eoi.md");
|
|
222
|
+
// The malformed backslash key is never presigned (so never PUT).
|
|
223
|
+
expect(presignKeysFor("put")).not.toContain("knowledge\\books-eoi.md");
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
it("uploadSymlink presigns a POSIX key for a Windows-style backslash path", async () => {
|
|
227
|
+
await uploadSymlink(makeCtx(), "../target.md", "data\\boots-accounts.json");
|
|
228
|
+
expect(presignKeysFor("put")).toContain("data/boots-accounts.json");
|
|
229
|
+
expect(presignKeysFor("put")).not.toContain("data\\boots-accounts.json");
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it("primeUploads canonicalizes keys so the primed PUT URL matches the upload lookup", async () => {
|
|
233
|
+
// Prime then upload the SAME Windows-origin key. Both must key the cache
|
|
234
|
+
// under the POSIX form: prime signs the POSIX PUT URL, and uploadFile's
|
|
235
|
+
// hasPrimedPut(toPosixKey(...)) reuses it instead of re-presigning — proving
|
|
236
|
+
// the primed fast-path can't strand a backslash key (and can't store one).
|
|
237
|
+
await primeUploads(makeCtx(), [
|
|
238
|
+
{ key: "knowledge\\books-eoi.md", localPath: tmpFile, isSymlink: false },
|
|
239
|
+
]);
|
|
240
|
+
// prime("get") + prime("put") both went out under the POSIX key only.
|
|
241
|
+
expect(presignKeysFor("get")).toEqual(["knowledge/books-eoi.md"]);
|
|
242
|
+
expect(presignKeysFor("put")).toEqual(["knowledge/books-eoi.md"]);
|
|
243
|
+
|
|
244
|
+
const putCountAfterPrime = presignKeysFor("put").length;
|
|
245
|
+
await uploadFile(makeCtx(), tmpFile, "knowledge\\books-eoi.md");
|
|
246
|
+
// No NEW put presign: uploadFile reused the primed POSIX URL (cache hit).
|
|
247
|
+
expect(presignKeysFor("put").length).toBe(putCountAfterPrime);
|
|
248
|
+
});
|
|
249
|
+
});
|
|
250
|
+
|
|
161
251
|
describe("uploadFile", () => {
|
|
162
252
|
let tmpFile: string;
|
|
163
253
|
|
package/src/s3.ts
CHANGED
|
@@ -339,9 +339,14 @@ export async function primeUploads(
|
|
|
339
339
|
if (!io.prime || items.length === 0) return;
|
|
340
340
|
|
|
341
341
|
// Prime GET first so each item's created-at HEAD reuses a cached URL.
|
|
342
|
+
// Canonicalize to POSIX here (one-canonical-form, matching the uploadFile /
|
|
343
|
+
// uploadSymlink boundary): a Windows-origin backslash key must cache under
|
|
344
|
+
// the SAME key uploadFile later looks up via hasPrimedPut(toPosixKey(...)),
|
|
345
|
+
// or the primed URL silently misses and the upload re-presigns. It also
|
|
346
|
+
// keeps the created-at HEAD pointed at the real (POSIX) object.
|
|
342
347
|
await io.prime(
|
|
343
348
|
"get",
|
|
344
|
-
items.map((i) => ({ key: i.key })),
|
|
349
|
+
items.map((i) => ({ key: toPosixKey(i.key) })),
|
|
345
350
|
);
|
|
346
351
|
|
|
347
352
|
// Build per-key PUT metadata with the SAME builders the upload path uses,
|
|
@@ -356,10 +361,15 @@ export async function primeUploads(
|
|
|
356
361
|
const worker = async (): Promise<void> => {
|
|
357
362
|
while (next < items.length) {
|
|
358
363
|
const it = items[next++];
|
|
359
|
-
|
|
364
|
+
// Same boundary guardrail as uploadFile/uploadSymlink: prime under the
|
|
365
|
+
// canonical POSIX key so the cached PUT URL is keyed identically to the
|
|
366
|
+
// hasPrimedPut/putObject lookup, and a backslash key can never be primed
|
|
367
|
+
// (let alone stored) as a non-POSIX vault key.
|
|
368
|
+
const key = toPosixKey(it.key);
|
|
369
|
+
const createdAt = await resolveCreatedAt(io, key, it.author);
|
|
360
370
|
if (it.isSymlink) {
|
|
361
371
|
putKeys.push({
|
|
362
|
-
key
|
|
372
|
+
key,
|
|
363
373
|
contentType: "application/octet-stream",
|
|
364
374
|
metadata: {
|
|
365
375
|
[SYMLINK_TARGET_META_KEY]: SYMLINK_MARKER_META_VALUE,
|
|
@@ -374,8 +384,8 @@ export async function primeUploads(
|
|
|
374
384
|
// raced rm / EPERM — leave stamps off (receiver umask default).
|
|
375
385
|
}
|
|
376
386
|
putKeys.push({
|
|
377
|
-
key
|
|
378
|
-
contentType: getMimeType(
|
|
387
|
+
key,
|
|
388
|
+
contentType: getMimeType(key),
|
|
379
389
|
metadata: {
|
|
380
390
|
...(it.author ? buildAuthorMetadata(it.author, createdAt) : {}),
|
|
381
391
|
...modeTime,
|