@indigoai-us/hq-cloud 6.11.5 → 6.11.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +16 -16
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +51 -41
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +108 -33
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +23 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +54 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +142 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +16 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +1 -62
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/tombstones.d.ts +43 -0
- package/dist/cli/tombstones.d.ts.map +1 -0
- package/dist/cli/tombstones.js +78 -0
- package/dist/cli/tombstones.js.map +1 -0
- package/dist/context.d.ts +6 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +57 -17
- package/dist/context.js.map +1 -1
- package/dist/context.test.js +113 -1
- package/dist/context.test.js.map +1 -1
- package/dist/entity-resolver.test.js +3 -3
- package/dist/entity-resolver.test.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +10 -0
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts +36 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +89 -1
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +143 -1
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/signals/get.d.ts.map +1 -1
- package/dist/signals/get.js +7 -11
- package/dist/signals/get.js.map +1 -1
- package/dist/signals/get.test.js +65 -1
- package/dist/signals/get.test.js.map +1 -1
- package/dist/signals/internals.d.ts +47 -3
- package/dist/signals/internals.d.ts.map +1 -1
- package/dist/signals/internals.js +110 -4
- package/dist/signals/internals.js.map +1 -1
- package/dist/signals/list.d.ts.map +1 -1
- package/dist/signals/list.js +16 -23
- package/dist/signals/list.js.map +1 -1
- package/dist/signals/list.test.js +84 -1
- package/dist/signals/list.test.js.map +1 -1
- package/dist/signals/types.d.ts +18 -1
- package/dist/signals/types.d.ts.map +1 -1
- package/dist/sources/get.d.ts.map +1 -1
- package/dist/sources/get.js +10 -22
- package/dist/sources/get.js.map +1 -1
- package/dist/sources/get.test.js +85 -1
- package/dist/sources/get.test.js.map +1 -1
- package/dist/sources/internals.d.ts +50 -3
- package/dist/sources/internals.d.ts.map +1 -1
- package/dist/sources/internals.js +113 -4
- package/dist/sources/internals.js.map +1 -1
- package/dist/sources/list.d.ts.map +1 -1
- package/dist/sources/list.js +16 -23
- package/dist/sources/list.js.map +1 -1
- package/dist/sources/list.test.js +101 -1
- package/dist/sources/list.test.js.map +1 -1
- package/dist/sources/types.d.ts +18 -1
- package/dist/sources/types.d.ts.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -7
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +6 -7
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/types.d.ts +33 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +14 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +20 -0
- package/dist/version.js.map +1 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +22 -1
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +29 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +131 -41
- package/src/bin/sync-runner.ts +56 -48
- package/src/cli/share.test.ts +169 -0
- package/src/cli/share.ts +81 -0
- package/src/cli/sync.ts +21 -88
- package/src/cli/tombstones.ts +106 -0
- package/src/context.test.ts +139 -1
- package/src/context.ts +59 -17
- package/src/entity-resolver.test.ts +3 -3
- package/src/index.ts +2 -0
- package/src/object-io.ts +12 -0
- package/src/personal-vault.test.ts +175 -0
- package/src/personal-vault.ts +86 -1
- package/src/signals/get.test.ts +83 -1
- package/src/signals/get.ts +9 -13
- package/src/signals/internals.ts +153 -4
- package/src/signals/list.test.ts +114 -1
- package/src/signals/list.ts +16 -29
- package/src/signals/types.ts +18 -1
- package/src/sources/get.test.ts +104 -1
- package/src/sources/get.ts +12 -24
- package/src/sources/internals.ts +156 -4
- package/src/sources/list.test.ts +132 -1
- package/src/sources/list.ts +16 -29
- package/src/sources/types.ts +18 -1
- package/src/sync/event-sync.ts +6 -7
- package/src/types.ts +33 -3
- package/src/version.ts +24 -0
- package/src/watcher.test.ts +41 -0
- package/src/watcher.ts +24 -1
package/src/bin/sync-runner.ts
CHANGED
|
@@ -98,8 +98,10 @@ import type { UploadAuthor } from "../s3.js";
|
|
|
98
98
|
import {
|
|
99
99
|
setObjectIOFactory,
|
|
100
100
|
presignObjectIOFactory,
|
|
101
|
+
type ObjectIOFactory,
|
|
101
102
|
type PresignTransportClient,
|
|
102
103
|
} from "../object-io.js";
|
|
104
|
+
import { HQ_CLOUD_VERSION } from "../version.js";
|
|
103
105
|
import { collectAndSendTelemetry } from "../telemetry.js";
|
|
104
106
|
import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
|
|
105
107
|
import { reindexAfterSync } from "../qmd-reindex.js";
|
|
@@ -227,34 +229,34 @@ export function resolveSkipPersonal(flag: boolean): boolean {
|
|
|
227
229
|
}
|
|
228
230
|
|
|
229
231
|
/**
|
|
230
|
-
*
|
|
232
|
+
* Resolve the object-transport factory for this sync session.
|
|
231
233
|
*
|
|
232
|
-
*
|
|
233
|
-
*
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
*
|
|
234
|
+
* Company vaults (`cmp_*`) ALWAYS use the presigned-URL transport: the client
|
|
235
|
+
* holds no raw AWS credentials (it fetches short-lived signed URLs) and every
|
|
236
|
+
* read/write is authorized server-side per-file, so it never hits the 2048-char
|
|
237
|
+
* STS session-policy ceiling that produced the HQ-59 lockout. The STS-direct-S3
|
|
238
|
+
* (`S3SdkObjectIO`) path for company vaults is RETIRED — there is no env
|
|
239
|
+
* override or rollback lever that can route a company vault back to direct S3.
|
|
240
|
+
* (The former `HQ_SYNC_PRESIGN_TRANSPORT` kill-switch is gone with this change.)
|
|
238
241
|
*
|
|
239
|
-
* `
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
* falls through to the GA default (on).
|
|
242
|
+
* Personal vaults (`prs_*`) KEEP the direct-S3/STS path: the membership-gated
|
|
243
|
+
* `list`/`presign` endpoints 403 for the membership-less vend-self model, and a
|
|
244
|
+
* single-owner personal vault has no ACL-scale problem. That cmp_/prs_ split
|
|
245
|
+
* lives inside {@link presignObjectIOFactory}.
|
|
244
246
|
*
|
|
245
|
-
* `
|
|
246
|
-
*
|
|
247
|
-
* site; it is intentionally unused while the transport is GA.
|
|
247
|
+
* Returns `null` only when the client predates the presign methods (never in a
|
|
248
|
+
* shipped build); the caller then resets to the SDK default factory.
|
|
248
249
|
*/
|
|
249
|
-
export function
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
250
|
+
export function selectObjectIOFactory(
|
|
251
|
+
client: Partial<PresignTransportClient>,
|
|
252
|
+
): ObjectIOFactory | null {
|
|
253
|
+
if (
|
|
254
|
+
typeof client.presign === "function" &&
|
|
255
|
+
typeof client.listFiles === "function"
|
|
256
|
+
) {
|
|
257
|
+
return presignObjectIOFactory(client as PresignTransportClient);
|
|
258
|
+
}
|
|
259
|
+
return null;
|
|
258
260
|
}
|
|
259
261
|
|
|
260
262
|
// Personal-vault scope (exclusion list + path computer) lives in
|
|
@@ -925,10 +927,18 @@ export async function runRunner(
|
|
|
925
927
|
}
|
|
926
928
|
|
|
927
929
|
// ---- vault client -----------------------------------------------------
|
|
930
|
+
// Stamp clientInfo so every vault request (incl. /sts/vend) carries
|
|
931
|
+
// x-hq-client-name=hq-sync + x-hq-client-version=<hq-cloud version>. The
|
|
932
|
+
// company-vend min-version gate (HQ-59, hq-pro) uses this to recognize a
|
|
933
|
+
// compliant (>= 6.11.6) sync-runner and NOT force-upgrade it — the
|
|
934
|
+
// belt-and-suspenders companion to skipping the cmp_ vend on the presign
|
|
935
|
+
// path. Version is the hq-cloud package version (what the gate compares to
|
|
936
|
+
// its floor), not the desktop app version.
|
|
928
937
|
const vaultConfig: VaultServiceConfig = {
|
|
929
938
|
apiUrl: DEFAULT_VAULT_API_URL,
|
|
930
939
|
authToken: getAccessToken,
|
|
931
940
|
region: DEFAULT_COGNITO.region,
|
|
941
|
+
clientInfo: { name: "hq-sync", version: HQ_CLOUD_VERSION },
|
|
932
942
|
};
|
|
933
943
|
const client =
|
|
934
944
|
deps.createVaultClient?.(vaultConfig) ?? new VaultClient(vaultConfig);
|
|
@@ -949,32 +959,29 @@ export async function runRunner(
|
|
|
949
959
|
? { userSub: claims.sub, email: claims.email }
|
|
950
960
|
: undefined;
|
|
951
961
|
|
|
952
|
-
// ---- transport selection (presigned-URL
|
|
953
|
-
//
|
|
954
|
-
//
|
|
955
|
-
//
|
|
956
|
-
//
|
|
957
|
-
//
|
|
958
|
-
//
|
|
959
|
-
//
|
|
960
|
-
// 0/false → force off) for testing or emergency rollback without a redeploy.
|
|
961
|
-
// Setting the factory unconditionally (even to the default) keeps the choice
|
|
962
|
+
// ---- transport selection (presigned-URL for cmp_, direct-S3/STS for prs_) -
|
|
963
|
+
// Company vaults ALWAYS use the presigned-URL transport now (HQ-59): the
|
|
964
|
+
// STS-direct-S3 path for cmp_ is retired — no env override routes a company
|
|
965
|
+
// vault back to direct S3. Personal vaults keep direct-S3 inside the factory
|
|
966
|
+
// (the cmp_/prs_ split lives in presignObjectIOFactory). selectObjectIOFactory
|
|
967
|
+
// runs ONCE here — every s3.ts call in this session's fanout resolves through
|
|
968
|
+
// the installed factory. Setting it unconditionally (even to null, which the
|
|
969
|
+
// default S3 SDK factory backs for a pre-presign client) keeps the choice
|
|
962
970
|
// deterministic if a prior run mutated module state.
|
|
963
971
|
const presignCapable = client as Partial<PresignTransportClient>;
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
}
|
|
972
|
+
const objectIOFactory = selectObjectIOFactory(presignCapable);
|
|
973
|
+
setObjectIOFactory(objectIOFactory);
|
|
974
|
+
// HQ-59 vend-skip: when company vaults run over the presign transport (the
|
|
975
|
+
// shipped case — selectObjectIOFactory only returns null for a pre-presign
|
|
976
|
+
// client), tell resolveEntityContext to SKIP `POST /sts/vend` for cmp_
|
|
977
|
+
// contexts. Two reasons, both load-bearing: (1) the presign transport never
|
|
978
|
+
// reads STS creds, so the vend is dead weight; (2) a compliant sync-runner
|
|
979
|
+
// must not call the company vend route at all — its presence is the
|
|
980
|
+
// pre-6.11.6 signal the hq-pro min-version gate denies on. Gated on the SAME
|
|
981
|
+
// value that picks the transport, so the vend-skip and the transport can
|
|
982
|
+
// never diverge. If the factory is null (S3 SDK fallback), cmp_ keeps vending
|
|
983
|
+
// because S3SdkObjectIO needs the creds. Personal vaults always vend self.
|
|
984
|
+
vaultConfig.companyVaultUsesPresign = objectIOFactory !== null;
|
|
978
985
|
|
|
979
986
|
// ---- resolve targets --------------------------------------------------
|
|
980
987
|
let memberships: Pick<Membership, "companyUid">[];
|
|
@@ -1235,6 +1242,7 @@ export async function runRunner(
|
|
|
1235
1242
|
filesTombstoned: 0,
|
|
1236
1243
|
filesRefusedStale: 0,
|
|
1237
1244
|
filesRefusedStalePaths: [],
|
|
1245
|
+
filesSuppressedByTombstone: 0,
|
|
1238
1246
|
filesExcludedByPolicy: 0,
|
|
1239
1247
|
filesExcludedByScope: 0,
|
|
1240
1248
|
conflictPaths: [],
|
package/src/cli/share.test.ts
CHANGED
|
@@ -336,6 +336,175 @@ describe("share", () => {
|
|
|
336
336
|
expect(uploadFile).toHaveBeenCalledWith(expect.anything(), testFile, "changed.md", undefined, expect.anything());
|
|
337
337
|
});
|
|
338
338
|
|
|
339
|
+
// ── Push-side FILE_TOMBSTONE consult (delete-resync, 3B) ────────────────────
|
|
340
|
+
// An authoritative delete (`hq files delete`) writes a FILE_TOMBSTONE and
|
|
341
|
+
// removes the S3 object. Without a push-side consult, a behind peer that still
|
|
342
|
+
// holds the deleted file RE-UPLOADS it on a skipUnchanged=false push (e.g.
|
|
343
|
+
// `hq cloud share <path>`); the re-upload post-dates the tombstone, so the
|
|
344
|
+
// pull planner's timestamp-only re-create heuristic treats it as a genuine
|
|
345
|
+
// re-create and resurrects the key for everyone. These tests pin the fix: a
|
|
346
|
+
// stale-baseline copy is suppressed, while genuine content (locally-changed or
|
|
347
|
+
// no-journal) still uploads. Differential proof: reverting the consult in
|
|
348
|
+
// share.ts makes the first test fail (uploadFile IS called → resurrection).
|
|
349
|
+
describe("push-side tombstone consult", () => {
|
|
350
|
+
function seedJournal(key: string, hash: string, size: number) {
|
|
351
|
+
fs.writeFileSync(
|
|
352
|
+
path.join(stateDir, "sync-journal.acme.json"),
|
|
353
|
+
JSON.stringify({
|
|
354
|
+
version: "1",
|
|
355
|
+
lastSync: new Date().toISOString(),
|
|
356
|
+
files: {
|
|
357
|
+
[key]: {
|
|
358
|
+
hash,
|
|
359
|
+
size,
|
|
360
|
+
syncedAt: new Date().toISOString(),
|
|
361
|
+
direction: "down",
|
|
362
|
+
},
|
|
363
|
+
},
|
|
364
|
+
}),
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
it("suppresses re-upload of a stale-baseline copy of a tombstoned key", async () => {
|
|
369
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
370
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
371
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
372
|
+
fs.writeFileSync(f, "shared content");
|
|
373
|
+
const { hashFile } = await import("../journal.js");
|
|
374
|
+
// Journal hash === current file hash → this machine holds the exact
|
|
375
|
+
// deleted baseline (a behind peer that never pulled the delete).
|
|
376
|
+
seedJournal("docs/shared.md", hashFile(f), 14);
|
|
377
|
+
|
|
378
|
+
const events: Array<{ type: string; path?: string }> = [];
|
|
379
|
+
const result = await share({
|
|
380
|
+
paths: [path.join(companyRoot, "docs")],
|
|
381
|
+
company: "acme",
|
|
382
|
+
vaultConfig: mockConfig,
|
|
383
|
+
hqRoot: tmpDir,
|
|
384
|
+
// skipUnchanged omitted (=false): models `hq cloud share <path>`, the
|
|
385
|
+
// path where a behind peer would otherwise re-upload the stale copy.
|
|
386
|
+
fileTombstones: new Map([
|
|
387
|
+
["docs/shared.md", { deletedAt: new Date().toISOString() }],
|
|
388
|
+
]),
|
|
389
|
+
onEvent: (e) => events.push(e as { type: string; path?: string }),
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
// The resurrection is blocked at the source: no upload for the tombstoned key.
|
|
393
|
+
expect(uploadFile).not.toHaveBeenCalled();
|
|
394
|
+
expect(result.filesUploaded).toBe(0);
|
|
395
|
+
expect(result.filesSuppressedByTombstone).toBe(1);
|
|
396
|
+
expect(
|
|
397
|
+
events.some(
|
|
398
|
+
(e) => e.type === "upload-suppressed-tombstone" && e.path === "docs/shared.md",
|
|
399
|
+
),
|
|
400
|
+
).toBe(true);
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
it("still uploads a LOCALLY-CHANGED file even when tombstoned (genuine edit/re-create)", async () => {
|
|
404
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
405
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
406
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
407
|
+
fs.writeFileSync(f, "edited-since-delete");
|
|
408
|
+
// Journal hash is stale (the file was edited after the delete) → genuine
|
|
409
|
+
// new content, must NOT be suppressed.
|
|
410
|
+
seedJournal("docs/shared.md", "stale-baseline-hash", 14);
|
|
411
|
+
|
|
412
|
+
const result = await share({
|
|
413
|
+
paths: [path.join(companyRoot, "docs")],
|
|
414
|
+
company: "acme",
|
|
415
|
+
vaultConfig: mockConfig,
|
|
416
|
+
hqRoot: tmpDir,
|
|
417
|
+
fileTombstones: new Map([
|
|
418
|
+
["docs/shared.md", { deletedAt: new Date().toISOString() }],
|
|
419
|
+
]),
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
expect(result.filesSuppressedByTombstone).toBe(0);
|
|
423
|
+
expect(result.filesUploaded).toBe(1);
|
|
424
|
+
expect(uploadFile).toHaveBeenCalledWith(
|
|
425
|
+
expect.anything(),
|
|
426
|
+
f,
|
|
427
|
+
"docs/shared.md",
|
|
428
|
+
undefined,
|
|
429
|
+
expect.anything(),
|
|
430
|
+
);
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
it("still uploads a tombstoned key with NO journal entry (genuine re-create)", async () => {
|
|
434
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
435
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
436
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
437
|
+
fs.writeFileSync(f, "freshly created");
|
|
438
|
+
// No journal entry seeded → indistinguishable from genuine new content;
|
|
439
|
+
// fail-open and upload (mirrors the pull side's `!journalEntry` branch).
|
|
440
|
+
|
|
441
|
+
const result = await share({
|
|
442
|
+
paths: [path.join(companyRoot, "docs")],
|
|
443
|
+
company: "acme",
|
|
444
|
+
vaultConfig: mockConfig,
|
|
445
|
+
hqRoot: tmpDir,
|
|
446
|
+
fileTombstones: new Map([
|
|
447
|
+
["docs/shared.md", { deletedAt: new Date().toISOString() }],
|
|
448
|
+
]),
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
expect(result.filesSuppressedByTombstone).toBe(0);
|
|
452
|
+
expect(result.filesUploaded).toBe(1);
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it("auto-fetches tombstones via vaultConfig (no injection) and suppresses", async () => {
|
|
456
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
457
|
+
fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
|
|
458
|
+
const f = path.join(companyRoot, "docs", "shared.md");
|
|
459
|
+
fs.writeFileSync(f, "shared content");
|
|
460
|
+
const { hashFile } = await import("../journal.js");
|
|
461
|
+
seedJournal("docs/shared.md", hashFile(f), 14);
|
|
462
|
+
|
|
463
|
+
// Re-stub fetch to also answer GET /v1/files/tombstones, proving share()
|
|
464
|
+
// fetches and consults tombstones on its own (the production wiring) when
|
|
465
|
+
// no map is injected.
|
|
466
|
+
const fetchMock = vi.fn().mockImplementation(async (url: string) => {
|
|
467
|
+
const u = String(url);
|
|
468
|
+
if (u.includes("/entity/check-slug/me")) {
|
|
469
|
+
return {
|
|
470
|
+
ok: true,
|
|
471
|
+
status: 200,
|
|
472
|
+
json: async () => ({ available: false, conflictingCompanyUid: mockEntity.uid }),
|
|
473
|
+
text: async () => "",
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
if (u.includes("/entity/by-slug/") || /\/entity\/cmp_/.test(u)) {
|
|
477
|
+
return { ok: true, status: 200, json: async () => ({ entity: mockEntity }), text: async () => "" };
|
|
478
|
+
}
|
|
479
|
+
if (u.includes("/sts/vend")) {
|
|
480
|
+
return { ok: true, status: 200, json: async () => mockVendResponse, text: async () => "" };
|
|
481
|
+
}
|
|
482
|
+
if (u.includes("/v1/files/tombstones")) {
|
|
483
|
+
return {
|
|
484
|
+
ok: true,
|
|
485
|
+
status: 200,
|
|
486
|
+
json: async () => ({
|
|
487
|
+
tombstones: [{ key: "docs/shared.md", deletedAt: new Date().toISOString() }],
|
|
488
|
+
}),
|
|
489
|
+
text: async () => "",
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
return { ok: false, status: 404, text: async () => "Not found" };
|
|
493
|
+
});
|
|
494
|
+
vi.stubGlobal("fetch", fetchMock);
|
|
495
|
+
|
|
496
|
+
const result = await share({
|
|
497
|
+
paths: [path.join(companyRoot, "docs")],
|
|
498
|
+
company: "acme",
|
|
499
|
+
vaultConfig: mockConfig,
|
|
500
|
+
hqRoot: tmpDir,
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
expect(uploadFile).not.toHaveBeenCalled();
|
|
504
|
+
expect(result.filesSuppressedByTombstone).toBe(1);
|
|
505
|
+
});
|
|
506
|
+
});
|
|
507
|
+
|
|
339
508
|
it("populates conflictPaths and emits a conflict event when both local and remote drifted from journal", async () => {
|
|
340
509
|
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
341
510
|
fs.mkdirSync(companyRoot, { recursive: true });
|
package/src/cli/share.ts
CHANGED
|
@@ -42,6 +42,10 @@ import {
|
|
|
42
42
|
import { resolveConflict } from "./conflict.js";
|
|
43
43
|
import type { ConflictStrategy } from "./conflict.js";
|
|
44
44
|
import type { SyncProgressEvent } from "./sync.js";
|
|
45
|
+
import {
|
|
46
|
+
fetchCompanyTombstones,
|
|
47
|
+
type CompanyTombstone,
|
|
48
|
+
} from "./tombstones.js";
|
|
45
49
|
import { isCoveredByAny, isDirInScope } from "../prefix-coalesce.js";
|
|
46
50
|
import {
|
|
47
51
|
buildConflictId,
|
|
@@ -634,6 +638,19 @@ export interface ShareOptions {
|
|
|
634
638
|
* path is out of scope (mirrors the pull side's `isCoveredByAny([])`).
|
|
635
639
|
*/
|
|
636
640
|
prefixSet?: string[];
|
|
641
|
+
/**
|
|
642
|
+
* Pre-fetched FILE_TOMBSTONE map (POSIX key → tombstone) for the push-side
|
|
643
|
+
* delete-resync consult. When omitted, share() fetches it itself via
|
|
644
|
+
* `fetchCompanyTombstones` for COMPANY vaults that have a `vaultConfig`
|
|
645
|
+
* (personal vaults and pre-vended `entityContext`-only callers without a
|
|
646
|
+
* `vaultConfig` degrade to no-suppression — the safe, legacy direction).
|
|
647
|
+
*
|
|
648
|
+
* Injection is an optimization + test seam: a sync run that already fetched
|
|
649
|
+
* tombstones for the pull leg can hand the same map to the push leg to avoid a
|
|
650
|
+
* second round-trip, and tests can supply a controlled map without stubbing
|
|
651
|
+
* the network. See the consult in the Stage-2 classification pass.
|
|
652
|
+
*/
|
|
653
|
+
fileTombstones?: Map<string, CompanyTombstone>;
|
|
637
654
|
}
|
|
638
655
|
|
|
639
656
|
export interface ShareResult {
|
|
@@ -678,6 +695,15 @@ export interface ShareResult {
|
|
|
678
695
|
* once the runner has folded them into the totals.
|
|
679
696
|
*/
|
|
680
697
|
filesRefusedStalePaths: string[];
|
|
698
|
+
/**
|
|
699
|
+
* Number of uploads suppressed by the push-side FILE_TOMBSTONE consult — keys
|
|
700
|
+
* an authoritative delete (`hq files delete`) tombstoned that this machine
|
|
701
|
+
* still held as the unchanged synced baseline. Skipping the upload is what
|
|
702
|
+
* stops a behind peer from resurrecting an authoritatively-deleted key. Always
|
|
703
|
+
* 0 when there are no tombstones for in-scope keys (the common case) or when
|
|
704
|
+
* the run can't load tombstones (personal vault / no `vaultConfig`).
|
|
705
|
+
*/
|
|
706
|
+
filesSuppressedByTombstone: number;
|
|
681
707
|
/**
|
|
682
708
|
* Number of paths blocked by `PERSONAL_VAULT_DEFAULT_EXCLUSIONS` during this
|
|
683
709
|
* run (push leg, personalMode=true). Includes both files that would have
|
|
@@ -919,6 +945,10 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
919
945
|
// propagateDeletes=false path.
|
|
920
946
|
let filesTombstoned = 0;
|
|
921
947
|
let filesRefusedStale = 0;
|
|
948
|
+
// Count of uploads suppressed by the push-side FILE_TOMBSTONE consult (a
|
|
949
|
+
// behind peer's stale copy of an authoritatively-deleted key). Always 0 when
|
|
950
|
+
// there are no tombstones for in-scope keys.
|
|
951
|
+
let filesSuppressedByTombstone = 0;
|
|
922
952
|
// Capped at 50 to bound event payload size — `newFiles` uses the same cap.
|
|
923
953
|
const REFUSED_STALE_PATH_CAP = 50;
|
|
924
954
|
const filesRefusedStalePaths: string[] = [];
|
|
@@ -1058,6 +1088,29 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1058
1088
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
1059
1089
|
})();
|
|
1060
1090
|
|
|
1091
|
+
// Push-side FILE_TOMBSTONE consult (delete-resync) — symmetric to the pull
|
|
1092
|
+
// planner's suppression in sync.ts. An authoritative delete
|
|
1093
|
+
// (`hq files delete <prefix>`) writes a FILE_TOMBSTONE and removes the S3
|
|
1094
|
+
// object; the pull side already honors it. But the PUSH side did not: a behind
|
|
1095
|
+
// peer who still holds the deleted file locally would re-upload it here, and
|
|
1096
|
+
// because the re-uploaded object post-dates the tombstone, the pull planner's
|
|
1097
|
+
// timestamp-only re-create heuristic (`isRemoteRecreateAfterTombstone`) treats
|
|
1098
|
+
// it as a genuine re-create and resurrects the key for EVERYONE — defeating the
|
|
1099
|
+
// authoritative delete. Consult the tombstones so a stale-baseline upload is
|
|
1100
|
+
// skipped at the source.
|
|
1101
|
+
//
|
|
1102
|
+
// Source: an injected `fileTombstones` (a sync run can hand the push leg the
|
|
1103
|
+
// map it already fetched for the pull leg), else a self-fetch for COMPANY
|
|
1104
|
+
// vaults that have a `vaultConfig`. Personal vaults have no company tombstones
|
|
1105
|
+
// (the pull side skips them too), and `entityContext`-only callers have no
|
|
1106
|
+
// auth to fetch with — both degrade to an empty map (no suppression), the
|
|
1107
|
+
// safe/legacy direction.
|
|
1108
|
+
const fileTombstones: Map<string, CompanyTombstone> =
|
|
1109
|
+
options.fileTombstones ??
|
|
1110
|
+
(!ctx.uid.startsWith("prs_") && vaultConfig
|
|
1111
|
+
? await fetchCompanyTombstones(vaultConfig, ctx.uid)
|
|
1112
|
+
: new Map<string, CompanyTombstone>());
|
|
1113
|
+
|
|
1061
1114
|
// Phase A: serial classification pass — handle skips inline, collect
|
|
1062
1115
|
// upload candidates for the parallel pool.
|
|
1063
1116
|
const uploadItems: Array<typeof plan.items[number] & { action: "upload" }> = [];
|
|
@@ -1071,6 +1124,30 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1071
1124
|
filesSkipped++;
|
|
1072
1125
|
continue;
|
|
1073
1126
|
}
|
|
1127
|
+
// Suppress the re-upload of an authoritatively-deleted key when the local
|
|
1128
|
+
// copy is still the deleted baseline. Discrimination mirrors the pull side's
|
|
1129
|
+
// `localChanged || !journalEntry` test: a journal entry whose hash matches
|
|
1130
|
+
// the upload's localHash means the file is byte-identical to what was last
|
|
1131
|
+
// synced — i.e. the deleted version a behind peer never pulled away — so
|
|
1132
|
+
// SKIP it. A locally-changed file (hash differs) or one with no journal
|
|
1133
|
+
// entry is genuine new content (a real re-create or a post-delete edit) and
|
|
1134
|
+
// uploads normally. The deleter's own stale local copy is removed by the
|
|
1135
|
+
// pull leg's `tombstone-delete`; this guard only stops the resurrection.
|
|
1136
|
+
if (item.action === "upload" && fileTombstones.size > 0) {
|
|
1137
|
+
const ts = fileTombstones.get(toPosixKey(item.relativePath));
|
|
1138
|
+
if (ts !== undefined) {
|
|
1139
|
+
const entry = journal.files[item.relativePath];
|
|
1140
|
+
if (entry && entry.hash === item.localHash) {
|
|
1141
|
+
filesSuppressedByTombstone++;
|
|
1142
|
+
emit({
|
|
1143
|
+
type: "upload-suppressed-tombstone",
|
|
1144
|
+
path: item.relativePath,
|
|
1145
|
+
deletedAt: ts.deletedAt,
|
|
1146
|
+
});
|
|
1147
|
+
continue;
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1074
1151
|
if (item.action === "skip-unchanged") {
|
|
1075
1152
|
// Refresh the journal's (mtimeMs, size) for a touched-but-identical file
|
|
1076
1153
|
// so the next sync's lstat fast-path matches and skips without re-hashing.
|
|
@@ -1546,6 +1623,9 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1546
1623
|
// defaulting fallback. Empty on the abort path because the
|
|
1547
1624
|
// delete-plan execution loop is short-circuited.
|
|
1548
1625
|
filesRefusedStalePaths,
|
|
1626
|
+
// Tombstone-suppressed uploads are classified in Phase A, which runs
|
|
1627
|
+
// before the upload pool can abort, so the count is meaningful here.
|
|
1628
|
+
filesSuppressedByTombstone,
|
|
1549
1629
|
// Exclusions are computed during the upload walk which has
|
|
1550
1630
|
// already completed by the time we hit a per-file conflict-
|
|
1551
1631
|
// abort, so the count is meaningful here. No event emit on
|
|
@@ -1717,6 +1797,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1717
1797
|
filesTombstoned,
|
|
1718
1798
|
filesRefusedStale,
|
|
1719
1799
|
filesRefusedStalePaths,
|
|
1800
|
+
filesSuppressedByTombstone,
|
|
1720
1801
|
filesExcludedByPolicy: excludedSet.size,
|
|
1721
1802
|
filesExcludedByScope: scopeExcludedSet.size,
|
|
1722
1803
|
conflictPaths,
|
package/src/cli/sync.ts
CHANGED
|
@@ -54,6 +54,10 @@ import {
|
|
|
54
54
|
} from "../lib/conflict-file.js";
|
|
55
55
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
56
56
|
import { reindex } from "./reindex.js";
|
|
57
|
+
import {
|
|
58
|
+
fetchCompanyTombstones,
|
|
59
|
+
type CompanyTombstone,
|
|
60
|
+
} from "./tombstones.js";
|
|
57
61
|
|
|
58
62
|
/**
|
|
59
63
|
* Per-file events emitted by `sync()` as it progresses.
|
|
@@ -265,6 +269,23 @@ export type SyncProgressEvent =
|
|
|
265
269
|
type: "scope-excluded";
|
|
266
270
|
count: number;
|
|
267
271
|
samplePaths: string[];
|
|
272
|
+
}
|
|
273
|
+
| {
|
|
274
|
+
/**
|
|
275
|
+
* Emitted by the PUSH leg (`share()`) once per key whose upload was
|
|
276
|
+
* suppressed because an authoritative FILE_TOMBSTONE marks it deleted and
|
|
277
|
+
* the local copy is still the deleted baseline (journal hash unchanged).
|
|
278
|
+
* Without this, a behind peer that still holds the file would re-upload it,
|
|
279
|
+
* and the pull planner's timestamp-only re-create heuristic
|
|
280
|
+
* (`isRemoteRecreateAfterTombstone`) would treat the re-upload as a genuine
|
|
281
|
+
* re-create and resurrect the key for everyone. The deleter's own stale
|
|
282
|
+
* local copy is cleaned by the pull leg's `tombstone-delete`; this event
|
|
283
|
+
* records that the push refused to resurrect it. `deletedAt` is the
|
|
284
|
+
* tombstone's delete time.
|
|
285
|
+
*/
|
|
286
|
+
type: "upload-suppressed-tombstone";
|
|
287
|
+
path: string;
|
|
288
|
+
deletedAt: string;
|
|
268
289
|
};
|
|
269
290
|
|
|
270
291
|
export interface SyncOptions {
|
|
@@ -539,94 +560,6 @@ async function reportNewFilesToNotify(
|
|
|
539
560
|
}
|
|
540
561
|
}
|
|
541
562
|
|
|
542
|
-
/** Timeout for the best-effort FILE_TOMBSTONE fetch (GET /v1/files/tombstones). */
|
|
543
|
-
const FETCH_TOMBSTONES_TIMEOUT_MS = 5000;
|
|
544
|
-
|
|
545
|
-
/**
|
|
546
|
-
* A FILE_TOMBSTONE as the pull planner needs it: the deleted key + when it was
|
|
547
|
-
* deleted. The `deletedAt` timestamp is the decisive precedence signal — a
|
|
548
|
-
* remote object newer than it is a genuine re-create (sync it), an object at or
|
|
549
|
-
* older than it is a stale resurrection of a deleted key (suppress it).
|
|
550
|
-
*/
|
|
551
|
-
interface CompanyTombstone {
|
|
552
|
-
deletedAt: string;
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
/**
|
|
556
|
-
* Fetch the company's FILE_TOMBSTONE rows from hq-pro (GET /v1/files/tombstones)
|
|
557
|
-
* and return them as a POSIX-keyed map the pull planner consults to avoid
|
|
558
|
-
* resurrecting an intentionally-deleted object (delete-resync). The endpoint is
|
|
559
|
-
* ACL-filtered server-side, so the map only ever contains keys this caller can
|
|
560
|
-
* read — exactly the keys that can appear in the (STS-scoped) remote LIST.
|
|
561
|
-
*
|
|
562
|
-
* Best-effort and bounded by a 5s timeout: a tombstone read that fails, times
|
|
563
|
-
* out, or returns non-2xx degrades to an EMPTY map — i.e. to the pre-fix
|
|
564
|
-
* behavior (no suppression). That is the safe failure direction: a missed
|
|
565
|
-
* tombstone re-pulls a deleted file (a known, recoverable annoyance), whereas a
|
|
566
|
-
* spurious tombstone would HIDE a file the user wants. The failure is logged
|
|
567
|
-
* (never silently swallowed) so a persistently-degraded read is visible.
|
|
568
|
-
*/
|
|
569
|
-
async function fetchCompanyTombstones(
|
|
570
|
-
vaultConfig: VaultServiceConfig,
|
|
571
|
-
companyUid: string,
|
|
572
|
-
): Promise<Map<string, CompanyTombstone>> {
|
|
573
|
-
const out = new Map<string, CompanyTombstone>();
|
|
574
|
-
try {
|
|
575
|
-
const token =
|
|
576
|
-
typeof vaultConfig.authToken === "function"
|
|
577
|
-
? await vaultConfig.authToken()
|
|
578
|
-
: vaultConfig.authToken;
|
|
579
|
-
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
580
|
-
const url = `${base}/v1/files/tombstones?company=${encodeURIComponent(
|
|
581
|
-
companyUid,
|
|
582
|
-
)}`;
|
|
583
|
-
const controller = new AbortController();
|
|
584
|
-
const timer = setTimeout(
|
|
585
|
-
() => controller.abort(),
|
|
586
|
-
FETCH_TOMBSTONES_TIMEOUT_MS,
|
|
587
|
-
);
|
|
588
|
-
try {
|
|
589
|
-
const res = await fetch(url, {
|
|
590
|
-
method: "GET",
|
|
591
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
592
|
-
signal: controller.signal,
|
|
593
|
-
});
|
|
594
|
-
if (!res.ok) {
|
|
595
|
-
// Non-2xx is non-fatal: log and degrade to no-suppression. A 404 means
|
|
596
|
-
// the endpoint is not deployed yet (hq-pro release lag) — the pull
|
|
597
|
-
// proceeds with the legacy behavior until it lands.
|
|
598
|
-
console.error(
|
|
599
|
-
`[hq-sync] tombstone fetch returned ${res.status} (degrading to no-suppression)`,
|
|
600
|
-
);
|
|
601
|
-
return out;
|
|
602
|
-
}
|
|
603
|
-
const body = (await res.json()) as {
|
|
604
|
-
tombstones?: Array<{ key?: string; deletedAt?: string }>;
|
|
605
|
-
};
|
|
606
|
-
for (const t of body.tombstones ?? []) {
|
|
607
|
-
if (typeof t.key === "string" && typeof t.deletedAt === "string") {
|
|
608
|
-
out.set(toPosixKey(t.key), { deletedAt: t.deletedAt });
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
} finally {
|
|
612
|
-
clearTimeout(timer);
|
|
613
|
-
}
|
|
614
|
-
} catch (err) {
|
|
615
|
-
// Best-effort: a failed tombstone read must never break the pull. Log
|
|
616
|
-
// (policy: never silently swallow) and degrade to no-suppression.
|
|
617
|
-
try {
|
|
618
|
-
console.error(
|
|
619
|
-
`[hq-sync] tombstone fetch failed (non-fatal, degrading to no-suppression): ${
|
|
620
|
-
err instanceof Error ? err.message : String(err)
|
|
621
|
-
}`,
|
|
622
|
-
);
|
|
623
|
-
} catch {
|
|
624
|
-
// swallow — logging must never break sync
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
|
-
return out;
|
|
628
|
-
}
|
|
629
|
-
|
|
630
563
|
/**
|
|
631
564
|
* Sync (pull) all allowed files from the entity vault.
|
|
632
565
|
*/
|