@indigoai-us/hq-cloud 6.11.11 → 6.11.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +231 -52
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +265 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +58 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.js +138 -15
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +178 -58
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +362 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +148 -29
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +8 -2
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +6 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +62 -13
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +189 -0
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +6 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +13 -15
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +36 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watcher.d.ts +36 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +152 -30
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +298 -11
- package/src/bin/sync-runner.ts +254 -52
- package/src/cli/rescue-classify-ordering.test.ts +61 -0
- package/src/cli/rescue-core.ts +174 -15
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +103 -34
- package/src/cli/sync.test.ts +435 -1
- package/src/cli/sync.ts +217 -64
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +182 -30
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +8 -2
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +205 -0
- package/src/remote-pull.ts +77 -14
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +38 -1
- package/src/sync/push-receiver.ts +15 -18
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +209 -33
package/src/cli/sync.ts
CHANGED
|
@@ -43,7 +43,11 @@ import {
|
|
|
43
43
|
ScopeShrinkLargePruneError,
|
|
44
44
|
type ScopeShrinkAdviceContext,
|
|
45
45
|
} from "../scope-shrink.js";
|
|
46
|
-
import {
|
|
46
|
+
import {
|
|
47
|
+
coalescePrefixes,
|
|
48
|
+
isCoveredByAny,
|
|
49
|
+
type ScopePrefixInput,
|
|
50
|
+
} from "../prefix-coalesce.js";
|
|
47
51
|
import { createIgnoreFilter } from "../ignore.js";
|
|
48
52
|
import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
49
53
|
import { resolveConflict } from "./conflict.js";
|
|
@@ -55,6 +59,7 @@ import {
|
|
|
55
59
|
} from "../lib/conflict-file.js";
|
|
56
60
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
57
61
|
import { reindex } from "./reindex.js";
|
|
62
|
+
import { withOperationLock } from "../operation-lock.js";
|
|
58
63
|
import {
|
|
59
64
|
fetchCompanyTombstones,
|
|
60
65
|
type CompanyTombstone,
|
|
@@ -369,7 +374,7 @@ export interface SyncOptions {
|
|
|
369
374
|
* (not empty `"shared"`) on any grant-resolution error, so a transient
|
|
370
375
|
* failure can never silently prune the local tree.
|
|
371
376
|
*/
|
|
372
|
-
prefixSet?:
|
|
377
|
+
prefixSet?: ScopePrefixInput[];
|
|
373
378
|
/**
|
|
374
379
|
* When the effective scope shrinks relative to the last pull and the shrink
|
|
375
380
|
* would orphan locally-modified ("dirty") files, `sync()` aborts with a
|
|
@@ -414,6 +419,11 @@ export interface SyncOptions {
|
|
|
414
419
|
* this and run `reindex()` once itself instead of per-company.
|
|
415
420
|
*/
|
|
416
421
|
skipReindex?: boolean;
|
|
422
|
+
/**
|
|
423
|
+
* Internal runner seam: true only when the caller already holds the
|
|
424
|
+
* per-root operation lock for this sync pass.
|
|
425
|
+
*/
|
|
426
|
+
operationLockAlreadyHeld?: boolean;
|
|
417
427
|
}
|
|
418
428
|
|
|
419
429
|
export interface SyncResult {
|
|
@@ -497,6 +507,16 @@ export function resolveAutoPruneCap(): number {
|
|
|
497
507
|
/** Max time to wait on the best-effort new-files notification POST. */
|
|
498
508
|
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
499
509
|
|
|
510
|
+
/**
|
|
511
|
+
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
512
|
+
* an oversized batch wholesale, so the client MUST split a large report into
|
|
513
|
+
* chunks at or under this size — otherwise a first sync with more than this many
|
|
514
|
+
* new files reports none of them, and the same oversized batch re-triggers every
|
|
515
|
+
* sync cycle (wasted work + dropped notifications). Keep in lockstep with the
|
|
516
|
+
* server-side limit.
|
|
517
|
+
*/
|
|
518
|
+
const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
|
|
519
|
+
|
|
500
520
|
/**
|
|
501
521
|
* Best-effort report of the files that were new to this drive during the sync,
|
|
502
522
|
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
@@ -505,22 +525,36 @@ const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
|
505
525
|
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
506
526
|
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
507
527
|
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
508
|
-
* 5s timeout so a hung endpoint can't stall sync completion. No-op
|
|
509
|
-
* are no new files.
|
|
528
|
+
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
529
|
+
* when there are no new files.
|
|
530
|
+
*
|
|
531
|
+
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
532
|
+
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
533
|
+
* best-effort, so one failing/oversized batch can never block the others or the
|
|
534
|
+
* sync. Exported only so the chunking can be unit-tested directly.
|
|
510
535
|
*/
|
|
511
|
-
async function reportNewFilesToNotify(
|
|
536
|
+
export async function reportNewFilesToNotify(
|
|
512
537
|
vaultConfig: VaultServiceConfig,
|
|
513
538
|
companyUid: string,
|
|
514
539
|
companySlug: string,
|
|
515
540
|
files: Array<{ path: string; bytes: number; addedBy: string | null }>,
|
|
516
541
|
): Promise<void> {
|
|
517
542
|
if (files.length === 0) return;
|
|
543
|
+
|
|
544
|
+
let token: string;
|
|
518
545
|
try {
|
|
519
|
-
|
|
546
|
+
token =
|
|
520
547
|
typeof vaultConfig.authToken === "function"
|
|
521
548
|
? await vaultConfig.authToken()
|
|
522
549
|
: vaultConfig.authToken;
|
|
523
|
-
|
|
550
|
+
} catch (err) {
|
|
551
|
+
logNotifyFailure(err);
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
555
|
+
|
|
556
|
+
for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
|
|
557
|
+
const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
|
|
524
558
|
const controller = new AbortController();
|
|
525
559
|
const timer = setTimeout(
|
|
526
560
|
() => controller.abort(),
|
|
@@ -536,7 +570,7 @@ async function reportNewFilesToNotify(
|
|
|
536
570
|
body: JSON.stringify({
|
|
537
571
|
companyUid,
|
|
538
572
|
companySlug,
|
|
539
|
-
files:
|
|
573
|
+
files: batch.map((f) => ({
|
|
540
574
|
path: f.path,
|
|
541
575
|
bytes: f.bytes,
|
|
542
576
|
...(f.addedBy ? { addedBy: f.addedBy } : {}),
|
|
@@ -544,20 +578,26 @@ async function reportNewFilesToNotify(
|
|
|
544
578
|
}),
|
|
545
579
|
signal: controller.signal,
|
|
546
580
|
});
|
|
581
|
+
} catch (err) {
|
|
582
|
+
// Best-effort per chunk: never let notification reporting affect the sync
|
|
583
|
+
// result, and a failed chunk must not abort the remaining chunks.
|
|
584
|
+
logNotifyFailure(err);
|
|
547
585
|
} finally {
|
|
548
586
|
clearTimeout(timer);
|
|
549
587
|
}
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
)
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/** Log a non-fatal notify failure without ever throwing out of the logger. */
|
|
592
|
+
function logNotifyFailure(err: unknown): void {
|
|
593
|
+
try {
|
|
594
|
+
console.error(
|
|
595
|
+
`[hq-sync] new-files notify report failed (non-fatal): ${
|
|
596
|
+
err instanceof Error ? err.message : String(err)
|
|
597
|
+
}`,
|
|
598
|
+
);
|
|
599
|
+
} catch {
|
|
600
|
+
// swallow — logging must never break sync
|
|
561
601
|
}
|
|
562
602
|
}
|
|
563
603
|
|
|
@@ -565,6 +605,17 @@ async function reportNewFilesToNotify(
|
|
|
565
605
|
* Sync (pull) all allowed files from the entity vault.
|
|
566
606
|
*/
|
|
567
607
|
export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
608
|
+
if (options.operationLockAlreadyHeld) {
|
|
609
|
+
return syncWithOperationLockHeld(options);
|
|
610
|
+
}
|
|
611
|
+
return withOperationLock(options.hqRoot, "sync", () =>
|
|
612
|
+
syncWithOperationLockHeld(options),
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
async function syncWithOperationLockHeld(
|
|
617
|
+
options: SyncOptions,
|
|
618
|
+
): Promise<SyncResult> {
|
|
568
619
|
const { company, onConflict, vaultConfig, hqRoot } = options;
|
|
569
620
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
570
621
|
|
|
@@ -898,16 +949,17 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
898
949
|
const tombstoneKey = item.remoteFile.key;
|
|
899
950
|
// Same Windows-backslash landmine guard as the journal-tombstone executor:
|
|
900
951
|
// a malformed key must never reach fs.unlinkSync (path.join collapses the
|
|
901
|
-
// backslashes onto a REAL POSIX file).
|
|
902
|
-
//
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
continue;
|
|
906
|
-
}
|
|
952
|
+
// backslashes onto a REAL POSIX file). Traversal keys are likewise
|
|
953
|
+
// refused before any local filesystem or journal mutation.
|
|
954
|
+
const tombstonePath = resolveContainedVaultPath(companyRoot, tombstoneKey);
|
|
955
|
+
if (tombstonePath === null) continue;
|
|
907
956
|
try {
|
|
908
|
-
const lstat = fs.lstatSync(
|
|
957
|
+
const lstat = fs.lstatSync(tombstonePath);
|
|
958
|
+
if (tombstoneTargetDiverged(journal, tombstoneKey, tombstonePath, lstat)) {
|
|
959
|
+
continue;
|
|
960
|
+
}
|
|
909
961
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
910
|
-
fs.unlinkSync(
|
|
962
|
+
fs.unlinkSync(tombstonePath);
|
|
911
963
|
}
|
|
912
964
|
// A directory at the key: don't recursively rm-rf the operator's dir;
|
|
913
965
|
// just drop the journal entry (safe-by-default, same as the other path).
|
|
@@ -985,11 +1037,22 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
985
1037
|
machineId,
|
|
986
1038
|
);
|
|
987
1039
|
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
1040
|
+
const conflictKey = toPosixKey(path.relative(companyRoot, conflictAbs));
|
|
1041
|
+
|
|
1042
|
+
if (!isDownloadWritePathStillContained(companyRoot, conflictKey, conflictAbs)) {
|
|
1043
|
+
filesSkipped++;
|
|
1044
|
+
emit({
|
|
1045
|
+
type: "error",
|
|
1046
|
+
path: remoteFile.key,
|
|
1047
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
1048
|
+
});
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
988
1051
|
|
|
989
1052
|
let remoteFetched = false;
|
|
990
1053
|
let converged = false;
|
|
991
1054
|
try {
|
|
992
|
-
await downloadFile(ctx, remoteFile.key, conflictAbs);
|
|
1055
|
+
const downloaded = await downloadFile(ctx, remoteFile.key, conflictAbs);
|
|
993
1056
|
remoteFetched = true;
|
|
994
1057
|
// Hash the fetched remote exactly the way the planner hashed local
|
|
995
1058
|
// (symlink-aware) so the two hashes are directly comparable. A
|
|
@@ -997,7 +1060,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
997
1060
|
// target string matches `hashSymlinkTarget(localPath)`.
|
|
998
1061
|
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
999
1062
|
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
1000
|
-
: hashFile(conflictAbs);
|
|
1063
|
+
: (downloaded.contentHash ?? hashFile(conflictAbs));
|
|
1001
1064
|
converged = remoteHash === item.localHash;
|
|
1002
1065
|
} catch (probeErr) {
|
|
1003
1066
|
// Couldn't fetch or hash the remote — fail safe by falling through to
|
|
@@ -1212,8 +1275,22 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1212
1275
|
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
1213
1276
|
}
|
|
1214
1277
|
|
|
1278
|
+
if (!isDownloadWritePathStillContained(companyRoot, remoteFile.key, localPath)) {
|
|
1279
|
+
filesSkipped++;
|
|
1280
|
+
emit({
|
|
1281
|
+
type: "error",
|
|
1282
|
+
path: remoteFile.key,
|
|
1283
|
+
message: "download skipped: local parent escaped the sync root",
|
|
1284
|
+
});
|
|
1285
|
+
return;
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1215
1288
|
try {
|
|
1216
|
-
const { metadata } = await downloadFile(
|
|
1289
|
+
const { metadata, contentHash, contentSize } = await downloadFile(
|
|
1290
|
+
ctx,
|
|
1291
|
+
remoteFile.key,
|
|
1292
|
+
localPath,
|
|
1293
|
+
);
|
|
1217
1294
|
const author = metadata?.["created-by"] ?? null;
|
|
1218
1295
|
// Author sub for the scope-shrink authorship guard — same field the
|
|
1219
1296
|
// upload side stamps, read straight off the GET response metadata.
|
|
@@ -1232,8 +1309,8 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1232
1309
|
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
1233
1310
|
const hash = isLocalSymlink
|
|
1234
1311
|
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
1235
|
-
: hashFile(localPath);
|
|
1236
|
-
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
1312
|
+
: (contentHash ?? hashFile(localPath));
|
|
1313
|
+
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
1237
1314
|
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
1238
1315
|
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
1239
1316
|
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
@@ -1430,21 +1507,16 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1430
1507
|
// converged. Failures are reported but non-fatal — the entry stays in
|
|
1431
1508
|
// the journal and the next run retries.
|
|
1432
1509
|
for (const key of plan.tombstones) {
|
|
1433
|
-
// Last line of defense
|
|
1434
|
-
//
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
// The planner already refuses to enqueue malformed keys; if one still
|
|
1438
|
-
// arrives, drop the poisoned journal entry without touching disk —
|
|
1439
|
-
// normalizeJournalKeys rewrites it to its POSIX form on load.
|
|
1440
|
-
if (isMalformedVaultKey(key)) {
|
|
1441
|
-
removeEntry(journal, key);
|
|
1442
|
-
continue;
|
|
1443
|
-
}
|
|
1444
|
-
const localPath = path.join(companyRoot, key);
|
|
1510
|
+
// Last line of defense: a malformed or traversal key must NEVER reach
|
|
1511
|
+
// fs.unlinkSync or journal mutation for a path outside the sync root.
|
|
1512
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
1513
|
+
if (localPath === null) continue;
|
|
1445
1514
|
let removedSomething = false;
|
|
1446
1515
|
try {
|
|
1447
1516
|
const lstat = fs.lstatSync(localPath);
|
|
1517
|
+
if (tombstoneTargetDiverged(journal, key, localPath, lstat)) {
|
|
1518
|
+
continue;
|
|
1519
|
+
}
|
|
1448
1520
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
1449
1521
|
fs.unlinkSync(localPath);
|
|
1450
1522
|
removedSomething = true;
|
|
@@ -1615,6 +1687,99 @@ function isRemoteRecreateAfterTombstone(
|
|
|
1615
1687
|
return remoteMs > deletedAtMs;
|
|
1616
1688
|
}
|
|
1617
1689
|
|
|
1690
|
+
function hasTraversalSegment(key: string): boolean {
|
|
1691
|
+
return key.split("/").some((segment) => segment === "..");
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
function isPathWithin(root: string, candidate: string): boolean {
|
|
1695
|
+
const relative = path.relative(root, candidate);
|
|
1696
|
+
return (
|
|
1697
|
+
relative === "" ||
|
|
1698
|
+
(!relative.startsWith("..") && !path.isAbsolute(relative))
|
|
1699
|
+
);
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
function deepestExistingAncestor(start: string): string | null {
|
|
1703
|
+
let current = start;
|
|
1704
|
+
for (;;) {
|
|
1705
|
+
try {
|
|
1706
|
+
fs.lstatSync(current);
|
|
1707
|
+
return current;
|
|
1708
|
+
} catch (err: unknown) {
|
|
1709
|
+
const code =
|
|
1710
|
+
err && typeof err === "object" && "code" in err
|
|
1711
|
+
? (err as { code?: string }).code
|
|
1712
|
+
: undefined;
|
|
1713
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") return null;
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
const parent = path.dirname(current);
|
|
1717
|
+
if (parent === current) return null;
|
|
1718
|
+
current = parent;
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
function resolveContainedVaultPath(root: string, key: string): string | null {
|
|
1723
|
+
if (isMalformedVaultKey(key) || hasTraversalSegment(key)) return null;
|
|
1724
|
+
|
|
1725
|
+
const resolvedRoot = path.resolve(root);
|
|
1726
|
+
const resolvedLocal = path.resolve(resolvedRoot, key);
|
|
1727
|
+
if (!isPathWithin(resolvedRoot, resolvedLocal)) return null;
|
|
1728
|
+
|
|
1729
|
+
let realRoot: string;
|
|
1730
|
+
try {
|
|
1731
|
+
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
1732
|
+
} catch {
|
|
1733
|
+
// If the vault root does not exist yet, no below-root symlink component can
|
|
1734
|
+
// already exist to redirect this key. Preserve first-pull behavior.
|
|
1735
|
+
return resolvedLocal;
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
1739
|
+
if (existingAncestor === null) return null;
|
|
1740
|
+
try {
|
|
1741
|
+
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
1742
|
+
if (!isPathWithin(realRoot, realAncestor)) return null;
|
|
1743
|
+
} catch {
|
|
1744
|
+
return null;
|
|
1745
|
+
}
|
|
1746
|
+
return resolvedLocal;
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
function isDownloadWritePathStillContained(
|
|
1750
|
+
root: string,
|
|
1751
|
+
key: string,
|
|
1752
|
+
localPath: string,
|
|
1753
|
+
): boolean {
|
|
1754
|
+
const resolved = resolveContainedVaultPath(root, key);
|
|
1755
|
+
return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
function tombstoneTargetDiverged(
|
|
1759
|
+
journal: SyncJournal,
|
|
1760
|
+
key: string,
|
|
1761
|
+
localPath: string,
|
|
1762
|
+
lstat: fs.Stats,
|
|
1763
|
+
): boolean {
|
|
1764
|
+
const journalEntry = journal.files[key];
|
|
1765
|
+
if (!journalEntry?.hash) {
|
|
1766
|
+
return lstat.isSymbolicLink() || lstat.isFile();
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
try {
|
|
1770
|
+
if (lstat.isSymbolicLink()) {
|
|
1771
|
+
return hashSymlinkTarget(fs.readlinkSync(localPath)) !== journalEntry.hash;
|
|
1772
|
+
}
|
|
1773
|
+
if (lstat.isFile()) {
|
|
1774
|
+
return hashFile(localPath) !== journalEntry.hash;
|
|
1775
|
+
}
|
|
1776
|
+
} catch {
|
|
1777
|
+
return true;
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1780
|
+
return false;
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1618
1783
|
/**
|
|
1619
1784
|
* Stage-1 classification for a single remote object. Each remote file falls
|
|
1620
1785
|
* into exactly one bucket; the executor in `sync()` switches on `action` to
|
|
@@ -1716,7 +1881,7 @@ function computePullPlan(
|
|
|
1716
1881
|
// Coalesced, company-relative prefixes the pull is scoped to (US-005).
|
|
1717
1882
|
// `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
|
|
1718
1883
|
// the scope filter below becomes a no-op and legacy behavior is preserved.
|
|
1719
|
-
prefixSet:
|
|
1884
|
+
prefixSet: readonly ScopePrefixInput[],
|
|
1720
1885
|
// FILE_TOMBSTONE records (POSIX-keyed) for the company — the durable
|
|
1721
1886
|
// "this key was intentionally deleted" signal the planner consults before
|
|
1722
1887
|
// re-downloading a key, so a deleted folder does not resync back in
|
|
@@ -1728,7 +1893,11 @@ function computePullPlan(
|
|
|
1728
1893
|
const items: PullPlanItem[] = [];
|
|
1729
1894
|
|
|
1730
1895
|
for (const remoteFile of remoteFiles) {
|
|
1731
|
-
const localPath =
|
|
1896
|
+
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
1897
|
+
if (localPath === null) {
|
|
1898
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
1899
|
+
continue;
|
|
1900
|
+
}
|
|
1732
1901
|
|
|
1733
1902
|
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
1734
1903
|
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
@@ -1741,17 +1910,6 @@ function computePullPlan(
|
|
|
1741
1910
|
continue;
|
|
1742
1911
|
}
|
|
1743
1912
|
|
|
1744
|
-
// Malformed-key filter — keys with backslash separators pushed by
|
|
1745
|
-
// pre-5.47.2 Windows clients. Downloading one materializes a junk local
|
|
1746
|
-
// file whose NAME contains backslashes (it is not a path on POSIX), which
|
|
1747
|
-
// then churns conflict mirrors forever. Refuse at planning time, same
|
|
1748
|
-
// policy bucket as the ephemeral filter above. The bogus keys themselves
|
|
1749
|
-
// are cleaned server-side; this keeps clean trees clean in the meantime.
|
|
1750
|
-
if (isMalformedVaultKey(remoteFile.key)) {
|
|
1751
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
1752
|
-
continue;
|
|
1753
|
-
}
|
|
1754
|
-
|
|
1755
1913
|
if (
|
|
1756
1914
|
personalMode &&
|
|
1757
1915
|
remoteFile.key.startsWith("companies/") &&
|
|
@@ -2112,12 +2270,8 @@ function computePullPlan(
|
|
|
2112
2270
|
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
2113
2271
|
const posixKey = toPosixKey(key);
|
|
2114
2272
|
if (remoteKeySet.has(posixKey)) continue;
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
// POSIX file and unlinks live data. Leave it for normalizeJournalKeys to
|
|
2118
|
-
// rewrite to POSIX on the next write; the canonical key is re-evaluated
|
|
2119
|
-
// (and correctly tombstoned if genuinely remote-deleted) on a later pull.
|
|
2120
|
-
if (isMalformedVaultKey(key)) continue;
|
|
2273
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
2274
|
+
if (localPath === null) continue;
|
|
2121
2275
|
// PersonalMode key gating — mirror the download branch.
|
|
2122
2276
|
if (personalMode && key.startsWith("companies/")) {
|
|
2123
2277
|
const slug = key.split("/")[1] ?? "";
|
|
@@ -2131,7 +2285,6 @@ function computePullPlan(
|
|
|
2131
2285
|
// Honor the current ignore filter — if a path was previously synced
|
|
2132
2286
|
// but is now ignored (operator edited .hqignore), do NOT delete
|
|
2133
2287
|
// the local copy. They're keeping it deliberately.
|
|
2134
|
-
const localPath = path.join(companyRoot, key);
|
|
2135
2288
|
if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) continue;
|
|
2136
2289
|
// Codex P1 (PR #24 round 3): detect local edits before tombstoning.
|
|
2137
2290
|
// Delete-vs-local-edit race: peer deleted the file remotely while
|
package/src/cognito-auth.test.ts
CHANGED
|
@@ -198,6 +198,83 @@ describe("getValidAccessToken stale-pool detection", () => {
|
|
|
198
198
|
});
|
|
199
199
|
});
|
|
200
200
|
|
|
201
|
+
// ---------------------------------------------------------------------------
|
|
202
|
+
// Machine identity cache isolation — machine mode must not reuse human tokens
|
|
203
|
+
// from the shared Cognito cache even when the app client matches.
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
|
|
206
|
+
describe("machine identity cache isolation", () => {
|
|
207
|
+
it("F06: machine mode re-mints instead of reusing a cached human token", async () => {
|
|
208
|
+
const { saveCachedTokens, getValidAccessToken } = await importModule();
|
|
209
|
+
|
|
210
|
+
const cachedHumanAccessToken = makeAccessToken({
|
|
211
|
+
token_use: "access",
|
|
212
|
+
client_id: PROD_CLIENT,
|
|
213
|
+
username: "human@example.com",
|
|
214
|
+
sub: "human-sub",
|
|
215
|
+
});
|
|
216
|
+
const cachedHumanIdToken = makeAccessToken({
|
|
217
|
+
token_use: "id",
|
|
218
|
+
aud: PROD_CLIENT,
|
|
219
|
+
email: "human@example.com",
|
|
220
|
+
"cognito:username": "human@example.com",
|
|
221
|
+
sub: "human-sub",
|
|
222
|
+
});
|
|
223
|
+
saveCachedTokens({
|
|
224
|
+
...baseTokens,
|
|
225
|
+
accessToken: cachedHumanAccessToken,
|
|
226
|
+
idToken: cachedHumanIdToken,
|
|
227
|
+
expiresAt: Date.now() + 60 * 60 * 1000,
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const machineUid = "agt_01JZ0000000000000000000000";
|
|
231
|
+
const machineCredsDir = path.join(tmpHome, ".hq-agent");
|
|
232
|
+
fs.mkdirSync(machineCredsDir, { recursive: true });
|
|
233
|
+
fs.writeFileSync(
|
|
234
|
+
path.join(machineCredsDir, "machine-creds.json"),
|
|
235
|
+
JSON.stringify({
|
|
236
|
+
username: "agt-01jz0000000000000000000000@agents.getindigo.ai",
|
|
237
|
+
secret: "machine-secret",
|
|
238
|
+
clientId: PROD_CLIENT,
|
|
239
|
+
region: "us-east-1",
|
|
240
|
+
}),
|
|
241
|
+
);
|
|
242
|
+
|
|
243
|
+
const mintedMachineAccessToken = makeAccessToken({
|
|
244
|
+
token_use: "access",
|
|
245
|
+
client_id: PROD_CLIENT,
|
|
246
|
+
username: "agt-01jz0000000000000000000000@agents.getindigo.ai",
|
|
247
|
+
sub: machineUid,
|
|
248
|
+
});
|
|
249
|
+
const mintedMachineIdToken = makeAccessToken({
|
|
250
|
+
token_use: "id",
|
|
251
|
+
aud: PROD_CLIENT,
|
|
252
|
+
"custom:entityType": "agent",
|
|
253
|
+
"custom:entityUid": machineUid,
|
|
254
|
+
"cognito:username": "agt-01jz0000000000000000000000@agents.getindigo.ai",
|
|
255
|
+
sub: machineUid,
|
|
256
|
+
});
|
|
257
|
+
const fetchMock = vi.fn(async () =>
|
|
258
|
+
new Response(
|
|
259
|
+
JSON.stringify({
|
|
260
|
+
AuthenticationResult: {
|
|
261
|
+
AccessToken: mintedMachineAccessToken,
|
|
262
|
+
IdToken: mintedMachineIdToken,
|
|
263
|
+
ExpiresIn: 3600,
|
|
264
|
+
},
|
|
265
|
+
}),
|
|
266
|
+
{ status: 200, headers: { "Content-Type": "application/json" } },
|
|
267
|
+
),
|
|
268
|
+
);
|
|
269
|
+
vi.stubGlobal("fetch", fetchMock);
|
|
270
|
+
|
|
271
|
+
await expect(
|
|
272
|
+
getValidAccessToken(baseConfig, { interactive: false }),
|
|
273
|
+
).resolves.toBe(mintedMachineIdToken);
|
|
274
|
+
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
275
|
+
});
|
|
276
|
+
});
|
|
277
|
+
|
|
201
278
|
// ---------------------------------------------------------------------------
|
|
202
279
|
// Round-trip: writers emit epoch-ms, readers read epoch-ms
|
|
203
280
|
// ---------------------------------------------------------------------------
|
package/src/cognito-auth.ts
CHANGED
|
@@ -134,19 +134,80 @@ export function isExpiring(tokens: CognitoTokens, bufferSeconds = 60): boolean {
|
|
|
134
134
|
* forcing a re-login is the only safe self-heal.
|
|
135
135
|
*/
|
|
136
136
|
export function decodeAccessTokenClientId(accessToken: string): string | null {
|
|
137
|
+
const claims = decodeJwtClaims(accessToken);
|
|
138
|
+
return typeof claims?.client_id === "string" ? claims.client_id : null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function decodeJwtClaims(token: string): Record<string, unknown> | null {
|
|
137
142
|
try {
|
|
138
|
-
const parts =
|
|
143
|
+
const parts = token.split(".");
|
|
139
144
|
if (parts.length < 2) return null;
|
|
140
145
|
const payloadB64 = parts[1];
|
|
141
|
-
const
|
|
146
|
+
const normalized = payloadB64.replace(/-/g, "+").replace(/_/g, "/");
|
|
147
|
+
const padded = normalized + "=".repeat((4 - (normalized.length % 4)) % 4);
|
|
142
148
|
const json = Buffer.from(padded, "base64").toString("utf-8");
|
|
143
|
-
const claims = JSON.parse(json)
|
|
144
|
-
return typeof claims
|
|
149
|
+
const claims = JSON.parse(json);
|
|
150
|
+
return claims && typeof claims === "object" && !Array.isArray(claims)
|
|
151
|
+
? (claims as Record<string, unknown>)
|
|
152
|
+
: null;
|
|
145
153
|
} catch {
|
|
146
154
|
return null;
|
|
147
155
|
}
|
|
148
156
|
}
|
|
149
157
|
|
|
158
|
+
function cachedTokensMatchMachineIdentity(
|
|
159
|
+
tokens: CognitoTokens,
|
|
160
|
+
creds: MachineCreds,
|
|
161
|
+
expectedClientId: string,
|
|
162
|
+
): boolean {
|
|
163
|
+
const accessClaims = decodeJwtClaims(tokens.accessToken);
|
|
164
|
+
const idClaims = decodeJwtClaims(tokens.idToken);
|
|
165
|
+
if (!accessClaims || !idClaims) return false;
|
|
166
|
+
|
|
167
|
+
const stringClaim = (
|
|
168
|
+
claims: Record<string, unknown>,
|
|
169
|
+
key: string,
|
|
170
|
+
): string | null => {
|
|
171
|
+
const value = claims[key];
|
|
172
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
const accessClientId = accessClaims.client_id;
|
|
176
|
+
const idAudience = idClaims.aud;
|
|
177
|
+
const accessUsername =
|
|
178
|
+
stringClaim(accessClaims, "username") ??
|
|
179
|
+
stringClaim(accessClaims, "cognito:username");
|
|
180
|
+
const accessSub = stringClaim(accessClaims, "sub");
|
|
181
|
+
const idUsername =
|
|
182
|
+
stringClaim(idClaims, "cognito:username") ?? stringClaim(idClaims, "username");
|
|
183
|
+
const idSub = stringClaim(idClaims, "sub");
|
|
184
|
+
const entityType = idClaims["custom:entityType"];
|
|
185
|
+
const entityUid = idClaims["custom:entityUid"];
|
|
186
|
+
const idTokenMatchesCreds =
|
|
187
|
+
idUsername === creds.username || idSub === creds.username;
|
|
188
|
+
const subjectBindings: boolean[] = [];
|
|
189
|
+
if (accessUsername !== null && idUsername !== null) {
|
|
190
|
+
subjectBindings.push(accessUsername === idUsername);
|
|
191
|
+
}
|
|
192
|
+
if (accessSub !== null && idSub !== null) {
|
|
193
|
+
subjectBindings.push(accessSub === idSub);
|
|
194
|
+
}
|
|
195
|
+
const tokensShareSubject =
|
|
196
|
+
subjectBindings.length > 0 && subjectBindings.every(Boolean);
|
|
197
|
+
|
|
198
|
+
return (
|
|
199
|
+
accessClaims.token_use === "access" &&
|
|
200
|
+
accessClientId === expectedClientId &&
|
|
201
|
+
idClaims.token_use === "id" &&
|
|
202
|
+
idAudience === expectedClientId &&
|
|
203
|
+
entityType === "agent" &&
|
|
204
|
+
typeof entityUid === "string" &&
|
|
205
|
+
entityUid.startsWith("agt_") &&
|
|
206
|
+
idTokenMatchesCreds &&
|
|
207
|
+
tokensShareSubject
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
|
|
150
211
|
// ---------------------------------------------------------------------------
|
|
151
212
|
// Machine identity (company agents)
|
|
152
213
|
// ---------------------------------------------------------------------------
|
|
@@ -309,17 +370,18 @@ export async function mintMachineTokens(
|
|
|
309
370
|
export async function getValidMachineTokens(
|
|
310
371
|
config: CognitoAuthConfig,
|
|
311
372
|
): Promise<CognitoTokens> {
|
|
373
|
+
const machineCreds = loadMachineCreds();
|
|
312
374
|
const cached = loadCachedTokens();
|
|
313
|
-
if (cached && !isExpiring(cached, 120)) {
|
|
314
|
-
|
|
315
|
-
//
|
|
316
|
-
//
|
|
317
|
-
const expectedClientId =
|
|
318
|
-
if (
|
|
375
|
+
if (machineCreds && cached && !isExpiring(cached, 120)) {
|
|
376
|
+
// Compare against the client we'd actually mint with (creds-file clientId
|
|
377
|
+
// wins over config), and require the cached ID token to prove this exact
|
|
378
|
+
// agent identity. Opaque/missing/human-shaped claims are treated as stale.
|
|
379
|
+
const expectedClientId = machineCreds.clientId ?? config.clientId;
|
|
380
|
+
if (cachedTokensMatchMachineIdentity(cached, machineCreds, expectedClientId)) {
|
|
319
381
|
return cached;
|
|
320
382
|
}
|
|
321
383
|
}
|
|
322
|
-
return mintMachineTokens(config);
|
|
384
|
+
return mintMachineTokens(config, machineCreds ?? undefined);
|
|
323
385
|
}
|
|
324
386
|
|
|
325
387
|
// ---------------------------------------------------------------------------
|
package/src/index.ts
CHANGED
|
@@ -62,6 +62,14 @@ export {
|
|
|
62
62
|
coalescePrefixes,
|
|
63
63
|
isCoveredByAny,
|
|
64
64
|
grantPathToPrefix,
|
|
65
|
+
grantPathToScopePrefix,
|
|
66
|
+
pathToScopePrefix,
|
|
67
|
+
toScopePrefixEntries,
|
|
68
|
+
} from "./prefix-coalesce.js";
|
|
69
|
+
export type {
|
|
70
|
+
ScopePrefixEntry,
|
|
71
|
+
ScopePrefixInput,
|
|
72
|
+
ScopePrefixMatch,
|
|
65
73
|
} from "./prefix-coalesce.js";
|
|
66
74
|
|
|
67
75
|
// Scope-shrink detection + application (US-005)
|