@indigoai-us/hq-cloud 6.11.11 → 6.11.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +231 -52
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +265 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +58 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.js +138 -15
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +178 -58
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +362 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +148 -29
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +8 -2
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +6 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +62 -13
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +189 -0
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +6 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +13 -15
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +36 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watcher.d.ts +36 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +152 -30
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +298 -11
- package/src/bin/sync-runner.ts +254 -52
- package/src/cli/rescue-classify-ordering.test.ts +61 -0
- package/src/cli/rescue-core.ts +174 -15
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +103 -34
- package/src/cli/sync.test.ts +435 -1
- package/src/cli/sync.ts +217 -64
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +182 -30
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +8 -2
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +205 -0
- package/src/remote-pull.ts +77 -14
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +38 -1
- package/src/sync/push-receiver.ts +15 -18
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +209 -33
package/dist/cli/sync.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { VaultServiceConfig } from "../types.js";
|
|
8
8
|
import type { SyncMode } from "../vault-client.js";
|
|
9
|
+
import { type ScopePrefixInput } from "../prefix-coalesce.js";
|
|
9
10
|
import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
|
|
10
11
|
/**
|
|
11
12
|
* Per-file events emitted by `sync()` as it progresses.
|
|
@@ -312,7 +313,7 @@ export interface SyncOptions {
|
|
|
312
313
|
* (not empty `"shared"`) on any grant-resolution error, so a transient
|
|
313
314
|
* failure can never silently prune the local tree.
|
|
314
315
|
*/
|
|
315
|
-
prefixSet?:
|
|
316
|
+
prefixSet?: ScopePrefixInput[];
|
|
316
317
|
/**
|
|
317
318
|
* When the effective scope shrinks relative to the last pull and the shrink
|
|
318
319
|
* would orphan locally-modified ("dirty") files, `sync()` aborts with a
|
|
@@ -357,6 +358,11 @@ export interface SyncOptions {
|
|
|
357
358
|
* this and run `reindex()` once itself instead of per-company.
|
|
358
359
|
*/
|
|
359
360
|
skipReindex?: boolean;
|
|
361
|
+
/**
|
|
362
|
+
* Internal runner seam: true only when the caller already holds the
|
|
363
|
+
* per-root operation lock for this sync pass.
|
|
364
|
+
*/
|
|
365
|
+
operationLockAlreadyHeld?: boolean;
|
|
360
366
|
}
|
|
361
367
|
export interface SyncResult {
|
|
362
368
|
filesDownloaded: number;
|
|
@@ -431,6 +437,27 @@ export interface SyncResult {
|
|
|
431
437
|
* `HQ_SYNC_MAX_AUTO_PRUNE`.
|
|
432
438
|
*/
|
|
433
439
|
export declare function resolveAutoPruneCap(): number;
|
|
440
|
+
/**
|
|
441
|
+
* Best-effort report of the files that were new to this drive during the sync,
|
|
442
|
+
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
443
|
+
*
|
|
444
|
+
* POSTs to `${apiUrl}/v1/notify/file-added`, which writes per-recipient
|
|
445
|
+
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
446
|
+
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
447
|
+
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
448
|
+
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
449
|
+
* when there are no new files.
|
|
450
|
+
*
|
|
451
|
+
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
452
|
+
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
453
|
+
* best-effort, so one failing/oversized batch can never block the others or the
|
|
454
|
+
* sync. Exported only so the chunking can be unit-tested directly.
|
|
455
|
+
*/
|
|
456
|
+
export declare function reportNewFilesToNotify(vaultConfig: VaultServiceConfig, companyUid: string, companySlug: string, files: Array<{
|
|
457
|
+
path: string;
|
|
458
|
+
bytes: number;
|
|
459
|
+
addedBy: string | null;
|
|
460
|
+
}>): Promise<void>;
|
|
434
461
|
/**
|
|
435
462
|
* Sync (pull) all allowed files from the entity vault.
|
|
436
463
|
*/
|
package/dist/cli/sync.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAmCnD,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAc1E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAChD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,UAAU,EAAE,kBAAkB,CAAC;CAChC,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACvE,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,YAAY,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;CAC5D,GACD;IACE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;OAeG;IACH,IAAI,EAAE,8BAA8B,CAAC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;OAWG;IACH,IAAI,EAAE,6BAA6B,CAAC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,2BAA2B;IAC3B,WAAW,EAAE,kBAAkB,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;;;;;;OAaG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;OAiBG;IACH,iBAAiB,CAAC,EAAE,OAAO,GAAG,cAAc,CAAC;IAC7C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAM5C;AAeD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,sBAAsB,CAC1C,WAAW,EAAE,kBAAkB,EAC/B,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,GACpE,OAAO,CAAC,IAAI,CAAC,CAgDf;AAeD;;GAEG;AACH,wBAAsB,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAOpE"}
|
package/dist/cli/sync.js
CHANGED
|
@@ -11,13 +11,14 @@ import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, to
|
|
|
11
11
|
import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
12
12
|
import { PERSONAL_VAULT_MANIFEST_KEY } from "../personal-vault.js";
|
|
13
13
|
import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
|
|
14
|
-
import { coalescePrefixes, isCoveredByAny } from "../prefix-coalesce.js";
|
|
14
|
+
import { coalescePrefixes, isCoveredByAny, } from "../prefix-coalesce.js";
|
|
15
15
|
import { createIgnoreFilter } from "../ignore.js";
|
|
16
16
|
import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
17
17
|
import { resolveConflict } from "./conflict.js";
|
|
18
18
|
import { buildConflictId, buildConflictPath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
19
19
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
20
20
|
import { reindex } from "./reindex.js";
|
|
21
|
+
import { withOperationLock } from "../operation-lock.js";
|
|
21
22
|
import { fetchCompanyTombstones, } from "./tombstones.js";
|
|
22
23
|
/**
|
|
23
24
|
* Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
|
|
@@ -36,6 +37,15 @@ export function resolveAutoPruneCap() {
|
|
|
36
37
|
}
|
|
37
38
|
/** Max time to wait on the best-effort new-files notification POST. */
|
|
38
39
|
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
40
|
+
/**
|
|
41
|
+
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
42
|
+
* an oversized batch wholesale, so the client MUST split a large report into
|
|
43
|
+
* chunks at or under this size — otherwise a first sync with more than this many
|
|
44
|
+
* new files reports none of them, and the same oversized batch re-triggers every
|
|
45
|
+
* sync cycle (wasted work + dropped notifications). Keep in lockstep with the
|
|
46
|
+
* server-side limit.
|
|
47
|
+
*/
|
|
48
|
+
const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
|
|
39
49
|
/**
|
|
40
50
|
* Best-effort report of the files that were new to this drive during the sync,
|
|
41
51
|
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
@@ -44,17 +54,31 @@ const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
|
44
54
|
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
45
55
|
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
46
56
|
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
47
|
-
* 5s timeout so a hung endpoint can't stall sync completion. No-op
|
|
48
|
-
* are no new files.
|
|
57
|
+
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
58
|
+
* when there are no new files.
|
|
59
|
+
*
|
|
60
|
+
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
61
|
+
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
62
|
+
* best-effort, so one failing/oversized batch can never block the others or the
|
|
63
|
+
* sync. Exported only so the chunking can be unit-tested directly.
|
|
49
64
|
*/
|
|
50
|
-
async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
|
|
65
|
+
export async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
|
|
51
66
|
if (files.length === 0)
|
|
52
67
|
return;
|
|
68
|
+
let token;
|
|
53
69
|
try {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
70
|
+
token =
|
|
71
|
+
typeof vaultConfig.authToken === "function"
|
|
72
|
+
? await vaultConfig.authToken()
|
|
73
|
+
: vaultConfig.authToken;
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
logNotifyFailure(err);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
80
|
+
for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
|
|
81
|
+
const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
|
|
58
82
|
const controller = new AbortController();
|
|
59
83
|
const timer = setTimeout(() => controller.abort(), NOTIFY_FILE_ADDED_TIMEOUT_MS);
|
|
60
84
|
try {
|
|
@@ -67,7 +91,7 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
|
|
|
67
91
|
body: JSON.stringify({
|
|
68
92
|
companyUid,
|
|
69
93
|
companySlug,
|
|
70
|
-
files:
|
|
94
|
+
files: batch.map((f) => ({
|
|
71
95
|
path: f.path,
|
|
72
96
|
bytes: f.bytes,
|
|
73
97
|
...(f.addedBy ? { addedBy: f.addedBy } : {}),
|
|
@@ -76,24 +100,35 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
|
|
|
76
100
|
signal: controller.signal,
|
|
77
101
|
});
|
|
78
102
|
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
// Best-effort per chunk: never let notification reporting affect the sync
|
|
105
|
+
// result, and a failed chunk must not abort the remaining chunks.
|
|
106
|
+
logNotifyFailure(err);
|
|
107
|
+
}
|
|
79
108
|
finally {
|
|
80
109
|
clearTimeout(timer);
|
|
81
110
|
}
|
|
82
111
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
112
|
+
}
|
|
113
|
+
/** Log a non-fatal notify failure without ever throwing out of the logger. */
|
|
114
|
+
function logNotifyFailure(err) {
|
|
115
|
+
try {
|
|
116
|
+
console.error(`[hq-sync] new-files notify report failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// swallow — logging must never break sync
|
|
91
120
|
}
|
|
92
121
|
}
|
|
93
122
|
/**
|
|
94
123
|
* Sync (pull) all allowed files from the entity vault.
|
|
95
124
|
*/
|
|
96
125
|
export async function sync(options) {
|
|
126
|
+
if (options.operationLockAlreadyHeld) {
|
|
127
|
+
return syncWithOperationLockHeld(options);
|
|
128
|
+
}
|
|
129
|
+
return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
|
|
130
|
+
}
|
|
131
|
+
async function syncWithOperationLockHeld(options) {
|
|
97
132
|
const { company, onConflict, vaultConfig, hqRoot } = options;
|
|
98
133
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
99
134
|
// Resolve company
|
|
@@ -377,16 +412,18 @@ export async function sync(options) {
|
|
|
377
412
|
const tombstoneKey = item.remoteFile.key;
|
|
378
413
|
// Same Windows-backslash landmine guard as the journal-tombstone executor:
|
|
379
414
|
// a malformed key must never reach fs.unlinkSync (path.join collapses the
|
|
380
|
-
// backslashes onto a REAL POSIX file).
|
|
381
|
-
//
|
|
382
|
-
|
|
383
|
-
|
|
415
|
+
// backslashes onto a REAL POSIX file). Traversal keys are likewise
|
|
416
|
+
// refused before any local filesystem or journal mutation.
|
|
417
|
+
const tombstonePath = resolveContainedVaultPath(companyRoot, tombstoneKey);
|
|
418
|
+
if (tombstonePath === null)
|
|
384
419
|
continue;
|
|
385
|
-
}
|
|
386
420
|
try {
|
|
387
|
-
const lstat = fs.lstatSync(
|
|
421
|
+
const lstat = fs.lstatSync(tombstonePath);
|
|
422
|
+
if (tombstoneTargetDiverged(journal, tombstoneKey, tombstonePath, lstat)) {
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
388
425
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
389
|
-
fs.unlinkSync(
|
|
426
|
+
fs.unlinkSync(tombstonePath);
|
|
390
427
|
}
|
|
391
428
|
// A directory at the key: don't recursively rm-rf the operator's dir;
|
|
392
429
|
// just drop the journal entry (safe-by-default, same as the other path).
|
|
@@ -454,10 +491,20 @@ export async function sync(options) {
|
|
|
454
491
|
const originalRelative = path.relative(hqRoot, localPath);
|
|
455
492
|
const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
|
|
456
493
|
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
494
|
+
const conflictKey = toPosixKey(path.relative(companyRoot, conflictAbs));
|
|
495
|
+
if (!isDownloadWritePathStillContained(companyRoot, conflictKey, conflictAbs)) {
|
|
496
|
+
filesSkipped++;
|
|
497
|
+
emit({
|
|
498
|
+
type: "error",
|
|
499
|
+
path: remoteFile.key,
|
|
500
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
501
|
+
});
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
457
504
|
let remoteFetched = false;
|
|
458
505
|
let converged = false;
|
|
459
506
|
try {
|
|
460
|
-
await downloadFile(ctx, remoteFile.key, conflictAbs);
|
|
507
|
+
const downloaded = await downloadFile(ctx, remoteFile.key, conflictAbs);
|
|
461
508
|
remoteFetched = true;
|
|
462
509
|
// Hash the fetched remote exactly the way the planner hashed local
|
|
463
510
|
// (symlink-aware) so the two hashes are directly comparable. A
|
|
@@ -465,7 +512,7 @@ export async function sync(options) {
|
|
|
465
512
|
// target string matches `hashSymlinkTarget(localPath)`.
|
|
466
513
|
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
467
514
|
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
468
|
-
: hashFile(conflictAbs);
|
|
515
|
+
: (downloaded.contentHash ?? hashFile(conflictAbs));
|
|
469
516
|
converged = remoteHash === item.localHash;
|
|
470
517
|
}
|
|
471
518
|
catch (probeErr) {
|
|
@@ -644,8 +691,17 @@ export async function sync(options) {
|
|
|
644
691
|
if (isExpiringSoon(ctx.expiresAt)) {
|
|
645
692
|
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
646
693
|
}
|
|
694
|
+
if (!isDownloadWritePathStillContained(companyRoot, remoteFile.key, localPath)) {
|
|
695
|
+
filesSkipped++;
|
|
696
|
+
emit({
|
|
697
|
+
type: "error",
|
|
698
|
+
path: remoteFile.key,
|
|
699
|
+
message: "download skipped: local parent escaped the sync root",
|
|
700
|
+
});
|
|
701
|
+
return;
|
|
702
|
+
}
|
|
647
703
|
try {
|
|
648
|
-
const { metadata } = await downloadFile(ctx, remoteFile.key, localPath);
|
|
704
|
+
const { metadata, contentHash, contentSize } = await downloadFile(ctx, remoteFile.key, localPath);
|
|
649
705
|
const author = metadata?.["created-by"] ?? null;
|
|
650
706
|
// Author sub for the scope-shrink authorship guard — same field the
|
|
651
707
|
// upload side stamps, read straight off the GET response metadata.
|
|
@@ -663,8 +719,8 @@ export async function sync(options) {
|
|
|
663
719
|
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
664
720
|
const hash = isLocalSymlink
|
|
665
721
|
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
666
|
-
: hashFile(localPath);
|
|
667
|
-
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
722
|
+
: (contentHash ?? hashFile(localPath));
|
|
723
|
+
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
668
724
|
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
669
725
|
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
670
726
|
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
@@ -840,21 +896,17 @@ export async function sync(options) {
|
|
|
840
896
|
// converged. Failures are reported but non-fatal — the entry stays in
|
|
841
897
|
// the journal and the next run retries.
|
|
842
898
|
for (const key of plan.tombstones) {
|
|
843
|
-
// Last line of defense
|
|
844
|
-
//
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
// The planner already refuses to enqueue malformed keys; if one still
|
|
848
|
-
// arrives, drop the poisoned journal entry without touching disk —
|
|
849
|
-
// normalizeJournalKeys rewrites it to its POSIX form on load.
|
|
850
|
-
if (isMalformedVaultKey(key)) {
|
|
851
|
-
removeEntry(journal, key);
|
|
899
|
+
// Last line of defense: a malformed or traversal key must NEVER reach
|
|
900
|
+
// fs.unlinkSync or journal mutation for a path outside the sync root.
|
|
901
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
902
|
+
if (localPath === null)
|
|
852
903
|
continue;
|
|
853
|
-
}
|
|
854
|
-
const localPath = path.join(companyRoot, key);
|
|
855
904
|
let removedSomething = false;
|
|
856
905
|
try {
|
|
857
906
|
const lstat = fs.lstatSync(localPath);
|
|
907
|
+
if (tombstoneTargetDiverged(journal, key, localPath, lstat)) {
|
|
908
|
+
continue;
|
|
909
|
+
}
|
|
858
910
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
859
911
|
fs.unlinkSync(localPath);
|
|
860
912
|
removedSomething = true;
|
|
@@ -1012,6 +1064,85 @@ function isRemoteRecreateAfterTombstone(remote, tombstone) {
|
|
|
1012
1064
|
return true; // no remote timestamp → don't suppress
|
|
1013
1065
|
return remoteMs > deletedAtMs;
|
|
1014
1066
|
}
|
|
1067
|
+
function hasTraversalSegment(key) {
|
|
1068
|
+
return key.split("/").some((segment) => segment === "..");
|
|
1069
|
+
}
|
|
1070
|
+
function isPathWithin(root, candidate) {
|
|
1071
|
+
const relative = path.relative(root, candidate);
|
|
1072
|
+
return (relative === "" ||
|
|
1073
|
+
(!relative.startsWith("..") && !path.isAbsolute(relative)));
|
|
1074
|
+
}
|
|
1075
|
+
function deepestExistingAncestor(start) {
|
|
1076
|
+
let current = start;
|
|
1077
|
+
for (;;) {
|
|
1078
|
+
try {
|
|
1079
|
+
fs.lstatSync(current);
|
|
1080
|
+
return current;
|
|
1081
|
+
}
|
|
1082
|
+
catch (err) {
|
|
1083
|
+
const code = err && typeof err === "object" && "code" in err
|
|
1084
|
+
? err.code
|
|
1085
|
+
: undefined;
|
|
1086
|
+
if (code !== "ENOENT" && code !== "ENOTDIR")
|
|
1087
|
+
return null;
|
|
1088
|
+
}
|
|
1089
|
+
const parent = path.dirname(current);
|
|
1090
|
+
if (parent === current)
|
|
1091
|
+
return null;
|
|
1092
|
+
current = parent;
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
function resolveContainedVaultPath(root, key) {
|
|
1096
|
+
if (isMalformedVaultKey(key) || hasTraversalSegment(key))
|
|
1097
|
+
return null;
|
|
1098
|
+
const resolvedRoot = path.resolve(root);
|
|
1099
|
+
const resolvedLocal = path.resolve(resolvedRoot, key);
|
|
1100
|
+
if (!isPathWithin(resolvedRoot, resolvedLocal))
|
|
1101
|
+
return null;
|
|
1102
|
+
let realRoot;
|
|
1103
|
+
try {
|
|
1104
|
+
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
1105
|
+
}
|
|
1106
|
+
catch {
|
|
1107
|
+
// If the vault root does not exist yet, no below-root symlink component can
|
|
1108
|
+
// already exist to redirect this key. Preserve first-pull behavior.
|
|
1109
|
+
return resolvedLocal;
|
|
1110
|
+
}
|
|
1111
|
+
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
1112
|
+
if (existingAncestor === null)
|
|
1113
|
+
return null;
|
|
1114
|
+
try {
|
|
1115
|
+
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
1116
|
+
if (!isPathWithin(realRoot, realAncestor))
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
catch {
|
|
1120
|
+
return null;
|
|
1121
|
+
}
|
|
1122
|
+
return resolvedLocal;
|
|
1123
|
+
}
|
|
1124
|
+
function isDownloadWritePathStillContained(root, key, localPath) {
|
|
1125
|
+
const resolved = resolveContainedVaultPath(root, key);
|
|
1126
|
+
return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
|
|
1127
|
+
}
|
|
1128
|
+
function tombstoneTargetDiverged(journal, key, localPath, lstat) {
|
|
1129
|
+
const journalEntry = journal.files[key];
|
|
1130
|
+
if (!journalEntry?.hash) {
|
|
1131
|
+
return lstat.isSymbolicLink() || lstat.isFile();
|
|
1132
|
+
}
|
|
1133
|
+
try {
|
|
1134
|
+
if (lstat.isSymbolicLink()) {
|
|
1135
|
+
return hashSymlinkTarget(fs.readlinkSync(localPath)) !== journalEntry.hash;
|
|
1136
|
+
}
|
|
1137
|
+
if (lstat.isFile()) {
|
|
1138
|
+
return hashFile(localPath) !== journalEntry.hash;
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
catch {
|
|
1142
|
+
return true;
|
|
1143
|
+
}
|
|
1144
|
+
return false;
|
|
1145
|
+
}
|
|
1015
1146
|
/**
|
|
1016
1147
|
* Stage-1 planning pass: classify every remote file into download / skip /
|
|
1017
1148
|
* conflict buckets without performing any S3 transfers. Local hashes are
|
|
@@ -1037,7 +1168,11 @@ prefixSet,
|
|
|
1037
1168
|
fileTombstones = new Map()) {
|
|
1038
1169
|
const items = [];
|
|
1039
1170
|
for (const remoteFile of remoteFiles) {
|
|
1040
|
-
const localPath =
|
|
1171
|
+
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
1172
|
+
if (localPath === null) {
|
|
1173
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
1174
|
+
continue;
|
|
1175
|
+
}
|
|
1041
1176
|
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
1042
1177
|
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
1043
1178
|
// mirrors since 5.33.0, but the pull side downloaded them freely from
|
|
@@ -1048,16 +1183,6 @@ fileTombstones = new Map()) {
|
|
|
1048
1183
|
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
1049
1184
|
continue;
|
|
1050
1185
|
}
|
|
1051
|
-
// Malformed-key filter — keys with backslash separators pushed by
|
|
1052
|
-
// pre-5.47.2 Windows clients. Downloading one materializes a junk local
|
|
1053
|
-
// file whose NAME contains backslashes (it is not a path on POSIX), which
|
|
1054
|
-
// then churns conflict mirrors forever. Refuse at planning time, same
|
|
1055
|
-
// policy bucket as the ephemeral filter above. The bogus keys themselves
|
|
1056
|
-
// are cleaned server-side; this keeps clean trees clean in the meantime.
|
|
1057
|
-
if (isMalformedVaultKey(remoteFile.key)) {
|
|
1058
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
1059
|
-
continue;
|
|
1060
|
-
}
|
|
1061
1186
|
if (personalMode &&
|
|
1062
1187
|
remoteFile.key.startsWith("companies/") &&
|
|
1063
1188
|
// EXEMPTION: companies/manifest.yaml is the routing source-of-truth
|
|
@@ -1403,12 +1528,8 @@ fileTombstones = new Map()) {
|
|
|
1403
1528
|
const posixKey = toPosixKey(key);
|
|
1404
1529
|
if (remoteKeySet.has(posixKey))
|
|
1405
1530
|
continue;
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
// POSIX file and unlinks live data. Leave it for normalizeJournalKeys to
|
|
1409
|
-
// rewrite to POSIX on the next write; the canonical key is re-evaluated
|
|
1410
|
-
// (and correctly tombstoned if genuinely remote-deleted) on a later pull.
|
|
1411
|
-
if (isMalformedVaultKey(key))
|
|
1531
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
1532
|
+
if (localPath === null)
|
|
1412
1533
|
continue;
|
|
1413
1534
|
// PersonalMode key gating — mirror the download branch.
|
|
1414
1535
|
if (personalMode && key.startsWith("companies/")) {
|
|
@@ -1424,7 +1545,6 @@ fileTombstones = new Map()) {
|
|
|
1424
1545
|
// Honor the current ignore filter — if a path was previously synced
|
|
1425
1546
|
// but is now ignored (operator edited .hqignore), do NOT delete
|
|
1426
1547
|
// the local copy. They're keeping it deliberately.
|
|
1427
|
-
const localPath = path.join(companyRoot, key);
|
|
1428
1548
|
if (!shouldSync(localPath, false) && !shouldSync(localPath, true))
|
|
1429
1549
|
continue;
|
|
1430
1550
|
// Codex P1 (PR #24 round 3): detect local edits before tombstoning.
|