@indigoai-us/hq-cloud 6.11.10 → 6.11.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +231 -52
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +330 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +16 -1
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -1
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +58 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.js +229 -15
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-exec-bit-preserve.test.d.ts +2 -0
- package/dist/cli/rescue-exec-bit-preserve.test.d.ts.map +1 -0
- package/dist/cli/rescue-exec-bit-preserve.test.js +169 -0
- package/dist/cli/rescue-exec-bit-preserve.test.js.map +1 -0
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +188 -59
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +487 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +148 -29
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts +8 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +17 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +6 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +62 -13
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +189 -0
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +6 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +13 -15
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +36 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watcher.d.ts +36 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +152 -30
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +396 -11
- package/src/bin/sync-runner.ts +254 -52
- package/src/cli/reindex.test.ts +47 -1
- package/src/cli/reindex.ts +17 -1
- package/src/cli/rescue-classify-ordering.test.ts +61 -0
- package/src/cli/rescue-core.ts +261 -15
- package/src/cli/rescue-exec-bit-preserve.test.ts +187 -0
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +103 -34
- package/src/cli/sync.test.ts +594 -1
- package/src/cli/sync.ts +229 -65
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +182 -30
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +18 -3
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +205 -0
- package/src/remote-pull.ts +77 -14
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +38 -1
- package/src/sync/push-receiver.ts +15 -18
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +209 -33
package/dist/cli/sync.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { VaultServiceConfig } from "../types.js";
|
|
8
8
|
import type { SyncMode } from "../vault-client.js";
|
|
9
|
+
import { type ScopePrefixInput } from "../prefix-coalesce.js";
|
|
9
10
|
import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
|
|
10
11
|
/**
|
|
11
12
|
* Per-file events emitted by `sync()` as it progresses.
|
|
@@ -312,7 +313,7 @@ export interface SyncOptions {
|
|
|
312
313
|
* (not empty `"shared"`) on any grant-resolution error, so a transient
|
|
313
314
|
* failure can never silently prune the local tree.
|
|
314
315
|
*/
|
|
315
|
-
prefixSet?:
|
|
316
|
+
prefixSet?: ScopePrefixInput[];
|
|
316
317
|
/**
|
|
317
318
|
* When the effective scope shrinks relative to the last pull and the shrink
|
|
318
319
|
* would orphan locally-modified ("dirty") files, `sync()` aborts with a
|
|
@@ -357,6 +358,11 @@ export interface SyncOptions {
|
|
|
357
358
|
* this and run `reindex()` once itself instead of per-company.
|
|
358
359
|
*/
|
|
359
360
|
skipReindex?: boolean;
|
|
361
|
+
/**
|
|
362
|
+
* Internal runner seam: true only when the caller already holds the
|
|
363
|
+
* per-root operation lock for this sync pass.
|
|
364
|
+
*/
|
|
365
|
+
operationLockAlreadyHeld?: boolean;
|
|
360
366
|
}
|
|
361
367
|
export interface SyncResult {
|
|
362
368
|
filesDownloaded: number;
|
|
@@ -431,6 +437,27 @@ export interface SyncResult {
|
|
|
431
437
|
* `HQ_SYNC_MAX_AUTO_PRUNE`.
|
|
432
438
|
*/
|
|
433
439
|
export declare function resolveAutoPruneCap(): number;
|
|
440
|
+
/**
|
|
441
|
+
* Best-effort report of the files that were new to this drive during the sync,
|
|
442
|
+
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
443
|
+
*
|
|
444
|
+
* POSTs to `${apiUrl}/v1/notify/file-added`, which writes per-recipient
|
|
445
|
+
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
446
|
+
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
447
|
+
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
448
|
+
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
449
|
+
* when there are no new files.
|
|
450
|
+
*
|
|
451
|
+
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
452
|
+
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
453
|
+
* best-effort, so one failing/oversized batch can never block the others or the
|
|
454
|
+
* sync. Exported only so the chunking can be unit-tested directly.
|
|
455
|
+
*/
|
|
456
|
+
export declare function reportNewFilesToNotify(vaultConfig: VaultServiceConfig, companyUid: string, companySlug: string, files: Array<{
|
|
457
|
+
path: string;
|
|
458
|
+
bytes: number;
|
|
459
|
+
addedBy: string | null;
|
|
460
|
+
}>): Promise<void>;
|
|
434
461
|
/**
|
|
435
462
|
* Sync (pull) all allowed files from the entity vault.
|
|
436
463
|
*/
|
package/dist/cli/sync.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAmCnD,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAc1E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAChD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,UAAU,EAAE,kBAAkB,CAAC;CAChC,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACvE,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,YAAY,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;CAC5D,GACD;IACE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;OAeG;IACH,IAAI,EAAE,8BAA8B,CAAC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;OAWG;IACH,IAAI,EAAE,6BAA6B,CAAC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,2BAA2B;IAC3B,WAAW,EAAE,kBAAkB,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;;;;;;OAaG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;OAiBG;IACH,iBAAiB,CAAC,EAAE,OAAO,GAAG,cAAc,CAAC;IAC7C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAM5C;AAeD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,sBAAsB,CAC1C,WAAW,EAAE,kBAAkB,EAC/B,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,GACpE,OAAO,CAAC,IAAI,CAAC,CAgDf;AAeD;;GAEG;AACH,wBAAsB,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAOpE"}
|
package/dist/cli/sync.js
CHANGED
|
@@ -9,14 +9,16 @@ import * as path from "path";
|
|
|
9
9
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
10
10
|
import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, } from "../s3.js";
|
|
11
11
|
import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
12
|
+
import { PERSONAL_VAULT_MANIFEST_KEY } from "../personal-vault.js";
|
|
12
13
|
import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
|
|
13
|
-
import { coalescePrefixes, isCoveredByAny } from "../prefix-coalesce.js";
|
|
14
|
+
import { coalescePrefixes, isCoveredByAny, } from "../prefix-coalesce.js";
|
|
14
15
|
import { createIgnoreFilter } from "../ignore.js";
|
|
15
16
|
import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
16
17
|
import { resolveConflict } from "./conflict.js";
|
|
17
18
|
import { buildConflictId, buildConflictPath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
18
19
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
19
20
|
import { reindex } from "./reindex.js";
|
|
21
|
+
import { withOperationLock } from "../operation-lock.js";
|
|
20
22
|
import { fetchCompanyTombstones, } from "./tombstones.js";
|
|
21
23
|
/**
|
|
22
24
|
* Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
|
|
@@ -35,6 +37,15 @@ export function resolveAutoPruneCap() {
|
|
|
35
37
|
}
|
|
36
38
|
/** Max time to wait on the best-effort new-files notification POST. */
|
|
37
39
|
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
40
|
+
/**
|
|
41
|
+
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
42
|
+
* an oversized batch wholesale, so the client MUST split a large report into
|
|
43
|
+
* chunks at or under this size — otherwise a first sync with more than this many
|
|
44
|
+
* new files reports none of them, and the same oversized batch re-triggers every
|
|
45
|
+
* sync cycle (wasted work + dropped notifications). Keep in lockstep with the
|
|
46
|
+
* server-side limit.
|
|
47
|
+
*/
|
|
48
|
+
const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
|
|
38
49
|
/**
|
|
39
50
|
* Best-effort report of the files that were new to this drive during the sync,
|
|
40
51
|
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
@@ -43,17 +54,31 @@ const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
|
43
54
|
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
44
55
|
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
45
56
|
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
46
|
-
* 5s timeout so a hung endpoint can't stall sync completion. No-op
|
|
47
|
-
* are no new files.
|
|
57
|
+
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
58
|
+
* when there are no new files.
|
|
59
|
+
*
|
|
60
|
+
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
61
|
+
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
62
|
+
* best-effort, so one failing/oversized batch can never block the others or the
|
|
63
|
+
* sync. Exported only so the chunking can be unit-tested directly.
|
|
48
64
|
*/
|
|
49
|
-
async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
|
|
65
|
+
export async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
|
|
50
66
|
if (files.length === 0)
|
|
51
67
|
return;
|
|
68
|
+
let token;
|
|
52
69
|
try {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
70
|
+
token =
|
|
71
|
+
typeof vaultConfig.authToken === "function"
|
|
72
|
+
? await vaultConfig.authToken()
|
|
73
|
+
: vaultConfig.authToken;
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
logNotifyFailure(err);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
80
|
+
for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
|
|
81
|
+
const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
|
|
57
82
|
const controller = new AbortController();
|
|
58
83
|
const timer = setTimeout(() => controller.abort(), NOTIFY_FILE_ADDED_TIMEOUT_MS);
|
|
59
84
|
try {
|
|
@@ -66,7 +91,7 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
|
|
|
66
91
|
body: JSON.stringify({
|
|
67
92
|
companyUid,
|
|
68
93
|
companySlug,
|
|
69
|
-
files:
|
|
94
|
+
files: batch.map((f) => ({
|
|
70
95
|
path: f.path,
|
|
71
96
|
bytes: f.bytes,
|
|
72
97
|
...(f.addedBy ? { addedBy: f.addedBy } : {}),
|
|
@@ -75,24 +100,35 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
|
|
|
75
100
|
signal: controller.signal,
|
|
76
101
|
});
|
|
77
102
|
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
// Best-effort per chunk: never let notification reporting affect the sync
|
|
105
|
+
// result, and a failed chunk must not abort the remaining chunks.
|
|
106
|
+
logNotifyFailure(err);
|
|
107
|
+
}
|
|
78
108
|
finally {
|
|
79
109
|
clearTimeout(timer);
|
|
80
110
|
}
|
|
81
111
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
112
|
+
}
|
|
113
|
+
/** Log a non-fatal notify failure without ever throwing out of the logger. */
|
|
114
|
+
function logNotifyFailure(err) {
|
|
115
|
+
try {
|
|
116
|
+
console.error(`[hq-sync] new-files notify report failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// swallow — logging must never break sync
|
|
90
120
|
}
|
|
91
121
|
}
|
|
92
122
|
/**
|
|
93
123
|
* Sync (pull) all allowed files from the entity vault.
|
|
94
124
|
*/
|
|
95
125
|
export async function sync(options) {
|
|
126
|
+
if (options.operationLockAlreadyHeld) {
|
|
127
|
+
return syncWithOperationLockHeld(options);
|
|
128
|
+
}
|
|
129
|
+
return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
|
|
130
|
+
}
|
|
131
|
+
async function syncWithOperationLockHeld(options) {
|
|
96
132
|
const { company, onConflict, vaultConfig, hqRoot } = options;
|
|
97
133
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
98
134
|
// Resolve company
|
|
@@ -376,16 +412,18 @@ export async function sync(options) {
|
|
|
376
412
|
const tombstoneKey = item.remoteFile.key;
|
|
377
413
|
// Same Windows-backslash landmine guard as the journal-tombstone executor:
|
|
378
414
|
// a malformed key must never reach fs.unlinkSync (path.join collapses the
|
|
379
|
-
// backslashes onto a REAL POSIX file).
|
|
380
|
-
//
|
|
381
|
-
|
|
382
|
-
|
|
415
|
+
// backslashes onto a REAL POSIX file). Traversal keys are likewise
|
|
416
|
+
// refused before any local filesystem or journal mutation.
|
|
417
|
+
const tombstonePath = resolveContainedVaultPath(companyRoot, tombstoneKey);
|
|
418
|
+
if (tombstonePath === null)
|
|
383
419
|
continue;
|
|
384
|
-
}
|
|
385
420
|
try {
|
|
386
|
-
const lstat = fs.lstatSync(
|
|
421
|
+
const lstat = fs.lstatSync(tombstonePath);
|
|
422
|
+
if (tombstoneTargetDiverged(journal, tombstoneKey, tombstonePath, lstat)) {
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
387
425
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
388
|
-
fs.unlinkSync(
|
|
426
|
+
fs.unlinkSync(tombstonePath);
|
|
389
427
|
}
|
|
390
428
|
// A directory at the key: don't recursively rm-rf the operator's dir;
|
|
391
429
|
// just drop the journal entry (safe-by-default, same as the other path).
|
|
@@ -453,10 +491,20 @@ export async function sync(options) {
|
|
|
453
491
|
const originalRelative = path.relative(hqRoot, localPath);
|
|
454
492
|
const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
|
|
455
493
|
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
494
|
+
const conflictKey = toPosixKey(path.relative(companyRoot, conflictAbs));
|
|
495
|
+
if (!isDownloadWritePathStillContained(companyRoot, conflictKey, conflictAbs)) {
|
|
496
|
+
filesSkipped++;
|
|
497
|
+
emit({
|
|
498
|
+
type: "error",
|
|
499
|
+
path: remoteFile.key,
|
|
500
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
501
|
+
});
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
456
504
|
let remoteFetched = false;
|
|
457
505
|
let converged = false;
|
|
458
506
|
try {
|
|
459
|
-
await downloadFile(ctx, remoteFile.key, conflictAbs);
|
|
507
|
+
const downloaded = await downloadFile(ctx, remoteFile.key, conflictAbs);
|
|
460
508
|
remoteFetched = true;
|
|
461
509
|
// Hash the fetched remote exactly the way the planner hashed local
|
|
462
510
|
// (symlink-aware) so the two hashes are directly comparable. A
|
|
@@ -464,7 +512,7 @@ export async function sync(options) {
|
|
|
464
512
|
// target string matches `hashSymlinkTarget(localPath)`.
|
|
465
513
|
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
466
514
|
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
467
|
-
: hashFile(conflictAbs);
|
|
515
|
+
: (downloaded.contentHash ?? hashFile(conflictAbs));
|
|
468
516
|
converged = remoteHash === item.localHash;
|
|
469
517
|
}
|
|
470
518
|
catch (probeErr) {
|
|
@@ -643,8 +691,17 @@ export async function sync(options) {
|
|
|
643
691
|
if (isExpiringSoon(ctx.expiresAt)) {
|
|
644
692
|
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
645
693
|
}
|
|
694
|
+
if (!isDownloadWritePathStillContained(companyRoot, remoteFile.key, localPath)) {
|
|
695
|
+
filesSkipped++;
|
|
696
|
+
emit({
|
|
697
|
+
type: "error",
|
|
698
|
+
path: remoteFile.key,
|
|
699
|
+
message: "download skipped: local parent escaped the sync root",
|
|
700
|
+
});
|
|
701
|
+
return;
|
|
702
|
+
}
|
|
646
703
|
try {
|
|
647
|
-
const { metadata } = await downloadFile(ctx, remoteFile.key, localPath);
|
|
704
|
+
const { metadata, contentHash, contentSize } = await downloadFile(ctx, remoteFile.key, localPath);
|
|
648
705
|
const author = metadata?.["created-by"] ?? null;
|
|
649
706
|
// Author sub for the scope-shrink authorship guard — same field the
|
|
650
707
|
// upload side stamps, read straight off the GET response metadata.
|
|
@@ -662,8 +719,8 @@ export async function sync(options) {
|
|
|
662
719
|
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
663
720
|
const hash = isLocalSymlink
|
|
664
721
|
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
665
|
-
: hashFile(localPath);
|
|
666
|
-
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
722
|
+
: (contentHash ?? hashFile(localPath));
|
|
723
|
+
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
667
724
|
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
668
725
|
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
669
726
|
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
@@ -839,21 +896,17 @@ export async function sync(options) {
|
|
|
839
896
|
// converged. Failures are reported but non-fatal — the entry stays in
|
|
840
897
|
// the journal and the next run retries.
|
|
841
898
|
for (const key of plan.tombstones) {
|
|
842
|
-
// Last line of defense
|
|
843
|
-
//
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
// The planner already refuses to enqueue malformed keys; if one still
|
|
847
|
-
// arrives, drop the poisoned journal entry without touching disk —
|
|
848
|
-
// normalizeJournalKeys rewrites it to its POSIX form on load.
|
|
849
|
-
if (isMalformedVaultKey(key)) {
|
|
850
|
-
removeEntry(journal, key);
|
|
899
|
+
// Last line of defense: a malformed or traversal key must NEVER reach
|
|
900
|
+
// fs.unlinkSync or journal mutation for a path outside the sync root.
|
|
901
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
902
|
+
if (localPath === null)
|
|
851
903
|
continue;
|
|
852
|
-
}
|
|
853
|
-
const localPath = path.join(companyRoot, key);
|
|
854
904
|
let removedSomething = false;
|
|
855
905
|
try {
|
|
856
906
|
const lstat = fs.lstatSync(localPath);
|
|
907
|
+
if (tombstoneTargetDiverged(journal, key, localPath, lstat)) {
|
|
908
|
+
continue;
|
|
909
|
+
}
|
|
857
910
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
858
911
|
fs.unlinkSync(localPath);
|
|
859
912
|
removedSomething = true;
|
|
@@ -1011,6 +1064,85 @@ function isRemoteRecreateAfterTombstone(remote, tombstone) {
|
|
|
1011
1064
|
return true; // no remote timestamp → don't suppress
|
|
1012
1065
|
return remoteMs > deletedAtMs;
|
|
1013
1066
|
}
|
|
1067
|
+
function hasTraversalSegment(key) {
|
|
1068
|
+
return key.split("/").some((segment) => segment === "..");
|
|
1069
|
+
}
|
|
1070
|
+
function isPathWithin(root, candidate) {
|
|
1071
|
+
const relative = path.relative(root, candidate);
|
|
1072
|
+
return (relative === "" ||
|
|
1073
|
+
(!relative.startsWith("..") && !path.isAbsolute(relative)));
|
|
1074
|
+
}
|
|
1075
|
+
function deepestExistingAncestor(start) {
|
|
1076
|
+
let current = start;
|
|
1077
|
+
for (;;) {
|
|
1078
|
+
try {
|
|
1079
|
+
fs.lstatSync(current);
|
|
1080
|
+
return current;
|
|
1081
|
+
}
|
|
1082
|
+
catch (err) {
|
|
1083
|
+
const code = err && typeof err === "object" && "code" in err
|
|
1084
|
+
? err.code
|
|
1085
|
+
: undefined;
|
|
1086
|
+
if (code !== "ENOENT" && code !== "ENOTDIR")
|
|
1087
|
+
return null;
|
|
1088
|
+
}
|
|
1089
|
+
const parent = path.dirname(current);
|
|
1090
|
+
if (parent === current)
|
|
1091
|
+
return null;
|
|
1092
|
+
current = parent;
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
function resolveContainedVaultPath(root, key) {
|
|
1096
|
+
if (isMalformedVaultKey(key) || hasTraversalSegment(key))
|
|
1097
|
+
return null;
|
|
1098
|
+
const resolvedRoot = path.resolve(root);
|
|
1099
|
+
const resolvedLocal = path.resolve(resolvedRoot, key);
|
|
1100
|
+
if (!isPathWithin(resolvedRoot, resolvedLocal))
|
|
1101
|
+
return null;
|
|
1102
|
+
let realRoot;
|
|
1103
|
+
try {
|
|
1104
|
+
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
1105
|
+
}
|
|
1106
|
+
catch {
|
|
1107
|
+
// If the vault root does not exist yet, no below-root symlink component can
|
|
1108
|
+
// already exist to redirect this key. Preserve first-pull behavior.
|
|
1109
|
+
return resolvedLocal;
|
|
1110
|
+
}
|
|
1111
|
+
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
1112
|
+
if (existingAncestor === null)
|
|
1113
|
+
return null;
|
|
1114
|
+
try {
|
|
1115
|
+
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
1116
|
+
if (!isPathWithin(realRoot, realAncestor))
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
catch {
|
|
1120
|
+
return null;
|
|
1121
|
+
}
|
|
1122
|
+
return resolvedLocal;
|
|
1123
|
+
}
|
|
1124
|
+
function isDownloadWritePathStillContained(root, key, localPath) {
|
|
1125
|
+
const resolved = resolveContainedVaultPath(root, key);
|
|
1126
|
+
return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
|
|
1127
|
+
}
|
|
1128
|
+
function tombstoneTargetDiverged(journal, key, localPath, lstat) {
|
|
1129
|
+
const journalEntry = journal.files[key];
|
|
1130
|
+
if (!journalEntry?.hash) {
|
|
1131
|
+
return lstat.isSymbolicLink() || lstat.isFile();
|
|
1132
|
+
}
|
|
1133
|
+
try {
|
|
1134
|
+
if (lstat.isSymbolicLink()) {
|
|
1135
|
+
return hashSymlinkTarget(fs.readlinkSync(localPath)) !== journalEntry.hash;
|
|
1136
|
+
}
|
|
1137
|
+
if (lstat.isFile()) {
|
|
1138
|
+
return hashFile(localPath) !== journalEntry.hash;
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
catch {
|
|
1142
|
+
return true;
|
|
1143
|
+
}
|
|
1144
|
+
return false;
|
|
1145
|
+
}
|
|
1014
1146
|
/**
|
|
1015
1147
|
* Stage-1 planning pass: classify every remote file into download / skip /
|
|
1016
1148
|
* conflict buckets without performing any S3 transfers. Local hashes are
|
|
@@ -1036,7 +1168,11 @@ prefixSet,
|
|
|
1036
1168
|
fileTombstones = new Map()) {
|
|
1037
1169
|
const items = [];
|
|
1038
1170
|
for (const remoteFile of remoteFiles) {
|
|
1039
|
-
const localPath =
|
|
1171
|
+
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
1172
|
+
if (localPath === null) {
|
|
1173
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
1174
|
+
continue;
|
|
1175
|
+
}
|
|
1040
1176
|
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
1041
1177
|
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
1042
1178
|
// mirrors since 5.33.0, but the pull side downloaded them freely from
|
|
@@ -1047,17 +1183,15 @@ fileTombstones = new Map()) {
|
|
|
1047
1183
|
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
1048
1184
|
continue;
|
|
1049
1185
|
}
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
}
|
|
1060
|
-
if (personalMode && remoteFile.key.startsWith("companies/")) {
|
|
1186
|
+
if (personalMode &&
|
|
1187
|
+
remoteFile.key.startsWith("companies/") &&
|
|
1188
|
+
// EXEMPTION: companies/manifest.yaml is the routing source-of-truth
|
|
1189
|
+
// carved INTO the personal vault on the push side
|
|
1190
|
+
// (computePersonalVaultPaths). It must round-trip on the pull leg too —
|
|
1191
|
+
// skipping it here leaves it forever unjournaled, which re-fires a
|
|
1192
|
+
// transient push-side conflict every sync (no journal baseline). Let it
|
|
1193
|
+
// fall through to download + journal like any personal file.
|
|
1194
|
+
remoteFile.key !== PERSONAL_VAULT_MANIFEST_KEY) {
|
|
1061
1195
|
// Default: drop every `companies/...` key — the legacy contract
|
|
1062
1196
|
// is that the personal bucket should never contain them.
|
|
1063
1197
|
//
|
|
@@ -1394,12 +1528,8 @@ fileTombstones = new Map()) {
|
|
|
1394
1528
|
const posixKey = toPosixKey(key);
|
|
1395
1529
|
if (remoteKeySet.has(posixKey))
|
|
1396
1530
|
continue;
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
// POSIX file and unlinks live data. Leave it for normalizeJournalKeys to
|
|
1400
|
-
// rewrite to POSIX on the next write; the canonical key is re-evaluated
|
|
1401
|
-
// (and correctly tombstoned if genuinely remote-deleted) on a later pull.
|
|
1402
|
-
if (isMalformedVaultKey(key))
|
|
1531
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
1532
|
+
if (localPath === null)
|
|
1403
1533
|
continue;
|
|
1404
1534
|
// PersonalMode key gating — mirror the download branch.
|
|
1405
1535
|
if (personalMode && key.startsWith("companies/")) {
|
|
@@ -1415,7 +1545,6 @@ fileTombstones = new Map()) {
|
|
|
1415
1545
|
// Honor the current ignore filter — if a path was previously synced
|
|
1416
1546
|
// but is now ignored (operator edited .hqignore), do NOT delete
|
|
1417
1547
|
// the local copy. They're keeping it deliberately.
|
|
1418
|
-
const localPath = path.join(companyRoot, key);
|
|
1419
1548
|
if (!shouldSync(localPath, false) && !shouldSync(localPath, true))
|
|
1420
1549
|
continue;
|
|
1421
1550
|
// Codex P1 (PR #24 round 3): detect local edits before tombstoning.
|