@indigoai-us/hq-cloud 6.11.11 → 6.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +35 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.js +290 -0
- package/dist/bin/sync-runner-company.js.map +1 -0
- package/dist/bin/sync-runner-events.d.ts +12 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -0
- package/dist/bin/sync-runner-events.js +12 -0
- package/dist/bin/sync-runner-events.js.map +1 -0
- package/dist/bin/sync-runner-planning.d.ts +53 -0
- package/dist/bin/sync-runner-planning.d.ts.map +1 -0
- package/dist/bin/sync-runner-planning.js +59 -0
- package/dist/bin/sync-runner-planning.js.map +1 -0
- package/dist/bin/sync-runner-rollup.d.ts +24 -0
- package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
- package/dist/bin/sync-runner-rollup.js +46 -0
- package/dist/bin/sync-runner-rollup.js.map +1 -0
- package/dist/bin/sync-runner-telemetry.d.ts +5 -0
- package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
- package/dist/bin/sync-runner-telemetry.js +5 -0
- package/dist/bin/sync-runner-telemetry.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-loop.js +372 -0
- package/dist/bin/sync-runner-watch-loop.js.map +1 -0
- package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-routes.js +74 -0
- package/dist/bin/sync-runner-watch-routes.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +5 -54
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +76 -978
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +265 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +34 -17
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -5
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +75 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +45 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +320 -170
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +276 -660
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +541 -748
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +382 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/daemon-worker.d.ts +2 -2
- package/dist/daemon-worker.js +3 -3
- package/dist/daemon-worker.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +149 -30
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +8 -2
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +8 -3
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +85 -16
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +213 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +12 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +45 -17
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +67 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync-core.d.ts +27 -0
- package/dist/sync-core.d.ts.map +1 -0
- package/dist/sync-core.js +54 -0
- package/dist/sync-core.js.map +1 -0
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +284 -36
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +59 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +38 -20
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +155 -143
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +350 -0
- package/src/bin/sync-runner-events.ts +25 -0
- package/src/bin/sync-runner-planning.ts +121 -0
- package/src/bin/sync-runner-rollup.ts +72 -0
- package/src/bin/sync-runner-telemetry.ts +8 -0
- package/src/bin/sync-runner-watch-loop.ts +443 -0
- package/src/bin/sync-runner-watch-routes.ts +86 -0
- package/src/bin/sync-runner.test.ts +298 -11
- package/src/bin/sync-runner.ts +99 -1054
- package/src/cli/reindex.test.ts +41 -3
- package/src/cli/reindex.ts +35 -19
- package/src/cli/rescue-classify-ordering.test.ts +81 -0
- package/src/cli/rescue-core.ts +400 -165
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +420 -693
- package/src/cli/sync.test.ts +460 -1
- package/src/cli/sync.ts +788 -825
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/daemon-worker.ts +3 -3
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +183 -31
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +8 -2
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +235 -1
- package/src/remote-pull.ts +106 -18
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +73 -1
- package/src/sync/push-receiver.ts +56 -20
- package/src/sync-core.ts +58 -0
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/vault-client.test.ts +74 -0
- package/src/vault-client.ts +395 -43
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +215 -174
package/src/cli/sync.ts
CHANGED
|
@@ -7,7 +7,11 @@
|
|
|
7
7
|
|
|
8
8
|
import * as fs from "fs";
|
|
9
9
|
import * as path from "path";
|
|
10
|
-
import type {
|
|
10
|
+
import type {
|
|
11
|
+
EntityContext,
|
|
12
|
+
VaultServiceConfig,
|
|
13
|
+
SyncJournal,
|
|
14
|
+
} from "../types.js";
|
|
11
15
|
import type { SyncMode } from "../vault-client.js";
|
|
12
16
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
13
17
|
import {
|
|
@@ -43,8 +47,18 @@ import {
|
|
|
43
47
|
ScopeShrinkLargePruneError,
|
|
44
48
|
type ScopeShrinkAdviceContext,
|
|
45
49
|
} from "../scope-shrink.js";
|
|
46
|
-
import {
|
|
50
|
+
import {
|
|
51
|
+
coalescePrefixes,
|
|
52
|
+
isCoveredByAny,
|
|
53
|
+
type ScopePrefixInput,
|
|
54
|
+
} from "../prefix-coalesce.js";
|
|
47
55
|
import { createIgnoreFilter } from "../ignore.js";
|
|
56
|
+
import {
|
|
57
|
+
hasRemoteChanged,
|
|
58
|
+
isAccessDenied,
|
|
59
|
+
resolveActiveCompany,
|
|
60
|
+
resolveTransferConcurrency,
|
|
61
|
+
} from "../sync-core.js";
|
|
48
62
|
import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
49
63
|
import { resolveConflict } from "./conflict.js";
|
|
50
64
|
import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
|
|
@@ -55,6 +69,7 @@ import {
|
|
|
55
69
|
} from "../lib/conflict-file.js";
|
|
56
70
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
57
71
|
import { reindex } from "./reindex.js";
|
|
72
|
+
import { withOperationLock } from "../operation-lock.js";
|
|
58
73
|
import {
|
|
59
74
|
fetchCompanyTombstones,
|
|
60
75
|
type CompanyTombstone,
|
|
@@ -369,7 +384,7 @@ export interface SyncOptions {
|
|
|
369
384
|
* (not empty `"shared"`) on any grant-resolution error, so a transient
|
|
370
385
|
* failure can never silently prune the local tree.
|
|
371
386
|
*/
|
|
372
|
-
prefixSet?:
|
|
387
|
+
prefixSet?: ScopePrefixInput[];
|
|
373
388
|
/**
|
|
374
389
|
* When the effective scope shrinks relative to the last pull and the shrink
|
|
375
390
|
* would orphan locally-modified ("dirty") files, `sync()` aborts with a
|
|
@@ -414,6 +429,11 @@ export interface SyncOptions {
|
|
|
414
429
|
* this and run `reindex()` once itself instead of per-company.
|
|
415
430
|
*/
|
|
416
431
|
skipReindex?: boolean;
|
|
432
|
+
/**
|
|
433
|
+
* Internal runner seam: true only when the caller already holds the
|
|
434
|
+
* per-root operation lock for this sync pass.
|
|
435
|
+
*/
|
|
436
|
+
operationLockAlreadyHeld?: boolean;
|
|
417
437
|
}
|
|
418
438
|
|
|
419
439
|
export interface SyncResult {
|
|
@@ -479,6 +499,52 @@ export interface SyncResult {
|
|
|
479
499
|
scopeOrphansBlocked: number;
|
|
480
500
|
}
|
|
481
501
|
|
|
502
|
+
type SyncEventEmitter = (event: SyncProgressEvent) => void;
|
|
503
|
+
|
|
504
|
+
type PullDownloadItem = Extract<PullPlanItem, { action: "download" }>;
|
|
505
|
+
|
|
506
|
+
interface PullRunContext {
|
|
507
|
+
options: SyncOptions;
|
|
508
|
+
companyRef: string;
|
|
509
|
+
vaultConfig: VaultServiceConfig;
|
|
510
|
+
hqRoot: string;
|
|
511
|
+
emit: SyncEventEmitter;
|
|
512
|
+
ctx: EntityContext;
|
|
513
|
+
companyRoot: string;
|
|
514
|
+
shouldSync: (filePath: string, isDir?: boolean) => boolean;
|
|
515
|
+
journalSlug: string;
|
|
516
|
+
startedAt: string;
|
|
517
|
+
journal: SyncJournal;
|
|
518
|
+
remoteFiles: RemoteFile[];
|
|
519
|
+
syncMode: SyncMode;
|
|
520
|
+
currentPrefixSet: string[];
|
|
521
|
+
fileTombstones: ReadonlyMap<string, CompanyTombstone>;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
interface PullCounters {
|
|
525
|
+
filesDownloaded: number;
|
|
526
|
+
bytesDownloaded: number;
|
|
527
|
+
filesSkipped: number;
|
|
528
|
+
conflicts: number;
|
|
529
|
+
filesTombstoned: number;
|
|
530
|
+
filesOutOfScope: number;
|
|
531
|
+
conflictPaths: string[];
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
interface ScopeShrinkPlanState {
|
|
535
|
+
lastRecord: ReturnType<typeof lastPullRecord>;
|
|
536
|
+
shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
|
|
537
|
+
autoRecover: boolean;
|
|
538
|
+
adviceContext: ScopeShrinkAdviceContext;
|
|
539
|
+
effectiveForce: boolean;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
interface ScopeShrinkRun {
|
|
543
|
+
shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
|
|
544
|
+
shrinkResult: ReturnType<typeof applyScopeShrink>;
|
|
545
|
+
scopeOrphansRemoved: number;
|
|
546
|
+
}
|
|
547
|
+
|
|
482
548
|
/**
|
|
483
549
|
* Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
|
|
484
550
|
* scope shrink that would delete more than this many CLEAN local files in one
|
|
@@ -497,6 +563,16 @@ export function resolveAutoPruneCap(): number {
|
|
|
497
563
|
/** Max time to wait on the best-effort new-files notification POST. */
|
|
498
564
|
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
499
565
|
|
|
566
|
+
/**
|
|
567
|
+
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
568
|
+
* an oversized batch wholesale, so the client MUST split a large report into
|
|
569
|
+
* chunks at or under this size — otherwise a first sync with more than this many
|
|
570
|
+
* new files reports none of them, and the same oversized batch re-triggers every
|
|
571
|
+
* sync cycle (wasted work + dropped notifications). Keep in lockstep with the
|
|
572
|
+
* server-side limit.
|
|
573
|
+
*/
|
|
574
|
+
const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
|
|
575
|
+
|
|
500
576
|
/**
|
|
501
577
|
* Best-effort report of the files that were new to this drive during the sync,
|
|
502
578
|
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
@@ -505,22 +581,36 @@ const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
|
505
581
|
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
506
582
|
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
507
583
|
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
508
|
-
* 5s timeout so a hung endpoint can't stall sync completion. No-op
|
|
509
|
-
* are no new files.
|
|
584
|
+
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
585
|
+
* when there are no new files.
|
|
586
|
+
*
|
|
587
|
+
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
588
|
+
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
589
|
+
* best-effort, so one failing/oversized batch can never block the others or the
|
|
590
|
+
* sync. Exported only so the chunking can be unit-tested directly.
|
|
510
591
|
*/
|
|
511
|
-
async function reportNewFilesToNotify(
|
|
592
|
+
export async function reportNewFilesToNotify(
|
|
512
593
|
vaultConfig: VaultServiceConfig,
|
|
513
594
|
companyUid: string,
|
|
514
595
|
companySlug: string,
|
|
515
596
|
files: Array<{ path: string; bytes: number; addedBy: string | null }>,
|
|
516
597
|
): Promise<void> {
|
|
517
598
|
if (files.length === 0) return;
|
|
599
|
+
|
|
600
|
+
let token: string;
|
|
518
601
|
try {
|
|
519
|
-
|
|
602
|
+
token =
|
|
520
603
|
typeof vaultConfig.authToken === "function"
|
|
521
604
|
? await vaultConfig.authToken()
|
|
522
605
|
: vaultConfig.authToken;
|
|
523
|
-
|
|
606
|
+
} catch (err) {
|
|
607
|
+
logNotifyFailure(err);
|
|
608
|
+
return;
|
|
609
|
+
}
|
|
610
|
+
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
611
|
+
|
|
612
|
+
for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
|
|
613
|
+
const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
|
|
524
614
|
const controller = new AbortController();
|
|
525
615
|
const timer = setTimeout(
|
|
526
616
|
() => controller.abort(),
|
|
@@ -536,7 +626,7 @@ async function reportNewFilesToNotify(
|
|
|
536
626
|
body: JSON.stringify({
|
|
537
627
|
companyUid,
|
|
538
628
|
companySlug,
|
|
539
|
-
files:
|
|
629
|
+
files: batch.map((f) => ({
|
|
540
630
|
path: f.path,
|
|
541
631
|
bytes: f.bytes,
|
|
542
632
|
...(f.addedBy ? { addedBy: f.addedBy } : {}),
|
|
@@ -544,20 +634,26 @@ async function reportNewFilesToNotify(
|
|
|
544
634
|
}),
|
|
545
635
|
signal: controller.signal,
|
|
546
636
|
});
|
|
637
|
+
} catch (err) {
|
|
638
|
+
// Best-effort per chunk: never let notification reporting affect the sync
|
|
639
|
+
// result, and a failed chunk must not abort the remaining chunks.
|
|
640
|
+
logNotifyFailure(err);
|
|
547
641
|
} finally {
|
|
548
642
|
clearTimeout(timer);
|
|
549
643
|
}
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
)
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/** Log a non-fatal notify failure without ever throwing out of the logger. */
|
|
648
|
+
function logNotifyFailure(err: unknown): void {
|
|
649
|
+
try {
|
|
650
|
+
console.error(
|
|
651
|
+
`[hq-sync] new-files notify report failed (non-fatal): ${
|
|
652
|
+
err instanceof Error ? err.message : String(err)
|
|
653
|
+
}`,
|
|
654
|
+
);
|
|
655
|
+
} catch {
|
|
656
|
+
// swallow — logging must never break sync
|
|
561
657
|
}
|
|
562
658
|
}
|
|
563
659
|
|
|
@@ -565,10 +661,55 @@ async function reportNewFilesToNotify(
|
|
|
565
661
|
* Sync (pull) all allowed files from the entity vault.
|
|
566
662
|
*/
|
|
567
663
|
export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
568
|
-
|
|
664
|
+
if (options.operationLockAlreadyHeld) {
|
|
665
|
+
return syncWithOperationLockHeld(options);
|
|
666
|
+
}
|
|
667
|
+
return withOperationLock(options.hqRoot, "sync", () =>
|
|
668
|
+
syncWithOperationLockHeld(options),
|
|
669
|
+
);
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
async function syncWithOperationLockHeld(
|
|
673
|
+
options: SyncOptions,
|
|
674
|
+
): Promise<SyncResult> {
|
|
675
|
+
const run = await buildPullContext(options);
|
|
676
|
+
const plan = planPull(run);
|
|
677
|
+
|
|
678
|
+
emitPullPlan(run.emit, plan);
|
|
679
|
+
|
|
680
|
+
const scopePlan = planScopeShrink(run);
|
|
681
|
+
const scopeRun = executeScopeShrink(run, scopePlan);
|
|
682
|
+
const counters = createPullCounters();
|
|
683
|
+
|
|
684
|
+
const transferConcurrency = resolveTransferConcurrency();
|
|
685
|
+
const conflictRun = await executeConflictExecutor(
|
|
686
|
+
run,
|
|
687
|
+
plan,
|
|
688
|
+
scopeRun,
|
|
689
|
+
counters,
|
|
690
|
+
);
|
|
691
|
+
if (conflictRun.abortResult) {
|
|
692
|
+
return conflictRun.abortResult;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
await executeDownloadExecutor(
|
|
696
|
+
run,
|
|
697
|
+
conflictRun.downloadItems,
|
|
698
|
+
transferConcurrency,
|
|
699
|
+
counters,
|
|
700
|
+
);
|
|
701
|
+
|
|
702
|
+
await emitAndReportNewFiles(run, plan);
|
|
703
|
+
await verifyPlannedJournalTombstones(run, plan);
|
|
704
|
+
executeJournalTombstoneDeletes(run, plan, counters);
|
|
705
|
+
|
|
706
|
+
return finalizePullRun(run, plan, scopeRun, counters);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
|
|
710
|
+
const { company, vaultConfig, hqRoot } = options;
|
|
569
711
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
570
712
|
|
|
571
|
-
// Resolve company
|
|
572
713
|
const companyRef = company ?? resolveActiveCompany(hqRoot);
|
|
573
714
|
if (!companyRef) {
|
|
574
715
|
throw new Error(
|
|
@@ -577,184 +718,131 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
577
718
|
);
|
|
578
719
|
}
|
|
579
720
|
|
|
580
|
-
|
|
581
|
-
let ctx = await resolveEntityContext(companyRef, vaultConfig);
|
|
582
|
-
// Every company's files land under companies/{slug}/ so fanning out multiple
|
|
583
|
-
// companies into the same hqRoot doesn't cross-clobber files with overlapping
|
|
584
|
-
// S3 keys (e.g. every company has a .hq/manifest.json). Remote keys stay
|
|
585
|
-
// company-relative; the prefix lives only on disk.
|
|
586
|
-
// In personalMode the journal slug + S3 keys are person-relative (e.g. "docs/foo.md");
|
|
587
|
-
// the local target is `hqRoot` directly, NOT `<hqRoot>/companies/<personSlug>/`. This
|
|
588
|
-
// keeps round-trip parity with the Rust personal first-push (Step 7) which sources
|
|
589
|
-
// `<hqRoot>/docs/foo.md`.
|
|
721
|
+
const ctx = await resolveEntityContext(companyRef, vaultConfig);
|
|
590
722
|
const companyRoot = options.personalMode === true
|
|
591
723
|
? hqRoot
|
|
592
724
|
: path.join(hqRoot, "companies", ctx.slug);
|
|
593
725
|
const shouldSync = createIgnoreFilter(hqRoot);
|
|
594
726
|
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
595
727
|
const startedAt = new Date().toISOString();
|
|
596
|
-
|
|
597
|
-
// the legacy `personal` file exists (mass re-download/etag churn). Seeding
|
|
598
|
-
// here — inside the engine — covers every consumer (sync-runner already
|
|
599
|
-
// seeds; hq-cli historically didn't, which split the vault's bookkeeping
|
|
600
|
-
// across two journal files and re-flagged synced files as conflicts).
|
|
728
|
+
|
|
601
729
|
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) migratePersonalVaultJournal();
|
|
602
|
-
// Migrate v1 → v2 in place so the scope-shrink / pull-record machinery has
|
|
603
|
-
// its fields, and GC any tombstones past the 30-day retention window before
|
|
604
|
-
// we re-evaluate orphans (so a long-pruned path can re-download cleanly).
|
|
605
730
|
const journal = migrateToV2(readJournal(journalSlug));
|
|
606
731
|
gcTombstones(journal, Date.now());
|
|
607
732
|
|
|
608
|
-
// ── Effective download scope (US-005) ─────────────────────────────────────
|
|
609
|
-
// `all` → prefixSet `[""]`, which `isCoveredByAny` treats as "covers
|
|
610
|
-
// everything" — so the download filter and the scope-shrink
|
|
611
|
-
// comparison both become no-ops, preserving legacy full-bucket
|
|
612
|
-
// behavior bit-for-bit.
|
|
613
|
-
// `shared`/`custom` → the coalesced, company-relative prefix set the runner
|
|
614
|
-
// resolved. An empty set means "nothing in scope" → download
|
|
615
|
-
// nothing (the runner falls back to `all` on resolution errors, so
|
|
616
|
-
// empty here is an intentional "nothing shared", never a failure).
|
|
617
733
|
const syncMode: SyncMode = options.syncMode ?? "all";
|
|
618
|
-
const currentPrefixSet =
|
|
734
|
+
const currentPrefixSet: string[] =
|
|
619
735
|
syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
|
|
620
|
-
// Authorship guard input (scope-shrink): the caller's own Cognito sub,
|
|
621
|
-
// injected by the entry point (the runner sources it from its decoded
|
|
622
|
-
// idToken claims — the same sub stamped onto uploads as `created-by-sub`).
|
|
623
|
-
// Undefined degrades safely: own-author files lose their special shield, but
|
|
624
|
-
// the `protectUnknownAuthors` conservative path below still prevents a
|
|
625
|
-
// routine sync from deleting anything it can't prove is foreign.
|
|
626
|
-
const callerSub = options.callerSub;
|
|
627
|
-
|
|
628
|
-
let filesDownloaded = 0;
|
|
629
|
-
let bytesDownloaded = 0;
|
|
630
|
-
let filesSkipped = 0;
|
|
631
|
-
let conflicts = 0;
|
|
632
|
-
let filesTombstoned = 0;
|
|
633
|
-
let filesOutOfScope = 0;
|
|
634
|
-
const conflictPaths: string[] = [];
|
|
635
736
|
|
|
636
|
-
// List all remote files (IAM session policy filters at the AWS layer)
|
|
637
737
|
const remoteFiles = await listRemoteFiles(ctx);
|
|
638
|
-
|
|
639
|
-
// Fetch the company's FILE_TOMBSTONE records so the planner can suppress
|
|
640
|
-
// resurrection of an intentionally-deleted object (delete-resync). Done in
|
|
641
|
-
// parallel intent with the LIST above conceptually, but kept serial here for
|
|
642
|
-
// a clean read of `ctx`; best-effort — a failed read degrades to an empty map
|
|
643
|
-
// (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
|
|
644
|
-
// companyUid the tombstone rows are keyed under.
|
|
645
|
-
//
|
|
646
|
-
// SKIP for the personal vault: its `ctx.uid` is a personUid (`prs_…`), but
|
|
647
|
-
// `GET /v1/files/tombstones?company=…` is COMPANY-scoped server-side
|
|
648
|
-
// (findCallerWithMembership), so a personal-vault request resolves
|
|
649
|
-
// `company=prs_…` to no membership and is correctly rejected with
|
|
650
|
-
// `403 "No active membership for caller in company prs_…"`. That 403 is
|
|
651
|
-
// benign for the pull (it already degrades to the empty map below), but
|
|
652
|
-
// hq-pro captures EVERY one as a Sentry warning — the per-personal-vault
|
|
653
|
-
// no-membership cluster (one Sentry issue per signed-in user). Personal-vault
|
|
654
|
-
// delete-resync was never a committed feature and there is no person-scoped
|
|
655
|
-
// tombstone path, so for the personal target we skip the fetch and use an
|
|
656
|
-
// empty map — byte-for-byte the current degraded behavior, minus the 403 spam.
|
|
657
|
-
// FUTURE FOLLOW-UP (not built here): if personal-vault delete-resync is
|
|
658
|
-
// wanted, it needs a real person-scoped tombstone endpoint + client read.
|
|
659
|
-
const tombstones =
|
|
738
|
+
const fileTombstones =
|
|
660
739
|
options.personalMode === true
|
|
661
740
|
? new Map<string, CompanyTombstone>()
|
|
662
741
|
: await fetchCompanyTombstones(vaultConfig, ctx.uid);
|
|
663
742
|
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
743
|
+
return {
|
|
744
|
+
options,
|
|
745
|
+
companyRef,
|
|
746
|
+
vaultConfig,
|
|
747
|
+
hqRoot,
|
|
748
|
+
emit,
|
|
749
|
+
ctx,
|
|
670
750
|
companyRoot,
|
|
671
751
|
shouldSync,
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
752
|
+
journalSlug,
|
|
753
|
+
startedAt,
|
|
754
|
+
journal,
|
|
755
|
+
remoteFiles,
|
|
756
|
+
syncMode,
|
|
675
757
|
currentPrefixSet,
|
|
676
|
-
|
|
758
|
+
fileTombstones,
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
function planPull(run: PullRunContext): PullPlan {
|
|
763
|
+
return computePullPlan(
|
|
764
|
+
run.remoteFiles,
|
|
765
|
+
run.journal,
|
|
766
|
+
run.companyRoot,
|
|
767
|
+
run.shouldSync,
|
|
768
|
+
run.options.personalMode === true,
|
|
769
|
+
run.options.includeLocalCompanies === true,
|
|
770
|
+
run.options.teamSyncedSlugs ?? null,
|
|
771
|
+
run.currentPrefixSet,
|
|
772
|
+
run.fileTombstones,
|
|
677
773
|
);
|
|
774
|
+
}
|
|
678
775
|
|
|
776
|
+
function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
|
|
679
777
|
emit({
|
|
680
778
|
type: "plan",
|
|
681
779
|
filesToDownload: plan.filesToDownload,
|
|
682
780
|
bytesToDownload: plan.bytesToDownload,
|
|
683
|
-
// sync() is pull-only; push counts are sourced from share()'s plan event.
|
|
684
781
|
filesToUpload: 0,
|
|
685
782
|
bytesToUpload: 0,
|
|
686
783
|
filesToSkip: plan.filesToSkip,
|
|
687
784
|
filesToConflict: plan.filesToConflict,
|
|
688
|
-
// Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
|
|
689
|
-
// deletes known at plan time; the journal-vs-LIST tombstones are
|
|
690
|
-
// HEAD-verified later and surfaced via the final filesTombstoned count.
|
|
691
785
|
filesToDelete: plan.filesToTombstoneDelete,
|
|
692
786
|
});
|
|
787
|
+
}
|
|
693
788
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
// AWS layer never narrows the pull. This client-side shrink is what makes
|
|
709
|
-
// `hq sync mode shared` actually stick across re-syncs for an owner.
|
|
710
|
-
const lastRecord = lastPullRecord(journal, ctx.uid);
|
|
711
|
-
// A missing record, or a v1-migrated record with an empty prefixSet, means
|
|
712
|
-
// "no recorded scope" → treat the last scope as full-bucket `all` (`[""]`),
|
|
713
|
-
// per the PullRecord.prefixSet contract in types.ts.
|
|
789
|
+
function createPullCounters(): PullCounters {
|
|
790
|
+
return {
|
|
791
|
+
filesDownloaded: 0,
|
|
792
|
+
bytesDownloaded: 0,
|
|
793
|
+
filesSkipped: 0,
|
|
794
|
+
conflicts: 0,
|
|
795
|
+
filesTombstoned: 0,
|
|
796
|
+
filesOutOfScope: 0,
|
|
797
|
+
conflictPaths: [],
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
function planScopeShrink(run: PullRunContext): ScopeShrinkPlanState {
|
|
802
|
+
const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
|
|
714
803
|
const lastPrefixSet =
|
|
715
804
|
lastRecord && lastRecord.prefixSet.length > 0
|
|
716
805
|
? lastRecord.prefixSet
|
|
717
806
|
: [""];
|
|
718
807
|
const shrinkPlan = buildScopeShrinkPlan({
|
|
719
|
-
journal,
|
|
720
|
-
hqRoot: companyRoot,
|
|
808
|
+
journal: run.journal,
|
|
809
|
+
hqRoot: run.companyRoot,
|
|
721
810
|
lastPrefixSet,
|
|
722
|
-
currentPrefixSet,
|
|
723
|
-
callerSub,
|
|
724
|
-
// Automatic pull: never auto-prune content the caller authored, and never
|
|
725
|
-
// make a destructive guess about unknown-author (legacy) orphans. The
|
|
726
|
-
// explicit `hq sync narrow` ritual opts out of the unknown-author shield.
|
|
811
|
+
currentPrefixSet: run.currentPrefixSet,
|
|
812
|
+
callerSub: run.options.callerSub,
|
|
727
813
|
protectUnknownAuthors: true,
|
|
728
814
|
});
|
|
729
|
-
|
|
730
|
-
// interactive flag, so it must never throw on a shrink — it self-heals
|
|
731
|
-
// non-destructively (dirty kept on disk + un-tracked, clean quarantined).
|
|
732
|
-
// A foreground `hq sync` ("block", the default) keeps the protective gate
|
|
733
|
-
// but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
|
|
734
|
-
// bypasses the bulk-prune cap (quarantine is non-destructive, so a large
|
|
735
|
-
// recovery move is safe). DEV-1768.
|
|
736
|
-
const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
|
|
815
|
+
const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
|
|
737
816
|
const autoRecover = scopeShrinkPolicy === "auto-recover";
|
|
738
817
|
const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
|
|
739
|
-
const effectiveForce = options.forceScopeShrink === true || autoRecover;
|
|
818
|
+
const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
|
|
819
|
+
|
|
820
|
+
return {
|
|
821
|
+
lastRecord,
|
|
822
|
+
shrinkPlan,
|
|
823
|
+
autoRecover,
|
|
824
|
+
adviceContext,
|
|
825
|
+
effectiveForce,
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
function executeScopeShrink(
|
|
830
|
+
run: PullRunContext,
|
|
831
|
+
scopePlan: ScopeShrinkPlanState,
|
|
832
|
+
): ScopeShrinkRun {
|
|
833
|
+
const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
|
|
740
834
|
|
|
741
835
|
if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
|
|
742
836
|
throw new ScopeShrinkBlockedError(
|
|
743
|
-
ctx.uid,
|
|
837
|
+
run.ctx.uid,
|
|
744
838
|
lastRecord?.syncMode ?? "unknown",
|
|
745
|
-
syncMode,
|
|
839
|
+
run.syncMode,
|
|
746
840
|
shrinkPlan.dirty,
|
|
747
841
|
shrinkPlan.clean,
|
|
748
842
|
adviceContext,
|
|
749
843
|
);
|
|
750
844
|
}
|
|
751
|
-
|
|
752
|
-
// a single foreground sync. A deliberate large narrow goes through
|
|
753
|
-
// `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
|
|
754
|
-
// raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
|
|
755
|
-
// under auto-recover — quarantine is non-destructive so a big recovery is
|
|
756
|
-
// safe, and the runner has no way to act on a thrown cap. The engine moves
|
|
757
|
-
// nothing when it throws here.
|
|
845
|
+
|
|
758
846
|
const autoPruneCap = resolveAutoPruneCap();
|
|
759
847
|
if (
|
|
760
848
|
!effectiveForce &&
|
|
@@ -762,40 +850,32 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
762
850
|
shrinkPlan.clean.length > autoPruneCap
|
|
763
851
|
) {
|
|
764
852
|
throw new ScopeShrinkLargePruneError(
|
|
765
|
-
ctx.uid,
|
|
766
|
-
syncMode,
|
|
853
|
+
run.ctx.uid,
|
|
854
|
+
run.syncMode,
|
|
767
855
|
shrinkPlan.clean.length,
|
|
768
856
|
autoPruneCap,
|
|
769
857
|
adviceContext,
|
|
770
858
|
);
|
|
771
859
|
}
|
|
772
|
-
|
|
773
|
-
// recoverable), never silently deleted — a background sync purging local
|
|
774
|
-
// files unannounced was DEV-1768 fix #3. The quarantine root lives under the
|
|
775
|
-
// real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
|
|
776
|
-
// files don't round-trip back through S3.
|
|
860
|
+
|
|
777
861
|
const scopeQuarantineRoot = path.join(
|
|
778
|
-
hqRoot,
|
|
862
|
+
run.hqRoot,
|
|
779
863
|
".hq",
|
|
780
864
|
"scope-quarantine",
|
|
781
|
-
journalSlug,
|
|
865
|
+
run.journalSlug,
|
|
782
866
|
);
|
|
783
867
|
const shrinkResult = applyScopeShrink({
|
|
784
|
-
journal,
|
|
868
|
+
journal: run.journal,
|
|
785
869
|
plan: shrinkPlan,
|
|
786
|
-
hqRoot: companyRoot,
|
|
870
|
+
hqRoot: run.companyRoot,
|
|
787
871
|
forceScopeShrink: effectiveForce,
|
|
788
872
|
reason: "scope_shrink",
|
|
789
873
|
cleanDisposition: "quarantine",
|
|
790
874
|
quarantineRoot: scopeQuarantineRoot,
|
|
791
875
|
});
|
|
792
|
-
|
|
793
|
-
// silent. Quarantined clean files render as `deleted: true` (removed from the
|
|
794
|
-
// working tree, recoverable in quarantine); dirty files KEPT on disk render
|
|
795
|
-
// as a non-deletion notice so the operator knows they were un-tracked, not
|
|
796
|
-
// removed. The Rust menubar parser already handles `deleted: true`.
|
|
876
|
+
|
|
797
877
|
for (const relPath of shrinkResult.quarantinedPaths) {
|
|
798
|
-
emit({
|
|
878
|
+
run.emit({
|
|
799
879
|
type: "progress",
|
|
800
880
|
path: relPath,
|
|
801
881
|
bytes: 0,
|
|
@@ -804,7 +884,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
804
884
|
});
|
|
805
885
|
}
|
|
806
886
|
for (const relPath of shrinkResult.removedPaths) {
|
|
807
|
-
emit({
|
|
887
|
+
run.emit({
|
|
808
888
|
type: "progress",
|
|
809
889
|
path: relPath,
|
|
810
890
|
bytes: 0,
|
|
@@ -813,7 +893,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
813
893
|
});
|
|
814
894
|
}
|
|
815
895
|
for (const relPath of shrinkResult.dirtyKeptPaths) {
|
|
816
|
-
emit({
|
|
896
|
+
run.emit({
|
|
817
897
|
type: "progress",
|
|
818
898
|
path: relPath,
|
|
819
899
|
bytes: 0,
|
|
@@ -821,521 +901,410 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
821
901
|
"scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
|
|
822
902
|
});
|
|
823
903
|
}
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
shrinkResult
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
const TRANSFER_CONCURRENCY = (() => {
|
|
849
|
-
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
850
|
-
if (raw === undefined || raw === "") return 16;
|
|
851
|
-
const parsed = Number.parseInt(raw, 10);
|
|
852
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
853
|
-
})();
|
|
854
|
-
|
|
855
|
-
// First pass: serial walk for non-download outcomes (skips + conflicts).
|
|
856
|
-
// Conflicts may set `aborted = true` and short-circuit the whole pull;
|
|
857
|
-
// we detect that and skip the parallel pass. Download items are
|
|
858
|
-
// collected into `downloadItems[]` for the pool pass below.
|
|
859
|
-
const downloadItems: Array<typeof plan.items[number] & { action: "download" }> = [];
|
|
860
|
-
let aborted = false;
|
|
861
|
-
let abortResult: SyncResult | null = null;
|
|
904
|
+
|
|
905
|
+
return {
|
|
906
|
+
shrinkPlan,
|
|
907
|
+
shrinkResult,
|
|
908
|
+
scopeOrphansRemoved:
|
|
909
|
+
shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
async function refreshRunContextIfExpiring(
|
|
914
|
+
run: PullRunContext,
|
|
915
|
+
): Promise<void> {
|
|
916
|
+
if (isExpiringSoon(run.ctx.expiresAt)) {
|
|
917
|
+
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
async function executeConflictExecutor(
|
|
922
|
+
run: PullRunContext,
|
|
923
|
+
plan: PullPlan,
|
|
924
|
+
scopeRun: ScopeShrinkRun,
|
|
925
|
+
counters: PullCounters,
|
|
926
|
+
): Promise<{ downloadItems: PullDownloadItem[]; abortResult: SyncResult | null }> {
|
|
927
|
+
const downloadItems: PullDownloadItem[] = [];
|
|
862
928
|
|
|
863
929
|
for (const item of plan.items) {
|
|
864
|
-
if (aborted) break;
|
|
865
930
|
if (
|
|
866
931
|
item.action === "skip-ignored" ||
|
|
867
932
|
item.action === "skip-personal-mode" ||
|
|
868
933
|
item.action === "skip-unchanged" ||
|
|
869
934
|
item.action === "skip-local-only"
|
|
870
935
|
) {
|
|
871
|
-
filesSkipped++;
|
|
936
|
+
counters.filesSkipped++;
|
|
872
937
|
continue;
|
|
873
938
|
}
|
|
874
939
|
if (item.action === "skip-excluded-policy") {
|
|
875
|
-
// Policy-excluded items count separately from `filesSkipped` so the
|
|
876
|
-
// pull result mirrors the push side's `filesExcludedByPolicy`
|
|
877
|
-
// counter — `filesSkipped` stays a measure of "unchanged on this
|
|
878
|
-
// run", not a catch-all for everything we didn't download.
|
|
879
940
|
continue;
|
|
880
941
|
}
|
|
881
942
|
if (item.action === "skip-out-of-scope") {
|
|
882
|
-
|
|
883
|
-
// axis so `filesSkipped` keeps meaning "unchanged on this run" — these
|
|
884
|
-
// are "deliberately not downloaded because of your sync scope".
|
|
885
|
-
filesOutOfScope++;
|
|
943
|
+
counters.filesOutOfScope++;
|
|
886
944
|
continue;
|
|
887
945
|
}
|
|
888
|
-
|
|
889
946
|
if (item.action === "tombstone-delete") {
|
|
890
|
-
|
|
891
|
-
// is present but a tombstone marks the key intentionally deleted and it is
|
|
892
|
-
// not a newer re-create. Delete any local copy and drop the journal entry
|
|
893
|
-
// so it stays gone — the mirror of the journal-vs-LIST tombstone executor
|
|
894
|
-
// below, but WITHOUT the HEAD-verify (the remote object is present by
|
|
895
|
-
// definition; the FILE_TOMBSTONE is the deletion authority). The planner
|
|
896
|
-
// already routed any divergent local copy to `conflict`, so a local file
|
|
897
|
-
// reaching here matches the deleted baseline and is safe to remove.
|
|
898
|
-
const tombstoneKey = item.remoteFile.key;
|
|
899
|
-
// Same Windows-backslash landmine guard as the journal-tombstone executor:
|
|
900
|
-
// a malformed key must never reach fs.unlinkSync (path.join collapses the
|
|
901
|
-
// backslashes onto a REAL POSIX file). Drop the poisoned journal entry
|
|
902
|
-
// without touching disk.
|
|
903
|
-
if (isMalformedVaultKey(tombstoneKey)) {
|
|
904
|
-
removeEntry(journal, tombstoneKey);
|
|
905
|
-
continue;
|
|
906
|
-
}
|
|
907
|
-
try {
|
|
908
|
-
const lstat = fs.lstatSync(item.localPath);
|
|
909
|
-
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
910
|
-
fs.unlinkSync(item.localPath);
|
|
911
|
-
}
|
|
912
|
-
// A directory at the key: don't recursively rm-rf the operator's dir;
|
|
913
|
-
// just drop the journal entry (safe-by-default, same as the other path).
|
|
914
|
-
} catch (err: unknown) {
|
|
915
|
-
const code =
|
|
916
|
-
err && typeof err === "object" && "code" in err
|
|
917
|
-
? (err as { code?: string }).code
|
|
918
|
-
: undefined;
|
|
919
|
-
// ENOENT → local already absent (the common case: a fresh machine that
|
|
920
|
-
// never held the file, or a prior pull already removed it) → drop the
|
|
921
|
-
// journal entry and converge. Other errors (EACCES/EPERM/…) leave the
|
|
922
|
-
// file in place; surface and KEEP the journal entry so the next sync
|
|
923
|
-
// retries rather than forgetting the delete.
|
|
924
|
-
if (code !== "ENOENT") {
|
|
925
|
-
emit({
|
|
926
|
-
type: "error",
|
|
927
|
-
path: tombstoneKey,
|
|
928
|
-
message: `tombstone-suppress unlink failed: ${
|
|
929
|
-
err instanceof Error ? err.message : String(err)
|
|
930
|
-
}`,
|
|
931
|
-
});
|
|
932
|
-
continue;
|
|
933
|
-
}
|
|
934
|
-
}
|
|
935
|
-
removeEntry(journal, tombstoneKey);
|
|
936
|
-
filesTombstoned++;
|
|
937
|
-
emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
947
|
+
executeFileTombstoneDelete(run, item, counters);
|
|
938
948
|
continue;
|
|
939
949
|
}
|
|
940
|
-
|
|
941
950
|
if (item.action === "download") {
|
|
942
951
|
downloadItems.push(item);
|
|
943
952
|
continue;
|
|
944
953
|
}
|
|
945
954
|
|
|
946
|
-
const
|
|
955
|
+
const abortResult = await executeConflictItem(
|
|
956
|
+
run,
|
|
957
|
+
plan,
|
|
958
|
+
scopeRun,
|
|
959
|
+
counters,
|
|
960
|
+
downloadItems,
|
|
961
|
+
item,
|
|
962
|
+
);
|
|
963
|
+
if (abortResult) {
|
|
964
|
+
return { downloadItems, abortResult };
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
return { downloadItems, abortResult: null };
|
|
969
|
+
}
|
|
947
970
|
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
971
|
+
function executeFileTombstoneDelete(
|
|
972
|
+
run: PullRunContext,
|
|
973
|
+
item: Extract<PullPlanItem, { action: "tombstone-delete" }>,
|
|
974
|
+
counters: PullCounters,
|
|
975
|
+
): void {
|
|
976
|
+
const tombstoneKey = item.remoteFile.key;
|
|
977
|
+
const tombstonePath = resolveContainedVaultPath(run.companyRoot, tombstoneKey);
|
|
978
|
+
if (tombstonePath === null) return;
|
|
979
|
+
try {
|
|
980
|
+
const lstat = fs.lstatSync(tombstonePath);
|
|
981
|
+
if (tombstoneTargetDiverged(run.journal, tombstoneKey, tombstonePath, lstat)) {
|
|
982
|
+
return;
|
|
983
|
+
}
|
|
984
|
+
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
985
|
+
fs.unlinkSync(tombstonePath);
|
|
952
986
|
}
|
|
987
|
+
} catch (err: unknown) {
|
|
988
|
+
const code =
|
|
989
|
+
err && typeof err === "object" && "code" in err
|
|
990
|
+
? (err as { code?: string }).code
|
|
991
|
+
: undefined;
|
|
992
|
+
if (code !== "ENOENT") {
|
|
993
|
+
run.emit({
|
|
994
|
+
type: "error",
|
|
995
|
+
path: tombstoneKey,
|
|
996
|
+
message: `tombstone-suppress unlink failed: ${
|
|
997
|
+
err instanceof Error ? err.message : String(err)
|
|
998
|
+
}`,
|
|
999
|
+
});
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
removeEntry(run.journal, tombstoneKey);
|
|
1004
|
+
counters.filesTombstoned++;
|
|
1005
|
+
run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
1006
|
+
}
|
|
953
1007
|
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1008
|
+
async function executeConflictItem(
|
|
1009
|
+
run: PullRunContext,
|
|
1010
|
+
plan: PullPlan,
|
|
1011
|
+
scopeRun: ScopeShrinkRun,
|
|
1012
|
+
counters: PullCounters,
|
|
1013
|
+
downloadItems: PullDownloadItem[],
|
|
1014
|
+
item: Extract<PullPlanItem, { action: "conflict" }>,
|
|
1015
|
+
): Promise<SyncResult | null> {
|
|
1016
|
+
const { remoteFile, localPath } = item;
|
|
1017
|
+
|
|
1018
|
+
await refreshRunContextIfExpiring(run);
|
|
1019
|
+
|
|
1020
|
+
const detectedAt = new Date().toISOString();
|
|
1021
|
+
const machineId = readShortMachineId(run.hqRoot);
|
|
1022
|
+
const originalRelative = path.relative(run.hqRoot, localPath);
|
|
1023
|
+
const conflictRelative = buildConflictPath(
|
|
1024
|
+
originalRelative,
|
|
1025
|
+
detectedAt,
|
|
1026
|
+
machineId,
|
|
1027
|
+
);
|
|
1028
|
+
const conflictAbs = path.join(run.hqRoot, conflictRelative);
|
|
1029
|
+
const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
|
|
1030
|
+
|
|
1031
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
|
|
1032
|
+
counters.filesSkipped++;
|
|
1033
|
+
run.emit({
|
|
1034
|
+
type: "error",
|
|
1035
|
+
path: remoteFile.key,
|
|
1036
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
1037
|
+
});
|
|
1038
|
+
return null;
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
let remoteFetched = false;
|
|
1042
|
+
let converged = false;
|
|
1043
|
+
try {
|
|
1044
|
+
const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
|
|
1045
|
+
remoteFetched = true;
|
|
1046
|
+
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
1047
|
+
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
1048
|
+
: (downloaded.contentHash ?? hashFile(conflictAbs));
|
|
1049
|
+
converged = remoteHash === item.localHash;
|
|
1050
|
+
} catch (probeErr) {
|
|
1051
|
+
run.emit({
|
|
1052
|
+
type: "error",
|
|
1053
|
+
path: remoteFile.key,
|
|
1054
|
+
message: `conflict convergence probe failed: ${
|
|
1055
|
+
probeErr instanceof Error ? probeErr.message : String(probeErr)
|
|
1056
|
+
}`,
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
if (converged) {
|
|
1061
|
+
if (remoteFetched) {
|
|
1062
|
+
try {
|
|
1063
|
+
fs.rmSync(conflictAbs, { force: true });
|
|
1064
|
+
} catch {
|
|
1065
|
+
/* best-effort cleanup; a stray identical mirror is harmless */
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
updateEntry(
|
|
1069
|
+
run.journal,
|
|
1070
|
+
remoteFile.key,
|
|
1071
|
+
item.localHash,
|
|
1072
|
+
item.localSize,
|
|
1073
|
+
"down",
|
|
1074
|
+
remoteFile.etag,
|
|
1075
|
+
item.localMtime.getTime(),
|
|
1076
|
+
);
|
|
1077
|
+
run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1078
|
+
counters.filesSkipped++;
|
|
1079
|
+
return null;
|
|
1080
|
+
}
|
|
988
1081
|
|
|
989
|
-
|
|
990
|
-
|
|
1082
|
+
counters.conflicts++;
|
|
1083
|
+
counters.conflictPaths.push(remoteFile.key);
|
|
1084
|
+
|
|
1085
|
+
const resolution = await resolveConflict(
|
|
1086
|
+
{
|
|
1087
|
+
path: remoteFile.key,
|
|
1088
|
+
localHash: item.localHash,
|
|
1089
|
+
remoteModified: remoteFile.lastModified,
|
|
1090
|
+
localModified: item.localMtime,
|
|
1091
|
+
direction: "pull",
|
|
1092
|
+
},
|
|
1093
|
+
run.options.onConflict,
|
|
1094
|
+
);
|
|
1095
|
+
|
|
1096
|
+
run.emit({
|
|
1097
|
+
type: "conflict",
|
|
1098
|
+
path: remoteFile.key,
|
|
1099
|
+
direction: "pull",
|
|
1100
|
+
resolution,
|
|
1101
|
+
});
|
|
1102
|
+
|
|
1103
|
+
if (resolution !== "abort" && resolution !== "overwrite") {
|
|
1104
|
+
if (remoteFetched) {
|
|
991
1105
|
try {
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
:
|
|
1001
|
-
|
|
1002
|
-
} catch (
|
|
1003
|
-
|
|
1004
|
-
// the conventional conflict path (converged stays false). No mirror
|
|
1005
|
-
// is on disk in this case.
|
|
1006
|
-
emit({
|
|
1106
|
+
appendConflictEntry(run.hqRoot, {
|
|
1107
|
+
id: buildConflictId(originalRelative, detectedAt),
|
|
1108
|
+
originalPath: originalRelative,
|
|
1109
|
+
conflictPath: conflictRelative,
|
|
1110
|
+
detectedAt,
|
|
1111
|
+
side: "pull",
|
|
1112
|
+
machineId,
|
|
1113
|
+
localHash: item.localHash,
|
|
1114
|
+
remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
|
|
1115
|
+
});
|
|
1116
|
+
} catch (mirrorErr) {
|
|
1117
|
+
run.emit({
|
|
1007
1118
|
type: "error",
|
|
1008
1119
|
path: remoteFile.key,
|
|
1009
|
-
message: `conflict
|
|
1010
|
-
|
|
1120
|
+
message: `conflict mirror index write failed: ${
|
|
1121
|
+
mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
|
|
1011
1122
|
}`,
|
|
1012
1123
|
});
|
|
1013
1124
|
}
|
|
1125
|
+
}
|
|
1126
|
+
} else if (remoteFetched) {
|
|
1127
|
+
try {
|
|
1128
|
+
fs.rmSync(conflictAbs, { force: true });
|
|
1129
|
+
} catch {
|
|
1130
|
+
/* best-effort; a leftover mirror is cosmetic, not corrupting */
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1014
1133
|
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
);
|
|
1035
|
-
emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1036
|
-
filesSkipped++;
|
|
1037
|
-
continue;
|
|
1038
|
-
}
|
|
1134
|
+
if (resolution === "abort") {
|
|
1135
|
+
run.emit({ type: "new-files", files: [] });
|
|
1136
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1137
|
+
return {
|
|
1138
|
+
filesDownloaded: counters.filesDownloaded,
|
|
1139
|
+
bytesDownloaded: counters.bytesDownloaded,
|
|
1140
|
+
filesSkipped: counters.filesSkipped,
|
|
1141
|
+
conflicts: counters.conflicts,
|
|
1142
|
+
conflictPaths: counters.conflictPaths,
|
|
1143
|
+
aborted: true,
|
|
1144
|
+
newFiles: plan.newFiles,
|
|
1145
|
+
newFilesCount: plan.newFilesCount,
|
|
1146
|
+
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1147
|
+
filesTombstoned: 0,
|
|
1148
|
+
filesOutOfScope: counters.filesOutOfScope,
|
|
1149
|
+
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1150
|
+
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1039
1153
|
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1154
|
+
if (resolution === "keep" || resolution === "skip") {
|
|
1155
|
+
counters.filesSkipped++;
|
|
1156
|
+
updateEntry(
|
|
1157
|
+
run.journal,
|
|
1158
|
+
remoteFile.key,
|
|
1159
|
+
item.localHash,
|
|
1160
|
+
item.localSize,
|
|
1161
|
+
"down",
|
|
1162
|
+
remoteFile.etag,
|
|
1163
|
+
item.localMtime.getTime(),
|
|
1164
|
+
);
|
|
1165
|
+
return null;
|
|
1166
|
+
}
|
|
1043
1167
|
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
localModified: item.localMtime,
|
|
1053
|
-
direction: "pull",
|
|
1054
|
-
},
|
|
1055
|
-
onConflict,
|
|
1056
|
-
);
|
|
1168
|
+
downloadItems.push({
|
|
1169
|
+
action: "download",
|
|
1170
|
+
remoteFile,
|
|
1171
|
+
localPath,
|
|
1172
|
+
isNew: false,
|
|
1173
|
+
});
|
|
1174
|
+
return null;
|
|
1175
|
+
}
|
|
1057
1176
|
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1177
|
+
async function executeDownloadExecutor(
|
|
1178
|
+
run: PullRunContext,
|
|
1179
|
+
downloadItems: PullDownloadItem[],
|
|
1180
|
+
transferConcurrency: number,
|
|
1181
|
+
counters: PullCounters,
|
|
1182
|
+
): Promise<void> {
|
|
1183
|
+
if (downloadItems.length === 0) return;
|
|
1064
1184
|
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
// redundant, so discard it. Best-effort: failure here only emits an
|
|
1071
|
-
// error, doesn't break the sync.
|
|
1072
|
-
if (resolution !== "abort" && resolution !== "overwrite") {
|
|
1073
|
-
if (remoteFetched) {
|
|
1074
|
-
try {
|
|
1075
|
-
appendConflictEntry(hqRoot, {
|
|
1076
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
1077
|
-
originalPath: originalRelative,
|
|
1078
|
-
conflictPath: conflictRelative,
|
|
1079
|
-
detectedAt,
|
|
1080
|
-
side: "pull",
|
|
1081
|
-
machineId,
|
|
1082
|
-
localHash: item.localHash,
|
|
1083
|
-
remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
|
|
1084
|
-
});
|
|
1085
|
-
} catch (mirrorErr) {
|
|
1086
|
-
emit({
|
|
1087
|
-
type: "error",
|
|
1088
|
-
path: remoteFile.key,
|
|
1089
|
-
message: `conflict mirror index write failed: ${
|
|
1090
|
-
mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
|
|
1091
|
-
}`,
|
|
1092
|
-
});
|
|
1093
|
-
}
|
|
1094
|
-
}
|
|
1095
|
-
// If the probe download failed (!remoteFetched) there is no mirror on
|
|
1096
|
-
// disk; the probe already emitted the error. The conflict is still
|
|
1097
|
-
// surfaced and journal-stamped below so it doesn't re-fire silently.
|
|
1098
|
-
} else if (remoteFetched) {
|
|
1099
|
-
try {
|
|
1100
|
-
fs.rmSync(conflictAbs, { force: true });
|
|
1101
|
-
} catch {
|
|
1102
|
-
/* best-effort; a leftover mirror is cosmetic, not corrupting */
|
|
1103
|
-
}
|
|
1104
|
-
}
|
|
1185
|
+
await primeObjectTransport(
|
|
1186
|
+
run.ctx,
|
|
1187
|
+
"get",
|
|
1188
|
+
downloadItems.map((d) => d.remoteFile.key),
|
|
1189
|
+
);
|
|
1105
1190
|
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
// Scope-shrink ran before execution, so its counts are real even on
|
|
1125
|
-
// a conflict abort. `filesOutOfScope` reflects how far the serial
|
|
1126
|
-
// pass got before the abort; that's acceptable for an abort result.
|
|
1127
|
-
filesOutOfScope,
|
|
1128
|
-
scopeOrphansRemoved,
|
|
1129
|
-
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1130
|
-
};
|
|
1131
|
-
break;
|
|
1132
|
-
}
|
|
1133
|
-
if (resolution === "keep" || resolution === "skip") {
|
|
1134
|
-
filesSkipped++;
|
|
1135
|
-
// Stamp the journal with the new baseline so the same conflict
|
|
1136
|
-
// doesn't re-fire on every subsequent sync. After "keep", local
|
|
1137
|
-
// wins — the user has accepted that the cloud version we just
|
|
1138
|
-
// mirrored is what cloud is at this etag, and they don't want
|
|
1139
|
-
// it. Recording (current localHash + current remoteEtag) tells
|
|
1140
|
-
// the next sync "no change on either side" until something new
|
|
1141
|
-
// diverges. Without this, both `localChanged` and `remoteChanged`
|
|
1142
|
-
// stay true forever and the conflict is sticky.
|
|
1143
|
-
// Stamp from planner-captured size (symlink-aware), NOT
|
|
1144
|
-
// statSync — which would follow a dangling symlink and
|
|
1145
|
-
// throw ENOENT, get swallowed, and leave the journal
|
|
1146
|
-
// stale so this conflict would re-fire on every sync
|
|
1147
|
-
// forever. localSize is sourced from the same lstat that
|
|
1148
|
-
// computed localMtime + localHash above.
|
|
1149
|
-
updateEntry(
|
|
1150
|
-
journal,
|
|
1151
|
-
remoteFile.key,
|
|
1152
|
-
item.localHash,
|
|
1153
|
-
item.localSize,
|
|
1154
|
-
"down",
|
|
1155
|
-
remoteFile.etag,
|
|
1156
|
-
item.localMtime.getTime(),
|
|
1157
|
-
);
|
|
1158
|
-
continue;
|
|
1159
|
-
}
|
|
1160
|
-
// "overwrite" falls through to download — re-route through the pool
|
|
1161
|
-
// so it benefits from parallelism too. Synthesize a download item
|
|
1162
|
-
// pointing at the same remoteFile/localPath; isNew=false because
|
|
1163
|
-
// there was a conflict-eligible local file present.
|
|
1164
|
-
downloadItems.push({
|
|
1165
|
-
action: "download",
|
|
1166
|
-
remoteFile,
|
|
1167
|
-
localPath,
|
|
1168
|
-
isNew: false,
|
|
1169
|
-
});
|
|
1170
|
-
continue;
|
|
1191
|
+
const queue = [...downloadItems];
|
|
1192
|
+
const inFlight: Set<Promise<unknown>> = new Set();
|
|
1193
|
+
const workerErrors: Error[] = [];
|
|
1194
|
+
|
|
1195
|
+
while (queue.length > 0 || inFlight.size > 0) {
|
|
1196
|
+
while (inFlight.size < transferConcurrency && queue.length > 0) {
|
|
1197
|
+
const downloadItem = queue.shift()!;
|
|
1198
|
+
const p: Promise<void> = downloadOne(run, downloadItem, counters)
|
|
1199
|
+
.catch((err: unknown) => {
|
|
1200
|
+
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
1201
|
+
})
|
|
1202
|
+
.finally(() => {
|
|
1203
|
+
inFlight.delete(p);
|
|
1204
|
+
});
|
|
1205
|
+
inFlight.add(p);
|
|
1206
|
+
}
|
|
1207
|
+
if (inFlight.size > 0) {
|
|
1208
|
+
await Promise.race(Array.from(inFlight));
|
|
1171
1209
|
}
|
|
1172
1210
|
}
|
|
1173
1211
|
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1212
|
+
if (workerErrors.length > 0) {
|
|
1213
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1214
|
+
const first = workerErrors[0]!;
|
|
1215
|
+
if (workerErrors.length > 1) {
|
|
1216
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
1217
|
+
}
|
|
1218
|
+
throw first;
|
|
1180
1219
|
}
|
|
1220
|
+
}
|
|
1181
1221
|
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
if (downloadItems.length > 0) {
|
|
1189
|
-
// Batch pre-mint GET URLs for every download in one shot (chunked server-
|
|
1190
|
-
// side) so the pool below — and the new-files HEAD enrichment that follows,
|
|
1191
|
-
// which re-reads the same keys — reuse them instead of presigning per file.
|
|
1192
|
-
// On a large initial pull this is the difference between ~ceil(N/100)
|
|
1193
|
-
// presign calls and N (which would 429 past the 100-req/hr limit). No-op
|
|
1194
|
-
// on the S3 SDK transport; best-effort (failure falls back to per-file).
|
|
1195
|
-
await primeObjectTransport(
|
|
1196
|
-
ctx,
|
|
1197
|
-
"get",
|
|
1198
|
-
downloadItems.map((d) => d.remoteFile.key),
|
|
1199
|
-
);
|
|
1200
|
-
|
|
1201
|
-
const queue = [...downloadItems];
|
|
1202
|
-
const inFlight: Set<Promise<unknown>> = new Set();
|
|
1203
|
-
|
|
1204
|
-
const downloadOne = async (
|
|
1205
|
-
downloadItem: typeof downloadItems[number],
|
|
1206
|
-
): Promise<void> => {
|
|
1207
|
-
const { remoteFile, localPath } = downloadItem;
|
|
1222
|
+
async function downloadOne(
|
|
1223
|
+
run: PullRunContext,
|
|
1224
|
+
downloadItem: PullDownloadItem,
|
|
1225
|
+
counters: PullCounters,
|
|
1226
|
+
): Promise<void> {
|
|
1227
|
+
const { remoteFile, localPath } = downloadItem;
|
|
1208
1228
|
|
|
1209
|
-
|
|
1210
|
-
// independently — refresh is idempotent on the same context object.
|
|
1211
|
-
if (isExpiringSoon(ctx.expiresAt)) {
|
|
1212
|
-
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
1213
|
-
}
|
|
1229
|
+
await refreshRunContextIfExpiring(run);
|
|
1214
1230
|
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
// mirrors what the push side would emit on the next tick:
|
|
1225
|
-
// hash = sha256(readlink target string)
|
|
1226
|
-
// size = 0
|
|
1227
|
-
// Without this check, hashFile would follow the link and stamp the
|
|
1228
|
-
// target file's contents — a value the next push would never
|
|
1229
|
-
// produce — which makes skipUnchanged perpetually re-upload every
|
|
1230
|
-
// symlink, defeating the point of the gate.
|
|
1231
|
-
const localLstat = fs.lstatSync(localPath);
|
|
1232
|
-
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
1233
|
-
const hash = isLocalSymlink
|
|
1234
|
-
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
1235
|
-
: hashFile(localPath);
|
|
1236
|
-
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
1237
|
-
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
1238
|
-
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
1239
|
-
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
1240
|
-
// SHA256 for this file without reading its bytes.
|
|
1241
|
-
//
|
|
1242
|
-
// 5.37.0 ordering invariant: downloadFile applies hq-mtime via
|
|
1243
|
-
// utimesSync AFTER its byte write but BEFORE returning, and this
|
|
1244
|
-
// lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
|
|
1245
|
-
// already reflects the source-stamped mtime, not the wall-clock
|
|
1246
|
-
// write-time. The journal therefore matches what the next push's
|
|
1247
|
-
// lstat fast-path will see, and the file is correctly skipped on
|
|
1248
|
-
// re-sync instead of being hashed every tick. Do not move this
|
|
1249
|
-
// lstat earlier; do not stamp the journal from any pre-download
|
|
1250
|
-
// mtime.
|
|
1251
|
-
updateEntry(
|
|
1252
|
-
journal,
|
|
1253
|
-
remoteFile.key,
|
|
1254
|
-
hash,
|
|
1255
|
-
size,
|
|
1256
|
-
"down",
|
|
1257
|
-
remoteFile.etag,
|
|
1258
|
-
localLstat.mtimeMs,
|
|
1259
|
-
createdBySub,
|
|
1260
|
-
);
|
|
1261
|
-
|
|
1262
|
-
// Attach message from the prior journal entry if present (set by a
|
|
1263
|
-
// previous `share` operation that included a --message).
|
|
1264
|
-
const priorEntry = getEntry(journal, remoteFile.key);
|
|
1265
|
-
const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
|
|
1266
|
-
emit({
|
|
1267
|
-
type: "progress",
|
|
1268
|
-
path: remoteFile.key,
|
|
1269
|
-
bytes: size,
|
|
1270
|
-
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
1271
|
-
...(author ? { author } : {}),
|
|
1272
|
-
});
|
|
1231
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
|
|
1232
|
+
counters.filesSkipped++;
|
|
1233
|
+
run.emit({
|
|
1234
|
+
type: "error",
|
|
1235
|
+
path: remoteFile.key,
|
|
1236
|
+
message: "download skipped: local parent escaped the sync root",
|
|
1237
|
+
});
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1273
1240
|
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1241
|
+
try {
|
|
1242
|
+
const { metadata, contentHash, contentSize } = await downloadFile(
|
|
1243
|
+
run.ctx,
|
|
1244
|
+
remoteFile.key,
|
|
1245
|
+
localPath,
|
|
1246
|
+
);
|
|
1247
|
+
const author = metadata?.["created-by"] ?? null;
|
|
1248
|
+
const createdBySub = metadata?.["created-by-sub"];
|
|
1249
|
+
|
|
1250
|
+
const localLstat = fs.lstatSync(localPath);
|
|
1251
|
+
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
1252
|
+
const hash = isLocalSymlink
|
|
1253
|
+
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
1254
|
+
: (contentHash ?? hashFile(localPath));
|
|
1255
|
+
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
1256
|
+
|
|
1257
|
+
updateEntry(
|
|
1258
|
+
run.journal,
|
|
1259
|
+
remoteFile.key,
|
|
1260
|
+
hash,
|
|
1261
|
+
size,
|
|
1262
|
+
"down",
|
|
1263
|
+
remoteFile.etag,
|
|
1264
|
+
localLstat.mtimeMs,
|
|
1265
|
+
createdBySub,
|
|
1266
|
+
);
|
|
1290
1267
|
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
// fully drains — see workerErrors throw below.
|
|
1301
|
-
const workerErrors: Error[] = [];
|
|
1302
|
-
while (queue.length > 0 || inFlight.size > 0) {
|
|
1303
|
-
while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
|
|
1304
|
-
const downloadItem = queue.shift()!;
|
|
1305
|
-
const p: Promise<void> = downloadOne(downloadItem)
|
|
1306
|
-
.catch((err: unknown) => {
|
|
1307
|
-
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
1308
|
-
})
|
|
1309
|
-
.finally(() => {
|
|
1310
|
-
inFlight.delete(p);
|
|
1311
|
-
});
|
|
1312
|
-
inFlight.add(p);
|
|
1313
|
-
}
|
|
1314
|
-
if (inFlight.size > 0) {
|
|
1315
|
-
// Wait for at least one in-flight task to settle before topping up
|
|
1316
|
-
// the pool. allSettled-style semantics via Promise.race — the
|
|
1317
|
-
// .catch wrap above guarantees no worker promise can reject.
|
|
1318
|
-
await Promise.race(Array.from(inFlight));
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1268
|
+
const priorEntry = getEntry(run.journal, remoteFile.key);
|
|
1269
|
+
const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
|
|
1270
|
+
run.emit({
|
|
1271
|
+
type: "progress",
|
|
1272
|
+
path: remoteFile.key,
|
|
1273
|
+
bytes: size,
|
|
1274
|
+
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
1275
|
+
...(author ? { author } : {}),
|
|
1276
|
+
});
|
|
1321
1277
|
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
if (
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1278
|
+
counters.filesDownloaded++;
|
|
1279
|
+
counters.bytesDownloaded += size;
|
|
1280
|
+
} catch (err) {
|
|
1281
|
+
if (isAccessDenied(err)) {
|
|
1282
|
+
counters.filesSkipped++;
|
|
1283
|
+
} else {
|
|
1284
|
+
run.emit({
|
|
1285
|
+
type: "error",
|
|
1286
|
+
path: remoteFile.key,
|
|
1287
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1288
|
+
});
|
|
1332
1289
|
}
|
|
1333
1290
|
}
|
|
1291
|
+
}
|
|
1334
1292
|
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1293
|
+
async function emitAndReportNewFiles(
|
|
1294
|
+
run: PullRunContext,
|
|
1295
|
+
plan: PullPlan,
|
|
1296
|
+
): Promise<void> {
|
|
1338
1297
|
const enrichedNewFiles: Array<{ path: string; bytes: number; addedBy: string | null }> = [];
|
|
1298
|
+
// Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
|
|
1299
|
+
// created-by HEADs below reuse the cache instead of each minting its own
|
|
1300
|
+
// presign. Without this, a big catch-up pull (hundreds of new files) bursts
|
|
1301
|
+
// the presign endpoint, trips the circuit breaker, and every enrichment HEAD
|
|
1302
|
+
// then fails. Mirrors the tombstone HEAD-verify pre-prime.
|
|
1303
|
+
await primeObjectTransport(
|
|
1304
|
+
run.ctx,
|
|
1305
|
+
"get",
|
|
1306
|
+
plan.newFiles.map((nf) => nf.path),
|
|
1307
|
+
);
|
|
1339
1308
|
const HEAD_CONCURRENCY = 5;
|
|
1340
1309
|
for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
|
|
1341
1310
|
const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
|
|
@@ -1343,13 +1312,11 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1343
1312
|
batch.map(async (nf) => {
|
|
1344
1313
|
let addedBy: string | null = null;
|
|
1345
1314
|
try {
|
|
1346
|
-
const head = await headRemoteFile(ctx, nf.path);
|
|
1315
|
+
const head = await headRemoteFile(run.ctx, nf.path);
|
|
1347
1316
|
if (head?.metadata?.["created-by"]) {
|
|
1348
1317
|
addedBy = head.metadata["created-by"];
|
|
1349
1318
|
}
|
|
1350
1319
|
} catch (headErr) {
|
|
1351
|
-
// Best-effort: log to console (Sentry captures via global handler)
|
|
1352
|
-
// and fall through with addedBy = null.
|
|
1353
1320
|
try {
|
|
1354
1321
|
console.error(
|
|
1355
1322
|
`[hq-sync] HeadObject failed for ${nf.path}: ${
|
|
@@ -1365,225 +1332,150 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1365
1332
|
);
|
|
1366
1333
|
enrichedNewFiles.push(...results);
|
|
1367
1334
|
}
|
|
1368
|
-
emit({ type: "new-files", files: enrichedNewFiles });
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
type: "error",
|
|
1410
|
-
path: key,
|
|
1411
|
-
message: `tombstone HEAD verify failed (deferring): ${
|
|
1412
|
-
err instanceof Error ? err.message : String(err)
|
|
1413
|
-
}`,
|
|
1414
|
-
});
|
|
1415
|
-
return null;
|
|
1416
|
-
}
|
|
1417
|
-
}),
|
|
1418
|
-
);
|
|
1419
|
-
for (const k of results) {
|
|
1420
|
-
if (k !== null) verified.push(k);
|
|
1421
|
-
}
|
|
1335
|
+
run.emit({ type: "new-files", files: enrichedNewFiles });
|
|
1336
|
+
await reportNewFilesToNotify(
|
|
1337
|
+
run.vaultConfig,
|
|
1338
|
+
run.ctx.uid,
|
|
1339
|
+
run.ctx.slug,
|
|
1340
|
+
enrichedNewFiles,
|
|
1341
|
+
);
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
async function verifyPlannedJournalTombstones(
|
|
1345
|
+
run: PullRunContext,
|
|
1346
|
+
plan: PullPlan,
|
|
1347
|
+
): Promise<void> {
|
|
1348
|
+
if (plan.tombstones.length === 0) return;
|
|
1349
|
+
|
|
1350
|
+
await primeObjectTransport(run.ctx, "get", plan.tombstones);
|
|
1351
|
+
|
|
1352
|
+
const HEAD_VERIFY_CONCURRENCY = 5;
|
|
1353
|
+
const verified: string[] = [];
|
|
1354
|
+
for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
|
|
1355
|
+
const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
|
|
1356
|
+
const results = await Promise.all(
|
|
1357
|
+
batch.map(async (key) => {
|
|
1358
|
+
try {
|
|
1359
|
+
const head = await headRemoteFile(run.ctx, key);
|
|
1360
|
+
return head === null ? key : null;
|
|
1361
|
+
} catch (err) {
|
|
1362
|
+
if (isAccessDenied(err)) return null;
|
|
1363
|
+
run.emit({
|
|
1364
|
+
type: "error",
|
|
1365
|
+
path: key,
|
|
1366
|
+
message: `tombstone HEAD verify failed (deferring): ${
|
|
1367
|
+
err instanceof Error ? err.message : String(err)
|
|
1368
|
+
}`,
|
|
1369
|
+
});
|
|
1370
|
+
return null;
|
|
1371
|
+
}
|
|
1372
|
+
}),
|
|
1373
|
+
);
|
|
1374
|
+
for (const k of results) {
|
|
1375
|
+
if (k !== null) verified.push(k);
|
|
1422
1376
|
}
|
|
1423
|
-
plan.tombstones = verified;
|
|
1424
1377
|
}
|
|
1378
|
+
plan.tombstones = verified;
|
|
1379
|
+
}
|
|
1425
1380
|
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
// the journal and the next run retries.
|
|
1381
|
+
function executeJournalTombstoneDeletes(
|
|
1382
|
+
run: PullRunContext,
|
|
1383
|
+
plan: PullPlan,
|
|
1384
|
+
counters: PullCounters,
|
|
1385
|
+
): void {
|
|
1432
1386
|
for (const key of plan.tombstones) {
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
// collapses the backslashes and resolves onto the REAL POSIX file, so
|
|
1436
|
-
// unlinking here destroys live data (ridge incident, feedback_b8d09d0f).
|
|
1437
|
-
// The planner already refuses to enqueue malformed keys; if one still
|
|
1438
|
-
// arrives, drop the poisoned journal entry without touching disk —
|
|
1439
|
-
// normalizeJournalKeys rewrites it to its POSIX form on load.
|
|
1440
|
-
if (isMalformedVaultKey(key)) {
|
|
1441
|
-
removeEntry(journal, key);
|
|
1442
|
-
continue;
|
|
1443
|
-
}
|
|
1444
|
-
const localPath = path.join(companyRoot, key);
|
|
1387
|
+
const localPath = resolveContainedVaultPath(run.companyRoot, key);
|
|
1388
|
+
if (localPath === null) continue;
|
|
1445
1389
|
let removedSomething = false;
|
|
1446
1390
|
try {
|
|
1447
1391
|
const lstat = fs.lstatSync(localPath);
|
|
1392
|
+
if (tombstoneTargetDiverged(run.journal, key, localPath, lstat)) {
|
|
1393
|
+
continue;
|
|
1394
|
+
}
|
|
1448
1395
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
1449
1396
|
fs.unlinkSync(localPath);
|
|
1450
1397
|
removedSomething = true;
|
|
1451
1398
|
} else if (lstat.isDirectory()) {
|
|
1452
|
-
// A dir at a key
|
|
1453
|
-
// state. Don't recursively rm-rf the operator's dir; just drop
|
|
1454
|
-
// the journal entry so we converge with reality.
|
|
1399
|
+
// A dir at a key is converged by dropping only the journal entry.
|
|
1455
1400
|
}
|
|
1456
1401
|
} catch (err: unknown) {
|
|
1457
1402
|
const code =
|
|
1458
1403
|
err && typeof err === "object" && "code" in err
|
|
1459
1404
|
? (err as { code?: string }).code
|
|
1460
1405
|
: undefined;
|
|
1461
|
-
// ENOENT → local already gone; safe to drop the journal entry.
|
|
1462
|
-
// Other errors (EACCES/EPERM/EBUSY/etc.) leave the local file in
|
|
1463
|
-
// place — if we dropped the journal entry anyway, the pull side
|
|
1464
|
-
// would forget the peer's delete and a later push could re-upload
|
|
1465
|
-
// the still-present local file, silently undoing the peer's delete.
|
|
1466
|
-
// Surface the error and KEEP the journal entry so the next sync
|
|
1467
|
-
// retries the unlink after the operator fixes the permission.
|
|
1468
1406
|
if (code !== "ENOENT") {
|
|
1469
|
-
emit({
|
|
1407
|
+
run.emit({
|
|
1470
1408
|
type: "error",
|
|
1471
1409
|
path: key,
|
|
1472
1410
|
message: `tombstone unlink failed: ${
|
|
1473
1411
|
err instanceof Error ? err.message : String(err)
|
|
1474
1412
|
}`,
|
|
1475
1413
|
});
|
|
1476
|
-
// Skip removeEntry / filesTombstoned / progress event — the
|
|
1477
|
-
// tombstone hasn't actually been honored. Next sync retries.
|
|
1478
1414
|
continue;
|
|
1479
1415
|
}
|
|
1480
1416
|
}
|
|
1481
|
-
removeEntry(journal, key);
|
|
1482
|
-
filesTombstoned++;
|
|
1483
|
-
emit({
|
|
1417
|
+
removeEntry(run.journal, key);
|
|
1418
|
+
counters.filesTombstoned++;
|
|
1419
|
+
run.emit({
|
|
1484
1420
|
type: "progress",
|
|
1485
1421
|
path: key,
|
|
1486
1422
|
bytes: 0,
|
|
1487
1423
|
deleted: true,
|
|
1488
|
-
// Suffix differentiates a tombstone from a normal delete in the
|
|
1489
|
-
// tty stream — matches the push-side `defaultConsoleLogger`
|
|
1490
|
-
// tombstone surface in share.ts.
|
|
1491
1424
|
message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
|
|
1492
1425
|
});
|
|
1493
1426
|
}
|
|
1427
|
+
}
|
|
1494
1428
|
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1429
|
+
function finalizePullRun(
|
|
1430
|
+
run: PullRunContext,
|
|
1431
|
+
plan: PullPlan,
|
|
1432
|
+
scopeRun: ScopeShrinkRun,
|
|
1433
|
+
counters: PullCounters,
|
|
1434
|
+
): SyncResult {
|
|
1435
|
+
appendPullRecord(run.journal, {
|
|
1500
1436
|
pullId: generatePullId(),
|
|
1501
|
-
companyUid: ctx.uid,
|
|
1502
|
-
startedAt,
|
|
1437
|
+
companyUid: run.ctx.uid,
|
|
1438
|
+
startedAt: run.startedAt,
|
|
1503
1439
|
completedAt: new Date().toISOString(),
|
|
1504
|
-
syncMode,
|
|
1505
|
-
prefixSet: currentPrefixSet,
|
|
1506
|
-
scopeChangeDetected: shrinkPlan.scopeChangeDetected,
|
|
1507
|
-
orphansRemoved: scopeOrphansRemoved,
|
|
1508
|
-
orphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1440
|
+
syncMode: run.syncMode,
|
|
1441
|
+
prefixSet: run.currentPrefixSet,
|
|
1442
|
+
scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
|
|
1443
|
+
orphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1444
|
+
orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1509
1445
|
});
|
|
1510
1446
|
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
// last file change, which is misleading.
|
|
1515
|
-
journal.lastSync = new Date().toISOString();
|
|
1516
|
-
writeJournal(journalSlug, journal);
|
|
1517
|
-
|
|
1518
|
-
// When the pull actually changed on-disk sources (new files, tombstoned
|
|
1519
|
-
// removals, or scope-orphan cleanups), refresh the generated skill wrappers,
|
|
1520
|
-
// personal-overlay mirrors, and workers registry. reindex is idempotent and
|
|
1521
|
-
// best-effort — it must never fail a sync, and is skipped on no-op syncs
|
|
1522
|
-
// (the common daemon case) and when the caller opts out via skipReindex.
|
|
1447
|
+
run.journal.lastSync = new Date().toISOString();
|
|
1448
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1449
|
+
|
|
1523
1450
|
const changedOnDisk =
|
|
1524
|
-
filesDownloaded > 0 ||
|
|
1525
|
-
filesTombstoned > 0 ||
|
|
1526
|
-
scopeOrphansRemoved > 0;
|
|
1527
|
-
if (!options.skipReindex && changedOnDisk) {
|
|
1451
|
+
counters.filesDownloaded > 0 ||
|
|
1452
|
+
counters.filesTombstoned > 0 ||
|
|
1453
|
+
scopeRun.scopeOrphansRemoved > 0;
|
|
1454
|
+
if (!run.options.skipReindex && changedOnDisk) {
|
|
1528
1455
|
try {
|
|
1529
|
-
|
|
1530
|
-
// lock; reindex re-acquiring would refuse against our own live PID.
|
|
1531
|
-
reindex({ repoRoot: hqRoot, skipLock: true });
|
|
1456
|
+
reindex({ repoRoot: run.hqRoot, skipLock: true });
|
|
1532
1457
|
} catch {
|
|
1533
1458
|
// best-effort: a post-sync refresh failure never fails the sync
|
|
1534
1459
|
}
|
|
1535
1460
|
}
|
|
1536
1461
|
|
|
1537
1462
|
return {
|
|
1538
|
-
filesDownloaded,
|
|
1539
|
-
bytesDownloaded,
|
|
1540
|
-
filesSkipped,
|
|
1541
|
-
conflicts,
|
|
1542
|
-
conflictPaths,
|
|
1463
|
+
filesDownloaded: counters.filesDownloaded,
|
|
1464
|
+
bytesDownloaded: counters.bytesDownloaded,
|
|
1465
|
+
filesSkipped: counters.filesSkipped,
|
|
1466
|
+
conflicts: counters.conflicts,
|
|
1467
|
+
conflictPaths: counters.conflictPaths,
|
|
1543
1468
|
aborted: false,
|
|
1544
1469
|
newFiles: plan.newFiles,
|
|
1545
1470
|
newFilesCount: plan.newFilesCount,
|
|
1546
1471
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1547
|
-
filesTombstoned,
|
|
1548
|
-
filesOutOfScope,
|
|
1549
|
-
scopeOrphansRemoved,
|
|
1550
|
-
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1472
|
+
filesTombstoned: counters.filesTombstoned,
|
|
1473
|
+
filesOutOfScope: counters.filesOutOfScope,
|
|
1474
|
+
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1475
|
+
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1551
1476
|
};
|
|
1552
1477
|
}
|
|
1553
1478
|
|
|
1554
|
-
/**
|
|
1555
|
-
* Resolve active company from .hq/config.json.
|
|
1556
|
-
*/
|
|
1557
|
-
function resolveActiveCompany(hqRoot: string): string | undefined {
|
|
1558
|
-
const configPath = path.join(hqRoot, ".hq", "config.json");
|
|
1559
|
-
if (fs.existsSync(configPath)) {
|
|
1560
|
-
try {
|
|
1561
|
-
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
|
|
1562
|
-
return config.activeCompany ?? config.companySlug;
|
|
1563
|
-
} catch {
|
|
1564
|
-
// Ignore parse errors
|
|
1565
|
-
}
|
|
1566
|
-
}
|
|
1567
|
-
return undefined;
|
|
1568
|
-
}
|
|
1569
|
-
|
|
1570
|
-
/**
|
|
1571
|
-
* Returns true when the remote object appears to have moved since the
|
|
1572
|
-
* journal entry's last-recorded sync. Prefers ETag equality; falls back to
|
|
1573
|
-
* `lastModified > syncedAt` for legacy entries written before remoteEtag
|
|
1574
|
-
* was tracked. Conservative on tie (`<=` skews "remote unchanged").
|
|
1575
|
-
*/
|
|
1576
|
-
function hasRemoteChanged(
|
|
1577
|
-
remote: { lastModified: Date; etag: string },
|
|
1578
|
-
entry: { syncedAt: string; remoteEtag?: string },
|
|
1579
|
-
): boolean {
|
|
1580
|
-
if (entry.remoteEtag) {
|
|
1581
|
-
return normalizeEtag(remote.etag) !== entry.remoteEtag;
|
|
1582
|
-
}
|
|
1583
|
-
const syncedAt = new Date(entry.syncedAt).getTime();
|
|
1584
|
-
return remote.lastModified.getTime() > syncedAt;
|
|
1585
|
-
}
|
|
1586
|
-
|
|
1587
1479
|
/**
|
|
1588
1480
|
* Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
|
|
1589
1481
|
* written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
|
|
@@ -1615,6 +1507,99 @@ function isRemoteRecreateAfterTombstone(
|
|
|
1615
1507
|
return remoteMs > deletedAtMs;
|
|
1616
1508
|
}
|
|
1617
1509
|
|
|
1510
|
+
function hasTraversalSegment(key: string): boolean {
|
|
1511
|
+
return key.split("/").some((segment) => segment === "..");
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
function isPathWithin(root: string, candidate: string): boolean {
|
|
1515
|
+
const relative = path.relative(root, candidate);
|
|
1516
|
+
return (
|
|
1517
|
+
relative === "" ||
|
|
1518
|
+
(!relative.startsWith("..") && !path.isAbsolute(relative))
|
|
1519
|
+
);
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
function deepestExistingAncestor(start: string): string | null {
|
|
1523
|
+
let current = start;
|
|
1524
|
+
for (;;) {
|
|
1525
|
+
try {
|
|
1526
|
+
fs.lstatSync(current);
|
|
1527
|
+
return current;
|
|
1528
|
+
} catch (err: unknown) {
|
|
1529
|
+
const code =
|
|
1530
|
+
err && typeof err === "object" && "code" in err
|
|
1531
|
+
? (err as { code?: string }).code
|
|
1532
|
+
: undefined;
|
|
1533
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") return null;
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
const parent = path.dirname(current);
|
|
1537
|
+
if (parent === current) return null;
|
|
1538
|
+
current = parent;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
|
|
1542
|
+
function resolveContainedVaultPath(root: string, key: string): string | null {
|
|
1543
|
+
if (isMalformedVaultKey(key) || hasTraversalSegment(key)) return null;
|
|
1544
|
+
|
|
1545
|
+
const resolvedRoot = path.resolve(root);
|
|
1546
|
+
const resolvedLocal = path.resolve(resolvedRoot, key);
|
|
1547
|
+
if (!isPathWithin(resolvedRoot, resolvedLocal)) return null;
|
|
1548
|
+
|
|
1549
|
+
let realRoot: string;
|
|
1550
|
+
try {
|
|
1551
|
+
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
1552
|
+
} catch {
|
|
1553
|
+
// If the vault root does not exist yet, no below-root symlink component can
|
|
1554
|
+
// already exist to redirect this key. Preserve first-pull behavior.
|
|
1555
|
+
return resolvedLocal;
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
1559
|
+
if (existingAncestor === null) return null;
|
|
1560
|
+
try {
|
|
1561
|
+
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
1562
|
+
if (!isPathWithin(realRoot, realAncestor)) return null;
|
|
1563
|
+
} catch {
|
|
1564
|
+
return null;
|
|
1565
|
+
}
|
|
1566
|
+
return resolvedLocal;
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
function isDownloadWritePathStillContained(
|
|
1570
|
+
root: string,
|
|
1571
|
+
key: string,
|
|
1572
|
+
localPath: string,
|
|
1573
|
+
): boolean {
|
|
1574
|
+
const resolved = resolveContainedVaultPath(root, key);
|
|
1575
|
+
return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
function tombstoneTargetDiverged(
|
|
1579
|
+
journal: SyncJournal,
|
|
1580
|
+
key: string,
|
|
1581
|
+
localPath: string,
|
|
1582
|
+
lstat: fs.Stats,
|
|
1583
|
+
): boolean {
|
|
1584
|
+
const journalEntry = journal.files[key];
|
|
1585
|
+
if (!journalEntry?.hash) {
|
|
1586
|
+
return lstat.isSymbolicLink() || lstat.isFile();
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
try {
|
|
1590
|
+
if (lstat.isSymbolicLink()) {
|
|
1591
|
+
return hashSymlinkTarget(fs.readlinkSync(localPath)) !== journalEntry.hash;
|
|
1592
|
+
}
|
|
1593
|
+
if (lstat.isFile()) {
|
|
1594
|
+
return hashFile(localPath) !== journalEntry.hash;
|
|
1595
|
+
}
|
|
1596
|
+
} catch {
|
|
1597
|
+
return true;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
return false;
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1618
1603
|
/**
|
|
1619
1604
|
* Stage-1 classification for a single remote object. Each remote file falls
|
|
1620
1605
|
* into exactly one bucket; the executor in `sync()` switches on `action` to
|
|
@@ -1716,7 +1701,7 @@ function computePullPlan(
|
|
|
1716
1701
|
// Coalesced, company-relative prefixes the pull is scoped to (US-005).
|
|
1717
1702
|
// `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
|
|
1718
1703
|
// the scope filter below becomes a no-op and legacy behavior is preserved.
|
|
1719
|
-
prefixSet:
|
|
1704
|
+
prefixSet: readonly ScopePrefixInput[],
|
|
1720
1705
|
// FILE_TOMBSTONE records (POSIX-keyed) for the company — the durable
|
|
1721
1706
|
// "this key was intentionally deleted" signal the planner consults before
|
|
1722
1707
|
// re-downloading a key, so a deleted folder does not resync back in
|
|
@@ -1728,7 +1713,11 @@ function computePullPlan(
|
|
|
1728
1713
|
const items: PullPlanItem[] = [];
|
|
1729
1714
|
|
|
1730
1715
|
for (const remoteFile of remoteFiles) {
|
|
1731
|
-
const localPath =
|
|
1716
|
+
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
1717
|
+
if (localPath === null) {
|
|
1718
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
1719
|
+
continue;
|
|
1720
|
+
}
|
|
1732
1721
|
|
|
1733
1722
|
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
1734
1723
|
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
@@ -1741,17 +1730,6 @@ function computePullPlan(
|
|
|
1741
1730
|
continue;
|
|
1742
1731
|
}
|
|
1743
1732
|
|
|
1744
|
-
// Malformed-key filter — keys with backslash separators pushed by
|
|
1745
|
-
// pre-5.47.2 Windows clients. Downloading one materializes a junk local
|
|
1746
|
-
// file whose NAME contains backslashes (it is not a path on POSIX), which
|
|
1747
|
-
// then churns conflict mirrors forever. Refuse at planning time, same
|
|
1748
|
-
// policy bucket as the ephemeral filter above. The bogus keys themselves
|
|
1749
|
-
// are cleaned server-side; this keeps clean trees clean in the meantime.
|
|
1750
|
-
if (isMalformedVaultKey(remoteFile.key)) {
|
|
1751
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
1752
|
-
continue;
|
|
1753
|
-
}
|
|
1754
|
-
|
|
1755
1733
|
if (
|
|
1756
1734
|
personalMode &&
|
|
1757
1735
|
remoteFile.key.startsWith("companies/") &&
|
|
@@ -2112,12 +2090,8 @@ function computePullPlan(
|
|
|
2112
2090
|
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
2113
2091
|
const posixKey = toPosixKey(key);
|
|
2114
2092
|
if (remoteKeySet.has(posixKey)) continue;
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
// POSIX file and unlinks live data. Leave it for normalizeJournalKeys to
|
|
2118
|
-
// rewrite to POSIX on the next write; the canonical key is re-evaluated
|
|
2119
|
-
// (and correctly tombstoned if genuinely remote-deleted) on a later pull.
|
|
2120
|
-
if (isMalformedVaultKey(key)) continue;
|
|
2093
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
2094
|
+
if (localPath === null) continue;
|
|
2121
2095
|
// PersonalMode key gating — mirror the download branch.
|
|
2122
2096
|
if (personalMode && key.startsWith("companies/")) {
|
|
2123
2097
|
const slug = key.split("/")[1] ?? "";
|
|
@@ -2131,7 +2105,6 @@ function computePullPlan(
|
|
|
2131
2105
|
// Honor the current ignore filter — if a path was previously synced
|
|
2132
2106
|
// but is now ignored (operator edited .hqignore), do NOT delete
|
|
2133
2107
|
// the local copy. They're keeping it deliberately.
|
|
2134
|
-
const localPath = path.join(companyRoot, key);
|
|
2135
2108
|
if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) continue;
|
|
2136
2109
|
// Codex P1 (PR #24 round 3): detect local edits before tombstoning.
|
|
2137
2110
|
// Delete-vs-local-edit race: peer deleted the file remotely while
|
|
@@ -2202,16 +2175,6 @@ function computePullPlan(
|
|
|
2202
2175
|
};
|
|
2203
2176
|
}
|
|
2204
2177
|
|
|
2205
|
-
/**
|
|
2206
|
-
* Check if an error is an S3 access denied (expected for filtered guests).
|
|
2207
|
-
*/
|
|
2208
|
-
function isAccessDenied(err: unknown): boolean {
|
|
2209
|
-
if (err && typeof err === "object" && "name" in err) {
|
|
2210
|
-
return err.name === "AccessDenied" || err.name === "Forbidden";
|
|
2211
|
-
}
|
|
2212
|
-
return false;
|
|
2213
|
-
}
|
|
2214
|
-
|
|
2215
2178
|
/**
|
|
2216
2179
|
* Default human-readable event rendering. Preserves the exact output format
|
|
2217
2180
|
* that `hq sync` emitted before SyncProgressEvent was introduced, so callers
|