@indigoai-us/hq-cloud 6.11.11 → 6.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +35 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.js +290 -0
- package/dist/bin/sync-runner-company.js.map +1 -0
- package/dist/bin/sync-runner-events.d.ts +12 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -0
- package/dist/bin/sync-runner-events.js +12 -0
- package/dist/bin/sync-runner-events.js.map +1 -0
- package/dist/bin/sync-runner-planning.d.ts +53 -0
- package/dist/bin/sync-runner-planning.d.ts.map +1 -0
- package/dist/bin/sync-runner-planning.js +59 -0
- package/dist/bin/sync-runner-planning.js.map +1 -0
- package/dist/bin/sync-runner-rollup.d.ts +24 -0
- package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
- package/dist/bin/sync-runner-rollup.js +46 -0
- package/dist/bin/sync-runner-rollup.js.map +1 -0
- package/dist/bin/sync-runner-telemetry.d.ts +5 -0
- package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
- package/dist/bin/sync-runner-telemetry.js +5 -0
- package/dist/bin/sync-runner-telemetry.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-loop.js +372 -0
- package/dist/bin/sync-runner-watch-loop.js.map +1 -0
- package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-routes.js +74 -0
- package/dist/bin/sync-runner-watch-routes.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +5 -54
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +76 -978
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +265 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +34 -17
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -5
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +75 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +45 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +320 -170
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +276 -660
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +541 -748
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +382 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/daemon-worker.d.ts +2 -2
- package/dist/daemon-worker.js +3 -3
- package/dist/daemon-worker.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +149 -30
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +8 -2
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +8 -3
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +85 -16
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +213 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +12 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +45 -17
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +67 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync-core.d.ts +27 -0
- package/dist/sync-core.d.ts.map +1 -0
- package/dist/sync-core.js +54 -0
- package/dist/sync-core.js.map +1 -0
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +284 -36
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +59 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +38 -20
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +155 -143
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +350 -0
- package/src/bin/sync-runner-events.ts +25 -0
- package/src/bin/sync-runner-planning.ts +121 -0
- package/src/bin/sync-runner-rollup.ts +72 -0
- package/src/bin/sync-runner-telemetry.ts +8 -0
- package/src/bin/sync-runner-watch-loop.ts +443 -0
- package/src/bin/sync-runner-watch-routes.ts +86 -0
- package/src/bin/sync-runner.test.ts +298 -11
- package/src/bin/sync-runner.ts +99 -1054
- package/src/cli/reindex.test.ts +41 -3
- package/src/cli/reindex.ts +35 -19
- package/src/cli/rescue-classify-ordering.test.ts +81 -0
- package/src/cli/rescue-core.ts +400 -165
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +420 -693
- package/src/cli/sync.test.ts +460 -1
- package/src/cli/sync.ts +788 -825
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/daemon-worker.ts +3 -3
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +183 -31
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +8 -2
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +235 -1
- package/src/remote-pull.ts +106 -18
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +73 -1
- package/src/sync/push-receiver.ts +56 -20
- package/src/sync-core.ts +58 -0
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/vault-client.test.ts +74 -0
- package/src/vault-client.ts +395 -43
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +215 -174
package/dist/cli/sync.js
CHANGED
|
@@ -11,13 +11,15 @@ import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, to
|
|
|
11
11
|
import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
12
12
|
import { PERSONAL_VAULT_MANIFEST_KEY } from "../personal-vault.js";
|
|
13
13
|
import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
|
|
14
|
-
import { coalescePrefixes, isCoveredByAny } from "../prefix-coalesce.js";
|
|
14
|
+
import { coalescePrefixes, isCoveredByAny, } from "../prefix-coalesce.js";
|
|
15
15
|
import { createIgnoreFilter } from "../ignore.js";
|
|
16
|
+
import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, } from "../sync-core.js";
|
|
16
17
|
import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
17
18
|
import { resolveConflict } from "./conflict.js";
|
|
18
19
|
import { buildConflictId, buildConflictPath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
19
20
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
20
21
|
import { reindex } from "./reindex.js";
|
|
22
|
+
import { withOperationLock } from "../operation-lock.js";
|
|
21
23
|
import { fetchCompanyTombstones, } from "./tombstones.js";
|
|
22
24
|
/**
|
|
23
25
|
* Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
|
|
@@ -36,6 +38,15 @@ export function resolveAutoPruneCap() {
|
|
|
36
38
|
}
|
|
37
39
|
/** Max time to wait on the best-effort new-files notification POST. */
|
|
38
40
|
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
41
|
+
/**
|
|
42
|
+
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
43
|
+
* an oversized batch wholesale, so the client MUST split a large report into
|
|
44
|
+
* chunks at or under this size — otherwise a first sync with more than this many
|
|
45
|
+
* new files reports none of them, and the same oversized batch re-triggers every
|
|
46
|
+
* sync cycle (wasted work + dropped notifications). Keep in lockstep with the
|
|
47
|
+
* server-side limit.
|
|
48
|
+
*/
|
|
49
|
+
const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
|
|
39
50
|
/**
|
|
40
51
|
* Best-effort report of the files that were new to this drive during the sync,
|
|
41
52
|
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
@@ -44,17 +55,31 @@ const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
|
44
55
|
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
45
56
|
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
46
57
|
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
47
|
-
* 5s timeout so a hung endpoint can't stall sync completion. No-op
|
|
48
|
-
* are no new files.
|
|
58
|
+
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
59
|
+
* when there are no new files.
|
|
60
|
+
*
|
|
61
|
+
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
62
|
+
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
63
|
+
* best-effort, so one failing/oversized batch can never block the others or the
|
|
64
|
+
* sync. Exported only so the chunking can be unit-tested directly.
|
|
49
65
|
*/
|
|
50
|
-
async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
|
|
66
|
+
export async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, files) {
|
|
51
67
|
if (files.length === 0)
|
|
52
68
|
return;
|
|
69
|
+
let token;
|
|
53
70
|
try {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
71
|
+
token =
|
|
72
|
+
typeof vaultConfig.authToken === "function"
|
|
73
|
+
? await vaultConfig.authToken()
|
|
74
|
+
: vaultConfig.authToken;
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
logNotifyFailure(err);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
81
|
+
for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
|
|
82
|
+
const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
|
|
58
83
|
const controller = new AbortController();
|
|
59
84
|
const timer = setTimeout(() => controller.abort(), NOTIFY_FILE_ADDED_TIMEOUT_MS);
|
|
60
85
|
try {
|
|
@@ -67,7 +92,7 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
|
|
|
67
92
|
body: JSON.stringify({
|
|
68
93
|
companyUid,
|
|
69
94
|
companySlug,
|
|
70
|
-
files:
|
|
95
|
+
files: batch.map((f) => ({
|
|
71
96
|
path: f.path,
|
|
72
97
|
bytes: f.bytes,
|
|
73
98
|
...(f.addedBy ? { addedBy: f.addedBy } : {}),
|
|
@@ -76,211 +101,169 @@ async function reportNewFilesToNotify(vaultConfig, companyUid, companySlug, file
|
|
|
76
101
|
signal: controller.signal,
|
|
77
102
|
});
|
|
78
103
|
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
// Best-effort per chunk: never let notification reporting affect the sync
|
|
106
|
+
// result, and a failed chunk must not abort the remaining chunks.
|
|
107
|
+
logNotifyFailure(err);
|
|
108
|
+
}
|
|
79
109
|
finally {
|
|
80
110
|
clearTimeout(timer);
|
|
81
111
|
}
|
|
82
112
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
113
|
+
}
|
|
114
|
+
/** Log a non-fatal notify failure without ever throwing out of the logger. */
|
|
115
|
+
function logNotifyFailure(err) {
|
|
116
|
+
try {
|
|
117
|
+
console.error(`[hq-sync] new-files notify report failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
// swallow — logging must never break sync
|
|
91
121
|
}
|
|
92
122
|
}
|
|
93
123
|
/**
|
|
94
124
|
* Sync (pull) all allowed files from the entity vault.
|
|
95
125
|
*/
|
|
96
126
|
export async function sync(options) {
|
|
97
|
-
|
|
127
|
+
if (options.operationLockAlreadyHeld) {
|
|
128
|
+
return syncWithOperationLockHeld(options);
|
|
129
|
+
}
|
|
130
|
+
return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
|
|
131
|
+
}
|
|
132
|
+
async function syncWithOperationLockHeld(options) {
|
|
133
|
+
const run = await buildPullContext(options);
|
|
134
|
+
const plan = planPull(run);
|
|
135
|
+
emitPullPlan(run.emit, plan);
|
|
136
|
+
const scopePlan = planScopeShrink(run);
|
|
137
|
+
const scopeRun = executeScopeShrink(run, scopePlan);
|
|
138
|
+
const counters = createPullCounters();
|
|
139
|
+
const transferConcurrency = resolveTransferConcurrency();
|
|
140
|
+
const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
|
|
141
|
+
if (conflictRun.abortResult) {
|
|
142
|
+
return conflictRun.abortResult;
|
|
143
|
+
}
|
|
144
|
+
await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
|
|
145
|
+
await emitAndReportNewFiles(run, plan);
|
|
146
|
+
await verifyPlannedJournalTombstones(run, plan);
|
|
147
|
+
executeJournalTombstoneDeletes(run, plan, counters);
|
|
148
|
+
return finalizePullRun(run, plan, scopeRun, counters);
|
|
149
|
+
}
|
|
150
|
+
async function buildPullContext(options) {
|
|
151
|
+
const { company, vaultConfig, hqRoot } = options;
|
|
98
152
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
99
|
-
// Resolve company
|
|
100
153
|
const companyRef = company ?? resolveActiveCompany(hqRoot);
|
|
101
154
|
if (!companyRef) {
|
|
102
155
|
throw new Error("No company specified and no active company found. " +
|
|
103
156
|
"Use --company <slug> or set up .hq/config.json.");
|
|
104
157
|
}
|
|
105
|
-
|
|
106
|
-
let ctx = await resolveEntityContext(companyRef, vaultConfig);
|
|
107
|
-
// Every company's files land under companies/{slug}/ so fanning out multiple
|
|
108
|
-
// companies into the same hqRoot doesn't cross-clobber files with overlapping
|
|
109
|
-
// S3 keys (e.g. every company has a .hq/manifest.json). Remote keys stay
|
|
110
|
-
// company-relative; the prefix lives only on disk.
|
|
111
|
-
// In personalMode the journal slug + S3 keys are person-relative (e.g. "docs/foo.md");
|
|
112
|
-
// the local target is `hqRoot` directly, NOT `<hqRoot>/companies/<personSlug>/`. This
|
|
113
|
-
// keeps round-trip parity with the Rust personal first-push (Step 7) which sources
|
|
114
|
-
// `<hqRoot>/docs/foo.md`.
|
|
158
|
+
const ctx = await resolveEntityContext(companyRef, vaultConfig);
|
|
115
159
|
const companyRoot = options.personalMode === true
|
|
116
160
|
? hqRoot
|
|
117
161
|
: path.join(hqRoot, "companies", ctx.slug);
|
|
118
162
|
const shouldSync = createIgnoreFilter(hqRoot);
|
|
119
163
|
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
120
164
|
const startedAt = new Date().toISOString();
|
|
121
|
-
// Personal-vault callers must never start from an empty journal when only
|
|
122
|
-
// the legacy `personal` file exists (mass re-download/etag churn). Seeding
|
|
123
|
-
// here — inside the engine — covers every consumer (sync-runner already
|
|
124
|
-
// seeds; hq-cli historically didn't, which split the vault's bookkeeping
|
|
125
|
-
// across two journal files and re-flagged synced files as conflicts).
|
|
126
165
|
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG)
|
|
127
166
|
migratePersonalVaultJournal();
|
|
128
|
-
// Migrate v1 → v2 in place so the scope-shrink / pull-record machinery has
|
|
129
|
-
// its fields, and GC any tombstones past the 30-day retention window before
|
|
130
|
-
// we re-evaluate orphans (so a long-pruned path can re-download cleanly).
|
|
131
167
|
const journal = migrateToV2(readJournal(journalSlug));
|
|
132
168
|
gcTombstones(journal, Date.now());
|
|
133
|
-
// ── Effective download scope (US-005) ─────────────────────────────────────
|
|
134
|
-
// `all` → prefixSet `[""]`, which `isCoveredByAny` treats as "covers
|
|
135
|
-
// everything" — so the download filter and the scope-shrink
|
|
136
|
-
// comparison both become no-ops, preserving legacy full-bucket
|
|
137
|
-
// behavior bit-for-bit.
|
|
138
|
-
// `shared`/`custom` → the coalesced, company-relative prefix set the runner
|
|
139
|
-
// resolved. An empty set means "nothing in scope" → download
|
|
140
|
-
// nothing (the runner falls back to `all` on resolution errors, so
|
|
141
|
-
// empty here is an intentional "nothing shared", never a failure).
|
|
142
169
|
const syncMode = options.syncMode ?? "all";
|
|
143
170
|
const currentPrefixSet = syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
|
|
144
|
-
// Authorship guard input (scope-shrink): the caller's own Cognito sub,
|
|
145
|
-
// injected by the entry point (the runner sources it from its decoded
|
|
146
|
-
// idToken claims — the same sub stamped onto uploads as `created-by-sub`).
|
|
147
|
-
// Undefined degrades safely: own-author files lose their special shield, but
|
|
148
|
-
// the `protectUnknownAuthors` conservative path below still prevents a
|
|
149
|
-
// routine sync from deleting anything it can't prove is foreign.
|
|
150
|
-
const callerSub = options.callerSub;
|
|
151
|
-
let filesDownloaded = 0;
|
|
152
|
-
let bytesDownloaded = 0;
|
|
153
|
-
let filesSkipped = 0;
|
|
154
|
-
let conflicts = 0;
|
|
155
|
-
let filesTombstoned = 0;
|
|
156
|
-
let filesOutOfScope = 0;
|
|
157
|
-
const conflictPaths = [];
|
|
158
|
-
// List all remote files (IAM session policy filters at the AWS layer)
|
|
159
171
|
const remoteFiles = await listRemoteFiles(ctx);
|
|
160
|
-
|
|
161
|
-
// resurrection of an intentionally-deleted object (delete-resync). Done in
|
|
162
|
-
// parallel intent with the LIST above conceptually, but kept serial here for
|
|
163
|
-
// a clean read of `ctx`; best-effort — a failed read degrades to an empty map
|
|
164
|
-
// (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
|
|
165
|
-
// companyUid the tombstone rows are keyed under.
|
|
166
|
-
//
|
|
167
|
-
// SKIP for the personal vault: its `ctx.uid` is a personUid (`prs_…`), but
|
|
168
|
-
// `GET /v1/files/tombstones?company=…` is COMPANY-scoped server-side
|
|
169
|
-
// (findCallerWithMembership), so a personal-vault request resolves
|
|
170
|
-
// `company=prs_…` to no membership and is correctly rejected with
|
|
171
|
-
// `403 "No active membership for caller in company prs_…"`. That 403 is
|
|
172
|
-
// benign for the pull (it already degrades to the empty map below), but
|
|
173
|
-
// hq-pro captures EVERY one as a Sentry warning — the per-personal-vault
|
|
174
|
-
// no-membership cluster (one Sentry issue per signed-in user). Personal-vault
|
|
175
|
-
// delete-resync was never a committed feature and there is no person-scoped
|
|
176
|
-
// tombstone path, so for the personal target we skip the fetch and use an
|
|
177
|
-
// empty map — byte-for-byte the current degraded behavior, minus the 403 spam.
|
|
178
|
-
// FUTURE FOLLOW-UP (not built here): if personal-vault delete-resync is
|
|
179
|
-
// wanted, it needs a real person-scoped tombstone endpoint + client read.
|
|
180
|
-
const tombstones = options.personalMode === true
|
|
172
|
+
const fileTombstones = options.personalMode === true
|
|
181
173
|
? new Map()
|
|
182
174
|
: await fetchCompanyTombstones(vaultConfig, ctx.uid);
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
175
|
+
return {
|
|
176
|
+
options,
|
|
177
|
+
companyRef,
|
|
178
|
+
vaultConfig,
|
|
179
|
+
hqRoot,
|
|
180
|
+
emit,
|
|
181
|
+
ctx,
|
|
182
|
+
companyRoot,
|
|
183
|
+
shouldSync,
|
|
184
|
+
journalSlug,
|
|
185
|
+
startedAt,
|
|
186
|
+
journal,
|
|
187
|
+
remoteFiles,
|
|
188
|
+
syncMode,
|
|
189
|
+
currentPrefixSet,
|
|
190
|
+
fileTombstones,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
function planPull(run) {
|
|
194
|
+
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.includeLocalCompanies === true, run.options.teamSyncedSlugs ?? null, run.currentPrefixSet, run.fileTombstones);
|
|
195
|
+
}
|
|
196
|
+
function emitPullPlan(emit, plan) {
|
|
187
197
|
emit({
|
|
188
198
|
type: "plan",
|
|
189
199
|
filesToDownload: plan.filesToDownload,
|
|
190
200
|
bytesToDownload: plan.bytesToDownload,
|
|
191
|
-
// sync() is pull-only; push counts are sourced from share()'s plan event.
|
|
192
201
|
filesToUpload: 0,
|
|
193
202
|
bytesToUpload: 0,
|
|
194
203
|
filesToSkip: plan.filesToSkip,
|
|
195
204
|
filesToConflict: plan.filesToConflict,
|
|
196
|
-
// Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
|
|
197
|
-
// deletes known at plan time; the journal-vs-LIST tombstones are
|
|
198
|
-
// HEAD-verified later and surfaced via the final filesTombstoned count.
|
|
199
205
|
filesToDelete: plan.filesToTombstoneDelete,
|
|
200
206
|
});
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
// AWS layer never narrows the pull. This client-side shrink is what makes
|
|
216
|
-
// `hq sync mode shared` actually stick across re-syncs for an owner.
|
|
217
|
-
const lastRecord = lastPullRecord(journal, ctx.uid);
|
|
218
|
-
// A missing record, or a v1-migrated record with an empty prefixSet, means
|
|
219
|
-
// "no recorded scope" → treat the last scope as full-bucket `all` (`[""]`),
|
|
220
|
-
// per the PullRecord.prefixSet contract in types.ts.
|
|
207
|
+
}
|
|
208
|
+
function createPullCounters() {
|
|
209
|
+
return {
|
|
210
|
+
filesDownloaded: 0,
|
|
211
|
+
bytesDownloaded: 0,
|
|
212
|
+
filesSkipped: 0,
|
|
213
|
+
conflicts: 0,
|
|
214
|
+
filesTombstoned: 0,
|
|
215
|
+
filesOutOfScope: 0,
|
|
216
|
+
conflictPaths: [],
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
function planScopeShrink(run) {
|
|
220
|
+
const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
|
|
221
221
|
const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
|
|
222
222
|
? lastRecord.prefixSet
|
|
223
223
|
: [""];
|
|
224
224
|
const shrinkPlan = buildScopeShrinkPlan({
|
|
225
|
-
journal,
|
|
226
|
-
hqRoot: companyRoot,
|
|
225
|
+
journal: run.journal,
|
|
226
|
+
hqRoot: run.companyRoot,
|
|
227
227
|
lastPrefixSet,
|
|
228
|
-
currentPrefixSet,
|
|
229
|
-
callerSub,
|
|
230
|
-
// Automatic pull: never auto-prune content the caller authored, and never
|
|
231
|
-
// make a destructive guess about unknown-author (legacy) orphans. The
|
|
232
|
-
// explicit `hq sync narrow` ritual opts out of the unknown-author shield.
|
|
228
|
+
currentPrefixSet: run.currentPrefixSet,
|
|
229
|
+
callerSub: run.options.callerSub,
|
|
233
230
|
protectUnknownAuthors: true,
|
|
234
231
|
});
|
|
235
|
-
|
|
236
|
-
// interactive flag, so it must never throw on a shrink — it self-heals
|
|
237
|
-
// non-destructively (dirty kept on disk + un-tracked, clean quarantined).
|
|
238
|
-
// A foreground `hq sync` ("block", the default) keeps the protective gate
|
|
239
|
-
// but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
|
|
240
|
-
// bypasses the bulk-prune cap (quarantine is non-destructive, so a large
|
|
241
|
-
// recovery move is safe). DEV-1768.
|
|
242
|
-
const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
|
|
232
|
+
const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
|
|
243
233
|
const autoRecover = scopeShrinkPolicy === "auto-recover";
|
|
244
234
|
const adviceContext = autoRecover ? "runner" : "cli";
|
|
245
|
-
const effectiveForce = options.forceScopeShrink === true || autoRecover;
|
|
235
|
+
const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
|
|
236
|
+
return {
|
|
237
|
+
lastRecord,
|
|
238
|
+
shrinkPlan,
|
|
239
|
+
autoRecover,
|
|
240
|
+
adviceContext,
|
|
241
|
+
effectiveForce,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
function executeScopeShrink(run, scopePlan) {
|
|
245
|
+
const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
|
|
246
246
|
if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
|
|
247
|
-
throw new ScopeShrinkBlockedError(ctx.uid, lastRecord?.syncMode ?? "unknown", syncMode, shrinkPlan.dirty, shrinkPlan.clean, adviceContext);
|
|
247
|
+
throw new ScopeShrinkBlockedError(run.ctx.uid, lastRecord?.syncMode ?? "unknown", run.syncMode, shrinkPlan.dirty, shrinkPlan.clean, adviceContext);
|
|
248
248
|
}
|
|
249
|
-
// Bulk guard: refuse to auto-move more than the safety cap of CLEAN files in
|
|
250
|
-
// a single foreground sync. A deliberate large narrow goes through
|
|
251
|
-
// `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
|
|
252
|
-
// raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
|
|
253
|
-
// under auto-recover — quarantine is non-destructive so a big recovery is
|
|
254
|
-
// safe, and the runner has no way to act on a thrown cap. The engine moves
|
|
255
|
-
// nothing when it throws here.
|
|
256
249
|
const autoPruneCap = resolveAutoPruneCap();
|
|
257
250
|
if (!effectiveForce &&
|
|
258
251
|
autoPruneCap > 0 &&
|
|
259
252
|
shrinkPlan.clean.length > autoPruneCap) {
|
|
260
|
-
throw new ScopeShrinkLargePruneError(ctx.uid, syncMode, shrinkPlan.clean.length, autoPruneCap, adviceContext);
|
|
253
|
+
throw new ScopeShrinkLargePruneError(run.ctx.uid, run.syncMode, shrinkPlan.clean.length, autoPruneCap, adviceContext);
|
|
261
254
|
}
|
|
262
|
-
|
|
263
|
-
// recoverable), never silently deleted — a background sync purging local
|
|
264
|
-
// files unannounced was DEV-1768 fix #3. The quarantine root lives under the
|
|
265
|
-
// real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
|
|
266
|
-
// files don't round-trip back through S3.
|
|
267
|
-
const scopeQuarantineRoot = path.join(hqRoot, ".hq", "scope-quarantine", journalSlug);
|
|
255
|
+
const scopeQuarantineRoot = path.join(run.hqRoot, ".hq", "scope-quarantine", run.journalSlug);
|
|
268
256
|
const shrinkResult = applyScopeShrink({
|
|
269
|
-
journal,
|
|
257
|
+
journal: run.journal,
|
|
270
258
|
plan: shrinkPlan,
|
|
271
|
-
hqRoot: companyRoot,
|
|
259
|
+
hqRoot: run.companyRoot,
|
|
272
260
|
forceScopeShrink: effectiveForce,
|
|
273
261
|
reason: "scope_shrink",
|
|
274
262
|
cleanDisposition: "quarantine",
|
|
275
263
|
quarantineRoot: scopeQuarantineRoot,
|
|
276
264
|
});
|
|
277
|
-
// Surface each affected orphan explicitly (named path) so the prune is never
|
|
278
|
-
// silent. Quarantined clean files render as `deleted: true` (removed from the
|
|
279
|
-
// working tree, recoverable in quarantine); dirty files KEPT on disk render
|
|
280
|
-
// as a non-deletion notice so the operator knows they were un-tracked, not
|
|
281
|
-
// removed. The Rust menubar parser already handles `deleted: true`.
|
|
282
265
|
for (const relPath of shrinkResult.quarantinedPaths) {
|
|
283
|
-
emit({
|
|
266
|
+
run.emit({
|
|
284
267
|
type: "progress",
|
|
285
268
|
path: relPath,
|
|
286
269
|
bytes: 0,
|
|
@@ -289,7 +272,7 @@ export async function sync(options) {
|
|
|
289
272
|
});
|
|
290
273
|
}
|
|
291
274
|
for (const relPath of shrinkResult.removedPaths) {
|
|
292
|
-
emit({
|
|
275
|
+
run.emit({
|
|
293
276
|
type: "progress",
|
|
294
277
|
path: relPath,
|
|
295
278
|
bytes: 0,
|
|
@@ -298,477 +281,314 @@ export async function sync(options) {
|
|
|
298
281
|
});
|
|
299
282
|
}
|
|
300
283
|
for (const relPath of shrinkResult.dirtyKeptPaths) {
|
|
301
|
-
emit({
|
|
284
|
+
run.emit({
|
|
302
285
|
type: "progress",
|
|
303
286
|
path: relPath,
|
|
304
287
|
bytes: 0,
|
|
305
288
|
message: "scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
|
|
306
289
|
});
|
|
307
290
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
// partition the plan items into "conflict (serial)" and "download
|
|
321
|
-
// (parallel)" buckets and run the serial pass first; the parallel pass
|
|
322
|
-
// only runs if no conflict aborted.
|
|
323
|
-
//
|
|
324
|
-
// Per-file `progress` events fire at the moment each individual download
|
|
325
|
-
// settles (inside the pool wrapper), NOT in plan-walk order. The cross-
|
|
326
|
-
// file interleave is acceptable: the menubar stream parser already
|
|
327
|
-
// handles per-company interleave, and the same shape applies within a
|
|
328
|
-
// single company's pool. Per-file event-count correctness is preserved
|
|
329
|
-
// (one progress per download, one error per failure).
|
|
330
|
-
const TRANSFER_CONCURRENCY = (() => {
|
|
331
|
-
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
332
|
-
if (raw === undefined || raw === "")
|
|
333
|
-
return 16;
|
|
334
|
-
const parsed = Number.parseInt(raw, 10);
|
|
335
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
336
|
-
})();
|
|
337
|
-
// First pass: serial walk for non-download outcomes (skips + conflicts).
|
|
338
|
-
// Conflicts may set `aborted = true` and short-circuit the whole pull;
|
|
339
|
-
// we detect that and skip the parallel pass. Download items are
|
|
340
|
-
// collected into `downloadItems[]` for the pool pass below.
|
|
291
|
+
return {
|
|
292
|
+
shrinkPlan,
|
|
293
|
+
shrinkResult,
|
|
294
|
+
scopeOrphansRemoved: shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
async function refreshRunContextIfExpiring(run) {
|
|
298
|
+
if (isExpiringSoon(run.ctx.expiresAt)) {
|
|
299
|
+
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
async function executeConflictExecutor(run, plan, scopeRun, counters) {
|
|
341
303
|
const downloadItems = [];
|
|
342
|
-
let aborted = false;
|
|
343
|
-
let abortResult = null;
|
|
344
304
|
for (const item of plan.items) {
|
|
345
|
-
if (aborted)
|
|
346
|
-
break;
|
|
347
305
|
if (item.action === "skip-ignored" ||
|
|
348
306
|
item.action === "skip-personal-mode" ||
|
|
349
307
|
item.action === "skip-unchanged" ||
|
|
350
308
|
item.action === "skip-local-only") {
|
|
351
|
-
filesSkipped++;
|
|
309
|
+
counters.filesSkipped++;
|
|
352
310
|
continue;
|
|
353
311
|
}
|
|
354
312
|
if (item.action === "skip-excluded-policy") {
|
|
355
|
-
// Policy-excluded items count separately from `filesSkipped` so the
|
|
356
|
-
// pull result mirrors the push side's `filesExcludedByPolicy`
|
|
357
|
-
// counter — `filesSkipped` stays a measure of "unchanged on this
|
|
358
|
-
// run", not a catch-all for everything we didn't download.
|
|
359
313
|
continue;
|
|
360
314
|
}
|
|
361
315
|
if (item.action === "skip-out-of-scope") {
|
|
362
|
-
|
|
363
|
-
// axis so `filesSkipped` keeps meaning "unchanged on this run" — these
|
|
364
|
-
// are "deliberately not downloaded because of your sync scope".
|
|
365
|
-
filesOutOfScope++;
|
|
316
|
+
counters.filesOutOfScope++;
|
|
366
317
|
continue;
|
|
367
318
|
}
|
|
368
319
|
if (item.action === "tombstone-delete") {
|
|
369
|
-
|
|
370
|
-
// is present but a tombstone marks the key intentionally deleted and it is
|
|
371
|
-
// not a newer re-create. Delete any local copy and drop the journal entry
|
|
372
|
-
// so it stays gone — the mirror of the journal-vs-LIST tombstone executor
|
|
373
|
-
// below, but WITHOUT the HEAD-verify (the remote object is present by
|
|
374
|
-
// definition; the FILE_TOMBSTONE is the deletion authority). The planner
|
|
375
|
-
// already routed any divergent local copy to `conflict`, so a local file
|
|
376
|
-
// reaching here matches the deleted baseline and is safe to remove.
|
|
377
|
-
const tombstoneKey = item.remoteFile.key;
|
|
378
|
-
// Same Windows-backslash landmine guard as the journal-tombstone executor:
|
|
379
|
-
// a malformed key must never reach fs.unlinkSync (path.join collapses the
|
|
380
|
-
// backslashes onto a REAL POSIX file). Drop the poisoned journal entry
|
|
381
|
-
// without touching disk.
|
|
382
|
-
if (isMalformedVaultKey(tombstoneKey)) {
|
|
383
|
-
removeEntry(journal, tombstoneKey);
|
|
384
|
-
continue;
|
|
385
|
-
}
|
|
386
|
-
try {
|
|
387
|
-
const lstat = fs.lstatSync(item.localPath);
|
|
388
|
-
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
389
|
-
fs.unlinkSync(item.localPath);
|
|
390
|
-
}
|
|
391
|
-
// A directory at the key: don't recursively rm-rf the operator's dir;
|
|
392
|
-
// just drop the journal entry (safe-by-default, same as the other path).
|
|
393
|
-
}
|
|
394
|
-
catch (err) {
|
|
395
|
-
const code = err && typeof err === "object" && "code" in err
|
|
396
|
-
? err.code
|
|
397
|
-
: undefined;
|
|
398
|
-
// ENOENT → local already absent (the common case: a fresh machine that
|
|
399
|
-
// never held the file, or a prior pull already removed it) → drop the
|
|
400
|
-
// journal entry and converge. Other errors (EACCES/EPERM/…) leave the
|
|
401
|
-
// file in place; surface and KEEP the journal entry so the next sync
|
|
402
|
-
// retries rather than forgetting the delete.
|
|
403
|
-
if (code !== "ENOENT") {
|
|
404
|
-
emit({
|
|
405
|
-
type: "error",
|
|
406
|
-
path: tombstoneKey,
|
|
407
|
-
message: `tombstone-suppress unlink failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
408
|
-
});
|
|
409
|
-
continue;
|
|
410
|
-
}
|
|
411
|
-
}
|
|
412
|
-
removeEntry(journal, tombstoneKey);
|
|
413
|
-
filesTombstoned++;
|
|
414
|
-
emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
320
|
+
executeFileTombstoneDelete(run, item, counters);
|
|
415
321
|
continue;
|
|
416
322
|
}
|
|
417
323
|
if (item.action === "download") {
|
|
418
324
|
downloadItems.push(item);
|
|
419
325
|
continue;
|
|
420
326
|
}
|
|
421
|
-
const
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
if (isExpiringSoon(ctx.expiresAt)) {
|
|
425
|
-
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
327
|
+
const abortResult = await executeConflictItem(run, plan, scopeRun, counters, downloadItems, item);
|
|
328
|
+
if (abortResult) {
|
|
329
|
+
return { downloadItems, abortResult };
|
|
426
330
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
const machineId = readShortMachineId(hqRoot);
|
|
454
|
-
const originalRelative = path.relative(hqRoot, localPath);
|
|
455
|
-
const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
|
|
456
|
-
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
457
|
-
let remoteFetched = false;
|
|
458
|
-
let converged = false;
|
|
459
|
-
try {
|
|
460
|
-
await downloadFile(ctx, remoteFile.key, conflictAbs);
|
|
461
|
-
remoteFetched = true;
|
|
462
|
-
// Hash the fetched remote exactly the way the planner hashed local
|
|
463
|
-
// (symlink-aware) so the two hashes are directly comparable. A
|
|
464
|
-
// symlink record round-trips to a symlink on disk; hashing its
|
|
465
|
-
// target string matches `hashSymlinkTarget(localPath)`.
|
|
466
|
-
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
467
|
-
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
468
|
-
: hashFile(conflictAbs);
|
|
469
|
-
converged = remoteHash === item.localHash;
|
|
470
|
-
}
|
|
471
|
-
catch (probeErr) {
|
|
472
|
-
// Couldn't fetch or hash the remote — fail safe by falling through to
|
|
473
|
-
// the conventional conflict path (converged stays false). No mirror
|
|
474
|
-
// is on disk in this case.
|
|
475
|
-
emit({
|
|
476
|
-
type: "error",
|
|
477
|
-
path: remoteFile.key,
|
|
478
|
-
message: `conflict convergence probe failed: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`,
|
|
479
|
-
});
|
|
480
|
-
}
|
|
481
|
-
if (converged) {
|
|
482
|
-
// False positive: remote == local. Drop the byte-identical mirror and
|
|
483
|
-
// re-stamp the baseline (current localHash + current remoteEtag) so
|
|
484
|
-
// the next sync sees "no change on either side". Counts as a skip.
|
|
485
|
-
if (remoteFetched) {
|
|
486
|
-
try {
|
|
487
|
-
fs.rmSync(conflictAbs, { force: true });
|
|
488
|
-
}
|
|
489
|
-
catch {
|
|
490
|
-
/* best-effort cleanup; a stray identical mirror is harmless */
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
updateEntry(journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
|
|
494
|
-
emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
495
|
-
filesSkipped++;
|
|
496
|
-
continue;
|
|
497
|
-
}
|
|
498
|
-
// ── Genuine divergence ───────────────────────────────────────────
|
|
499
|
-
conflicts++;
|
|
500
|
-
conflictPaths.push(remoteFile.key);
|
|
501
|
-
const resolution = await resolveConflict({
|
|
502
|
-
path: remoteFile.key,
|
|
503
|
-
localHash: item.localHash,
|
|
504
|
-
remoteModified: remoteFile.lastModified,
|
|
505
|
-
// Use the lstat-mtime captured by the planner — statSync
|
|
506
|
-
// here would follow a dangling symlink and throw ENOENT,
|
|
507
|
-
// aborting the pull before resolveConflict could prompt.
|
|
508
|
-
localModified: item.localMtime,
|
|
509
|
-
direction: "pull",
|
|
510
|
-
}, onConflict);
|
|
511
|
-
emit({
|
|
512
|
-
type: "conflict",
|
|
513
|
-
path: remoteFile.key,
|
|
514
|
-
direction: "pull",
|
|
515
|
-
resolution,
|
|
516
|
-
});
|
|
517
|
-
// The remote bytes were already fetched to `conflictAbs` by the
|
|
518
|
-
// convergence probe. For "keep"/"skip" they become the
|
|
519
|
-
// `<original>.conflict-<ts>-<machine>.<ext>` inspection mirror — just
|
|
520
|
-
// index it (no second download). For "abort" (user gave up) and
|
|
521
|
-
// "overwrite" (cloud bytes are about to replace local) the mirror is
|
|
522
|
-
// redundant, so discard it. Best-effort: failure here only emits an
|
|
523
|
-
// error, doesn't break the sync.
|
|
524
|
-
if (resolution !== "abort" && resolution !== "overwrite") {
|
|
525
|
-
if (remoteFetched) {
|
|
526
|
-
try {
|
|
527
|
-
appendConflictEntry(hqRoot, {
|
|
528
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
529
|
-
originalPath: originalRelative,
|
|
530
|
-
conflictPath: conflictRelative,
|
|
531
|
-
detectedAt,
|
|
532
|
-
side: "pull",
|
|
533
|
-
machineId,
|
|
534
|
-
localHash: item.localHash,
|
|
535
|
-
remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
|
|
536
|
-
});
|
|
537
|
-
}
|
|
538
|
-
catch (mirrorErr) {
|
|
539
|
-
emit({
|
|
540
|
-
type: "error",
|
|
541
|
-
path: remoteFile.key,
|
|
542
|
-
message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
|
|
543
|
-
});
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
// If the probe download failed (!remoteFetched) there is no mirror on
|
|
547
|
-
// disk; the probe already emitted the error. The conflict is still
|
|
548
|
-
// surfaced and journal-stamped below so it doesn't re-fire silently.
|
|
549
|
-
}
|
|
550
|
-
else if (remoteFetched) {
|
|
551
|
-
try {
|
|
552
|
-
fs.rmSync(conflictAbs, { force: true });
|
|
553
|
-
}
|
|
554
|
-
catch {
|
|
555
|
-
/* best-effort; a leftover mirror is cosmetic, not corrupting */
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
if (resolution === "abort") {
|
|
559
|
-
emit({ type: "new-files", files: [] });
|
|
560
|
-
writeJournal(journalSlug, journal);
|
|
561
|
-
aborted = true;
|
|
562
|
-
abortResult = {
|
|
563
|
-
filesDownloaded,
|
|
564
|
-
bytesDownloaded,
|
|
565
|
-
filesSkipped,
|
|
566
|
-
conflicts,
|
|
567
|
-
conflictPaths,
|
|
568
|
-
aborted: true,
|
|
569
|
-
newFiles: plan.newFiles,
|
|
570
|
-
newFilesCount: plan.newFilesCount,
|
|
571
|
-
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
572
|
-
// Abort short-circuits before the tombstone loop runs; report
|
|
573
|
-
// 0 so the field shape stays stable for consumers that
|
|
574
|
-
// destructure it.
|
|
575
|
-
filesTombstoned: 0,
|
|
576
|
-
// Scope-shrink ran before execution, so its counts are real even on
|
|
577
|
-
// a conflict abort. `filesOutOfScope` reflects how far the serial
|
|
578
|
-
// pass got before the abort; that's acceptable for an abort result.
|
|
579
|
-
filesOutOfScope,
|
|
580
|
-
scopeOrphansRemoved,
|
|
581
|
-
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
582
|
-
};
|
|
583
|
-
break;
|
|
584
|
-
}
|
|
585
|
-
if (resolution === "keep" || resolution === "skip") {
|
|
586
|
-
filesSkipped++;
|
|
587
|
-
// Stamp the journal with the new baseline so the same conflict
|
|
588
|
-
// doesn't re-fire on every subsequent sync. After "keep", local
|
|
589
|
-
// wins — the user has accepted that the cloud version we just
|
|
590
|
-
// mirrored is what cloud is at this etag, and they don't want
|
|
591
|
-
// it. Recording (current localHash + current remoteEtag) tells
|
|
592
|
-
// the next sync "no change on either side" until something new
|
|
593
|
-
// diverges. Without this, both `localChanged` and `remoteChanged`
|
|
594
|
-
// stay true forever and the conflict is sticky.
|
|
595
|
-
// Stamp from planner-captured size (symlink-aware), NOT
|
|
596
|
-
// statSync — which would follow a dangling symlink and
|
|
597
|
-
// throw ENOENT, get swallowed, and leave the journal
|
|
598
|
-
// stale so this conflict would re-fire on every sync
|
|
599
|
-
// forever. localSize is sourced from the same lstat that
|
|
600
|
-
// computed localMtime + localHash above.
|
|
601
|
-
updateEntry(journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
|
|
602
|
-
continue;
|
|
603
|
-
}
|
|
604
|
-
// "overwrite" falls through to download — re-route through the pool
|
|
605
|
-
// so it benefits from parallelism too. Synthesize a download item
|
|
606
|
-
// pointing at the same remoteFile/localPath; isNew=false because
|
|
607
|
-
// there was a conflict-eligible local file present.
|
|
608
|
-
downloadItems.push({
|
|
609
|
-
action: "download",
|
|
610
|
-
remoteFile,
|
|
611
|
-
localPath,
|
|
612
|
-
isNew: false,
|
|
331
|
+
}
|
|
332
|
+
return { downloadItems, abortResult: null };
|
|
333
|
+
}
|
|
334
|
+
function executeFileTombstoneDelete(run, item, counters) {
|
|
335
|
+
const tombstoneKey = item.remoteFile.key;
|
|
336
|
+
const tombstonePath = resolveContainedVaultPath(run.companyRoot, tombstoneKey);
|
|
337
|
+
if (tombstonePath === null)
|
|
338
|
+
return;
|
|
339
|
+
try {
|
|
340
|
+
const lstat = fs.lstatSync(tombstonePath);
|
|
341
|
+
if (tombstoneTargetDiverged(run.journal, tombstoneKey, tombstonePath, lstat)) {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
345
|
+
fs.unlinkSync(tombstonePath);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
catch (err) {
|
|
349
|
+
const code = err && typeof err === "object" && "code" in err
|
|
350
|
+
? err.code
|
|
351
|
+
: undefined;
|
|
352
|
+
if (code !== "ENOENT") {
|
|
353
|
+
run.emit({
|
|
354
|
+
type: "error",
|
|
355
|
+
path: tombstoneKey,
|
|
356
|
+
message: `tombstone-suppress unlink failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
613
357
|
});
|
|
614
|
-
|
|
358
|
+
return;
|
|
615
359
|
}
|
|
616
360
|
}
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
361
|
+
removeEntry(run.journal, tombstoneKey);
|
|
362
|
+
counters.filesTombstoned++;
|
|
363
|
+
run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
364
|
+
}
|
|
365
|
+
async function executeConflictItem(run, plan, scopeRun, counters, downloadItems, item) {
|
|
366
|
+
const { remoteFile, localPath } = item;
|
|
367
|
+
await refreshRunContextIfExpiring(run);
|
|
368
|
+
const detectedAt = new Date().toISOString();
|
|
369
|
+
const machineId = readShortMachineId(run.hqRoot);
|
|
370
|
+
const originalRelative = path.relative(run.hqRoot, localPath);
|
|
371
|
+
const conflictRelative = buildConflictPath(originalRelative, detectedAt, machineId);
|
|
372
|
+
const conflictAbs = path.join(run.hqRoot, conflictRelative);
|
|
373
|
+
const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
|
|
374
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
|
|
375
|
+
counters.filesSkipped++;
|
|
376
|
+
run.emit({
|
|
377
|
+
type: "error",
|
|
378
|
+
path: remoteFile.key,
|
|
379
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
380
|
+
});
|
|
381
|
+
return null;
|
|
382
|
+
}
|
|
383
|
+
let remoteFetched = false;
|
|
384
|
+
let converged = false;
|
|
385
|
+
try {
|
|
386
|
+
const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
|
|
387
|
+
remoteFetched = true;
|
|
388
|
+
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
389
|
+
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
390
|
+
: (downloaded.contentHash ?? hashFile(conflictAbs));
|
|
391
|
+
converged = remoteHash === item.localHash;
|
|
623
392
|
}
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
// which re-reads the same keys — reuse them instead of presigning per file.
|
|
634
|
-
// On a large initial pull this is the difference between ~ceil(N/100)
|
|
635
|
-
// presign calls and N (which would 429 past the 100-req/hr limit). No-op
|
|
636
|
-
// on the S3 SDK transport; best-effort (failure falls back to per-file).
|
|
637
|
-
await primeObjectTransport(ctx, "get", downloadItems.map((d) => d.remoteFile.key));
|
|
638
|
-
const queue = [...downloadItems];
|
|
639
|
-
const inFlight = new Set();
|
|
640
|
-
const downloadOne = async (downloadItem) => {
|
|
641
|
-
const { remoteFile, localPath } = downloadItem;
|
|
642
|
-
// Auto-refresh context if credentials expiring. Each task checks
|
|
643
|
-
// independently — refresh is idempotent on the same context object.
|
|
644
|
-
if (isExpiringSoon(ctx.expiresAt)) {
|
|
645
|
-
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
646
|
-
}
|
|
393
|
+
catch (probeErr) {
|
|
394
|
+
run.emit({
|
|
395
|
+
type: "error",
|
|
396
|
+
path: remoteFile.key,
|
|
397
|
+
message: `conflict convergence probe failed: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}`,
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
if (converged) {
|
|
401
|
+
if (remoteFetched) {
|
|
647
402
|
try {
|
|
648
|
-
|
|
649
|
-
const author = metadata?.["created-by"] ?? null;
|
|
650
|
-
// Author sub for the scope-shrink authorship guard — same field the
|
|
651
|
-
// upload side stamps, read straight off the GET response metadata.
|
|
652
|
-
const createdBySub = metadata?.["created-by-sub"];
|
|
653
|
-
// Symlink records materialize as real symlinks on disk. lstat
|
|
654
|
-
// (does not follow) lets us detect that case so the journal stamp
|
|
655
|
-
// mirrors what the push side would emit on the next tick:
|
|
656
|
-
// hash = sha256(readlink target string)
|
|
657
|
-
// size = 0
|
|
658
|
-
// Without this check, hashFile would follow the link and stamp the
|
|
659
|
-
// target file's contents — a value the next push would never
|
|
660
|
-
// produce — which makes skipUnchanged perpetually re-upload every
|
|
661
|
-
// symlink, defeating the point of the gate.
|
|
662
|
-
const localLstat = fs.lstatSync(localPath);
|
|
663
|
-
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
664
|
-
const hash = isLocalSymlink
|
|
665
|
-
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
666
|
-
: hashFile(localPath);
|
|
667
|
-
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
668
|
-
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
669
|
-
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
670
|
-
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
671
|
-
// SHA256 for this file without reading its bytes.
|
|
672
|
-
//
|
|
673
|
-
// 5.37.0 ordering invariant: downloadFile applies hq-mtime via
|
|
674
|
-
// utimesSync AFTER its byte write but BEFORE returning, and this
|
|
675
|
-
// lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
|
|
676
|
-
// already reflects the source-stamped mtime, not the wall-clock
|
|
677
|
-
// write-time. The journal therefore matches what the next push's
|
|
678
|
-
// lstat fast-path will see, and the file is correctly skipped on
|
|
679
|
-
// re-sync instead of being hashed every tick. Do not move this
|
|
680
|
-
// lstat earlier; do not stamp the journal from any pre-download
|
|
681
|
-
// mtime.
|
|
682
|
-
updateEntry(journal, remoteFile.key, hash, size, "down", remoteFile.etag, localLstat.mtimeMs, createdBySub);
|
|
683
|
-
// Attach message from the prior journal entry if present (set by a
|
|
684
|
-
// previous `share` operation that included a --message).
|
|
685
|
-
const priorEntry = getEntry(journal, remoteFile.key);
|
|
686
|
-
const remoteJournalMessage = priorEntry?.message;
|
|
687
|
-
emit({
|
|
688
|
-
type: "progress",
|
|
689
|
-
path: remoteFile.key,
|
|
690
|
-
bytes: size,
|
|
691
|
-
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
692
|
-
...(author ? { author } : {}),
|
|
693
|
-
});
|
|
694
|
-
filesDownloaded++;
|
|
695
|
-
bytesDownloaded += size;
|
|
403
|
+
fs.rmSync(conflictAbs, { force: true });
|
|
696
404
|
}
|
|
697
|
-
catch
|
|
698
|
-
|
|
699
|
-
// for guest members with allowedPrefixes
|
|
700
|
-
if (isAccessDenied(err)) {
|
|
701
|
-
filesSkipped++;
|
|
702
|
-
}
|
|
703
|
-
else {
|
|
704
|
-
emit({
|
|
705
|
-
type: "error",
|
|
706
|
-
path: remoteFile.key,
|
|
707
|
-
message: err instanceof Error ? err.message : String(err),
|
|
708
|
-
});
|
|
709
|
-
}
|
|
405
|
+
catch {
|
|
406
|
+
/* best-effort cleanup; a stray identical mirror is harmless */
|
|
710
407
|
}
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
408
|
+
}
|
|
409
|
+
updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
|
|
410
|
+
run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
411
|
+
counters.filesSkipped++;
|
|
412
|
+
return null;
|
|
413
|
+
}
|
|
414
|
+
counters.conflicts++;
|
|
415
|
+
counters.conflictPaths.push(remoteFile.key);
|
|
416
|
+
const resolution = await resolveConflict({
|
|
417
|
+
path: remoteFile.key,
|
|
418
|
+
localHash: item.localHash,
|
|
419
|
+
remoteModified: remoteFile.lastModified,
|
|
420
|
+
localModified: item.localMtime,
|
|
421
|
+
direction: "pull",
|
|
422
|
+
}, run.options.onConflict);
|
|
423
|
+
run.emit({
|
|
424
|
+
type: "conflict",
|
|
425
|
+
path: remoteFile.key,
|
|
426
|
+
direction: "pull",
|
|
427
|
+
resolution,
|
|
428
|
+
});
|
|
429
|
+
if (resolution !== "abort" && resolution !== "overwrite") {
|
|
430
|
+
if (remoteFetched) {
|
|
431
|
+
try {
|
|
432
|
+
appendConflictEntry(run.hqRoot, {
|
|
433
|
+
id: buildConflictId(originalRelative, detectedAt),
|
|
434
|
+
originalPath: originalRelative,
|
|
435
|
+
conflictPath: conflictRelative,
|
|
436
|
+
detectedAt,
|
|
437
|
+
side: "pull",
|
|
438
|
+
machineId,
|
|
439
|
+
localHash: item.localHash,
|
|
440
|
+
remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
|
|
732
441
|
});
|
|
733
|
-
inFlight.add(p);
|
|
734
442
|
}
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
443
|
+
catch (mirrorErr) {
|
|
444
|
+
run.emit({
|
|
445
|
+
type: "error",
|
|
446
|
+
path: remoteFile.key,
|
|
447
|
+
message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
|
|
448
|
+
});
|
|
740
449
|
}
|
|
741
450
|
}
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
451
|
+
}
|
|
452
|
+
else if (remoteFetched) {
|
|
453
|
+
try {
|
|
454
|
+
fs.rmSync(conflictAbs, { force: true });
|
|
455
|
+
}
|
|
456
|
+
catch {
|
|
457
|
+
/* best-effort; a leftover mirror is cosmetic, not corrupting */
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
if (resolution === "abort") {
|
|
461
|
+
run.emit({ type: "new-files", files: [] });
|
|
462
|
+
writeJournal(run.journalSlug, run.journal);
|
|
463
|
+
return {
|
|
464
|
+
filesDownloaded: counters.filesDownloaded,
|
|
465
|
+
bytesDownloaded: counters.bytesDownloaded,
|
|
466
|
+
filesSkipped: counters.filesSkipped,
|
|
467
|
+
conflicts: counters.conflicts,
|
|
468
|
+
conflictPaths: counters.conflictPaths,
|
|
469
|
+
aborted: true,
|
|
470
|
+
newFiles: plan.newFiles,
|
|
471
|
+
newFilesCount: plan.newFilesCount,
|
|
472
|
+
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
473
|
+
filesTombstoned: 0,
|
|
474
|
+
filesOutOfScope: counters.filesOutOfScope,
|
|
475
|
+
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
476
|
+
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
if (resolution === "keep" || resolution === "skip") {
|
|
480
|
+
counters.filesSkipped++;
|
|
481
|
+
updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", remoteFile.etag, item.localMtime.getTime());
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
484
|
+
downloadItems.push({
|
|
485
|
+
action: "download",
|
|
486
|
+
remoteFile,
|
|
487
|
+
localPath,
|
|
488
|
+
isNew: false,
|
|
489
|
+
});
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
|
|
493
|
+
if (downloadItems.length === 0)
|
|
494
|
+
return;
|
|
495
|
+
await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
|
|
496
|
+
const queue = [...downloadItems];
|
|
497
|
+
const inFlight = new Set();
|
|
498
|
+
const workerErrors = [];
|
|
499
|
+
while (queue.length > 0 || inFlight.size > 0) {
|
|
500
|
+
while (inFlight.size < transferConcurrency && queue.length > 0) {
|
|
501
|
+
const downloadItem = queue.shift();
|
|
502
|
+
const p = downloadOne(run, downloadItem, counters)
|
|
503
|
+
.catch((err) => {
|
|
504
|
+
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
505
|
+
})
|
|
506
|
+
.finally(() => {
|
|
507
|
+
inFlight.delete(p);
|
|
508
|
+
});
|
|
509
|
+
inFlight.add(p);
|
|
510
|
+
}
|
|
511
|
+
if (inFlight.size > 0) {
|
|
512
|
+
await Promise.race(Array.from(inFlight));
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
if (workerErrors.length > 0) {
|
|
516
|
+
writeJournal(run.journalSlug, run.journal);
|
|
517
|
+
const first = workerErrors[0];
|
|
518
|
+
if (workerErrors.length > 1) {
|
|
519
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
520
|
+
}
|
|
521
|
+
throw first;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
async function downloadOne(run, downloadItem, counters) {
|
|
525
|
+
const { remoteFile, localPath } = downloadItem;
|
|
526
|
+
await refreshRunContextIfExpiring(run);
|
|
527
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
|
|
528
|
+
counters.filesSkipped++;
|
|
529
|
+
run.emit({
|
|
530
|
+
type: "error",
|
|
531
|
+
path: remoteFile.key,
|
|
532
|
+
message: "download skipped: local parent escaped the sync root",
|
|
533
|
+
});
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
try {
|
|
537
|
+
const { metadata, contentHash, contentSize } = await downloadFile(run.ctx, remoteFile.key, localPath);
|
|
538
|
+
const author = metadata?.["created-by"] ?? null;
|
|
539
|
+
const createdBySub = metadata?.["created-by-sub"];
|
|
540
|
+
const localLstat = fs.lstatSync(localPath);
|
|
541
|
+
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
542
|
+
const hash = isLocalSymlink
|
|
543
|
+
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
544
|
+
: (contentHash ?? hashFile(localPath));
|
|
545
|
+
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
546
|
+
updateEntry(run.journal, remoteFile.key, hash, size, "down", remoteFile.etag, localLstat.mtimeMs, createdBySub);
|
|
547
|
+
const priorEntry = getEntry(run.journal, remoteFile.key);
|
|
548
|
+
const remoteJournalMessage = priorEntry?.message;
|
|
549
|
+
run.emit({
|
|
550
|
+
type: "progress",
|
|
551
|
+
path: remoteFile.key,
|
|
552
|
+
bytes: size,
|
|
553
|
+
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
554
|
+
...(author ? { author } : {}),
|
|
555
|
+
});
|
|
556
|
+
counters.filesDownloaded++;
|
|
557
|
+
counters.bytesDownloaded += size;
|
|
558
|
+
}
|
|
559
|
+
catch (err) {
|
|
560
|
+
if (isAccessDenied(err)) {
|
|
561
|
+
counters.filesSkipped++;
|
|
562
|
+
}
|
|
563
|
+
else {
|
|
564
|
+
run.emit({
|
|
565
|
+
type: "error",
|
|
566
|
+
path: remoteFile.key,
|
|
567
|
+
message: err instanceof Error ? err.message : String(err),
|
|
568
|
+
});
|
|
752
569
|
}
|
|
753
570
|
}
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
// calls are best-effort and capped at 5 concurrent to avoid hammering S3.
|
|
571
|
+
}
|
|
572
|
+
async function emitAndReportNewFiles(run, plan) {
|
|
757
573
|
const enrichedNewFiles = [];
|
|
574
|
+
// Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
|
|
575
|
+
// created-by HEADs below reuse the cache instead of each minting its own
|
|
576
|
+
// presign. Without this, a big catch-up pull (hundreds of new files) bursts
|
|
577
|
+
// the presign endpoint, trips the circuit breaker, and every enrichment HEAD
|
|
578
|
+
// then fails. Mirrors the tombstone HEAD-verify pre-prime.
|
|
579
|
+
await primeObjectTransport(run.ctx, "get", plan.newFiles.map((nf) => nf.path));
|
|
758
580
|
const HEAD_CONCURRENCY = 5;
|
|
759
581
|
for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
|
|
760
582
|
const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
|
|
761
583
|
const results = await Promise.all(batch.map(async (nf) => {
|
|
762
584
|
let addedBy = null;
|
|
763
585
|
try {
|
|
764
|
-
const head = await headRemoteFile(ctx, nf.path);
|
|
586
|
+
const head = await headRemoteFile(run.ctx, nf.path);
|
|
765
587
|
if (head?.metadata?.["created-by"]) {
|
|
766
588
|
addedBy = head.metadata["created-by"];
|
|
767
589
|
}
|
|
768
590
|
}
|
|
769
591
|
catch (headErr) {
|
|
770
|
-
// Best-effort: log to console (Sentry captures via global handler)
|
|
771
|
-
// and fall through with addedBy = null.
|
|
772
592
|
try {
|
|
773
593
|
console.error(`[hq-sync] HeadObject failed for ${nf.path}: ${headErr instanceof Error ? headErr.message : String(headErr)}`);
|
|
774
594
|
}
|
|
@@ -780,210 +600,124 @@ export async function sync(options) {
|
|
|
780
600
|
}));
|
|
781
601
|
enrichedNewFiles.push(...results);
|
|
782
602
|
}
|
|
783
|
-
emit({ type: "new-files", files: enrichedNewFiles });
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
// - HEAD throws AccessDenied → can't tell → defensive skip; journal
|
|
802
|
-
// stays so next sync (with broader scope) can re-evaluate.
|
|
803
|
-
// - HEAD throws transient → defensive skip + emit error.
|
|
804
|
-
// Bounded concurrency mirrors the new-files attribution pass above.
|
|
805
|
-
if (plan.tombstones.length > 0) {
|
|
806
|
-
// Pre-mint GET URLs for the tombstone HEAD-verify probes below (headRemote
|
|
807
|
-
// File presigns a GET), so a large delete set doesn't add N presign calls.
|
|
808
|
-
await primeObjectTransport(ctx, "get", plan.tombstones);
|
|
809
|
-
const HEAD_VERIFY_CONCURRENCY = 5;
|
|
810
|
-
const verified = [];
|
|
811
|
-
for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
|
|
812
|
-
const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
|
|
813
|
-
const results = await Promise.all(batch.map(async (key) => {
|
|
814
|
-
try {
|
|
815
|
-
const head = await headRemoteFile(ctx, key);
|
|
816
|
-
return head === null ? key : null;
|
|
817
|
-
}
|
|
818
|
-
catch (err) {
|
|
819
|
-
if (isAccessDenied(err))
|
|
820
|
-
return null;
|
|
821
|
-
emit({
|
|
822
|
-
type: "error",
|
|
823
|
-
path: key,
|
|
824
|
-
message: `tombstone HEAD verify failed (deferring): ${err instanceof Error ? err.message : String(err)}`,
|
|
825
|
-
});
|
|
603
|
+
run.emit({ type: "new-files", files: enrichedNewFiles });
|
|
604
|
+
await reportNewFilesToNotify(run.vaultConfig, run.ctx.uid, run.ctx.slug, enrichedNewFiles);
|
|
605
|
+
}
|
|
606
|
+
async function verifyPlannedJournalTombstones(run, plan) {
|
|
607
|
+
if (plan.tombstones.length === 0)
|
|
608
|
+
return;
|
|
609
|
+
await primeObjectTransport(run.ctx, "get", plan.tombstones);
|
|
610
|
+
const HEAD_VERIFY_CONCURRENCY = 5;
|
|
611
|
+
const verified = [];
|
|
612
|
+
for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
|
|
613
|
+
const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
|
|
614
|
+
const results = await Promise.all(batch.map(async (key) => {
|
|
615
|
+
try {
|
|
616
|
+
const head = await headRemoteFile(run.ctx, key);
|
|
617
|
+
return head === null ? key : null;
|
|
618
|
+
}
|
|
619
|
+
catch (err) {
|
|
620
|
+
if (isAccessDenied(err))
|
|
826
621
|
return null;
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
622
|
+
run.emit({
|
|
623
|
+
type: "error",
|
|
624
|
+
path: key,
|
|
625
|
+
message: `tombstone HEAD verify failed (deferring): ${err instanceof Error ? err.message : String(err)}`,
|
|
626
|
+
});
|
|
627
|
+
return null;
|
|
832
628
|
}
|
|
629
|
+
}));
|
|
630
|
+
for (const k of results) {
|
|
631
|
+
if (k !== null)
|
|
632
|
+
verified.push(k);
|
|
833
633
|
}
|
|
834
|
-
plan.tombstones = verified;
|
|
835
634
|
}
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
// remnant) and drop the journal entry so the next sync's planner stays
|
|
840
|
-
// converged. Failures are reported but non-fatal — the entry stays in
|
|
841
|
-
// the journal and the next run retries.
|
|
635
|
+
plan.tombstones = verified;
|
|
636
|
+
}
|
|
637
|
+
function executeJournalTombstoneDeletes(run, plan, counters) {
|
|
842
638
|
for (const key of plan.tombstones) {
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
// collapses the backslashes and resolves onto the REAL POSIX file, so
|
|
846
|
-
// unlinking here destroys live data (ridge incident, feedback_b8d09d0f).
|
|
847
|
-
// The planner already refuses to enqueue malformed keys; if one still
|
|
848
|
-
// arrives, drop the poisoned journal entry without touching disk —
|
|
849
|
-
// normalizeJournalKeys rewrites it to its POSIX form on load.
|
|
850
|
-
if (isMalformedVaultKey(key)) {
|
|
851
|
-
removeEntry(journal, key);
|
|
639
|
+
const localPath = resolveContainedVaultPath(run.companyRoot, key);
|
|
640
|
+
if (localPath === null)
|
|
852
641
|
continue;
|
|
853
|
-
}
|
|
854
|
-
const localPath = path.join(companyRoot, key);
|
|
855
642
|
let removedSomething = false;
|
|
856
643
|
try {
|
|
857
644
|
const lstat = fs.lstatSync(localPath);
|
|
645
|
+
if (tombstoneTargetDiverged(run.journal, key, localPath, lstat)) {
|
|
646
|
+
continue;
|
|
647
|
+
}
|
|
858
648
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
859
649
|
fs.unlinkSync(localPath);
|
|
860
650
|
removedSomething = true;
|
|
861
651
|
}
|
|
862
652
|
else if (lstat.isDirectory()) {
|
|
863
|
-
// A dir at a key
|
|
864
|
-
// state. Don't recursively rm-rf the operator's dir; just drop
|
|
865
|
-
// the journal entry so we converge with reality.
|
|
653
|
+
// A dir at a key is converged by dropping only the journal entry.
|
|
866
654
|
}
|
|
867
655
|
}
|
|
868
656
|
catch (err) {
|
|
869
657
|
const code = err && typeof err === "object" && "code" in err
|
|
870
658
|
? err.code
|
|
871
659
|
: undefined;
|
|
872
|
-
// ENOENT → local already gone; safe to drop the journal entry.
|
|
873
|
-
// Other errors (EACCES/EPERM/EBUSY/etc.) leave the local file in
|
|
874
|
-
// place — if we dropped the journal entry anyway, the pull side
|
|
875
|
-
// would forget the peer's delete and a later push could re-upload
|
|
876
|
-
// the still-present local file, silently undoing the peer's delete.
|
|
877
|
-
// Surface the error and KEEP the journal entry so the next sync
|
|
878
|
-
// retries the unlink after the operator fixes the permission.
|
|
879
660
|
if (code !== "ENOENT") {
|
|
880
|
-
emit({
|
|
661
|
+
run.emit({
|
|
881
662
|
type: "error",
|
|
882
663
|
path: key,
|
|
883
664
|
message: `tombstone unlink failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
884
665
|
});
|
|
885
|
-
// Skip removeEntry / filesTombstoned / progress event — the
|
|
886
|
-
// tombstone hasn't actually been honored. Next sync retries.
|
|
887
666
|
continue;
|
|
888
667
|
}
|
|
889
668
|
}
|
|
890
|
-
removeEntry(journal, key);
|
|
891
|
-
filesTombstoned++;
|
|
892
|
-
emit({
|
|
669
|
+
removeEntry(run.journal, key);
|
|
670
|
+
counters.filesTombstoned++;
|
|
671
|
+
run.emit({
|
|
893
672
|
type: "progress",
|
|
894
673
|
path: key,
|
|
895
674
|
bytes: 0,
|
|
896
675
|
deleted: true,
|
|
897
|
-
// Suffix differentiates a tombstone from a normal delete in the
|
|
898
|
-
// tty stream — matches the push-side `defaultConsoleLogger`
|
|
899
|
-
// tombstone surface in share.ts.
|
|
900
676
|
message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
|
|
901
677
|
});
|
|
902
678
|
}
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
// the journal keys; `all` mode records `[""]` (covers everything).
|
|
907
|
-
appendPullRecord(journal, {
|
|
679
|
+
}
|
|
680
|
+
function finalizePullRun(run, plan, scopeRun, counters) {
|
|
681
|
+
appendPullRecord(run.journal, {
|
|
908
682
|
pullId: generatePullId(),
|
|
909
|
-
companyUid: ctx.uid,
|
|
910
|
-
startedAt,
|
|
683
|
+
companyUid: run.ctx.uid,
|
|
684
|
+
startedAt: run.startedAt,
|
|
911
685
|
completedAt: new Date().toISOString(),
|
|
912
|
-
syncMode,
|
|
913
|
-
prefixSet: currentPrefixSet,
|
|
914
|
-
scopeChangeDetected: shrinkPlan.scopeChangeDetected,
|
|
915
|
-
orphansRemoved: scopeOrphansRemoved,
|
|
916
|
-
orphansBlocked: shrinkResult.dirtyTombstoned,
|
|
686
|
+
syncMode: run.syncMode,
|
|
687
|
+
prefixSet: run.currentPrefixSet,
|
|
688
|
+
scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
|
|
689
|
+
orphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
690
|
+
orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
917
691
|
});
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
// When the pull actually changed on-disk sources (new files, tombstoned
|
|
925
|
-
// removals, or scope-orphan cleanups), refresh the generated skill wrappers,
|
|
926
|
-
// personal-overlay mirrors, and workers registry. reindex is idempotent and
|
|
927
|
-
// best-effort — it must never fail a sync, and is skipped on no-op syncs
|
|
928
|
-
// (the common daemon case) and when the caller opts out via skipReindex.
|
|
929
|
-
const changedOnDisk = filesDownloaded > 0 ||
|
|
930
|
-
filesTombstoned > 0 ||
|
|
931
|
-
scopeOrphansRemoved > 0;
|
|
932
|
-
if (!options.skipReindex && changedOnDisk) {
|
|
692
|
+
run.journal.lastSync = new Date().toISOString();
|
|
693
|
+
writeJournal(run.journalSlug, run.journal);
|
|
694
|
+
const changedOnDisk = counters.filesDownloaded > 0 ||
|
|
695
|
+
counters.filesTombstoned > 0 ||
|
|
696
|
+
scopeRun.scopeOrphansRemoved > 0;
|
|
697
|
+
if (!run.options.skipReindex && changedOnDisk) {
|
|
933
698
|
try {
|
|
934
|
-
|
|
935
|
-
// lock; reindex re-acquiring would refuse against our own live PID.
|
|
936
|
-
reindex({ repoRoot: hqRoot, skipLock: true });
|
|
699
|
+
reindex({ repoRoot: run.hqRoot, skipLock: true });
|
|
937
700
|
}
|
|
938
701
|
catch {
|
|
939
702
|
// best-effort: a post-sync refresh failure never fails the sync
|
|
940
703
|
}
|
|
941
704
|
}
|
|
942
705
|
return {
|
|
943
|
-
filesDownloaded,
|
|
944
|
-
bytesDownloaded,
|
|
945
|
-
filesSkipped,
|
|
946
|
-
conflicts,
|
|
947
|
-
conflictPaths,
|
|
706
|
+
filesDownloaded: counters.filesDownloaded,
|
|
707
|
+
bytesDownloaded: counters.bytesDownloaded,
|
|
708
|
+
filesSkipped: counters.filesSkipped,
|
|
709
|
+
conflicts: counters.conflicts,
|
|
710
|
+
conflictPaths: counters.conflictPaths,
|
|
948
711
|
aborted: false,
|
|
949
712
|
newFiles: plan.newFiles,
|
|
950
713
|
newFilesCount: plan.newFilesCount,
|
|
951
714
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
952
|
-
filesTombstoned,
|
|
953
|
-
filesOutOfScope,
|
|
954
|
-
scopeOrphansRemoved,
|
|
955
|
-
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
715
|
+
filesTombstoned: counters.filesTombstoned,
|
|
716
|
+
filesOutOfScope: counters.filesOutOfScope,
|
|
717
|
+
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
718
|
+
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
956
719
|
};
|
|
957
720
|
}
|
|
958
|
-
/**
|
|
959
|
-
* Resolve active company from .hq/config.json.
|
|
960
|
-
*/
|
|
961
|
-
function resolveActiveCompany(hqRoot) {
|
|
962
|
-
const configPath = path.join(hqRoot, ".hq", "config.json");
|
|
963
|
-
if (fs.existsSync(configPath)) {
|
|
964
|
-
try {
|
|
965
|
-
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
|
|
966
|
-
return config.activeCompany ?? config.companySlug;
|
|
967
|
-
}
|
|
968
|
-
catch {
|
|
969
|
-
// Ignore parse errors
|
|
970
|
-
}
|
|
971
|
-
}
|
|
972
|
-
return undefined;
|
|
973
|
-
}
|
|
974
|
-
/**
|
|
975
|
-
* Returns true when the remote object appears to have moved since the
|
|
976
|
-
* journal entry's last-recorded sync. Prefers ETag equality; falls back to
|
|
977
|
-
* `lastModified > syncedAt` for legacy entries written before remoteEtag
|
|
978
|
-
* was tracked. Conservative on tie (`<=` skews "remote unchanged").
|
|
979
|
-
*/
|
|
980
|
-
function hasRemoteChanged(remote, entry) {
|
|
981
|
-
if (entry.remoteEtag) {
|
|
982
|
-
return normalizeEtag(remote.etag) !== entry.remoteEtag;
|
|
983
|
-
}
|
|
984
|
-
const syncedAt = new Date(entry.syncedAt).getTime();
|
|
985
|
-
return remote.lastModified.getTime() > syncedAt;
|
|
986
|
-
}
|
|
987
721
|
/**
|
|
988
722
|
* Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
|
|
989
723
|
* written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
|
|
@@ -1012,6 +746,85 @@ function isRemoteRecreateAfterTombstone(remote, tombstone) {
|
|
|
1012
746
|
return true; // no remote timestamp → don't suppress
|
|
1013
747
|
return remoteMs > deletedAtMs;
|
|
1014
748
|
}
|
|
749
|
+
function hasTraversalSegment(key) {
|
|
750
|
+
return key.split("/").some((segment) => segment === "..");
|
|
751
|
+
}
|
|
752
|
+
function isPathWithin(root, candidate) {
|
|
753
|
+
const relative = path.relative(root, candidate);
|
|
754
|
+
return (relative === "" ||
|
|
755
|
+
(!relative.startsWith("..") && !path.isAbsolute(relative)));
|
|
756
|
+
}
|
|
757
|
+
function deepestExistingAncestor(start) {
|
|
758
|
+
let current = start;
|
|
759
|
+
for (;;) {
|
|
760
|
+
try {
|
|
761
|
+
fs.lstatSync(current);
|
|
762
|
+
return current;
|
|
763
|
+
}
|
|
764
|
+
catch (err) {
|
|
765
|
+
const code = err && typeof err === "object" && "code" in err
|
|
766
|
+
? err.code
|
|
767
|
+
: undefined;
|
|
768
|
+
if (code !== "ENOENT" && code !== "ENOTDIR")
|
|
769
|
+
return null;
|
|
770
|
+
}
|
|
771
|
+
const parent = path.dirname(current);
|
|
772
|
+
if (parent === current)
|
|
773
|
+
return null;
|
|
774
|
+
current = parent;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
function resolveContainedVaultPath(root, key) {
|
|
778
|
+
if (isMalformedVaultKey(key) || hasTraversalSegment(key))
|
|
779
|
+
return null;
|
|
780
|
+
const resolvedRoot = path.resolve(root);
|
|
781
|
+
const resolvedLocal = path.resolve(resolvedRoot, key);
|
|
782
|
+
if (!isPathWithin(resolvedRoot, resolvedLocal))
|
|
783
|
+
return null;
|
|
784
|
+
let realRoot;
|
|
785
|
+
try {
|
|
786
|
+
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
787
|
+
}
|
|
788
|
+
catch {
|
|
789
|
+
// If the vault root does not exist yet, no below-root symlink component can
|
|
790
|
+
// already exist to redirect this key. Preserve first-pull behavior.
|
|
791
|
+
return resolvedLocal;
|
|
792
|
+
}
|
|
793
|
+
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
794
|
+
if (existingAncestor === null)
|
|
795
|
+
return null;
|
|
796
|
+
try {
|
|
797
|
+
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
798
|
+
if (!isPathWithin(realRoot, realAncestor))
|
|
799
|
+
return null;
|
|
800
|
+
}
|
|
801
|
+
catch {
|
|
802
|
+
return null;
|
|
803
|
+
}
|
|
804
|
+
return resolvedLocal;
|
|
805
|
+
}
|
|
806
|
+
function isDownloadWritePathStillContained(root, key, localPath) {
|
|
807
|
+
const resolved = resolveContainedVaultPath(root, key);
|
|
808
|
+
return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
|
|
809
|
+
}
|
|
810
|
+
function tombstoneTargetDiverged(journal, key, localPath, lstat) {
|
|
811
|
+
const journalEntry = journal.files[key];
|
|
812
|
+
if (!journalEntry?.hash) {
|
|
813
|
+
return lstat.isSymbolicLink() || lstat.isFile();
|
|
814
|
+
}
|
|
815
|
+
try {
|
|
816
|
+
if (lstat.isSymbolicLink()) {
|
|
817
|
+
return hashSymlinkTarget(fs.readlinkSync(localPath)) !== journalEntry.hash;
|
|
818
|
+
}
|
|
819
|
+
if (lstat.isFile()) {
|
|
820
|
+
return hashFile(localPath) !== journalEntry.hash;
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
catch {
|
|
824
|
+
return true;
|
|
825
|
+
}
|
|
826
|
+
return false;
|
|
827
|
+
}
|
|
1015
828
|
/**
|
|
1016
829
|
* Stage-1 planning pass: classify every remote file into download / skip /
|
|
1017
830
|
* conflict buckets without performing any S3 transfers. Local hashes are
|
|
@@ -1037,7 +850,11 @@ prefixSet,
|
|
|
1037
850
|
fileTombstones = new Map()) {
|
|
1038
851
|
const items = [];
|
|
1039
852
|
for (const remoteFile of remoteFiles) {
|
|
1040
|
-
const localPath =
|
|
853
|
+
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
854
|
+
if (localPath === null) {
|
|
855
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
856
|
+
continue;
|
|
857
|
+
}
|
|
1041
858
|
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
1042
859
|
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
1043
860
|
// mirrors since 5.33.0, but the pull side downloaded them freely from
|
|
@@ -1048,16 +865,6 @@ fileTombstones = new Map()) {
|
|
|
1048
865
|
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
1049
866
|
continue;
|
|
1050
867
|
}
|
|
1051
|
-
// Malformed-key filter — keys with backslash separators pushed by
|
|
1052
|
-
// pre-5.47.2 Windows clients. Downloading one materializes a junk local
|
|
1053
|
-
// file whose NAME contains backslashes (it is not a path on POSIX), which
|
|
1054
|
-
// then churns conflict mirrors forever. Refuse at planning time, same
|
|
1055
|
-
// policy bucket as the ephemeral filter above. The bogus keys themselves
|
|
1056
|
-
// are cleaned server-side; this keeps clean trees clean in the meantime.
|
|
1057
|
-
if (isMalformedVaultKey(remoteFile.key)) {
|
|
1058
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
1059
|
-
continue;
|
|
1060
|
-
}
|
|
1061
868
|
if (personalMode &&
|
|
1062
869
|
remoteFile.key.startsWith("companies/") &&
|
|
1063
870
|
// EXEMPTION: companies/manifest.yaml is the routing source-of-truth
|
|
@@ -1403,12 +1210,8 @@ fileTombstones = new Map()) {
|
|
|
1403
1210
|
const posixKey = toPosixKey(key);
|
|
1404
1211
|
if (remoteKeySet.has(posixKey))
|
|
1405
1212
|
continue;
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
// POSIX file and unlinks live data. Leave it for normalizeJournalKeys to
|
|
1409
|
-
// rewrite to POSIX on the next write; the canonical key is re-evaluated
|
|
1410
|
-
// (and correctly tombstoned if genuinely remote-deleted) on a later pull.
|
|
1411
|
-
if (isMalformedVaultKey(key))
|
|
1213
|
+
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
1214
|
+
if (localPath === null)
|
|
1412
1215
|
continue;
|
|
1413
1216
|
// PersonalMode key gating — mirror the download branch.
|
|
1414
1217
|
if (personalMode && key.startsWith("companies/")) {
|
|
@@ -1424,7 +1227,6 @@ fileTombstones = new Map()) {
|
|
|
1424
1227
|
// Honor the current ignore filter — if a path was previously synced
|
|
1425
1228
|
// but is now ignored (operator edited .hqignore), do NOT delete
|
|
1426
1229
|
// the local copy. They're keeping it deliberately.
|
|
1427
|
-
const localPath = path.join(companyRoot, key);
|
|
1428
1230
|
if (!shouldSync(localPath, false) && !shouldSync(localPath, true))
|
|
1429
1231
|
continue;
|
|
1430
1232
|
// Codex P1 (PR #24 round 3): detect local edits before tombstoning.
|
|
@@ -1497,15 +1299,6 @@ fileTombstones = new Map()) {
|
|
|
1497
1299
|
tombstones,
|
|
1498
1300
|
};
|
|
1499
1301
|
}
|
|
1500
|
-
/**
|
|
1501
|
-
* Check if an error is an S3 access denied (expected for filtered guests).
|
|
1502
|
-
*/
|
|
1503
|
-
function isAccessDenied(err) {
|
|
1504
|
-
if (err && typeof err === "object" && "name" in err) {
|
|
1505
|
-
return err.name === "AccessDenied" || err.name === "Forbidden";
|
|
1506
|
-
}
|
|
1507
|
-
return false;
|
|
1508
|
-
}
|
|
1509
1302
|
/**
|
|
1510
1303
|
* Default human-readable event rendering. Preserves the exact output format
|
|
1511
1304
|
* that `hq sync` emitted before SyncProgressEvent was introduced, so callers
|