@indigoai-us/hq-cloud 6.18.75 → 6.18.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-codex-instructions.d.ts.map +1 -1
- package/dist/agent-codex-instructions.js +4 -1
- package/dist/agent-codex-instructions.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +16 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-manifest.d.ts.map +1 -1
- package/dist/bin/sync-runner-manifest.js +17 -2
- package/dist/bin/sync-runner-manifest.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +2 -1
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +11 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/reindex-knowledge.d.ts.map +1 -1
- package/dist/cli/reindex-knowledge.js +6 -1
- package/dist/cli/reindex-knowledge.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +77 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +467 -14
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +137 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +88 -17
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +71 -6
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +330 -24
- package/dist/cli/sync.js.map +1 -1
- package/dist/cloud-placeholder.d.ts +6 -0
- package/dist/cloud-placeholder.d.ts.map +1 -1
- package/dist/cloud-placeholder.js +16 -7
- package/dist/cloud-placeholder.js.map +1 -1
- package/dist/ignore.d.ts +31 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +68 -2
- package/dist/ignore.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -2
- package/dist/index.js.map +1 -1
- package/dist/lib/board-merge.d.ts +25 -0
- package/dist/lib/board-merge.d.ts.map +1 -0
- package/dist/lib/board-merge.js +201 -0
- package/dist/lib/board-merge.js.map +1 -0
- package/dist/lib/readlink-safe.d.ts +24 -1
- package/dist/lib/readlink-safe.d.ts.map +1 -1
- package/dist/lib/readlink-safe.js +29 -2
- package/dist/lib/readlink-safe.js.map +1 -1
- package/dist/manifest/build-manifest.d.ts.map +1 -1
- package/dist/manifest/build-manifest.js +2 -1
- package/dist/manifest/build-manifest.js.map +1 -1
- package/dist/manifest/upload-manifest.d.ts +11 -0
- package/dist/manifest/upload-manifest.d.ts.map +1 -1
- package/dist/manifest/upload-manifest.js +60 -3
- package/dist/manifest/upload-manifest.js.map +1 -1
- package/dist/multipart-upload.d.ts +96 -0
- package/dist/multipart-upload.d.ts.map +1 -0
- package/dist/multipart-upload.js +309 -0
- package/dist/multipart-upload.js.map +1 -0
- package/dist/offload-journal.d.ts +22 -0
- package/dist/offload-journal.d.ts.map +1 -0
- package/dist/offload-journal.js +55 -0
- package/dist/offload-journal.js.map +1 -0
- package/dist/s3.d.ts +15 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +22 -6
- package/dist/s3.js.map +1 -1
- package/dist/scope-shrink.d.ts +86 -3
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +177 -24
- package/dist/scope-shrink.js.map +1 -1
- package/dist/session-log-capture.d.ts.map +1 -1
- package/dist/session-log-capture.js +18 -2
- package/dist/session-log-capture.js.map +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +10 -5
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +4 -2
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +16 -4
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/pull-scope.d.ts +7 -0
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +2 -0
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/scope-inventory-client.d.ts.map +1 -1
- package/dist/sync/scope-inventory-client.js +2 -1
- package/dist/sync/scope-inventory-client.js.map +1 -1
- package/dist/types.d.ts +20 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +23 -0
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/sync.js
CHANGED
|
@@ -6,11 +6,11 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "fs";
|
|
8
8
|
import * as path from "path";
|
|
9
|
-
import { cloudPlaceholderPathFor, formatCloudPlaceholder, isCloudOnlyLocally, markCloudPlaceholderOrphaned, readCloudPlaceholder, writeCloudPlaceholder, } from "../cloud-placeholder.js";
|
|
9
|
+
import { cloudPlaceholderPathFor, formatCloudPlaceholder, hasCloudPlaceholderSibling, isCloudOnlyLocally, markCloudPlaceholderOrphaned, readCloudPlaceholder, writeCloudPlaceholder, } from "../cloud-placeholder.js";
|
|
10
10
|
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
11
11
|
import { getActiveFullPassDecodeWindow, getActiveFullPassUnitSize, } from "../sync/full-pass-caps.js";
|
|
12
12
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
13
|
-
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
13
|
+
import { readlinkOrNull, readlinkPortableSync } from "../lib/readlink-safe.js";
|
|
14
14
|
import { describeError } from "../lib/describe-error.js";
|
|
15
15
|
import { noteSwallowedError } from "../best-effort-error.js";
|
|
16
16
|
import { appendSyncDecision } from "../sync-decision-log.js";
|
|
@@ -30,21 +30,22 @@ import { readJournalForPass, readQuarantinedJournalPaths, writeJournal, setJourn
|
|
|
30
30
|
import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
|
|
31
31
|
import { isPersonalVaultExcluded, isSafePersonalMcpConfigFile } from "../personal-vault-exclusions.js";
|
|
32
32
|
import { resolvePersonalConnectorSettingsSync } from "../lib/personal-connector-settings-sync.js";
|
|
33
|
-
import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
|
|
33
|
+
import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, readPendingMoves, } from "../scope-shrink.js";
|
|
34
34
|
import { coalescePrefixes, compileScopeMatcher, intersectPrefixSets, pathToScopePrefix, toScopePrefixEntries, } from "../prefix-coalesce.js";
|
|
35
|
-
import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
35
|
+
import { SESSION_LOGS_PUSH_ONLY_PREFIX, readPinnedPrefixes } from "../sync/pull-scope.js";
|
|
36
36
|
import { contentHashMatchesRawSha256 } from "../sync/content-hash.js";
|
|
37
37
|
import { isKms10CopyOfJournalBase } from "../kms10-etag-safe-pull.js";
|
|
38
38
|
import { isTargetedUpsertExactListEnabled } from "../sync/targeted-upsert-exact-list.js";
|
|
39
39
|
import { resolveSymlinkTargetRefusalFlag } from "../sync/symlink-target-refusal-flag.js";
|
|
40
40
|
import { coalesceDenseTargetedPullPaths } from "../sync/targeted-pull-prefixes.js";
|
|
41
41
|
import { listRemoteForScope, POST_FILTER_THRESHOLD } from "../remote-pull.js";
|
|
42
|
-
import { createIgnoreFilter, isPathExcludedFromSync } from "../ignore.js";
|
|
42
|
+
import { createIgnoreFilter, isPathExcludedFromSync, SYNC_MAX_FILE_BYTES, } from "../ignore.js";
|
|
43
43
|
import { isSecretBearingVaultPath, isSkipSecretPathsEnabled, } from "../sync/secret-path-policy.js";
|
|
44
44
|
import { hasRemoteChanged, isAccessDenied, isNotFound, resolveActiveCompany, resolveTransferConcurrency, readHostLoad, } from "../sync-core.js";
|
|
45
45
|
import { formatIgnoreExcludedWarningLines, isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
46
46
|
import { pickWinner, resolveConflict, usesVersionAwareWinner } from "./conflict.js";
|
|
47
47
|
import { appendIfUnchanged, isAppendOnlyPath, mergeAppendOnly } from "../lib/append-only-merge.js";
|
|
48
|
+
import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/board-merge.js";
|
|
48
49
|
import { createHash } from "node:crypto";
|
|
49
50
|
import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
50
51
|
import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
|
|
@@ -196,20 +197,157 @@ function overlayPullJournalDelta(run) {
|
|
|
196
197
|
live.set(key, structuredClone(entry));
|
|
197
198
|
}
|
|
198
199
|
}
|
|
200
|
+
/** Same bypass and truthy parsing as share.ts `isBulkAsymmetryOverride`. */
|
|
201
|
+
function isScopeShrinkBulkOverride() {
|
|
202
|
+
const v = (process.env.HQ_SYNC_DELETE_BULK_OVERRIDE ?? "").toLowerCase();
|
|
203
|
+
return v === "1" || v === "true" || v === "yes";
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Whole-set ratio guard thresholds for the automatic scope shrink, the same
|
|
207
|
+
* defaults as the push-side bulk-asymmetry breaker (share.ts
|
|
208
|
+
* `BULK_ASYMMETRY_RATIO` / `BULK_ASYMMETRY_MIN_ABS`, policy
|
|
209
|
+
* sync-delete-bulk-asymmetry-guard): refuse when the orphans are at least
|
|
210
|
+
* 10% of the live in-scope `direction: down` rows AND at least 10 files.
|
|
211
|
+
*/
|
|
212
|
+
const SCOPE_SHRINK_BULK_RATIO = 0.10;
|
|
213
|
+
const SCOPE_SHRINK_BULK_MIN_ABS = 10;
|
|
214
|
+
const SCOPE_SHRINK_SAMPLE_CAP = 10;
|
|
215
|
+
/**
|
|
216
|
+
* Bounded auto-recovery for a `bulk-ratio` refusal. A shrink is a local,
|
|
217
|
+
* recoverable move (not a remote delete), and a real revocation is
|
|
218
|
+
* persistent while a bad grant response is usually transient. So once the
|
|
219
|
+
* SAME narrowed scope has been refused on at least
|
|
220
|
+
* `SCOPE_SHRINK_CONFIRM_PASSES` consecutive pulls AND the first of those
|
|
221
|
+
* refusals is at least this long ago, the shrink proceeds without a human.
|
|
222
|
+
* Any pull that sees a different scope (or is not refused) resets the count.
|
|
223
|
+
* Override the window with `HQ_SYNC_SCOPE_SHRINK_CONFIRM_MS` (non-negative
|
|
224
|
+
* integer milliseconds); the pass count is fixed.
|
|
225
|
+
*/
|
|
226
|
+
const SCOPE_SHRINK_CONFIRM_MS_DEFAULT = 30 * 60 * 1000;
|
|
227
|
+
const SCOPE_SHRINK_CONFIRM_PASSES = 3;
|
|
228
|
+
function resolveScopeShrinkConfirmMs() {
|
|
229
|
+
const raw = process.env.HQ_SYNC_SCOPE_SHRINK_CONFIRM_MS;
|
|
230
|
+
if (raw === undefined || raw.trim() === "")
|
|
231
|
+
return SCOPE_SHRINK_CONFIRM_MS_DEFAULT;
|
|
232
|
+
const parsed = Number(raw);
|
|
233
|
+
return Number.isInteger(parsed) && parsed >= 0 ? parsed : SCOPE_SHRINK_CONFIRM_MS_DEFAULT;
|
|
234
|
+
}
|
|
235
|
+
function sameScope(a, b) {
|
|
236
|
+
if (a.syncMode !== b.syncMode)
|
|
237
|
+
return false;
|
|
238
|
+
const left = [...a.prefixSet].sort();
|
|
239
|
+
const right = [...b.prefixSet].sort();
|
|
240
|
+
return left.length === right.length && left.every((p, i) => p === right[i]);
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Number of scope prefixes that came from the membership itself (explicit
|
|
244
|
+
* grants for `shared`, `customPaths` for `custom`), excluding pins. The
|
|
245
|
+
* runner passes the resolver's count; other callers fall back to removing
|
|
246
|
+
* this machine's pinned prefixes from the coalesced scope.
|
|
247
|
+
*/
|
|
248
|
+
function grantDerivedPrefixCount(run) {
|
|
249
|
+
if (run.options.scopeGrantPrefixCount !== undefined) {
|
|
250
|
+
return run.options.scopeGrantPrefixCount;
|
|
251
|
+
}
|
|
252
|
+
const pins = new Set(coalescePrefixes(readPinnedPrefixes(run.hqRoot, run.ctx.slug).map(pathToScopePrefix)));
|
|
253
|
+
return coalescePrefixes(run.currentPrefixSet).filter((p) => !pins.has(p)).length;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Whole-set guards for the automatic scope shrink. Returns a refusal (the
|
|
257
|
+
* shrink must not run this pass) or null.
|
|
258
|
+
*
|
|
259
|
+
* - `empty-grants`: a scoped (`shared`/`custom`) membership whose grant
|
|
260
|
+
* list resolved to nothing, ignoring pins. Refused on every pass until a
|
|
261
|
+
* human passes `forceScopeShrink` or sets HQ_SYNC_DELETE_BULK_OVERRIDE=1.
|
|
262
|
+
* - `bulk-ratio`: orphans >= 10% of in-scope rows and >= 10 files. Refused
|
|
263
|
+
* until the same narrowed scope has been seen for the confirm window, or
|
|
264
|
+
* either bypass above is used.
|
|
265
|
+
*/
|
|
266
|
+
function evaluateScopeShrinkGuard(run, scopePlan) {
|
|
267
|
+
const { shrinkPlan, lastRecord } = scopePlan;
|
|
268
|
+
const orphans = [...shrinkPlan.clean, ...shrinkPlan.dirty];
|
|
269
|
+
if (orphans.length === 0)
|
|
270
|
+
return null;
|
|
271
|
+
if (run.options.forceScopeShrink === true)
|
|
272
|
+
return null;
|
|
273
|
+
if (isScopeShrinkBulkOverride() && !scopeShrinkLockDisabled(run))
|
|
274
|
+
return null;
|
|
275
|
+
const inScope = Math.max(shrinkPlan.inScopeDownRows, orphans.length);
|
|
276
|
+
const ratio = inScope > 0 ? orphans.length / inScope : 1;
|
|
277
|
+
const scoped = run.syncMode === "shared" || run.syncMode === "custom";
|
|
278
|
+
let reason = null;
|
|
279
|
+
if (scopeShrinkLockDisabled(run)) {
|
|
280
|
+
reason = "lock-disabled";
|
|
281
|
+
}
|
|
282
|
+
else if (scoped && grantDerivedPrefixCount(run) === 0) {
|
|
283
|
+
reason = "empty-grants";
|
|
284
|
+
}
|
|
285
|
+
else if (orphans.length >= SCOPE_SHRINK_BULK_MIN_ABS &&
|
|
286
|
+
ratio >= SCOPE_SHRINK_BULK_RATIO) {
|
|
287
|
+
reason = "bulk-ratio";
|
|
288
|
+
}
|
|
289
|
+
if (!reason)
|
|
290
|
+
return null;
|
|
291
|
+
const current = { syncMode: run.syncMode, prefixSet: run.currentPrefixSet };
|
|
292
|
+
const prior = lastRecord?.pendingScopeShrink;
|
|
293
|
+
const nowMs = Date.now();
|
|
294
|
+
const samePending = prior !== undefined && prior.reason === reason && sameScope(prior, current);
|
|
295
|
+
const firstSeenAt = samePending ? prior.firstSeenAt : new Date(nowMs).toISOString();
|
|
296
|
+
const priorRefusedPasses = samePending ? (prior.refusedPasses ?? 1) : 0;
|
|
297
|
+
const confirmMs = resolveScopeShrinkConfirmMs();
|
|
298
|
+
if (reason === "bulk-ratio" &&
|
|
299
|
+
samePending &&
|
|
300
|
+
priorRefusedPasses >= SCOPE_SHRINK_CONFIRM_PASSES &&
|
|
301
|
+
nowMs - Date.parse(firstSeenAt) >= confirmMs) {
|
|
302
|
+
return null; // bounded auto-recovery: the narrowed scope is stable
|
|
303
|
+
}
|
|
304
|
+
const confirmAfter = reason === "bulk-ratio"
|
|
305
|
+
? new Date(Date.parse(firstSeenAt) + confirmMs).toISOString()
|
|
306
|
+
: undefined;
|
|
307
|
+
return {
|
|
308
|
+
event: {
|
|
309
|
+
reason,
|
|
310
|
+
candidates: orphans.length,
|
|
311
|
+
inScope,
|
|
312
|
+
ratio,
|
|
313
|
+
samplePaths: orphans.slice(0, SCOPE_SHRINK_SAMPLE_CAP).map((o) => o.path),
|
|
314
|
+
firstSeenAt,
|
|
315
|
+
refusedPasses: priorRefusedPasses + 1,
|
|
316
|
+
...(confirmAfter ? { confirmAfter } : {}),
|
|
317
|
+
},
|
|
318
|
+
pending: {
|
|
319
|
+
reason,
|
|
320
|
+
syncMode: run.syncMode,
|
|
321
|
+
prefixSet: [...run.currentPrefixSet],
|
|
322
|
+
firstSeenAt,
|
|
323
|
+
candidates: orphans.length,
|
|
324
|
+
refusedPasses: priorRefusedPasses + 1,
|
|
325
|
+
},
|
|
326
|
+
};
|
|
327
|
+
}
|
|
199
328
|
/**
|
|
200
|
-
* Resolve the auto-prune
|
|
201
|
-
* scope
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
329
|
+
* Resolve the optional auto-prune cap. An automatic scope shrink MOVES clean
|
|
330
|
+
* out-of-scope files into the per-company scope-quarantine directory under
|
|
331
|
+
* the HQ root's `.hq` folder (recoverable, never deleted, never propagated to
|
|
332
|
+
* the vault), so by default there is no cap: a member who loses access to a
|
|
333
|
+
* large folder has every unchanged copy quarantined in one sync. Set
|
|
334
|
+
* `HQ_SYNC_MAX_AUTO_PRUNE` to a positive integer to opt into a cap; a shrink
|
|
335
|
+
* over it is refused with `ScopeShrinkLargePruneError`. Unset, empty, `0`,
|
|
336
|
+
* or negative values mean unlimited (returns 0). Anything that is not a
|
|
337
|
+
* whole number (for example the typo `1OO`) also means unlimited, with a
|
|
338
|
+
* warning on stderr so the misconfiguration is not silent.
|
|
205
339
|
*/
|
|
206
340
|
export function resolveAutoPruneCap() {
|
|
207
341
|
const raw = process.env.HQ_SYNC_MAX_AUTO_PRUNE;
|
|
208
|
-
if (raw === undefined || raw === "")
|
|
209
|
-
return
|
|
342
|
+
if (raw === undefined || raw.trim() === "")
|
|
343
|
+
return 0;
|
|
344
|
+
if (!/^\s*[+-]?\d+\s*$/.test(raw)) {
|
|
345
|
+
warnOnce(`max-auto-prune:${raw}`, `hq-cloud: HQ_SYNC_MAX_AUTO_PRUNE=${JSON.stringify(raw)} is not a whole number; ignoring it (no scope-shrink cap). Set a positive integer to cap, or unset it.`);
|
|
346
|
+
return 0;
|
|
347
|
+
}
|
|
210
348
|
const parsed = Number.parseInt(raw, 10);
|
|
211
|
-
//
|
|
212
|
-
return
|
|
349
|
+
// 0 or negative → explicitly unlimited.
|
|
350
|
+
return parsed > 0 ? parsed : 0;
|
|
213
351
|
}
|
|
214
352
|
/** Max time to wait on the best-effort new-files notification POST. */
|
|
215
353
|
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
@@ -1066,6 +1204,58 @@ function createPullCounters() {
|
|
|
1066
1204
|
function reportableNewFiles(_run, plan) {
|
|
1067
1205
|
return plan.newFiles;
|
|
1068
1206
|
}
|
|
1207
|
+
/** Sidecar listing the moves an in-flight scope shrink has not finished. */
|
|
1208
|
+
function scopeShrinkPendingMovesFile(run) {
|
|
1209
|
+
return path.join(run.hqRoot, ".hq", "scope-quarantine", `${run.journalSlug}.pending-moves.json`);
|
|
1210
|
+
}
|
|
1211
|
+
/**
|
|
1212
|
+
* The sync operation lock serializes shrink passes (and therefore the
|
|
1213
|
+
* pending-moves sidecar). With the HQ_DISABLE_OP_LOCK escape hatch on and no
|
|
1214
|
+
* caller-held lock, two pulls could interleave their sidecar writes, so the
|
|
1215
|
+
* automatic shrink is refused (an explicit forceScopeShrink still runs).
|
|
1216
|
+
*/
|
|
1217
|
+
function scopeShrinkLockDisabled(run) {
|
|
1218
|
+
return (process.env.HQ_DISABLE_OP_LOCK === "1" &&
|
|
1219
|
+
run.options.operationLockAlreadyHeld !== true);
|
|
1220
|
+
}
|
|
1221
|
+
const warnedOnce = new Set();
|
|
1222
|
+
/** Emit a stderr warning at most once per process for `key`. */
|
|
1223
|
+
function warnOnce(key, message) {
|
|
1224
|
+
if (warnedOnce.has(key))
|
|
1225
|
+
return;
|
|
1226
|
+
warnedOnce.add(key);
|
|
1227
|
+
console.warn(message);
|
|
1228
|
+
}
|
|
1229
|
+
/** Test seam: forget which once-per-process warnings were already printed. */
|
|
1230
|
+
export function resetSyncWarnOnceForTests() {
|
|
1231
|
+
warnedOnce.clear();
|
|
1232
|
+
}
|
|
1233
|
+
/**
|
|
1234
|
+
* Pending moves an interrupted shrink left for recovery, or an empty set.
|
|
1235
|
+
* - corrupt sidecar: treated as unknown; no recovery pickup, warn once.
|
|
1236
|
+
* - sidecar written for a different scope: stale, removed.
|
|
1237
|
+
* - no listed path is still a scope_shrink tombstone: finished, removed.
|
|
1238
|
+
*/
|
|
1239
|
+
function resolvePendingMoves(run) {
|
|
1240
|
+
const file = scopeShrinkPendingMovesFile(run);
|
|
1241
|
+
const read = readPendingMoves(file);
|
|
1242
|
+
if (read.status === "missing")
|
|
1243
|
+
return new Set();
|
|
1244
|
+
if (read.status === "corrupt") {
|
|
1245
|
+
warnOnce(`pending-moves-corrupt:${file}`, `hq-cloud: scope-shrink pending-moves list ${file} is unreadable (${read.error}); unfinished quarantine moves will not be resumed automatically. Files are still on disk; run \`hq sync narrow --apply\` to finish them, then delete the file.`);
|
|
1246
|
+
return new Set();
|
|
1247
|
+
}
|
|
1248
|
+
const current = { syncMode: run.syncMode, prefixSet: run.currentPrefixSet };
|
|
1249
|
+
const stillPending = [...read.paths].some((p) => {
|
|
1250
|
+
const entry = journalRows(run.journal).get(p);
|
|
1251
|
+
return entry?.removedReason === "scope_shrink";
|
|
1252
|
+
});
|
|
1253
|
+
if (!read.scope || !sameScope(read.scope, current) || !stillPending) {
|
|
1254
|
+
fs.rmSync(file, { force: true });
|
|
1255
|
+
return new Set();
|
|
1256
|
+
}
|
|
1257
|
+
return read.paths;
|
|
1258
|
+
}
|
|
1069
1259
|
function planScopeShrink(run) {
|
|
1070
1260
|
const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
|
|
1071
1261
|
const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
|
|
@@ -1085,6 +1275,7 @@ function planScopeShrink(run) {
|
|
|
1085
1275
|
callerSub: run.options.callerSub,
|
|
1086
1276
|
protectUnknownAuthors: true,
|
|
1087
1277
|
...(run.skipSecretPaths ? { preservePath: isSecretBearingVaultPath } : {}),
|
|
1278
|
+
pendingMoves: resolvePendingMoves(run),
|
|
1088
1279
|
});
|
|
1089
1280
|
const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
|
|
1090
1281
|
const autoRecover = scopeShrinkPolicy === "auto-recover";
|
|
@@ -1100,6 +1291,14 @@ function planScopeShrink(run) {
|
|
|
1100
1291
|
}
|
|
1101
1292
|
function executeScopeShrink(run, scopePlan) {
|
|
1102
1293
|
const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
|
|
1294
|
+
// Whole-set guards run BEFORE the dirty check, so a bad grant response can
|
|
1295
|
+
// never surface as a dirty-file error whose advice (`--force-scope-shrink`)
|
|
1296
|
+
// would also bypass these guards.
|
|
1297
|
+
const refusal = evaluateScopeShrinkGuard(run, scopePlan);
|
|
1298
|
+
if (refusal) {
|
|
1299
|
+
run.emit({ type: "scope-shrink-refused", ...refusal.event });
|
|
1300
|
+
return { ...emptyTargetedScopeRun(run), refused: refusal.pending };
|
|
1301
|
+
}
|
|
1103
1302
|
if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
|
|
1104
1303
|
throw new ScopeShrinkBlockedError(run.ctx.uid, lastRecord?.syncMode ?? "unknown", run.syncMode, shrinkPlan.dirty, shrinkPlan.clean, adviceContext);
|
|
1105
1304
|
}
|
|
@@ -1118,6 +1317,12 @@ function executeScopeShrink(run, scopePlan) {
|
|
|
1118
1317
|
reason: "scope_shrink",
|
|
1119
1318
|
cleanDisposition: "quarantine",
|
|
1120
1319
|
quarantineRoot: scopeQuarantineRoot,
|
|
1320
|
+
// Durable tombstones before any file moves: an abort or throw later in
|
|
1321
|
+
// this pull must never leave live direction:down rows for moved files,
|
|
1322
|
+
// which the next push leg could plan as remote deletes.
|
|
1323
|
+
persistTombstones: () => persistPullJournal(run),
|
|
1324
|
+
pendingMovesFile: scopeShrinkPendingMovesFile(run),
|
|
1325
|
+
pendingMovesScope: { syncMode: run.syncMode, prefixSet: run.currentPrefixSet },
|
|
1121
1326
|
});
|
|
1122
1327
|
for (const relPath of shrinkResult.quarantinedPaths) {
|
|
1123
1328
|
run.emit({
|
|
@@ -1221,6 +1426,10 @@ async function executeConflictExecutor(run, plan, scopeRun, counters) {
|
|
|
1221
1426
|
executeCloudPlaceholderUpdate(run, item, counters);
|
|
1222
1427
|
continue;
|
|
1223
1428
|
}
|
|
1429
|
+
if (item.action === "create-cloud-placeholder") {
|
|
1430
|
+
executeCloudPlaceholderCreate(run, item, counters);
|
|
1431
|
+
continue;
|
|
1432
|
+
}
|
|
1224
1433
|
if (item.action === "orphan-cloud-placeholder") {
|
|
1225
1434
|
executeCloudPlaceholderOrphan(run, item, counters);
|
|
1226
1435
|
continue;
|
|
@@ -1328,6 +1537,40 @@ function executeCloudPlaceholderUpdate(run, item, counters) {
|
|
|
1328
1537
|
});
|
|
1329
1538
|
}
|
|
1330
1539
|
}
|
|
1540
|
+
/**
|
|
1541
|
+
* A remote object over the sync size limit that this machine never held.
|
|
1542
|
+
* Write a cloud-only placeholder for it instead of downloading. No journal row
|
|
1543
|
+
* is written: the placeholder's `remoteEtag` is the baseline the next pull
|
|
1544
|
+
* compares against, and `hq storage fetch` records the row once the bytes are
|
|
1545
|
+
* here and hash-verified. Re-checks the path first so a file or placeholder
|
|
1546
|
+
* that appeared since planning is never overwritten.
|
|
1547
|
+
*/
|
|
1548
|
+
function executeCloudPlaceholderCreate(run, item, counters) {
|
|
1549
|
+
const key = item.remoteFile.key;
|
|
1550
|
+
counters.filesSkipped++;
|
|
1551
|
+
if (fs.existsSync(item.localPath) || hasCloudPlaceholderSibling(item.localPath))
|
|
1552
|
+
return;
|
|
1553
|
+
try {
|
|
1554
|
+
fs.mkdirSync(path.dirname(item.localPath), { recursive: true });
|
|
1555
|
+
writeCloudPlaceholder(cloudPlaceholderPathFor(item.localPath), {
|
|
1556
|
+
path: path.relative(run.hqRoot, item.localPath).split(path.sep).join("/"),
|
|
1557
|
+
size: item.remoteFile.size,
|
|
1558
|
+
sha256: null,
|
|
1559
|
+
vault: run.options.personalMode === true ? "personal" : "company",
|
|
1560
|
+
key,
|
|
1561
|
+
remoteEtag: normalizeEtag(item.remoteFile.etag),
|
|
1562
|
+
});
|
|
1563
|
+
}
|
|
1564
|
+
catch (err) {
|
|
1565
|
+
run.emit({
|
|
1566
|
+
type: "error",
|
|
1567
|
+
path: key,
|
|
1568
|
+
message: `cloud placeholder create failed: ${describeError(err)}`,
|
|
1569
|
+
});
|
|
1570
|
+
return;
|
|
1571
|
+
}
|
|
1572
|
+
run.emit({ type: "skip-cloud-only", path: key, bytes: item.remoteFile.size });
|
|
1573
|
+
}
|
|
1331
1574
|
/**
|
|
1332
1575
|
* The remote object behind a cloud-only placeholder was deleted (FILE_TOMBSTONE
|
|
1333
1576
|
* or this machine's tombstone). Keep the placeholder but mark it orphaned so
|
|
@@ -1566,7 +1809,12 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1566
1809
|
// the REMOTE hash/etag) and skip the conflict path entirely. This removes both
|
|
1567
1810
|
// the `.conflict-` mirror loop AND the journal false-stamp (remote etag over
|
|
1568
1811
|
// divergent local content) that the keep path produced for these files.
|
|
1569
|
-
|
|
1812
|
+
// board.json is client-written as well as cloud-listed: a divergent board is
|
|
1813
|
+
// merged by entry id below instead of replaced from the cloud, so a partial
|
|
1814
|
+
// cloud board can never wipe the entries this machine holds.
|
|
1815
|
+
const boardMerge = isBoardJsonPath(vaultKeyForLocalPath(run.hqRoot, localPath)) &&
|
|
1816
|
+
usesVersionAwareWinner(run.options.onConflict);
|
|
1817
|
+
if (!boardMerge && isCloudAuthoritative(vaultKeyForLocalPath(run.hqRoot, localPath))) {
|
|
1570
1818
|
const currentSnapshot = snapshotLocalState(localPath);
|
|
1571
1819
|
if (currentSnapshot === null) {
|
|
1572
1820
|
counters.filesSkipped++;
|
|
@@ -1909,7 +2157,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1909
2157
|
remoteFetched &&
|
|
1910
2158
|
executionLocal?.kind === "file" &&
|
|
1911
2159
|
!probeIsSymlink &&
|
|
1912
|
-
isAppendOnlyPath(remoteFile.key)) {
|
|
2160
|
+
(isAppendOnlyPath(remoteFile.key) || boardMerge)) {
|
|
1913
2161
|
let localBody = null;
|
|
1914
2162
|
let remoteBody = null;
|
|
1915
2163
|
try {
|
|
@@ -1921,18 +2169,20 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1921
2169
|
noteSwallowedError((line) => process.stderr.write(line), "hq-cloud.append_only_read_failed", error, { quietOnAbsence: false });
|
|
1922
2170
|
localBody = null;
|
|
1923
2171
|
}
|
|
1924
|
-
const
|
|
2172
|
+
const merge = boardMerge ? mergeBoardJson : mergeAppendOnly;
|
|
2173
|
+
const writeMerged = boardMerge ? replaceIfUnchanged : appendIfUnchanged;
|
|
2174
|
+
const merged = localBody && remoteBody ? merge(localBody, remoteBody) : null;
|
|
1925
2175
|
if (localBody && remoteBody && merged) {
|
|
1926
2176
|
let appended;
|
|
1927
2177
|
try {
|
|
1928
|
-
appended =
|
|
2178
|
+
appended = writeMerged(localPath, localBody, merged);
|
|
1929
2179
|
}
|
|
1930
2180
|
catch (error) {
|
|
1931
2181
|
// A real write failure, not a race: say so instead of skipping quietly.
|
|
1932
2182
|
run.emit({
|
|
1933
2183
|
type: "error",
|
|
1934
2184
|
path: remoteFile.key,
|
|
1935
|
-
message:
|
|
2185
|
+
message: `${boardMerge ? "could not merge the cloud board entries into the local board" : "could not add the cloud's lines to the local file"}: ${describeError(error)}`,
|
|
1936
2186
|
});
|
|
1937
2187
|
counters.filesSkipped++;
|
|
1938
2188
|
return null;
|
|
@@ -2956,15 +3206,33 @@ async function executeJournalTombstoneDeletes(run, plan, counters) {
|
|
|
2956
3206
|
});
|
|
2957
3207
|
}
|
|
2958
3208
|
}
|
|
3209
|
+
/** Scope of the last recorded pull (what the local mirror still holds). */
|
|
3210
|
+
function scopePlanPreviousMode(run) {
|
|
3211
|
+
return lastPullRecord(run.journal, run.ctx.uid)?.syncMode ?? "all";
|
|
3212
|
+
}
|
|
3213
|
+
function scopePlanPreviousPrefixSet(run) {
|
|
3214
|
+
return [...(lastPullRecord(run.journal, run.ctx.uid)?.prefixSet ?? [])];
|
|
3215
|
+
}
|
|
2959
3216
|
function finalizePullRun(run, plan, scopeRun, counters) {
|
|
2960
3217
|
appendPullRecord(run.journal, {
|
|
2961
3218
|
pullId: generatePullId(),
|
|
2962
3219
|
companyUid: run.ctx.uid,
|
|
2963
3220
|
startedAt: run.startedAt,
|
|
2964
3221
|
completedAt: new Date().toISOString(),
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
3222
|
+
// A refused shrink keeps the previous scope on the record: the local
|
|
3223
|
+
// mirror still holds it, and the next pull must see the same orphans.
|
|
3224
|
+
...(scopeRun.refused
|
|
3225
|
+
? {
|
|
3226
|
+
syncMode: scopePlanPreviousMode(run),
|
|
3227
|
+
prefixSet: scopePlanPreviousPrefixSet(run),
|
|
3228
|
+
scopeChangeDetected: true,
|
|
3229
|
+
pendingScopeShrink: scopeRun.refused,
|
|
3230
|
+
}
|
|
3231
|
+
: {
|
|
3232
|
+
syncMode: run.syncMode,
|
|
3233
|
+
prefixSet: run.currentPrefixSet,
|
|
3234
|
+
scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
|
|
3235
|
+
}),
|
|
2968
3236
|
orphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
2969
3237
|
orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
2970
3238
|
});
|
|
@@ -3152,7 +3420,7 @@ function snapshotLocalState(localPath) {
|
|
|
3152
3420
|
try {
|
|
3153
3421
|
const lstat = fs.lstatSync(localPath);
|
|
3154
3422
|
if (lstat.isSymbolicLink()) {
|
|
3155
|
-
return { kind: "symlink", hash: hashSymlinkTarget(
|
|
3423
|
+
return { kind: "symlink", hash: hashSymlinkTarget(readlinkPortableSync(localPath)) };
|
|
3156
3424
|
}
|
|
3157
3425
|
if (lstat.isFile()) {
|
|
3158
3426
|
return { kind: "file", hash: hashFile(localPath) };
|
|
@@ -3343,6 +3611,7 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
3343
3611
|
item.action === "skip-kms10-copy" ||
|
|
3344
3612
|
item.action === "tombstone-delete" ||
|
|
3345
3613
|
item.action === "update-cloud-placeholder" ||
|
|
3614
|
+
item.action === "create-cloud-placeholder" ||
|
|
3346
3615
|
item.action === "orphan-cloud-placeholder" ||
|
|
3347
3616
|
item.action === "skip-junk-key-spelling" ||
|
|
3348
3617
|
item.action === "skip-unreadable-link") {
|
|
@@ -3873,6 +4142,26 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
3873
4142
|
}
|
|
3874
4143
|
continue;
|
|
3875
4144
|
}
|
|
4145
|
+
// Any other `.hqcloud` sibling (malformed, oversized, or naming a
|
|
4146
|
+
// different file) is a fail-closed hold: never read it as a local
|
|
4147
|
+
// delete, never tombstone, never download over it. readCloudPlaceholder
|
|
4148
|
+
// above already logged the one-time warning.
|
|
4149
|
+
if (hasCloudPlaceholderSibling(localPath)) {
|
|
4150
|
+
items.push({ action: "skip-unchanged", remoteFile, localPath });
|
|
4151
|
+
continue;
|
|
4152
|
+
}
|
|
4153
|
+
// An object over the sync size limit reached the vault through
|
|
4154
|
+
// `hq storage offload`, never through sync. This machine has never
|
|
4155
|
+
// held it, so it arrives cloud-only: write a placeholder and leave
|
|
4156
|
+
// the bytes for `hq storage fetch`. Deleted objects keep their
|
|
4157
|
+
// tombstone handling below.
|
|
4158
|
+
if ((journalEntry === undefined || isTombstone(journalEntry)) &&
|
|
4159
|
+
!tombstoneSuppresses &&
|
|
4160
|
+
!localTombstoneSuppresses &&
|
|
4161
|
+
remoteFile.size > SYNC_MAX_FILE_BYTES) {
|
|
4162
|
+
items.push({ action: "create-cloud-placeholder", remoteFile, localPath });
|
|
4163
|
+
continue;
|
|
4164
|
+
}
|
|
3876
4165
|
}
|
|
3877
4166
|
if (localExists) {
|
|
3878
4167
|
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
@@ -4489,7 +4778,7 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4489
4778
|
// readlink hash against the journal baseline; defer on
|
|
4490
4779
|
// divergence. readlink errors (race / EACCES) also defer.
|
|
4491
4780
|
try {
|
|
4492
|
-
const localHash = hashSymlinkTarget(
|
|
4781
|
+
const localHash = hashSymlinkTarget(readlinkPortableSync(localPath));
|
|
4493
4782
|
const journalEntry = journalRows(journal).get(key);
|
|
4494
4783
|
if (journalEntry?.hash && journalEntry.hash !== localHash) {
|
|
4495
4784
|
continue;
|
|
@@ -4652,10 +4941,27 @@ function defaultConsoleLogger(event) {
|
|
|
4652
4941
|
const pct = Math.round((event.attempted / Math.max(1, event.totalRows)) * 100);
|
|
4653
4942
|
console.warn(` ! skipped a mass journal purge — ${event.attempted}/${event.totalRows} rows (${pct}%) would have been dropped in one pass; journal kept intact (likely a transient scope fault, not a real change)`);
|
|
4654
4943
|
}
|
|
4944
|
+
else if (event.type === "skip-cloud-only") {
|
|
4945
|
+
const mb = (event.bytes / (1024 * 1024)).toFixed(1);
|
|
4946
|
+
console.log(` · ${event.path} left in the cloud (${mb} MB, over the 50 MB sync limit); run hq storage fetch to download it`);
|
|
4947
|
+
}
|
|
4655
4948
|
else if (event.type === "skip-size-limit") {
|
|
4656
4949
|
const mb = (event.bytes / (1024 * 1024)).toFixed(1);
|
|
4657
4950
|
console.warn(` ! ${event.path} skipped — ${mb} MB exceeds the 50 MB sync size limit`);
|
|
4658
4951
|
}
|
|
4952
|
+
else if (event.type === "scope-shrink-refused") {
|
|
4953
|
+
const why = event.reason === "empty-grants"
|
|
4954
|
+
? "your sync scope came back with no shared folders, which usually means a server or grant problem"
|
|
4955
|
+
: event.reason === "lock-disabled"
|
|
4956
|
+
? "HQ_DISABLE_OP_LOCK=1 is set, so another sync could run at the same time"
|
|
4957
|
+
: `${event.candidates} of ${event.inScope} synced files would leave your sync scope at once`;
|
|
4958
|
+
const next = event.reason === "bulk-ratio" && event.confirmAfter
|
|
4959
|
+
? `It will proceed automatically if the scope is unchanged after ${event.confirmAfter}.`
|
|
4960
|
+
: "It will be retried on every sync.";
|
|
4961
|
+
console.warn(` ⚠ scope-shrink guard: kept ${event.candidates} local file(s) in place because ${why}. ${next} If you really lost access, re-run with --force-scope-shrink or set HQ_SYNC_DELETE_BULK_OVERRIDE=1.`);
|
|
4962
|
+
for (const samplePath of event.samplePaths.slice(0, 5))
|
|
4963
|
+
console.warn(` · ${samplePath}`);
|
|
4964
|
+
}
|
|
4659
4965
|
else if (event.type === "delete-refused-mass-guard") {
|
|
4660
4966
|
const explanation = event.reason === "absolute-volume"
|
|
4661
4967
|
? `${event.candidates} planned deletes reached the absolute ceiling of ${event.threshold ?? MASS_DELETE_ABSOLUTE_CEILING}`
|