@indigoai-us/hq-cloud 6.18.109 → 6.18.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/area-ledger-migration.d.ts.map +1 -1
- package/dist/bin/area-ledger-migration.js +5 -3
- package/dist/bin/area-ledger-migration.js.map +1 -1
- package/dist/bin/backup-prune-runner.js +2 -2
- package/dist/bin/backup-prune-runner.js.map +1 -1
- package/dist/bin/runner-output-guard.d.ts +39 -0
- package/dist/bin/runner-output-guard.d.ts.map +1 -0
- package/dist/bin/runner-output-guard.js +104 -0
- package/dist/bin/runner-output-guard.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +20 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +134 -6
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +42 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +37 -3
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/conflict.d.ts +30 -1
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +69 -3
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/reconcile-skip-reason.d.ts +13 -3
- package/dist/cli/reconcile-skip-reason.d.ts.map +1 -1
- package/dist/cli/reconcile-skip-reason.js +18 -0
- package/dist/cli/reconcile-skip-reason.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +231 -44
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +98 -2
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +501 -76
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-hq-registrations.d.ts +45 -0
- package/dist/cli/rescue-hq-registrations.d.ts.map +1 -0
- package/dist/cli/rescue-hq-registrations.js +252 -0
- package/dist/cli/rescue-hq-registrations.js.map +1 -0
- package/dist/cli/rescue.d.ts +8 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +64 -17
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/share.d.ts +35 -8
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +491 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +70 -5
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +445 -79
- package/dist/cli/sync.js.map +1 -1
- package/dist/cognito-auth.d.ts +10 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +162 -1
- package/dist/cognito-auth.js.map +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +43 -21
- package/dist/ignore.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +3 -0
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -0
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-decoder.js +2 -0
- package/dist/journal-snapshot-decoder.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -1
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal.d.ts +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +20 -20
- package/dist/journal.js.map +1 -1
- package/dist/lib/append-log-merge.d.ts +43 -0
- package/dist/lib/append-log-merge.d.ts.map +1 -0
- package/dist/lib/append-log-merge.js +424 -0
- package/dist/lib/append-log-merge.js.map +1 -0
- package/dist/lib/same-path-upload-lock.d.ts +27 -1
- package/dist/lib/same-path-upload-lock.d.ts.map +1 -1
- package/dist/lib/same-path-upload-lock.js +32 -1
- package/dist/lib/same-path-upload-lock.js.map +1 -1
- package/dist/object-io.d.ts +36 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +97 -13
- package/dist/object-io.js.map +1 -1
- package/dist/operation-lock.d.ts +27 -1
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +94 -8
- package/dist/operation-lock.js.map +1 -1
- package/dist/outposts/client.d.ts +19 -6
- package/dist/outposts/client.d.ts.map +1 -1
- package/dist/outposts/client.js +16 -5
- package/dist/outposts/client.js.map +1 -1
- package/dist/outposts/command.d.ts +2 -1
- package/dist/outposts/command.d.ts.map +1 -1
- package/dist/outposts/command.js +81 -13
- package/dist/outposts/command.js.map +1 -1
- package/dist/outposts/self-deploy.d.ts.map +1 -1
- package/dist/outposts/self-deploy.js +3 -0
- package/dist/outposts/self-deploy.js.map +1 -1
- package/dist/private-folder-transport.d.ts +0 -2
- package/dist/private-folder-transport.d.ts.map +1 -1
- package/dist/private-folder-transport.js +17 -16
- package/dist/private-folder-transport.js.map +1 -1
- package/dist/push-identity-fence.d.ts +116 -3
- package/dist/push-identity-fence.d.ts.map +1 -1
- package/dist/push-identity-fence.js +625 -62
- package/dist/push-identity-fence.js.map +1 -1
- package/dist/qmd-reindex.d.ts.map +1 -1
- package/dist/qmd-reindex.js +3 -2
- package/dist/qmd-reindex.js.map +1 -1
- package/dist/s3.d.ts +30 -9
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +110 -82
- package/dist/s3.js.map +1 -1
- package/dist/session-log-retention.d.ts +86 -0
- package/dist/session-log-retention.d.ts.map +1 -1
- package/dist/session-log-retention.js +127 -19
- package/dist/session-log-retention.js.map +1 -1
- package/dist/sync/area-ledger-migration.d.ts +17 -0
- package/dist/sync/area-ledger-migration.d.ts.map +1 -1
- package/dist/sync/area-ledger-migration.js +37 -3
- package/dist/sync/area-ledger-migration.js.map +1 -1
- package/dist/sync/scattered-delete-burst.d.ts +61 -0
- package/dist/sync/scattered-delete-burst.d.ts.map +1 -0
- package/dist/sync/scattered-delete-burst.js +111 -0
- package/dist/sync/scattered-delete-burst.js.map +1 -0
- package/dist/sync/unattended-delete-hold.d.ts +102 -0
- package/dist/sync/unattended-delete-hold.d.ts.map +1 -0
- package/dist/sync/unattended-delete-hold.js +164 -0
- package/dist/sync/unattended-delete-hold.js.map +1 -0
- package/dist/sync-core.d.ts +5 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -6
- package/dist/sync-core.js.map +1 -1
- package/dist/types.d.ts +13 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts +6 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +53 -8
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-key-grammar.d.ts +44 -0
- package/dist/vault-key-grammar.d.ts.map +1 -0
- package/dist/vault-key-grammar.js +158 -0
- package/dist/vault-key-grammar.js.map +1 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +4 -1
- package/dist/watcher.js.map +1 -1
- package/dist/windows-extended-path.d.ts +22 -0
- package/dist/windows-extended-path.d.ts.map +1 -0
- package/dist/windows-extended-path.js +55 -0
- package/dist/windows-extended-path.js.map +1 -0
- package/package.json +1 -1
package/dist/cli/sync.js
CHANGED
|
@@ -9,7 +9,7 @@ import { localLinkWarning, rememberLinkWarning, brokenLinkSkipEvent, LinkWarning
|
|
|
9
9
|
import * as fs from "fs";
|
|
10
10
|
import * as path from "path";
|
|
11
11
|
import { isDeepStrictEqual } from "node:util";
|
|
12
|
-
import { cloudPlaceholderPathFor, formatCloudPlaceholder, hasCloudPlaceholderSibling, hasLocalOriginalForMissingPlaceholder, isCloudOnlyLocally, markCloudPlaceholderOrphaned, readCloudPlaceholder, writeCloudPlaceholder, } from "../cloud-placeholder.js";
|
|
12
|
+
import { CLOUD_PLACEHOLDER_SUFFIX, cloudPlaceholderPathFor, formatCloudPlaceholder, hasCloudPlaceholderSibling, hasLocalOriginalForMissingPlaceholder, isCloudOnlyLocally, markCloudPlaceholderOrphaned, readCloudPlaceholder, writeCloudPlaceholder, } from "../cloud-placeholder.js";
|
|
13
13
|
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
14
14
|
import { getActiveFullPassDecodeWindow, getActiveFullPassUnitSize, } from "../sync/full-pass-caps.js";
|
|
15
15
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
@@ -45,12 +45,14 @@ import { resolveSymlinkTargetRefusalFlag } from "../sync/symlink-target-refusal-
|
|
|
45
45
|
import { coalesceDenseTargetedPullPaths } from "../sync/targeted-pull-prefixes.js";
|
|
46
46
|
import { listRemoteForScope, POST_FILTER_THRESHOLD } from "../remote-pull.js";
|
|
47
47
|
import { createIgnoreFilter, isPathExcludedFromSync, SYNC_MAX_FILE_BYTES, } from "../ignore.js";
|
|
48
|
+
import { formatSizeLimitSkipLine } from "../session-log-retention.js";
|
|
48
49
|
import { claimNeverSyncedLocalNotices, formatSecretsNotSyncedLine, isNeverSyncedVaultPath, isSecretBearingVaultPath, isSkipSecretPathsEnabled, isWalRejectedDeltaRecoveryEnabled, } from "../sync/secret-path-policy.js";
|
|
49
50
|
import { hasRemoteChanged, isAccessDenied, isNotFound, resolveActiveCompany, resolveTransferConcurrency, readHostLoad, } from "../sync-core.js";
|
|
50
51
|
import { formatIgnoreExcludedWarningLines, isEphemeralPath, isMalformedVaultKey, rebindContextToLocalFolder } from "./share.js";
|
|
51
52
|
import { createHumanSkipLineLog, formatReconcileSkipLine, } from "./reconcile-skip-reason.js";
|
|
52
|
-
import { CONTINUITY_POINTER_KEY, parseContinuityCreatedAt, pickWinner, resolveConflict, usesVersionAwareWinner, } from "./conflict.js";
|
|
53
|
+
import { CONTINUITY_POINTER_KEY, countManifestCompanies, parseContinuityCreatedAt, pickWinner, resolveConflict, usesVersionAwareWinner, } from "./conflict.js";
|
|
53
54
|
import { appendIfUnchanged, isAppendOnlyPath, mergeAppendOnly } from "../lib/append-only-merge.js";
|
|
55
|
+
import { isAppendLogPath, mergeAppendLog, writeAppendLogMerge } from "../lib/append-log-merge.js";
|
|
54
56
|
import { findUploadIntent } from "../lib/same-path-upload-lock.js";
|
|
55
57
|
import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/board-merge.js";
|
|
56
58
|
import { createHash } from "node:crypto";
|
|
@@ -58,6 +60,8 @@ import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readS
|
|
|
58
60
|
import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
|
|
59
61
|
import { isCloudAuthoritative, isPullReconciledServerOwned, } from "../lib/cloud-authoritative.js";
|
|
60
62
|
import { isMassDeleteGuardOverrideEnabled, MASS_DELETE_ABSOLUTE_CEILING, MASS_DELETE_SAMPLE_CAP, massDeleteGuardReason, resolveMassDeleteGuards, } from "../sync/mass-delete-guards.js";
|
|
63
|
+
import { detectScatteredDeleteBurst } from "../sync/scattered-delete-burst.js";
|
|
64
|
+
import { clearUnattendedHeldRow, resolveUnattendedDeleteHold, UNKNOWN_CREATOR, unattendedPullDeleteDecision, } from "../sync/unattended-delete-hold.js";
|
|
61
65
|
import { markScopePullSettled, noteCompanyRebind, scopeIdentitySwitchPending } from "../push-identity-fence.js";
|
|
62
66
|
import { reindex } from "./reindex.js";
|
|
63
67
|
import { withOperationLock } from "../operation-lock.js";
|
|
@@ -1185,7 +1189,7 @@ async function planPull(run) {
|
|
|
1185
1189
|
if (!rememberLinkWarning(run.linkWarnings, warning))
|
|
1186
1190
|
return;
|
|
1187
1191
|
run.emit(brokenLinkSkipEvent(warning));
|
|
1188
|
-
}, getJournalRecoveryMarker(run.journalSlug) !== undefined);
|
|
1192
|
+
}, getJournalRecoveryMarker(run.journalSlug) !== undefined, await pullUnattendedDeleteHold(run));
|
|
1189
1193
|
if (run.options.holdLocalKeys?.length) {
|
|
1190
1194
|
holdPushFailedKeys(plan, new Set(run.options.holdLocalKeys));
|
|
1191
1195
|
}
|
|
@@ -1224,6 +1228,9 @@ async function planPull(run) {
|
|
|
1224
1228
|
samplePaths: localDeleteKeys.slice(0, MASS_DELETE_SAMPLE_CAP),
|
|
1225
1229
|
});
|
|
1226
1230
|
}
|
|
1231
|
+
if (plan.scatteredDeleteRestore) {
|
|
1232
|
+
run.emit({ type: "pull-restored-scattered-deletes", ...plan.scatteredDeleteRestore });
|
|
1233
|
+
}
|
|
1227
1234
|
applyAccountSwitchDeleteGuard(run, plan);
|
|
1228
1235
|
return plan;
|
|
1229
1236
|
}
|
|
@@ -1276,6 +1283,33 @@ function quarantineScopeSegment(journalSlug) {
|
|
|
1276
1283
|
*
|
|
1277
1284
|
* The state clears after the first clean full pull (see finalizePullRun).
|
|
1278
1285
|
*/
|
|
1286
|
+
/**
|
|
1287
|
+
* feedback_65ee9e51: the pull leg's unattended delete hold for this run, or
|
|
1288
|
+
* undefined for an interactive run. Each held file is reported with the same
|
|
1289
|
+
* `delete-refused-stale-etag` event the push leg uses.
|
|
1290
|
+
*/
|
|
1291
|
+
async function pullUnattendedDeleteHold(run) {
|
|
1292
|
+
const hold = await resolveUnattendedDeleteHold({
|
|
1293
|
+
personalMode: run.options.personalMode === true,
|
|
1294
|
+
knownSubs: [run.options.callerSub],
|
|
1295
|
+
authToken: run.vaultConfig.authToken,
|
|
1296
|
+
});
|
|
1297
|
+
if (!hold)
|
|
1298
|
+
return undefined;
|
|
1299
|
+
return {
|
|
1300
|
+
...hold,
|
|
1301
|
+
onHold: (key, journalEtag, remoteEtag, createdBy) => {
|
|
1302
|
+
run.emit({
|
|
1303
|
+
type: "delete-refused-stale-etag",
|
|
1304
|
+
path: key,
|
|
1305
|
+
journalEtag,
|
|
1306
|
+
remoteEtag,
|
|
1307
|
+
reason: "unattended-foreign-creator",
|
|
1308
|
+
createdBy,
|
|
1309
|
+
});
|
|
1310
|
+
},
|
|
1311
|
+
};
|
|
1312
|
+
}
|
|
1279
1313
|
function applyAccountSwitchDeleteGuard(run, plan) {
|
|
1280
1314
|
const keys = Array.from(new Set([
|
|
1281
1315
|
...plan.tombstones.map((tombstone) => tombstone.key),
|
|
@@ -1853,11 +1887,11 @@ function executeFileTombstoneDelete(run, item, counters, caseAliasedKeys) {
|
|
|
1853
1887
|
* Record a teammate's newer remote version in a cloud-only placeholder without
|
|
1854
1888
|
* downloading it. The new version's sha256 is unknown until fetched, so the
|
|
1855
1889
|
* placeholder points at the vault object and carries its ETag. The journal
|
|
1856
|
-
* row
|
|
1857
|
-
*
|
|
1858
|
-
*
|
|
1859
|
-
* fetched the file or edited the
|
|
1860
|
-
* written.
|
|
1890
|
+
* row for the key is dropped, because its hash belongs to an earlier revision
|
|
1891
|
+
* and must not be read as the baseline of the new one; the placeholder's ETag
|
|
1892
|
+
* is the baseline until `hq storage fetch` records a verified row. The
|
|
1893
|
+
* placeholder is re-read first: if the user fetched the file or edited the
|
|
1894
|
+
* placeholder since planning, nothing is written.
|
|
1861
1895
|
*/
|
|
1862
1896
|
function executeCloudPlaceholderUpdate(run, item, counters) {
|
|
1863
1897
|
const key = item.remoteFile.key;
|
|
@@ -1889,13 +1923,16 @@ function executeCloudPlaceholderUpdate(run, item, counters) {
|
|
|
1889
1923
|
});
|
|
1890
1924
|
return;
|
|
1891
1925
|
}
|
|
1926
|
+
// The row's hash describes the revision this machine held before the
|
|
1927
|
+
// offload, not the teammate's revision the placeholder now points at. Keeping
|
|
1928
|
+
// the row with a moved ETag and the old hash would let the ETag-first check
|
|
1929
|
+
// in `hasRemoteChanged` call that old hash current. Drop it, as
|
|
1930
|
+
// `executeCloudPlaceholderCreate` never writes one: the placeholder's
|
|
1931
|
+
// `remoteEtag` is the baseline, and `hq storage fetch` records a new row once
|
|
1932
|
+
// the bytes are here and hash-verified. A tombstone row is left alone.
|
|
1892
1933
|
const entry = journalRows(run.journal).get(key);
|
|
1893
1934
|
if (entry && !isTombstone(entry)) {
|
|
1894
|
-
journalRows(run.journal).
|
|
1895
|
-
...entry,
|
|
1896
|
-
remoteEtag: normalizeEtag(item.remoteFile.etag),
|
|
1897
|
-
size: item.remoteFile.size,
|
|
1898
|
-
});
|
|
1935
|
+
journalRows(run.journal).delete(key);
|
|
1899
1936
|
}
|
|
1900
1937
|
}
|
|
1901
1938
|
/**
|
|
@@ -2546,7 +2583,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2546
2583
|
remoteProbeHash !== null &&
|
|
2547
2584
|
journalEntryAtConflict?.localDiverges !== true) {
|
|
2548
2585
|
const rowWrittenAt = journalEntryAtConflict ? Date.parse(journalEntryAtConflict.syncedAt) : Number.NaN;
|
|
2549
|
-
const ownUpload = findUploadIntent(run.journalSlug, remoteFile.key, remoteProbeHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined);
|
|
2586
|
+
const ownUpload = findUploadIntent(run.journalSlug, remoteFile.key, remoteProbeHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined, remoteFile.lastModified && !remoteFile.lastModifiedUnknown ? remoteFile.lastModified.getTime() : null);
|
|
2550
2587
|
if (ownUpload) {
|
|
2551
2588
|
updateEntry(run.journal, remoteFile.key, remoteProbeHash, remoteFile.size, "up", localPath, { remoteEtag: remoteFile.etag, kind: "file" });
|
|
2552
2589
|
run.emit({
|
|
@@ -2566,11 +2603,17 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2566
2603
|
// cloud body, the entry is held with the honesty flag so the push leg
|
|
2567
2604
|
// publishes it under a fence on this etag; otherwise it converges on the
|
|
2568
2605
|
// cloud body.
|
|
2606
|
+
//
|
|
2607
|
+
// Markdown and text logs (PLANNING.md and the like) are merged by entry,
|
|
2608
|
+
// but only when both sides appended to the body this machine last synced
|
|
2609
|
+
// (the journal hash); any other divergence takes the ordinary conflict
|
|
2610
|
+
// rule below, which parks the losing body (feedback_5f5f3ee1).
|
|
2611
|
+
const appendLogMerge = !boardMerge && !isAppendOnlyPath(remoteFile.key) && isAppendLogPath(remoteFile.key);
|
|
2569
2612
|
if (usesVersionAwareWinner(run.options.onConflict) &&
|
|
2570
2613
|
remoteFetched &&
|
|
2571
2614
|
executionLocal?.kind === "file" &&
|
|
2572
2615
|
!probeIsSymlink &&
|
|
2573
|
-
(isAppendOnlyPath(remoteFile.key) || boardMerge)) {
|
|
2616
|
+
(isAppendOnlyPath(remoteFile.key) || boardMerge || appendLogMerge)) {
|
|
2574
2617
|
let localBody = null;
|
|
2575
2618
|
let remoteBody = null;
|
|
2576
2619
|
try {
|
|
@@ -2582,8 +2625,17 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2582
2625
|
noteSwallowedError((line) => process.stderr.write(line), "hq-cloud.append_only_read_failed", error, { quietOnAbsence: false });
|
|
2583
2626
|
localBody = null;
|
|
2584
2627
|
}
|
|
2585
|
-
const
|
|
2586
|
-
const
|
|
2628
|
+
const journalBaseHash = journalEntryAtConflict?.hash ?? null;
|
|
2629
|
+
const merge = boardMerge
|
|
2630
|
+
? mergeBoardJson
|
|
2631
|
+
: appendLogMerge
|
|
2632
|
+
? (l, r) => mergeAppendLog(l, r, journalBaseHash, remoteFile.key)
|
|
2633
|
+
: mergeAppendOnly;
|
|
2634
|
+
const writeMerged = boardMerge
|
|
2635
|
+
? replaceIfUnchanged
|
|
2636
|
+
: appendLogMerge
|
|
2637
|
+
? writeAppendLogMerge
|
|
2638
|
+
: appendIfUnchanged;
|
|
2587
2639
|
const merged = localBody && remoteBody ? merge(localBody, remoteBody) : null;
|
|
2588
2640
|
if (localBody && remoteBody && merged) {
|
|
2589
2641
|
let appended;
|
|
@@ -2595,7 +2647,11 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2595
2647
|
run.emit({
|
|
2596
2648
|
type: "error",
|
|
2597
2649
|
path: remoteFile.key,
|
|
2598
|
-
message: `${boardMerge
|
|
2650
|
+
message: `${boardMerge
|
|
2651
|
+
? "could not merge the cloud board entries into the local board"
|
|
2652
|
+
: appendLogMerge
|
|
2653
|
+
? "could not merge the cloud's log entries into the local file"
|
|
2654
|
+
: "could not add the cloud's lines to the local file"}: ${describeError(error)}`,
|
|
2599
2655
|
});
|
|
2600
2656
|
counters.filesSkipped++;
|
|
2601
2657
|
return null;
|
|
@@ -2612,20 +2668,28 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2612
2668
|
});
|
|
2613
2669
|
return null;
|
|
2614
2670
|
}
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2671
|
+
if (!appendLogMerge || merged.equals(remoteBody)) {
|
|
2672
|
+
const mergedStat = fs.lstatSync(localPath);
|
|
2673
|
+
const mergedHash = createHash("sha256").update(merged).digest("hex");
|
|
2674
|
+
updateEntry(run.journal, remoteFile.key, mergedHash, merged.length, "down", localPath, {
|
|
2675
|
+
remoteEtag: remoteFile.etag,
|
|
2676
|
+
mtimeMs: mergedStat.mtimeMs,
|
|
2677
|
+
ctimeMs: mergedStat.ctimeMs,
|
|
2678
|
+
kind: "file",
|
|
2679
|
+
});
|
|
2680
|
+
if (!merged.equals(remoteBody)) {
|
|
2681
|
+
const mergedEntry = getEntry(run.journal, remoteFile.key);
|
|
2682
|
+
if (mergedEntry) {
|
|
2683
|
+
journalRows(run.journal).set(remoteFile.key, { ...mergedEntry, localDiverges: true });
|
|
2684
|
+
}
|
|
2627
2685
|
}
|
|
2628
2686
|
}
|
|
2687
|
+
// A merged text log that differs from the cloud body leaves the
|
|
2688
|
+
// journal on the common base. Journalling sha256(merged) would make
|
|
2689
|
+
// the next merge-base check fail when a third machine pushes first
|
|
2690
|
+
// (the merged body is not a prefix of its cloud body), so it would
|
|
2691
|
+
// park instead of merge. The push leg sees local changed and cloud
|
|
2692
|
+
// changed, merges against the same base and publishes under a fence.
|
|
2629
2693
|
if (!merged.equals(localBody))
|
|
2630
2694
|
counters.changedPaths.push(remoteFile.key);
|
|
2631
2695
|
run.objectEvents.conflict({
|
|
@@ -2678,6 +2742,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2678
2742
|
remoteMtime: remoteFile.lastModified,
|
|
2679
2743
|
noJournalBaseline: journalEntryAtConflict === undefined,
|
|
2680
2744
|
path: remoteFile.key,
|
|
2745
|
+
personalVault: run.options.personalMode === true,
|
|
2681
2746
|
});
|
|
2682
2747
|
}
|
|
2683
2748
|
catch (readErr) {
|
|
@@ -2750,7 +2815,8 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2750
2815
|
};
|
|
2751
2816
|
/** This is called only when the park helper returned a confirmed backup key. */
|
|
2752
2817
|
const emitConflictParkedAfterConfirmedBackup = (backupPath, sideKept) => {
|
|
2753
|
-
const winnerReason = decision?.reason === "version" || decision?.reason === "mtime" || decision?.reason === "created_at"
|
|
2818
|
+
const winnerReason = decision?.reason === "version" || decision?.reason === "mtime" || decision?.reason === "created_at" ||
|
|
2819
|
+
decision?.reason === "manifest-populated"
|
|
2754
2820
|
? sideKept === "local" ? "local-newer" : "remote-newer"
|
|
2755
2821
|
: "remote-changed-since-journal";
|
|
2756
2822
|
const remoteEtagForNotice = remoteFile.etag ? normalizeEtag(remoteFile.etag) : "";
|
|
@@ -2847,32 +2913,13 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2847
2913
|
}
|
|
2848
2914
|
return holdLocal();
|
|
2849
2915
|
}
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
// The local body is the newer version (higher frontmatter `version:`
|
|
2858
|
-
// or newer mtime). Keep it live; park the cloud body as the backup so
|
|
2859
|
-
// the decision stays auditable; hold with the honesty flag so the push
|
|
2860
|
-
// leg publishes it under a fence on the very etag observed here.
|
|
2861
|
-
const conflictPath = parkProbeAsBackup(localHash);
|
|
2862
|
-
if (!conflictPath) {
|
|
2863
|
-
counters.filesSkipped++;
|
|
2864
|
-
return null;
|
|
2865
|
-
}
|
|
2866
|
-
recordConflict(conflictPath, "remote");
|
|
2867
|
-
return holdLocal();
|
|
2868
|
-
}
|
|
2869
|
-
// The cloud body wins (higher version / newer mtime, or no comparable
|
|
2870
|
-
// bodies — the historical `keep` outcome). Adopt it at the live path and
|
|
2871
|
-
// preserve the displaced local body OUT of the tree under
|
|
2872
|
-
// `.hq/conflict-backups/`, never as a sibling twin. Rename first, then
|
|
2873
|
-
// reuse the ordinary tested download path for the replacement (symlink
|
|
2874
|
-
// materialization, path codec, mtime and etag journalling) instead of a
|
|
2875
|
-
// hand-rolled swap from the probe.
|
|
2916
|
+
// Adopt the cloud body at the live path and preserve the displaced local
|
|
2917
|
+
// body OUT of the tree under `.hq/conflict-backups/`, never as a sibling
|
|
2918
|
+
// twin, with a conflict-index row and notice. Rename first, then reuse the
|
|
2919
|
+
// ordinary tested download path for the replacement (symlink
|
|
2920
|
+
// materialization, path codec, mtime and etag journalling) instead of a
|
|
2921
|
+
// hand-rolled swap from the probe.
|
|
2922
|
+
const adoptCloudParkingLocal = () => {
|
|
2876
2923
|
const downloadItem = {
|
|
2877
2924
|
action: "download",
|
|
2878
2925
|
remoteFile,
|
|
@@ -2917,16 +2964,33 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2917
2964
|
};
|
|
2918
2965
|
downloadItems.push(downloadItem);
|
|
2919
2966
|
return null;
|
|
2967
|
+
};
|
|
2968
|
+
if (resolution === "keep") {
|
|
2969
|
+
if (!remoteFetched) {
|
|
2970
|
+
// The probe never produced a body (its failure is already emitted above).
|
|
2971
|
+
// Hold rather than move aside a file we have nothing to replace it with.
|
|
2972
|
+
return holdLocal();
|
|
2973
|
+
}
|
|
2974
|
+
if (decision?.winner === "local") {
|
|
2975
|
+
// The local body is the newer version (higher frontmatter `version:`
|
|
2976
|
+
// or newer mtime). Keep it live; park the cloud body as the backup so
|
|
2977
|
+
// the decision stays auditable; hold with the honesty flag so the push
|
|
2978
|
+
// leg publishes it under a fence on the very etag observed here.
|
|
2979
|
+
const conflictPath = parkProbeAsBackup(localHash);
|
|
2980
|
+
if (!conflictPath) {
|
|
2981
|
+
counters.filesSkipped++;
|
|
2982
|
+
return null;
|
|
2983
|
+
}
|
|
2984
|
+
recordConflict(conflictPath, "remote");
|
|
2985
|
+
return holdLocal();
|
|
2986
|
+
}
|
|
2987
|
+
// The cloud body wins (higher version / newer mtime, or no comparable
|
|
2988
|
+
// bodies — the historical `keep` outcome).
|
|
2989
|
+
return adoptCloudParkingLocal();
|
|
2920
2990
|
}
|
|
2921
|
-
// `overwrite`: the cloud body replaces local
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
remoteFile,
|
|
2925
|
-
localPath,
|
|
2926
|
-
isNew: false,
|
|
2927
|
-
localSnapshot,
|
|
2928
|
-
});
|
|
2929
|
-
return null;
|
|
2991
|
+
// `overwrite`: the cloud body replaces local, but the local body it
|
|
2992
|
+
// displaces is still kept as a reported conflict copy (feedback_5f5f3ee1).
|
|
2993
|
+
return adoptCloudParkingLocal();
|
|
2930
2994
|
}
|
|
2931
2995
|
finally {
|
|
2932
2996
|
if (!probeParked)
|
|
@@ -3097,8 +3161,12 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
3097
3161
|
// The continuity pointer is downloaded to a stage first so an older vault
|
|
3098
3162
|
// copy can be compared with the local pointer before it replaces it.
|
|
3099
3163
|
const guardContinuityPointer = remoteFile.key === CONTINUITY_POINTER_KEY;
|
|
3164
|
+
// The companies manifest is staged too, so a blank template in the vault is
|
|
3165
|
+
// compared with a populated local manifest before it can replace it, even
|
|
3166
|
+
// when the local file is unchanged since the journal (feedback_09af2875).
|
|
3167
|
+
const guardCompanyManifest = run.options.personalMode === true && remoteFile.key === PERSONAL_VAULT_MANIFEST_KEY;
|
|
3100
3168
|
const stagePath = stageOptions?.staged?.stagePath ??
|
|
3101
|
-
(stageOptions?.stage || validatePersonalMcp || guardContinuityPointer
|
|
3169
|
+
(stageOptions?.stage || validatePersonalMcp || guardContinuityPointer || guardCompanyManifest
|
|
3102
3170
|
? createDownloadStagingPath(localPath)
|
|
3103
3171
|
: undefined);
|
|
3104
3172
|
if (!stageOptions?.staged)
|
|
@@ -3134,6 +3202,17 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
3134
3202
|
if (guardContinuityPointer && holdNewerLocalContinuityPointer(run, remoteFile, localPath, stagePath, counters)) {
|
|
3135
3203
|
return;
|
|
3136
3204
|
}
|
|
3205
|
+
if (guardCompanyManifest) {
|
|
3206
|
+
if (await holdPopulatedLocalManifest(run, remoteFile, localPath, stagePath, counters))
|
|
3207
|
+
return;
|
|
3208
|
+
// The hold can wait on the conflict prompt; the local manifest may
|
|
3209
|
+
// change meanwhile, so check it again before replacing it, as the
|
|
3210
|
+
// unstaged branch does.
|
|
3211
|
+
if (!matchesLocalSnapshot(localPath, downloadItem.localSnapshot)) {
|
|
3212
|
+
discardDownloadStage(stagePath);
|
|
3213
|
+
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
3214
|
+
}
|
|
3215
|
+
}
|
|
3137
3216
|
downloadItem.beforeReplace?.();
|
|
3138
3217
|
replaceStagedPath(stagePath, localPath);
|
|
3139
3218
|
downloaded = stageOptions.staged.result;
|
|
@@ -3189,8 +3268,17 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
3189
3268
|
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
3190
3269
|
}
|
|
3191
3270
|
}
|
|
3192
|
-
if (
|
|
3271
|
+
if (guardCompanyManifest && stagePath) {
|
|
3272
|
+
if (await holdPopulatedLocalManifest(run, remoteFile, localPath, stagePath, counters))
|
|
3273
|
+
return;
|
|
3274
|
+
if (!matchesLocalSnapshot(localPath, downloadItem.localSnapshot)) {
|
|
3275
|
+
discardDownloadStage(stagePath);
|
|
3276
|
+
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
3277
|
+
}
|
|
3278
|
+
}
|
|
3279
|
+
if ((validatePersonalMcp || guardContinuityPointer || guardCompanyManifest) && stagePath) {
|
|
3193
3280
|
replaceStagedPath(stagePath, localPath);
|
|
3281
|
+
}
|
|
3194
3282
|
}
|
|
3195
3283
|
const { metadata, contentHash, contentSize, modeWarnings } = downloaded;
|
|
3196
3284
|
reportModeWarnings(run, modeWarnings);
|
|
@@ -4169,6 +4257,126 @@ function holdNewerLocalContinuityPointer(run, remoteFile, localPath, stagePath,
|
|
|
4169
4257
|
});
|
|
4170
4258
|
return true;
|
|
4171
4259
|
}
|
|
4260
|
+
/**
|
|
4261
|
+
* feedback_09af2875: a blank companies manifest (the hq-core template) in the
|
|
4262
|
+
* personal vault never replaces a local `companies/manifest.yaml` that lists
|
|
4263
|
+
* companies, even on a plain pull where the local file is unchanged since the
|
|
4264
|
+
* journal. The local file stays; the incoming blank copy is saved as a
|
|
4265
|
+
* conflict backup and recorded in the conflict index; the journal row is
|
|
4266
|
+
* marked `localDiverges` against the vault ETag, so the next push takes the
|
|
4267
|
+
* conflict path, where the populated local manifest wins and is published
|
|
4268
|
+
* over the blank one under a fence. Returns false (download proceeds) when
|
|
4269
|
+
* either body is not a readable manifest, the vault copy lists companies, or
|
|
4270
|
+
* the caller chose the cloud copy: an explicit `--on-conflict overwrite`, or
|
|
4271
|
+
* `overwrite` at the interactive prompt (strategy unset on a TTY). That is how
|
|
4272
|
+
* a person takes an emptied list from the vault on purpose.
|
|
4273
|
+
*/
|
|
4274
|
+
async function holdPopulatedLocalManifest(run, remoteFile, localPath, stagePath, counters) {
|
|
4275
|
+
let localBytes;
|
|
4276
|
+
let remoteBytes;
|
|
4277
|
+
try {
|
|
4278
|
+
localBytes = fs.readFileSync(localPath);
|
|
4279
|
+
remoteBytes = fs.readFileSync(stagePath);
|
|
4280
|
+
}
|
|
4281
|
+
catch {
|
|
4282
|
+
return false;
|
|
4283
|
+
}
|
|
4284
|
+
const localCount = countManifestCompanies(localBytes);
|
|
4285
|
+
const remoteCount = countManifestCompanies(remoteBytes);
|
|
4286
|
+
if (localCount === null || remoteCount === null || localCount === 0 || remoteCount !== 0) {
|
|
4287
|
+
return false;
|
|
4288
|
+
}
|
|
4289
|
+
const localHash = createHash("sha256").update(localBytes).digest("hex");
|
|
4290
|
+
if (!usesVersionAwareWinner(run.options.onConflict)) {
|
|
4291
|
+
// Same gate as the push leg: an explicit strategy (or the person at the
|
|
4292
|
+
// prompt) decides. Only `overwrite` takes the blank cloud copy.
|
|
4293
|
+
const resolution = await resolveConflict({
|
|
4294
|
+
path: remoteFile.key,
|
|
4295
|
+
localHash,
|
|
4296
|
+
...(remoteFile.lastModified ? { remoteModified: remoteFile.lastModified } : {}),
|
|
4297
|
+
direction: "pull",
|
|
4298
|
+
headline: `The vault copy lists no companies; the local manifest lists ${localCount}`,
|
|
4299
|
+
}, run.options.onConflict);
|
|
4300
|
+
if (resolution === "overwrite")
|
|
4301
|
+
return false;
|
|
4302
|
+
}
|
|
4303
|
+
const remoteEtag = remoteFile.etag ? normalizeEtag(remoteFile.etag) : "";
|
|
4304
|
+
const detectedAt = new Date().toISOString();
|
|
4305
|
+
const machineId = readShortMachineId(run.hqRoot);
|
|
4306
|
+
const originalRelative = vaultKeyForLocalPath(run.hqRoot, localPath);
|
|
4307
|
+
const backupKey = buildConflictBackupPath(originalRelative, detectedAt, machineId);
|
|
4308
|
+
let savedBackup;
|
|
4309
|
+
try {
|
|
4310
|
+
const backupAbs = localPathForVaultKey(run.hqRoot, backupKey);
|
|
4311
|
+
fs.mkdirSync(path.dirname(backupAbs), { recursive: true });
|
|
4312
|
+
fs.renameSync(stagePath, backupAbs);
|
|
4313
|
+
savedBackup = backupKey;
|
|
4314
|
+
}
|
|
4315
|
+
catch (error) {
|
|
4316
|
+
discardDownloadStage(stagePath);
|
|
4317
|
+
run.emit({
|
|
4318
|
+
type: "error",
|
|
4319
|
+
path: remoteFile.key,
|
|
4320
|
+
message: `kept the local companies manifest; could not save the blank vault copy as a conflict backup: ${describeError(error)}`,
|
|
4321
|
+
});
|
|
4322
|
+
}
|
|
4323
|
+
let entry = getEntry(run.journal, remoteFile.key);
|
|
4324
|
+
if (!entry) {
|
|
4325
|
+
// No journal row yet: record the kept local body so the `localDiverges`
|
|
4326
|
+
// flag below has a row to live on and the push leg sees the divergence.
|
|
4327
|
+
try {
|
|
4328
|
+
const stat = fs.statSync(localPath);
|
|
4329
|
+
updateEntry(run.journal, remoteFile.key, localHash, stat.size, "down", localPath, {
|
|
4330
|
+
...(remoteFile.etag ? { remoteEtag: remoteFile.etag } : {}),
|
|
4331
|
+
mtimeMs: stat.mtimeMs,
|
|
4332
|
+
kind: "file",
|
|
4333
|
+
});
|
|
4334
|
+
}
|
|
4335
|
+
catch {
|
|
4336
|
+
// The local manifest vanished between the read and the stat.
|
|
4337
|
+
}
|
|
4338
|
+
entry = getEntry(run.journal, remoteFile.key);
|
|
4339
|
+
}
|
|
4340
|
+
if (entry) {
|
|
4341
|
+
journalRows(run.journal).set(remoteFile.key, {
|
|
4342
|
+
...entry,
|
|
4343
|
+
...(remoteFile.etag ? { remoteEtag: remoteFile.etag } : {}),
|
|
4344
|
+
localDiverges: true,
|
|
4345
|
+
});
|
|
4346
|
+
}
|
|
4347
|
+
if (savedBackup !== undefined) {
|
|
4348
|
+
try {
|
|
4349
|
+
appendConflictEntry(run.hqRoot, {
|
|
4350
|
+
id: buildConflictId(originalRelative, detectedAt),
|
|
4351
|
+
originalPath: originalRelative,
|
|
4352
|
+
conflictPath: savedBackup,
|
|
4353
|
+
detectedAt,
|
|
4354
|
+
side: "pull",
|
|
4355
|
+
machineId,
|
|
4356
|
+
localHash,
|
|
4357
|
+
remoteHash: remoteEtag,
|
|
4358
|
+
preserved: "remote",
|
|
4359
|
+
decision: { winner: "local", reason: "manifest-populated" },
|
|
4360
|
+
});
|
|
4361
|
+
}
|
|
4362
|
+
catch (indexErr) {
|
|
4363
|
+
run.emit({
|
|
4364
|
+
type: "error",
|
|
4365
|
+
path: remoteFile.key,
|
|
4366
|
+
message: `conflict index write failed: ${describeError(indexErr)}`,
|
|
4367
|
+
});
|
|
4368
|
+
}
|
|
4369
|
+
}
|
|
4370
|
+
process.stderr.write(`hq-cloud: kept local ${remoteFile.key} (${localCount} ${localCount === 1 ? "company" : "companies"}); the vault copy is blank and was not applied${savedBackup ? `, saved to ${savedBackup}` : ""}\n`);
|
|
4371
|
+
counters.filesSkipped++;
|
|
4372
|
+
run.emit({
|
|
4373
|
+
type: "reconciled",
|
|
4374
|
+
path: remoteFile.key,
|
|
4375
|
+
direction: "pull",
|
|
4376
|
+
reason: "local_manifest_populated",
|
|
4377
|
+
});
|
|
4378
|
+
return true;
|
|
4379
|
+
}
|
|
4172
4380
|
function discardDownloadStage(stagePath) {
|
|
4173
4381
|
try {
|
|
4174
4382
|
fs.unlinkSync(stagePath);
|
|
@@ -4253,6 +4461,59 @@ export function win32SymlinkFlavorBroken(localPath, win32 = process.platform ===
|
|
|
4253
4461
|
return true; // target exists but the link cannot traverse: wrong flavor
|
|
4254
4462
|
}
|
|
4255
4463
|
}
|
|
4464
|
+
/**
|
|
4465
|
+
* Valid, not-yet-orphaned `.hqcloud` placeholders under `companyRoot`, keyed by
|
|
4466
|
+
* POSIX vault key. Skips unreadable directories, symlinks, and anything the
|
|
4467
|
+
* ignore filter or scope excludes. Used to orphan a placeholder whose object is
|
|
4468
|
+
* absent from a complete remote listing, independent of any journal row.
|
|
4469
|
+
*/
|
|
4470
|
+
function collectLivePlaceholderCandidates(companyRoot, shouldSync, inScope, onHeartbeat) {
|
|
4471
|
+
const found = new Map();
|
|
4472
|
+
const root = path.resolve(companyRoot);
|
|
4473
|
+
const visit = (dir) => {
|
|
4474
|
+
// The walk is synchronous, so it cannot yield; a heartbeat per directory
|
|
4475
|
+
// keeps the watchdog fed on a large tree.
|
|
4476
|
+
onHeartbeat?.();
|
|
4477
|
+
let entries;
|
|
4478
|
+
try {
|
|
4479
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
4480
|
+
}
|
|
4481
|
+
catch {
|
|
4482
|
+
return;
|
|
4483
|
+
}
|
|
4484
|
+
for (const entry of entries) {
|
|
4485
|
+
const full = path.join(dir, entry.name);
|
|
4486
|
+
if (entry.isDirectory()) {
|
|
4487
|
+
if (entry.name === ".git" || entry.name === "node_modules" || entry.name === ".hq")
|
|
4488
|
+
continue;
|
|
4489
|
+
if (!shouldSync(full, true))
|
|
4490
|
+
continue;
|
|
4491
|
+
visit(full);
|
|
4492
|
+
}
|
|
4493
|
+
else if (entry.isFile() && entry.name.endsWith(CLOUD_PLACEHOLDER_SUFFIX)) {
|
|
4494
|
+
const localPath = full.slice(0, -CLOUD_PLACEHOLDER_SUFFIX.length);
|
|
4495
|
+
const key = toPosixKey(path.relative(root, localPath));
|
|
4496
|
+
if (key === "" || key.startsWith("..") || !inScope(key))
|
|
4497
|
+
continue;
|
|
4498
|
+
if (isPathExcludedFromSync(shouldSync, localPath))
|
|
4499
|
+
continue;
|
|
4500
|
+
const placeholder = readCloudPlaceholder(full, localPath);
|
|
4501
|
+
if (!placeholder || placeholder.orphaned)
|
|
4502
|
+
continue;
|
|
4503
|
+
// Only a placeholder that points at the file's own vault key can be
|
|
4504
|
+
// judged by the listing. One that points at an offload blob or another
|
|
4505
|
+
// vault's object says nothing about this key's presence.
|
|
4506
|
+
if (placeholder.vault !== "company" || placeholder.key !== key)
|
|
4507
|
+
continue;
|
|
4508
|
+
if (!isCloudOnlyLocally(localPath))
|
|
4509
|
+
continue;
|
|
4510
|
+
found.set(key, localPath);
|
|
4511
|
+
}
|
|
4512
|
+
}
|
|
4513
|
+
};
|
|
4514
|
+
visit(root);
|
|
4515
|
+
return found;
|
|
4516
|
+
}
|
|
4256
4517
|
async function computePullPlan(remoteFiles, journal, companyRoot, shouldSync, personalMode, teamSyncedSlugs,
|
|
4257
4518
|
// Coalesced, company-relative prefixes the pull is scoped to (US-005).
|
|
4258
4519
|
// `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
|
|
@@ -4280,7 +4541,11 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4280
4541
|
// so they are not evidence that a local file is a clean synced copy. While
|
|
4281
4542
|
// set, pull never deletes a local file and never replaces an existing local
|
|
4282
4543
|
// file without the conflict executor's byte comparison (feedback_b088e391).
|
|
4283
|
-
recoveryHold = false
|
|
4544
|
+
recoveryHold = false,
|
|
4545
|
+
// feedback_65ee9e51: set for unattended runs. A missing file the journal
|
|
4546
|
+
// does not show the caller created is downloaded back instead of becoming
|
|
4547
|
+
// a `local-delete` tombstone; `onHold` reports each one.
|
|
4548
|
+
unattendedDeleteHold) {
|
|
4284
4549
|
const prefixMatcher = compileScopeMatcher(prefixSet);
|
|
4285
4550
|
const excludeMatcher = compileScopeMatcher(excludePrefixes);
|
|
4286
4551
|
const items = [];
|
|
@@ -4366,6 +4631,16 @@ recoveryHold = false) {
|
|
|
4366
4631
|
})(), (key) => unseenJournalKeys.has(key) || fileTombstones.has(key))
|
|
4367
4632
|
: null;
|
|
4368
4633
|
const caseAliasedKeys = new Set();
|
|
4634
|
+
// Placeholder-driven orphaning: every live placeholder on disk starts as a
|
|
4635
|
+
// candidate and is cleared when its key shows up in the listing. What is left
|
|
4636
|
+
// after the whole listing streamed was deleted from the vault, even when no
|
|
4637
|
+
// journal row or FILE_TOMBSTONE remembers it (a placeholder updated to a
|
|
4638
|
+
// peer's revision has no row). The listing is complete here: a failed or
|
|
4639
|
+
// partial listing throws before this planner finishes, and keys outside the
|
|
4640
|
+
// listed scope are never candidates.
|
|
4641
|
+
const orphanCandidates = collectLivePlaceholderCandidates(companyRoot, shouldSync, (key) => prefixMatcher.coversPath(key), onHeartbeat);
|
|
4642
|
+
// Keys already planned for orphaning, so the leftover pass never repeats one.
|
|
4643
|
+
const orphanPlannedKeys = new Set();
|
|
4369
4644
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
4370
4645
|
onHeartbeat?.();
|
|
4371
4646
|
const posixJournaledKey = toPosixKey(journaledKey);
|
|
@@ -4473,6 +4748,9 @@ recoveryHold = false) {
|
|
|
4473
4748
|
else if (plannedRows % decodeWindowSize() === 0) {
|
|
4474
4749
|
dropJournalDecodeWindow();
|
|
4475
4750
|
}
|
|
4751
|
+
// A key the listing shows is present, whatever happens to it next
|
|
4752
|
+
// (quarantine, backslash spelling, skip), so it is never an orphan.
|
|
4753
|
+
orphanCandidates.delete(toPosixKey(remoteFile.key));
|
|
4476
4754
|
if (quarantinedPaths.has(remoteFile.key)) {
|
|
4477
4755
|
preplannedFilesSkipped++;
|
|
4478
4756
|
continue;
|
|
@@ -4481,6 +4759,7 @@ recoveryHold = false) {
|
|
|
4481
4759
|
// The key can prove its POSIX comparison twin is present, but it must
|
|
4482
4760
|
// never be used to derive a local path or overwrite the slash spelling.
|
|
4483
4761
|
const comparisonKey = toPosixKey(remoteFile.key);
|
|
4762
|
+
orphanCandidates.delete(comparisonKey);
|
|
4484
4763
|
unseenJournalKeys.delete(comparisonKey);
|
|
4485
4764
|
for (const legacyKey of legacyJournalKeysByPosix.get(comparisonKey) ?? []) {
|
|
4486
4765
|
unseenJournalKeys.delete(legacyKey);
|
|
@@ -4498,6 +4777,7 @@ recoveryHold = false) {
|
|
|
4498
4777
|
continue;
|
|
4499
4778
|
}
|
|
4500
4779
|
const posixRemoteKey = toPosixKey(remoteFile.key);
|
|
4780
|
+
orphanCandidates.delete(posixRemoteKey);
|
|
4501
4781
|
// Canonical slash keys clear both canonical journal rows and any legacy
|
|
4502
4782
|
// backslash spelling, on the comparison path only.
|
|
4503
4783
|
if (remoteFile.key === posixRemoteKey) {
|
|
@@ -4817,6 +5097,10 @@ recoveryHold = false) {
|
|
|
4817
5097
|
}
|
|
4818
5098
|
}
|
|
4819
5099
|
const localExists = localLstat !== null;
|
|
5100
|
+
// feedback_65ee9e51: a held file back on disk syncs as an ordinary file.
|
|
5101
|
+
if (localExists && journalEntry?.unattendedHeldBy !== undefined) {
|
|
5102
|
+
clearUnattendedHeldRow(journal, remoteFile.key);
|
|
5103
|
+
}
|
|
4820
5104
|
if (localPathBlockedByFileAncestor) {
|
|
4821
5105
|
// Symmetric counterpart to the (local-dir, cloud-file) warning below.
|
|
4822
5106
|
// Emit the same "manual reconciliation required" surface so the
|
|
@@ -4843,12 +5127,15 @@ recoveryHold = false) {
|
|
|
4843
5127
|
// pointer that a later fetch + push would resurrect.
|
|
4844
5128
|
if ((tombstoneSuppresses || localTombstoneSuppresses) &&
|
|
4845
5129
|
journalEntry?.removedReason !== "local-delete") {
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4851
|
-
|
|
5130
|
+
if (!orphanPlannedKeys.has(remoteFile.key)) {
|
|
5131
|
+
orphanPlannedKeys.add(remoteFile.key);
|
|
5132
|
+
items.push({
|
|
5133
|
+
action: "orphan-cloud-placeholder",
|
|
5134
|
+
remoteFile,
|
|
5135
|
+
localPath,
|
|
5136
|
+
reason: "deleted from the vault by another device or teammate",
|
|
5137
|
+
});
|
|
5138
|
+
}
|
|
4852
5139
|
continue;
|
|
4853
5140
|
}
|
|
4854
5141
|
const baselineEtag = placeholder.remoteEtag ??
|
|
@@ -4858,7 +5145,9 @@ recoveryHold = false) {
|
|
|
4858
5145
|
const remoteChanged = baselineEtag
|
|
4859
5146
|
? normalizeEtag(remoteFile.etag) !== baselineEtag
|
|
4860
5147
|
: remoteFile.size !== placeholder.size;
|
|
4861
|
-
|
|
5148
|
+
// An orphaned placeholder whose key is listed again (same ETag) is
|
|
5149
|
+
// live again: the update executor clears the orphan mark.
|
|
5150
|
+
if (remoteChanged || placeholder.orphaned) {
|
|
4862
5151
|
items.push({
|
|
4863
5152
|
action: "update-cloud-placeholder",
|
|
4864
5153
|
remoteFile,
|
|
@@ -5318,6 +5607,10 @@ recoveryHold = false) {
|
|
|
5318
5607
|
// it to propagate the S3 delete. Skip the re-download without dropping the
|
|
5319
5608
|
// entry. Other tombstone kinds (scope-shrink / FILE_TOMBSTONE) drop it.
|
|
5320
5609
|
if (journalEntry?.removedReason === "local-delete") {
|
|
5610
|
+
// Earlier pulls left these pending for push. Reconsider the whole
|
|
5611
|
+
// burst, unless the server says the object was actually deleted.
|
|
5612
|
+
if (!tombstoneSuppresses)
|
|
5613
|
+
intentionalDeleteCandidates.push({ remoteFile, localPath });
|
|
5321
5614
|
continue;
|
|
5322
5615
|
}
|
|
5323
5616
|
items.push({
|
|
@@ -5342,6 +5635,8 @@ recoveryHold = false) {
|
|
|
5342
5635
|
journalEntry !== undefined &&
|
|
5343
5636
|
!isTombstone(journalEntry) &&
|
|
5344
5637
|
!journalEntry.localDiverges &&
|
|
5638
|
+
// feedback_65ee9e51: an unattended push held this delete; restore it.
|
|
5639
|
+
journalEntry.unattendedHeldBy === undefined &&
|
|
5345
5640
|
(hasCurrentLocalDeleteIntent(journalEntry) ||
|
|
5346
5641
|
propagateDeletePolicy === "currency-gated") &&
|
|
5347
5642
|
normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag) {
|
|
@@ -5376,11 +5671,51 @@ recoveryHold = false) {
|
|
|
5376
5671
|
totalJournalEntries > 0 &&
|
|
5377
5672
|
intentionalDeleteCandidates.length / totalJournalEntries >=
|
|
5378
5673
|
PULL_INTENTIONAL_DELETE_RATIO;
|
|
5674
|
+
// feedback_65089095: the push leg's scattered-delete breaker refuses a
|
|
5675
|
+
// burst of files missing from folders that still exist, even under the
|
|
5676
|
+
// ratio. Apply the same rule here, with the same bypasses, so this pull
|
|
5677
|
+
// downloads those files back instead of stamping `local-delete`
|
|
5678
|
+
// tombstones that would suppress the download and re-trip every push.
|
|
5679
|
+
// Count the same set the push leg counts: under owned-only the push only
|
|
5680
|
+
// considers rows this machine uploaded (direction "up").
|
|
5681
|
+
const scatteredRows = journalRows(journal);
|
|
5682
|
+
const scatteredDeleteRestore = !intentionalDeleteIsBulk && !isMassDeleteGuardOverrideEnabled(propagateDeletePolicy)
|
|
5683
|
+
? detectScatteredDeleteBurst(intentionalDeleteCandidates.flatMap((cand) => propagateDeletePolicy === "owned-only" &&
|
|
5684
|
+
scatteredRows.get(cand.remoteFile.key)?.direction !== "up"
|
|
5685
|
+
? []
|
|
5686
|
+
: [cand.remoteFile.key]), journal, companyRoot, new Set(), { countLocalDeleteTombstones: true })
|
|
5687
|
+
: undefined;
|
|
5688
|
+
const intentionalDeleteIsScattered = scatteredDeleteRestore !== undefined;
|
|
5379
5689
|
for (const cand of intentionalDeleteCandidates) {
|
|
5690
|
+
// Unattended hold (feedback_65ee9e51): only a file the caller created
|
|
5691
|
+
// may become a delete. Another member's file, or one that came down
|
|
5692
|
+
// with no creator, is restored. A file the caller uploaded with no
|
|
5693
|
+
// recorded creator is left for the push leg, which reads `created-by-sub`
|
|
5694
|
+
// from HEAD and deletes or holds it.
|
|
5695
|
+
//
|
|
5380
5696
|
// Recovery hold: a missing local file is not proof of a user delete when
|
|
5381
5697
|
// the journal row may be stale, and a local-delete tombstone would let
|
|
5382
5698
|
// the push leg delete the vault copy once the marker clears. Restore it.
|
|
5383
|
-
|
|
5699
|
+
// The breakers and the recovery hold restore first, so a "defer" can
|
|
5700
|
+
// never leave a candidate they would have restored missing.
|
|
5701
|
+
const forcedRestore = intentionalDeleteIsBulk || intentionalDeleteIsScattered || recoveryHold;
|
|
5702
|
+
const unattendedDecision = unattendedDeleteHold && !forcedRestore
|
|
5703
|
+
? unattendedPullDeleteDecision(unattendedDeleteHold, scatteredRows.get(cand.remoteFile.key) ?? { direction: "down" })
|
|
5704
|
+
: "tombstone";
|
|
5705
|
+
if (unattendedDecision === "defer")
|
|
5706
|
+
continue;
|
|
5707
|
+
if (unattendedDecision === "restore" && unattendedDeleteHold) {
|
|
5708
|
+
const row = scatteredRows.get(cand.remoteFile.key);
|
|
5709
|
+
unattendedDeleteHold.onHold(cand.remoteFile.key, row?.remoteEtag ?? "<not-checked>", normalizeEtag(cand.remoteFile.etag), row?.createdBySub ?? UNKNOWN_CREATOR);
|
|
5710
|
+
}
|
|
5711
|
+
if (forcedRestore || unattendedDecision === "restore") {
|
|
5712
|
+
// Pending local-delete rows are candidates again, so a restore has to
|
|
5713
|
+
// drop the tombstone or the next push still deletes the vault copy.
|
|
5714
|
+
const row = scatteredRows.get(cand.remoteFile.key);
|
|
5715
|
+
if (row?.removedReason === "local-delete") {
|
|
5716
|
+
const { removedAt: _removedAt, removedReason: _removedReason, localDeleteIntent: _intent, ...rest } = row;
|
|
5717
|
+
scatteredRows.set(cand.remoteFile.key, rest);
|
|
5718
|
+
}
|
|
5384
5719
|
items.push({
|
|
5385
5720
|
action: "download",
|
|
5386
5721
|
remoteFile: cand.remoteFile,
|
|
@@ -5393,6 +5728,17 @@ recoveryHold = false) {
|
|
|
5393
5728
|
tombstoneEntry(journal, cand.remoteFile.key, "local-delete");
|
|
5394
5729
|
}
|
|
5395
5730
|
}
|
|
5731
|
+
for (const [key, localPath] of orphanCandidates) {
|
|
5732
|
+
if (orphanPlannedKeys.has(key))
|
|
5733
|
+
continue;
|
|
5734
|
+
orphanPlannedKeys.add(key);
|
|
5735
|
+
items.push({
|
|
5736
|
+
action: "orphan-cloud-placeholder",
|
|
5737
|
+
remoteFile: { key, size: 0, lastModified: new Date(0), etag: "" },
|
|
5738
|
+
localPath,
|
|
5739
|
+
reason: "deleted from the vault by another device or teammate",
|
|
5740
|
+
});
|
|
5741
|
+
}
|
|
5396
5742
|
let filesToDownload = 0;
|
|
5397
5743
|
let bytesToDownload = 0;
|
|
5398
5744
|
let filesToSkip = 0;
|
|
@@ -5617,6 +5963,15 @@ recoveryHold = false) {
|
|
|
5617
5963
|
tombstones,
|
|
5618
5964
|
invalidRemoteKeys,
|
|
5619
5965
|
caseAliasedKeys,
|
|
5966
|
+
...(scatteredDeleteRestore
|
|
5967
|
+
? {
|
|
5968
|
+
scatteredDeleteRestore: {
|
|
5969
|
+
candidates: scatteredDeleteRestore.keys.length,
|
|
5970
|
+
folders: scatteredDeleteRestore.areas,
|
|
5971
|
+
samplePaths: scatteredDeleteRestore.samplePaths,
|
|
5972
|
+
},
|
|
5973
|
+
}
|
|
5974
|
+
: {}),
|
|
5620
5975
|
};
|
|
5621
5976
|
}
|
|
5622
5977
|
/**
|
|
@@ -5631,6 +5986,9 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5631
5986
|
else if (event.type === "identity-switch-push-held" && event.reason === "company-rebind") {
|
|
5632
5987
|
console.warn(` ! this folder was linked to a different company, so ${event.count} local file${event.count === 1 ? " was" : "s were"} not uploaded and nothing was deleted in the cloud. The hold clears after a full \`hq sync\` pull that finishes cleanly: no refused deletes, and every file it sets aside moved into .hq/switch-quarantine (which must be a folder you can write to; fix that first if a pull reported it). HQ_SYNC_DELETE_BULK_OVERRIDE=1 skips only the bulk-delete refusal; files are still moved into quarantine, not deleted.`);
|
|
5633
5988
|
}
|
|
5989
|
+
else if (event.type === "identity-switch-push-held" && event.message) {
|
|
5990
|
+
console.warn(` ! ${event.message}`);
|
|
5991
|
+
}
|
|
5634
5992
|
else if (event.type === "identity-switch-push-held" && event.count === 0) {
|
|
5635
5993
|
console.warn(" ! someone else was signed in on this computer, so uploads from here are held until you review and release them.");
|
|
5636
5994
|
}
|
|
@@ -5764,8 +6122,16 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5764
6122
|
console.log(` · ${event.path} left in the cloud (${mb} MB, over the 50 MB sync limit); run hq storage fetch to download it`);
|
|
5765
6123
|
}
|
|
5766
6124
|
else if (event.type === "skip-size-limit") {
|
|
5767
|
-
const
|
|
5768
|
-
|
|
6125
|
+
const line = formatSizeLimitSkipLine(event.path, event.bytes);
|
|
6126
|
+
if (humanSkipLines)
|
|
6127
|
+
humanSkipLines.noteSizeLimit(line);
|
|
6128
|
+
else
|
|
6129
|
+
console.warn(line);
|
|
6130
|
+
}
|
|
6131
|
+
else if (event.type === "pull-restored-scattered-deletes") {
|
|
6132
|
+
console.warn(` ⚠ scattered-delete guard: downloading ${event.candidates} missing file(s) again instead of deleting them; they vanished from ${event.folders} top-level folders that still exist, which looks like a rollback rather than a cleanup. To delete them on purpose, re-run with HQ_SYNC_DELETE_BULK_OVERRIDE=1.`);
|
|
6133
|
+
for (const samplePath of event.samplePaths.slice(0, 5))
|
|
6134
|
+
console.warn(` · ${samplePath}`);
|
|
5769
6135
|
}
|
|
5770
6136
|
else if (event.type === "scope-shrink-refused") {
|
|
5771
6137
|
const why = event.reason === "empty-grants"
|