@indigoai-us/hq-cloud 6.18.108 → 6.18.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/area-ledger-migration.d.ts.map +1 -1
- package/dist/bin/area-ledger-migration.js +5 -3
- package/dist/bin/area-ledger-migration.js.map +1 -1
- package/dist/bin/backup-prune-runner.js +2 -2
- package/dist/bin/backup-prune-runner.js.map +1 -1
- package/dist/bin/runner-output-guard.d.ts +39 -0
- package/dist/bin/runner-output-guard.d.ts.map +1 -0
- package/dist/bin/runner-output-guard.js +104 -0
- package/dist/bin/runner-output-guard.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +9 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +76 -6
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +37 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +37 -3
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/conflict.d.ts +30 -1
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +69 -3
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/reconcile-skip-reason.d.ts +1 -1
- package/dist/cli/reconcile-skip-reason.d.ts.map +1 -1
- package/dist/cli/reconcile-skip-reason.js +2 -0
- package/dist/cli/reconcile-skip-reason.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +228 -43
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +160 -1
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +681 -162
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-native-copy.d.ts +88 -0
- package/dist/cli/rescue-native-copy.d.ts.map +1 -0
- package/dist/cli/rescue-native-copy.js +823 -0
- package/dist/cli/rescue-native-copy.js.map +1 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +37 -9
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/share.d.ts +35 -8
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +437 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +70 -5
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +439 -77
- package/dist/cli/sync.js.map +1 -1
- package/dist/cognito-auth.d.ts +10 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +162 -1
- package/dist/cognito-auth.js.map +1 -1
- package/dist/ignore.d.ts +5 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +75 -23
- package/dist/ignore.js.map +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +3 -0
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -0
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-decoder.js +2 -0
- package/dist/journal-snapshot-decoder.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -1
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal.d.ts +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +20 -20
- package/dist/journal.js.map +1 -1
- package/dist/lib/append-log-merge.d.ts +43 -0
- package/dist/lib/append-log-merge.d.ts.map +1 -0
- package/dist/lib/append-log-merge.js +424 -0
- package/dist/lib/append-log-merge.js.map +1 -0
- package/dist/lib/same-path-upload-lock.d.ts +27 -1
- package/dist/lib/same-path-upload-lock.d.ts.map +1 -1
- package/dist/lib/same-path-upload-lock.js +32 -1
- package/dist/lib/same-path-upload-lock.js.map +1 -1
- package/dist/object-io.d.ts +36 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +93 -12
- package/dist/object-io.js.map +1 -1
- package/dist/operation-lock.d.ts +27 -1
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +94 -8
- package/dist/operation-lock.js.map +1 -1
- package/dist/outposts/client.d.ts +19 -6
- package/dist/outposts/client.d.ts.map +1 -1
- package/dist/outposts/client.js +16 -5
- package/dist/outposts/client.js.map +1 -1
- package/dist/outposts/command.d.ts +2 -1
- package/dist/outposts/command.d.ts.map +1 -1
- package/dist/outposts/command.js +81 -13
- package/dist/outposts/command.js.map +1 -1
- package/dist/outposts/self-deploy.d.ts.map +1 -1
- package/dist/outposts/self-deploy.js +3 -0
- package/dist/outposts/self-deploy.js.map +1 -1
- package/dist/private-folder-transport.d.ts +0 -2
- package/dist/private-folder-transport.d.ts.map +1 -1
- package/dist/private-folder-transport.js +17 -16
- package/dist/private-folder-transport.js.map +1 -1
- package/dist/push-identity-fence.d.ts +116 -3
- package/dist/push-identity-fence.d.ts.map +1 -1
- package/dist/push-identity-fence.js +625 -62
- package/dist/push-identity-fence.js.map +1 -1
- package/dist/qmd-reindex.d.ts.map +1 -1
- package/dist/qmd-reindex.js +3 -2
- package/dist/qmd-reindex.js.map +1 -1
- package/dist/s3.d.ts +30 -9
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +110 -82
- package/dist/s3.js.map +1 -1
- package/dist/sync/area-ledger-migration.d.ts +17 -0
- package/dist/sync/area-ledger-migration.d.ts.map +1 -1
- package/dist/sync/area-ledger-migration.js +37 -3
- package/dist/sync/area-ledger-migration.js.map +1 -1
- package/dist/sync/scattered-delete-burst.d.ts +61 -0
- package/dist/sync/scattered-delete-burst.d.ts.map +1 -0
- package/dist/sync/scattered-delete-burst.js +111 -0
- package/dist/sync/scattered-delete-burst.js.map +1 -0
- package/dist/sync/unattended-delete-hold.d.ts +102 -0
- package/dist/sync/unattended-delete-hold.d.ts.map +1 -0
- package/dist/sync/unattended-delete-hold.js +164 -0
- package/dist/sync/unattended-delete-hold.js.map +1 -0
- package/dist/sync-core.d.ts +5 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -6
- package/dist/sync-core.js.map +1 -1
- package/dist/types.d.ts +13 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts +6 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +53 -8
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-key-grammar.d.ts +44 -0
- package/dist/vault-key-grammar.d.ts.map +1 -0
- package/dist/vault-key-grammar.js +158 -0
- package/dist/vault-key-grammar.js.map +1 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +4 -1
- package/dist/watcher.js.map +1 -1
- package/dist/windows-extended-path.d.ts +22 -0
- package/dist/windows-extended-path.d.ts.map +1 -0
- package/dist/windows-extended-path.js +55 -0
- package/dist/windows-extended-path.js.map +1 -0
- package/package.json +1 -1
package/dist/cli/sync.js
CHANGED
|
@@ -9,7 +9,7 @@ import { localLinkWarning, rememberLinkWarning, brokenLinkSkipEvent, LinkWarning
|
|
|
9
9
|
import * as fs from "fs";
|
|
10
10
|
import * as path from "path";
|
|
11
11
|
import { isDeepStrictEqual } from "node:util";
|
|
12
|
-
import { cloudPlaceholderPathFor, formatCloudPlaceholder, hasCloudPlaceholderSibling, hasLocalOriginalForMissingPlaceholder, isCloudOnlyLocally, markCloudPlaceholderOrphaned, readCloudPlaceholder, writeCloudPlaceholder, } from "../cloud-placeholder.js";
|
|
12
|
+
import { CLOUD_PLACEHOLDER_SUFFIX, cloudPlaceholderPathFor, formatCloudPlaceholder, hasCloudPlaceholderSibling, hasLocalOriginalForMissingPlaceholder, isCloudOnlyLocally, markCloudPlaceholderOrphaned, readCloudPlaceholder, writeCloudPlaceholder, } from "../cloud-placeholder.js";
|
|
13
13
|
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
14
14
|
import { getActiveFullPassDecodeWindow, getActiveFullPassUnitSize, } from "../sync/full-pass-caps.js";
|
|
15
15
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
@@ -49,8 +49,9 @@ import { claimNeverSyncedLocalNotices, formatSecretsNotSyncedLine, isNeverSynced
|
|
|
49
49
|
import { hasRemoteChanged, isAccessDenied, isNotFound, resolveActiveCompany, resolveTransferConcurrency, readHostLoad, } from "../sync-core.js";
|
|
50
50
|
import { formatIgnoreExcludedWarningLines, isEphemeralPath, isMalformedVaultKey, rebindContextToLocalFolder } from "./share.js";
|
|
51
51
|
import { createHumanSkipLineLog, formatReconcileSkipLine, } from "./reconcile-skip-reason.js";
|
|
52
|
-
import { CONTINUITY_POINTER_KEY, parseContinuityCreatedAt, pickWinner, resolveConflict, usesVersionAwareWinner, } from "./conflict.js";
|
|
52
|
+
import { CONTINUITY_POINTER_KEY, countManifestCompanies, parseContinuityCreatedAt, pickWinner, resolveConflict, usesVersionAwareWinner, } from "./conflict.js";
|
|
53
53
|
import { appendIfUnchanged, isAppendOnlyPath, mergeAppendOnly } from "../lib/append-only-merge.js";
|
|
54
|
+
import { isAppendLogPath, mergeAppendLog, writeAppendLogMerge } from "../lib/append-log-merge.js";
|
|
54
55
|
import { findUploadIntent } from "../lib/same-path-upload-lock.js";
|
|
55
56
|
import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/board-merge.js";
|
|
56
57
|
import { createHash } from "node:crypto";
|
|
@@ -58,6 +59,8 @@ import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readS
|
|
|
58
59
|
import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
|
|
59
60
|
import { isCloudAuthoritative, isPullReconciledServerOwned, } from "../lib/cloud-authoritative.js";
|
|
60
61
|
import { isMassDeleteGuardOverrideEnabled, MASS_DELETE_ABSOLUTE_CEILING, MASS_DELETE_SAMPLE_CAP, massDeleteGuardReason, resolveMassDeleteGuards, } from "../sync/mass-delete-guards.js";
|
|
62
|
+
import { detectScatteredDeleteBurst } from "../sync/scattered-delete-burst.js";
|
|
63
|
+
import { clearUnattendedHeldRow, resolveUnattendedDeleteHold, UNKNOWN_CREATOR, unattendedPullDeleteDecision, } from "../sync/unattended-delete-hold.js";
|
|
61
64
|
import { markScopePullSettled, noteCompanyRebind, scopeIdentitySwitchPending } from "../push-identity-fence.js";
|
|
62
65
|
import { reindex } from "./reindex.js";
|
|
63
66
|
import { withOperationLock } from "../operation-lock.js";
|
|
@@ -1185,7 +1188,7 @@ async function planPull(run) {
|
|
|
1185
1188
|
if (!rememberLinkWarning(run.linkWarnings, warning))
|
|
1186
1189
|
return;
|
|
1187
1190
|
run.emit(brokenLinkSkipEvent(warning));
|
|
1188
|
-
}, getJournalRecoveryMarker(run.journalSlug) !== undefined);
|
|
1191
|
+
}, getJournalRecoveryMarker(run.journalSlug) !== undefined, await pullUnattendedDeleteHold(run));
|
|
1189
1192
|
if (run.options.holdLocalKeys?.length) {
|
|
1190
1193
|
holdPushFailedKeys(plan, new Set(run.options.holdLocalKeys));
|
|
1191
1194
|
}
|
|
@@ -1224,6 +1227,9 @@ async function planPull(run) {
|
|
|
1224
1227
|
samplePaths: localDeleteKeys.slice(0, MASS_DELETE_SAMPLE_CAP),
|
|
1225
1228
|
});
|
|
1226
1229
|
}
|
|
1230
|
+
if (plan.scatteredDeleteRestore) {
|
|
1231
|
+
run.emit({ type: "pull-restored-scattered-deletes", ...plan.scatteredDeleteRestore });
|
|
1232
|
+
}
|
|
1227
1233
|
applyAccountSwitchDeleteGuard(run, plan);
|
|
1228
1234
|
return plan;
|
|
1229
1235
|
}
|
|
@@ -1276,6 +1282,33 @@ function quarantineScopeSegment(journalSlug) {
|
|
|
1276
1282
|
*
|
|
1277
1283
|
* The state clears after the first clean full pull (see finalizePullRun).
|
|
1278
1284
|
*/
|
|
1285
|
+
/**
|
|
1286
|
+
* feedback_65ee9e51: the pull leg's unattended delete hold for this run, or
|
|
1287
|
+
* undefined for an interactive run. Each held file is reported with the same
|
|
1288
|
+
* `delete-refused-stale-etag` event the push leg uses.
|
|
1289
|
+
*/
|
|
1290
|
+
async function pullUnattendedDeleteHold(run) {
|
|
1291
|
+
const hold = await resolveUnattendedDeleteHold({
|
|
1292
|
+
personalMode: run.options.personalMode === true,
|
|
1293
|
+
knownSubs: [run.options.callerSub],
|
|
1294
|
+
authToken: run.vaultConfig.authToken,
|
|
1295
|
+
});
|
|
1296
|
+
if (!hold)
|
|
1297
|
+
return undefined;
|
|
1298
|
+
return {
|
|
1299
|
+
...hold,
|
|
1300
|
+
onHold: (key, journalEtag, remoteEtag, createdBy) => {
|
|
1301
|
+
run.emit({
|
|
1302
|
+
type: "delete-refused-stale-etag",
|
|
1303
|
+
path: key,
|
|
1304
|
+
journalEtag,
|
|
1305
|
+
remoteEtag,
|
|
1306
|
+
reason: "unattended-foreign-creator",
|
|
1307
|
+
createdBy,
|
|
1308
|
+
});
|
|
1309
|
+
},
|
|
1310
|
+
};
|
|
1311
|
+
}
|
|
1279
1312
|
function applyAccountSwitchDeleteGuard(run, plan) {
|
|
1280
1313
|
const keys = Array.from(new Set([
|
|
1281
1314
|
...plan.tombstones.map((tombstone) => tombstone.key),
|
|
@@ -1853,11 +1886,11 @@ function executeFileTombstoneDelete(run, item, counters, caseAliasedKeys) {
|
|
|
1853
1886
|
* Record a teammate's newer remote version in a cloud-only placeholder without
|
|
1854
1887
|
* downloading it. The new version's sha256 is unknown until fetched, so the
|
|
1855
1888
|
* placeholder points at the vault object and carries its ETag. The journal
|
|
1856
|
-
* row
|
|
1857
|
-
*
|
|
1858
|
-
*
|
|
1859
|
-
* fetched the file or edited the
|
|
1860
|
-
* written.
|
|
1889
|
+
* row for the key is dropped, because its hash belongs to an earlier revision
|
|
1890
|
+
* and must not be read as the baseline of the new one; the placeholder's ETag
|
|
1891
|
+
* is the baseline until `hq storage fetch` records a verified row. The
|
|
1892
|
+
* placeholder is re-read first: if the user fetched the file or edited the
|
|
1893
|
+
* placeholder since planning, nothing is written.
|
|
1861
1894
|
*/
|
|
1862
1895
|
function executeCloudPlaceholderUpdate(run, item, counters) {
|
|
1863
1896
|
const key = item.remoteFile.key;
|
|
@@ -1889,13 +1922,16 @@ function executeCloudPlaceholderUpdate(run, item, counters) {
|
|
|
1889
1922
|
});
|
|
1890
1923
|
return;
|
|
1891
1924
|
}
|
|
1925
|
+
// The row's hash describes the revision this machine held before the
|
|
1926
|
+
// offload, not the teammate's revision the placeholder now points at. Keeping
|
|
1927
|
+
// the row with a moved ETag and the old hash would let the ETag-first check
|
|
1928
|
+
// in `hasRemoteChanged` call that old hash current. Drop it, as
|
|
1929
|
+
// `executeCloudPlaceholderCreate` never writes one: the placeholder's
|
|
1930
|
+
// `remoteEtag` is the baseline, and `hq storage fetch` records a new row once
|
|
1931
|
+
// the bytes are here and hash-verified. A tombstone row is left alone.
|
|
1892
1932
|
const entry = journalRows(run.journal).get(key);
|
|
1893
1933
|
if (entry && !isTombstone(entry)) {
|
|
1894
|
-
journalRows(run.journal).
|
|
1895
|
-
...entry,
|
|
1896
|
-
remoteEtag: normalizeEtag(item.remoteFile.etag),
|
|
1897
|
-
size: item.remoteFile.size,
|
|
1898
|
-
});
|
|
1934
|
+
journalRows(run.journal).delete(key);
|
|
1899
1935
|
}
|
|
1900
1936
|
}
|
|
1901
1937
|
/**
|
|
@@ -2546,7 +2582,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2546
2582
|
remoteProbeHash !== null &&
|
|
2547
2583
|
journalEntryAtConflict?.localDiverges !== true) {
|
|
2548
2584
|
const rowWrittenAt = journalEntryAtConflict ? Date.parse(journalEntryAtConflict.syncedAt) : Number.NaN;
|
|
2549
|
-
const ownUpload = findUploadIntent(run.journalSlug, remoteFile.key, remoteProbeHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined);
|
|
2585
|
+
const ownUpload = findUploadIntent(run.journalSlug, remoteFile.key, remoteProbeHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined, remoteFile.lastModified && !remoteFile.lastModifiedUnknown ? remoteFile.lastModified.getTime() : null);
|
|
2550
2586
|
if (ownUpload) {
|
|
2551
2587
|
updateEntry(run.journal, remoteFile.key, remoteProbeHash, remoteFile.size, "up", localPath, { remoteEtag: remoteFile.etag, kind: "file" });
|
|
2552
2588
|
run.emit({
|
|
@@ -2566,11 +2602,17 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2566
2602
|
// cloud body, the entry is held with the honesty flag so the push leg
|
|
2567
2603
|
// publishes it under a fence on this etag; otherwise it converges on the
|
|
2568
2604
|
// cloud body.
|
|
2605
|
+
//
|
|
2606
|
+
// Markdown and text logs (PLANNING.md and the like) are merged by entry,
|
|
2607
|
+
// but only when both sides appended to the body this machine last synced
|
|
2608
|
+
// (the journal hash); any other divergence takes the ordinary conflict
|
|
2609
|
+
// rule below, which parks the losing body (feedback_5f5f3ee1).
|
|
2610
|
+
const appendLogMerge = !boardMerge && !isAppendOnlyPath(remoteFile.key) && isAppendLogPath(remoteFile.key);
|
|
2569
2611
|
if (usesVersionAwareWinner(run.options.onConflict) &&
|
|
2570
2612
|
remoteFetched &&
|
|
2571
2613
|
executionLocal?.kind === "file" &&
|
|
2572
2614
|
!probeIsSymlink &&
|
|
2573
|
-
(isAppendOnlyPath(remoteFile.key) || boardMerge)) {
|
|
2615
|
+
(isAppendOnlyPath(remoteFile.key) || boardMerge || appendLogMerge)) {
|
|
2574
2616
|
let localBody = null;
|
|
2575
2617
|
let remoteBody = null;
|
|
2576
2618
|
try {
|
|
@@ -2582,8 +2624,17 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2582
2624
|
noteSwallowedError((line) => process.stderr.write(line), "hq-cloud.append_only_read_failed", error, { quietOnAbsence: false });
|
|
2583
2625
|
localBody = null;
|
|
2584
2626
|
}
|
|
2585
|
-
const
|
|
2586
|
-
const
|
|
2627
|
+
const journalBaseHash = journalEntryAtConflict?.hash ?? null;
|
|
2628
|
+
const merge = boardMerge
|
|
2629
|
+
? mergeBoardJson
|
|
2630
|
+
: appendLogMerge
|
|
2631
|
+
? (l, r) => mergeAppendLog(l, r, journalBaseHash, remoteFile.key)
|
|
2632
|
+
: mergeAppendOnly;
|
|
2633
|
+
const writeMerged = boardMerge
|
|
2634
|
+
? replaceIfUnchanged
|
|
2635
|
+
: appendLogMerge
|
|
2636
|
+
? writeAppendLogMerge
|
|
2637
|
+
: appendIfUnchanged;
|
|
2587
2638
|
const merged = localBody && remoteBody ? merge(localBody, remoteBody) : null;
|
|
2588
2639
|
if (localBody && remoteBody && merged) {
|
|
2589
2640
|
let appended;
|
|
@@ -2595,7 +2646,11 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2595
2646
|
run.emit({
|
|
2596
2647
|
type: "error",
|
|
2597
2648
|
path: remoteFile.key,
|
|
2598
|
-
message: `${boardMerge
|
|
2649
|
+
message: `${boardMerge
|
|
2650
|
+
? "could not merge the cloud board entries into the local board"
|
|
2651
|
+
: appendLogMerge
|
|
2652
|
+
? "could not merge the cloud's log entries into the local file"
|
|
2653
|
+
: "could not add the cloud's lines to the local file"}: ${describeError(error)}`,
|
|
2599
2654
|
});
|
|
2600
2655
|
counters.filesSkipped++;
|
|
2601
2656
|
return null;
|
|
@@ -2612,20 +2667,28 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2612
2667
|
});
|
|
2613
2668
|
return null;
|
|
2614
2669
|
}
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2670
|
+
if (!appendLogMerge || merged.equals(remoteBody)) {
|
|
2671
|
+
const mergedStat = fs.lstatSync(localPath);
|
|
2672
|
+
const mergedHash = createHash("sha256").update(merged).digest("hex");
|
|
2673
|
+
updateEntry(run.journal, remoteFile.key, mergedHash, merged.length, "down", localPath, {
|
|
2674
|
+
remoteEtag: remoteFile.etag,
|
|
2675
|
+
mtimeMs: mergedStat.mtimeMs,
|
|
2676
|
+
ctimeMs: mergedStat.ctimeMs,
|
|
2677
|
+
kind: "file",
|
|
2678
|
+
});
|
|
2679
|
+
if (!merged.equals(remoteBody)) {
|
|
2680
|
+
const mergedEntry = getEntry(run.journal, remoteFile.key);
|
|
2681
|
+
if (mergedEntry) {
|
|
2682
|
+
journalRows(run.journal).set(remoteFile.key, { ...mergedEntry, localDiverges: true });
|
|
2683
|
+
}
|
|
2627
2684
|
}
|
|
2628
2685
|
}
|
|
2686
|
+
// A merged text log that differs from the cloud body leaves the
|
|
2687
|
+
// journal on the common base. Journalling sha256(merged) would make
|
|
2688
|
+
// the next merge-base check fail when a third machine pushes first
|
|
2689
|
+
// (the merged body is not a prefix of its cloud body), so it would
|
|
2690
|
+
// park instead of merge. The push leg sees local changed and cloud
|
|
2691
|
+
// changed, merges against the same base and publishes under a fence.
|
|
2629
2692
|
if (!merged.equals(localBody))
|
|
2630
2693
|
counters.changedPaths.push(remoteFile.key);
|
|
2631
2694
|
run.objectEvents.conflict({
|
|
@@ -2678,6 +2741,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2678
2741
|
remoteMtime: remoteFile.lastModified,
|
|
2679
2742
|
noJournalBaseline: journalEntryAtConflict === undefined,
|
|
2680
2743
|
path: remoteFile.key,
|
|
2744
|
+
personalVault: run.options.personalMode === true,
|
|
2681
2745
|
});
|
|
2682
2746
|
}
|
|
2683
2747
|
catch (readErr) {
|
|
@@ -2750,7 +2814,8 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2750
2814
|
};
|
|
2751
2815
|
/** This is called only when the park helper returned a confirmed backup key. */
|
|
2752
2816
|
const emitConflictParkedAfterConfirmedBackup = (backupPath, sideKept) => {
|
|
2753
|
-
const winnerReason = decision?.reason === "version" || decision?.reason === "mtime" || decision?.reason === "created_at"
|
|
2817
|
+
const winnerReason = decision?.reason === "version" || decision?.reason === "mtime" || decision?.reason === "created_at" ||
|
|
2818
|
+
decision?.reason === "manifest-populated"
|
|
2754
2819
|
? sideKept === "local" ? "local-newer" : "remote-newer"
|
|
2755
2820
|
: "remote-changed-since-journal";
|
|
2756
2821
|
const remoteEtagForNotice = remoteFile.etag ? normalizeEtag(remoteFile.etag) : "";
|
|
@@ -2847,32 +2912,13 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2847
2912
|
}
|
|
2848
2913
|
return holdLocal();
|
|
2849
2914
|
}
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
// The local body is the newer version (higher frontmatter `version:`
|
|
2858
|
-
// or newer mtime). Keep it live; park the cloud body as the backup so
|
|
2859
|
-
// the decision stays auditable; hold with the honesty flag so the push
|
|
2860
|
-
// leg publishes it under a fence on the very etag observed here.
|
|
2861
|
-
const conflictPath = parkProbeAsBackup(localHash);
|
|
2862
|
-
if (!conflictPath) {
|
|
2863
|
-
counters.filesSkipped++;
|
|
2864
|
-
return null;
|
|
2865
|
-
}
|
|
2866
|
-
recordConflict(conflictPath, "remote");
|
|
2867
|
-
return holdLocal();
|
|
2868
|
-
}
|
|
2869
|
-
// The cloud body wins (higher version / newer mtime, or no comparable
|
|
2870
|
-
// bodies — the historical `keep` outcome). Adopt it at the live path and
|
|
2871
|
-
// preserve the displaced local body OUT of the tree under
|
|
2872
|
-
// `.hq/conflict-backups/`, never as a sibling twin. Rename first, then
|
|
2873
|
-
// reuse the ordinary tested download path for the replacement (symlink
|
|
2874
|
-
// materialization, path codec, mtime and etag journalling) instead of a
|
|
2875
|
-
// hand-rolled swap from the probe.
|
|
2915
|
+
// Adopt the cloud body at the live path and preserve the displaced local
|
|
2916
|
+
// body OUT of the tree under `.hq/conflict-backups/`, never as a sibling
|
|
2917
|
+
// twin, with a conflict-index row and notice. Rename first, then reuse the
|
|
2918
|
+
// ordinary tested download path for the replacement (symlink
|
|
2919
|
+
// materialization, path codec, mtime and etag journalling) instead of a
|
|
2920
|
+
// hand-rolled swap from the probe.
|
|
2921
|
+
const adoptCloudParkingLocal = () => {
|
|
2876
2922
|
const downloadItem = {
|
|
2877
2923
|
action: "download",
|
|
2878
2924
|
remoteFile,
|
|
@@ -2917,16 +2963,33 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2917
2963
|
};
|
|
2918
2964
|
downloadItems.push(downloadItem);
|
|
2919
2965
|
return null;
|
|
2966
|
+
};
|
|
2967
|
+
if (resolution === "keep") {
|
|
2968
|
+
if (!remoteFetched) {
|
|
2969
|
+
// The probe never produced a body (its failure is already emitted above).
|
|
2970
|
+
// Hold rather than move aside a file we have nothing to replace it with.
|
|
2971
|
+
return holdLocal();
|
|
2972
|
+
}
|
|
2973
|
+
if (decision?.winner === "local") {
|
|
2974
|
+
// The local body is the newer version (higher frontmatter `version:`
|
|
2975
|
+
// or newer mtime). Keep it live; park the cloud body as the backup so
|
|
2976
|
+
// the decision stays auditable; hold with the honesty flag so the push
|
|
2977
|
+
// leg publishes it under a fence on the very etag observed here.
|
|
2978
|
+
const conflictPath = parkProbeAsBackup(localHash);
|
|
2979
|
+
if (!conflictPath) {
|
|
2980
|
+
counters.filesSkipped++;
|
|
2981
|
+
return null;
|
|
2982
|
+
}
|
|
2983
|
+
recordConflict(conflictPath, "remote");
|
|
2984
|
+
return holdLocal();
|
|
2985
|
+
}
|
|
2986
|
+
// The cloud body wins (higher version / newer mtime, or no comparable
|
|
2987
|
+
// bodies — the historical `keep` outcome).
|
|
2988
|
+
return adoptCloudParkingLocal();
|
|
2920
2989
|
}
|
|
2921
|
-
// `overwrite`: the cloud body replaces local
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
remoteFile,
|
|
2925
|
-
localPath,
|
|
2926
|
-
isNew: false,
|
|
2927
|
-
localSnapshot,
|
|
2928
|
-
});
|
|
2929
|
-
return null;
|
|
2990
|
+
// `overwrite`: the cloud body replaces local, but the local body it
|
|
2991
|
+
// displaces is still kept as a reported conflict copy (feedback_5f5f3ee1).
|
|
2992
|
+
return adoptCloudParkingLocal();
|
|
2930
2993
|
}
|
|
2931
2994
|
finally {
|
|
2932
2995
|
if (!probeParked)
|
|
@@ -3097,8 +3160,12 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
3097
3160
|
// The continuity pointer is downloaded to a stage first so an older vault
|
|
3098
3161
|
// copy can be compared with the local pointer before it replaces it.
|
|
3099
3162
|
const guardContinuityPointer = remoteFile.key === CONTINUITY_POINTER_KEY;
|
|
3163
|
+
// The companies manifest is staged too, so a blank template in the vault is
|
|
3164
|
+
// compared with a populated local manifest before it can replace it, even
|
|
3165
|
+
// when the local file is unchanged since the journal (feedback_09af2875).
|
|
3166
|
+
const guardCompanyManifest = run.options.personalMode === true && remoteFile.key === PERSONAL_VAULT_MANIFEST_KEY;
|
|
3100
3167
|
const stagePath = stageOptions?.staged?.stagePath ??
|
|
3101
|
-
(stageOptions?.stage || validatePersonalMcp || guardContinuityPointer
|
|
3168
|
+
(stageOptions?.stage || validatePersonalMcp || guardContinuityPointer || guardCompanyManifest
|
|
3102
3169
|
? createDownloadStagingPath(localPath)
|
|
3103
3170
|
: undefined);
|
|
3104
3171
|
if (!stageOptions?.staged)
|
|
@@ -3134,6 +3201,17 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
3134
3201
|
if (guardContinuityPointer && holdNewerLocalContinuityPointer(run, remoteFile, localPath, stagePath, counters)) {
|
|
3135
3202
|
return;
|
|
3136
3203
|
}
|
|
3204
|
+
if (guardCompanyManifest) {
|
|
3205
|
+
if (await holdPopulatedLocalManifest(run, remoteFile, localPath, stagePath, counters))
|
|
3206
|
+
return;
|
|
3207
|
+
// The hold can wait on the conflict prompt; the local manifest may
|
|
3208
|
+
// change meanwhile, so check it again before replacing it, as the
|
|
3209
|
+
// unstaged branch does.
|
|
3210
|
+
if (!matchesLocalSnapshot(localPath, downloadItem.localSnapshot)) {
|
|
3211
|
+
discardDownloadStage(stagePath);
|
|
3212
|
+
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3137
3215
|
downloadItem.beforeReplace?.();
|
|
3138
3216
|
replaceStagedPath(stagePath, localPath);
|
|
3139
3217
|
downloaded = stageOptions.staged.result;
|
|
@@ -3189,8 +3267,17 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
3189
3267
|
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
3190
3268
|
}
|
|
3191
3269
|
}
|
|
3192
|
-
if (
|
|
3270
|
+
if (guardCompanyManifest && stagePath) {
|
|
3271
|
+
if (await holdPopulatedLocalManifest(run, remoteFile, localPath, stagePath, counters))
|
|
3272
|
+
return;
|
|
3273
|
+
if (!matchesLocalSnapshot(localPath, downloadItem.localSnapshot)) {
|
|
3274
|
+
discardDownloadStage(stagePath);
|
|
3275
|
+
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
3276
|
+
}
|
|
3277
|
+
}
|
|
3278
|
+
if ((validatePersonalMcp || guardContinuityPointer || guardCompanyManifest) && stagePath) {
|
|
3193
3279
|
replaceStagedPath(stagePath, localPath);
|
|
3280
|
+
}
|
|
3194
3281
|
}
|
|
3195
3282
|
const { metadata, contentHash, contentSize, modeWarnings } = downloaded;
|
|
3196
3283
|
reportModeWarnings(run, modeWarnings);
|
|
@@ -4169,6 +4256,126 @@ function holdNewerLocalContinuityPointer(run, remoteFile, localPath, stagePath,
|
|
|
4169
4256
|
});
|
|
4170
4257
|
return true;
|
|
4171
4258
|
}
|
|
4259
|
+
/**
|
|
4260
|
+
* feedback_09af2875: a blank companies manifest (the hq-core template) in the
|
|
4261
|
+
* personal vault never replaces a local `companies/manifest.yaml` that lists
|
|
4262
|
+
* companies, even on a plain pull where the local file is unchanged since the
|
|
4263
|
+
* journal. The local file stays; the incoming blank copy is saved as a
|
|
4264
|
+
* conflict backup and recorded in the conflict index; the journal row is
|
|
4265
|
+
* marked `localDiverges` against the vault ETag, so the next push takes the
|
|
4266
|
+
* conflict path, where the populated local manifest wins and is published
|
|
4267
|
+
* over the blank one under a fence. Returns false (download proceeds) when
|
|
4268
|
+
* either body is not a readable manifest, the vault copy lists companies, or
|
|
4269
|
+
* the caller chose the cloud copy: an explicit `--on-conflict overwrite`, or
|
|
4270
|
+
* `overwrite` at the interactive prompt (strategy unset on a TTY). That is how
|
|
4271
|
+
* a person takes an emptied list from the vault on purpose.
|
|
4272
|
+
*/
|
|
4273
|
+
async function holdPopulatedLocalManifest(run, remoteFile, localPath, stagePath, counters) {
|
|
4274
|
+
let localBytes;
|
|
4275
|
+
let remoteBytes;
|
|
4276
|
+
try {
|
|
4277
|
+
localBytes = fs.readFileSync(localPath);
|
|
4278
|
+
remoteBytes = fs.readFileSync(stagePath);
|
|
4279
|
+
}
|
|
4280
|
+
catch {
|
|
4281
|
+
return false;
|
|
4282
|
+
}
|
|
4283
|
+
const localCount = countManifestCompanies(localBytes);
|
|
4284
|
+
const remoteCount = countManifestCompanies(remoteBytes);
|
|
4285
|
+
if (localCount === null || remoteCount === null || localCount === 0 || remoteCount !== 0) {
|
|
4286
|
+
return false;
|
|
4287
|
+
}
|
|
4288
|
+
const localHash = createHash("sha256").update(localBytes).digest("hex");
|
|
4289
|
+
if (!usesVersionAwareWinner(run.options.onConflict)) {
|
|
4290
|
+
// Same gate as the push leg: an explicit strategy (or the person at the
|
|
4291
|
+
// prompt) decides. Only `overwrite` takes the blank cloud copy.
|
|
4292
|
+
const resolution = await resolveConflict({
|
|
4293
|
+
path: remoteFile.key,
|
|
4294
|
+
localHash,
|
|
4295
|
+
...(remoteFile.lastModified ? { remoteModified: remoteFile.lastModified } : {}),
|
|
4296
|
+
direction: "pull",
|
|
4297
|
+
headline: `The vault copy lists no companies; the local manifest lists ${localCount}`,
|
|
4298
|
+
}, run.options.onConflict);
|
|
4299
|
+
if (resolution === "overwrite")
|
|
4300
|
+
return false;
|
|
4301
|
+
}
|
|
4302
|
+
const remoteEtag = remoteFile.etag ? normalizeEtag(remoteFile.etag) : "";
|
|
4303
|
+
const detectedAt = new Date().toISOString();
|
|
4304
|
+
const machineId = readShortMachineId(run.hqRoot);
|
|
4305
|
+
const originalRelative = vaultKeyForLocalPath(run.hqRoot, localPath);
|
|
4306
|
+
const backupKey = buildConflictBackupPath(originalRelative, detectedAt, machineId);
|
|
4307
|
+
let savedBackup;
|
|
4308
|
+
try {
|
|
4309
|
+
const backupAbs = localPathForVaultKey(run.hqRoot, backupKey);
|
|
4310
|
+
fs.mkdirSync(path.dirname(backupAbs), { recursive: true });
|
|
4311
|
+
fs.renameSync(stagePath, backupAbs);
|
|
4312
|
+
savedBackup = backupKey;
|
|
4313
|
+
}
|
|
4314
|
+
catch (error) {
|
|
4315
|
+
discardDownloadStage(stagePath);
|
|
4316
|
+
run.emit({
|
|
4317
|
+
type: "error",
|
|
4318
|
+
path: remoteFile.key,
|
|
4319
|
+
message: `kept the local companies manifest; could not save the blank vault copy as a conflict backup: ${describeError(error)}`,
|
|
4320
|
+
});
|
|
4321
|
+
}
|
|
4322
|
+
let entry = getEntry(run.journal, remoteFile.key);
|
|
4323
|
+
if (!entry) {
|
|
4324
|
+
// No journal row yet: record the kept local body so the `localDiverges`
|
|
4325
|
+
// flag below has a row to live on and the push leg sees the divergence.
|
|
4326
|
+
try {
|
|
4327
|
+
const stat = fs.statSync(localPath);
|
|
4328
|
+
updateEntry(run.journal, remoteFile.key, localHash, stat.size, "down", localPath, {
|
|
4329
|
+
...(remoteFile.etag ? { remoteEtag: remoteFile.etag } : {}),
|
|
4330
|
+
mtimeMs: stat.mtimeMs,
|
|
4331
|
+
kind: "file",
|
|
4332
|
+
});
|
|
4333
|
+
}
|
|
4334
|
+
catch {
|
|
4335
|
+
// The local manifest vanished between the read and the stat.
|
|
4336
|
+
}
|
|
4337
|
+
entry = getEntry(run.journal, remoteFile.key);
|
|
4338
|
+
}
|
|
4339
|
+
if (entry) {
|
|
4340
|
+
journalRows(run.journal).set(remoteFile.key, {
|
|
4341
|
+
...entry,
|
|
4342
|
+
...(remoteFile.etag ? { remoteEtag: remoteFile.etag } : {}),
|
|
4343
|
+
localDiverges: true,
|
|
4344
|
+
});
|
|
4345
|
+
}
|
|
4346
|
+
if (savedBackup !== undefined) {
|
|
4347
|
+
try {
|
|
4348
|
+
appendConflictEntry(run.hqRoot, {
|
|
4349
|
+
id: buildConflictId(originalRelative, detectedAt),
|
|
4350
|
+
originalPath: originalRelative,
|
|
4351
|
+
conflictPath: savedBackup,
|
|
4352
|
+
detectedAt,
|
|
4353
|
+
side: "pull",
|
|
4354
|
+
machineId,
|
|
4355
|
+
localHash,
|
|
4356
|
+
remoteHash: remoteEtag,
|
|
4357
|
+
preserved: "remote",
|
|
4358
|
+
decision: { winner: "local", reason: "manifest-populated" },
|
|
4359
|
+
});
|
|
4360
|
+
}
|
|
4361
|
+
catch (indexErr) {
|
|
4362
|
+
run.emit({
|
|
4363
|
+
type: "error",
|
|
4364
|
+
path: remoteFile.key,
|
|
4365
|
+
message: `conflict index write failed: ${describeError(indexErr)}`,
|
|
4366
|
+
});
|
|
4367
|
+
}
|
|
4368
|
+
}
|
|
4369
|
+
process.stderr.write(`hq-cloud: kept local ${remoteFile.key} (${localCount} ${localCount === 1 ? "company" : "companies"}); the vault copy is blank and was not applied${savedBackup ? `, saved to ${savedBackup}` : ""}\n`);
|
|
4370
|
+
counters.filesSkipped++;
|
|
4371
|
+
run.emit({
|
|
4372
|
+
type: "reconciled",
|
|
4373
|
+
path: remoteFile.key,
|
|
4374
|
+
direction: "pull",
|
|
4375
|
+
reason: "local_manifest_populated",
|
|
4376
|
+
});
|
|
4377
|
+
return true;
|
|
4378
|
+
}
|
|
4172
4379
|
function discardDownloadStage(stagePath) {
|
|
4173
4380
|
try {
|
|
4174
4381
|
fs.unlinkSync(stagePath);
|
|
@@ -4253,6 +4460,59 @@ export function win32SymlinkFlavorBroken(localPath, win32 = process.platform ===
|
|
|
4253
4460
|
return true; // target exists but the link cannot traverse: wrong flavor
|
|
4254
4461
|
}
|
|
4255
4462
|
}
|
|
4463
|
+
/**
|
|
4464
|
+
* Valid, not-yet-orphaned `.hqcloud` placeholders under `companyRoot`, keyed by
|
|
4465
|
+
* POSIX vault key. Skips unreadable directories, symlinks, and anything the
|
|
4466
|
+
* ignore filter or scope excludes. Used to orphan a placeholder whose object is
|
|
4467
|
+
* absent from a complete remote listing, independent of any journal row.
|
|
4468
|
+
*/
|
|
4469
|
+
function collectLivePlaceholderCandidates(companyRoot, shouldSync, inScope, onHeartbeat) {
|
|
4470
|
+
const found = new Map();
|
|
4471
|
+
const root = path.resolve(companyRoot);
|
|
4472
|
+
const visit = (dir) => {
|
|
4473
|
+
// The walk is synchronous, so it cannot yield; a heartbeat per directory
|
|
4474
|
+
// keeps the watchdog fed on a large tree.
|
|
4475
|
+
onHeartbeat?.();
|
|
4476
|
+
let entries;
|
|
4477
|
+
try {
|
|
4478
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
4479
|
+
}
|
|
4480
|
+
catch {
|
|
4481
|
+
return;
|
|
4482
|
+
}
|
|
4483
|
+
for (const entry of entries) {
|
|
4484
|
+
const full = path.join(dir, entry.name);
|
|
4485
|
+
if (entry.isDirectory()) {
|
|
4486
|
+
if (entry.name === ".git" || entry.name === "node_modules" || entry.name === ".hq")
|
|
4487
|
+
continue;
|
|
4488
|
+
if (!shouldSync(full, true))
|
|
4489
|
+
continue;
|
|
4490
|
+
visit(full);
|
|
4491
|
+
}
|
|
4492
|
+
else if (entry.isFile() && entry.name.endsWith(CLOUD_PLACEHOLDER_SUFFIX)) {
|
|
4493
|
+
const localPath = full.slice(0, -CLOUD_PLACEHOLDER_SUFFIX.length);
|
|
4494
|
+
const key = toPosixKey(path.relative(root, localPath));
|
|
4495
|
+
if (key === "" || key.startsWith("..") || !inScope(key))
|
|
4496
|
+
continue;
|
|
4497
|
+
if (isPathExcludedFromSync(shouldSync, localPath))
|
|
4498
|
+
continue;
|
|
4499
|
+
const placeholder = readCloudPlaceholder(full, localPath);
|
|
4500
|
+
if (!placeholder || placeholder.orphaned)
|
|
4501
|
+
continue;
|
|
4502
|
+
// Only a placeholder that points at the file's own vault key can be
|
|
4503
|
+
// judged by the listing. One that points at an offload blob or another
|
|
4504
|
+
// vault's object says nothing about this key's presence.
|
|
4505
|
+
if (placeholder.vault !== "company" || placeholder.key !== key)
|
|
4506
|
+
continue;
|
|
4507
|
+
if (!isCloudOnlyLocally(localPath))
|
|
4508
|
+
continue;
|
|
4509
|
+
found.set(key, localPath);
|
|
4510
|
+
}
|
|
4511
|
+
}
|
|
4512
|
+
};
|
|
4513
|
+
visit(root);
|
|
4514
|
+
return found;
|
|
4515
|
+
}
|
|
4256
4516
|
async function computePullPlan(remoteFiles, journal, companyRoot, shouldSync, personalMode, teamSyncedSlugs,
|
|
4257
4517
|
// Coalesced, company-relative prefixes the pull is scoped to (US-005).
|
|
4258
4518
|
// `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
|
|
@@ -4280,7 +4540,11 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4280
4540
|
// so they are not evidence that a local file is a clean synced copy. While
|
|
4281
4541
|
// set, pull never deletes a local file and never replaces an existing local
|
|
4282
4542
|
// file without the conflict executor's byte comparison (feedback_b088e391).
|
|
4283
|
-
recoveryHold = false
|
|
4543
|
+
recoveryHold = false,
|
|
4544
|
+
// feedback_65ee9e51: set for unattended runs. A missing file the journal
|
|
4545
|
+
// does not show the caller created is downloaded back instead of becoming
|
|
4546
|
+
// a `local-delete` tombstone; `onHold` reports each one.
|
|
4547
|
+
unattendedDeleteHold) {
|
|
4284
4548
|
const prefixMatcher = compileScopeMatcher(prefixSet);
|
|
4285
4549
|
const excludeMatcher = compileScopeMatcher(excludePrefixes);
|
|
4286
4550
|
const items = [];
|
|
@@ -4366,6 +4630,16 @@ recoveryHold = false) {
|
|
|
4366
4630
|
})(), (key) => unseenJournalKeys.has(key) || fileTombstones.has(key))
|
|
4367
4631
|
: null;
|
|
4368
4632
|
const caseAliasedKeys = new Set();
|
|
4633
|
+
// Placeholder-driven orphaning: every live placeholder on disk starts as a
|
|
4634
|
+
// candidate and is cleared when its key shows up in the listing. What is left
|
|
4635
|
+
// after the whole listing streamed was deleted from the vault, even when no
|
|
4636
|
+
// journal row or FILE_TOMBSTONE remembers it (a placeholder updated to a
|
|
4637
|
+
// peer's revision has no row). The listing is complete here: a failed or
|
|
4638
|
+
// partial listing throws before this planner finishes, and keys outside the
|
|
4639
|
+
// listed scope are never candidates.
|
|
4640
|
+
const orphanCandidates = collectLivePlaceholderCandidates(companyRoot, shouldSync, (key) => prefixMatcher.coversPath(key), onHeartbeat);
|
|
4641
|
+
// Keys already planned for orphaning, so the leftover pass never repeats one.
|
|
4642
|
+
const orphanPlannedKeys = new Set();
|
|
4369
4643
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
4370
4644
|
onHeartbeat?.();
|
|
4371
4645
|
const posixJournaledKey = toPosixKey(journaledKey);
|
|
@@ -4473,6 +4747,9 @@ recoveryHold = false) {
|
|
|
4473
4747
|
else if (plannedRows % decodeWindowSize() === 0) {
|
|
4474
4748
|
dropJournalDecodeWindow();
|
|
4475
4749
|
}
|
|
4750
|
+
// A key the listing shows is present, whatever happens to it next
|
|
4751
|
+
// (quarantine, backslash spelling, skip), so it is never an orphan.
|
|
4752
|
+
orphanCandidates.delete(toPosixKey(remoteFile.key));
|
|
4476
4753
|
if (quarantinedPaths.has(remoteFile.key)) {
|
|
4477
4754
|
preplannedFilesSkipped++;
|
|
4478
4755
|
continue;
|
|
@@ -4481,6 +4758,7 @@ recoveryHold = false) {
|
|
|
4481
4758
|
// The key can prove its POSIX comparison twin is present, but it must
|
|
4482
4759
|
// never be used to derive a local path or overwrite the slash spelling.
|
|
4483
4760
|
const comparisonKey = toPosixKey(remoteFile.key);
|
|
4761
|
+
orphanCandidates.delete(comparisonKey);
|
|
4484
4762
|
unseenJournalKeys.delete(comparisonKey);
|
|
4485
4763
|
for (const legacyKey of legacyJournalKeysByPosix.get(comparisonKey) ?? []) {
|
|
4486
4764
|
unseenJournalKeys.delete(legacyKey);
|
|
@@ -4498,6 +4776,7 @@ recoveryHold = false) {
|
|
|
4498
4776
|
continue;
|
|
4499
4777
|
}
|
|
4500
4778
|
const posixRemoteKey = toPosixKey(remoteFile.key);
|
|
4779
|
+
orphanCandidates.delete(posixRemoteKey);
|
|
4501
4780
|
// Canonical slash keys clear both canonical journal rows and any legacy
|
|
4502
4781
|
// backslash spelling, on the comparison path only.
|
|
4503
4782
|
if (remoteFile.key === posixRemoteKey) {
|
|
@@ -4817,6 +5096,10 @@ recoveryHold = false) {
|
|
|
4817
5096
|
}
|
|
4818
5097
|
}
|
|
4819
5098
|
const localExists = localLstat !== null;
|
|
5099
|
+
// feedback_65ee9e51: a held file back on disk syncs as an ordinary file.
|
|
5100
|
+
if (localExists && journalEntry?.unattendedHeldBy !== undefined) {
|
|
5101
|
+
clearUnattendedHeldRow(journal, remoteFile.key);
|
|
5102
|
+
}
|
|
4820
5103
|
if (localPathBlockedByFileAncestor) {
|
|
4821
5104
|
// Symmetric counterpart to the (local-dir, cloud-file) warning below.
|
|
4822
5105
|
// Emit the same "manual reconciliation required" surface so the
|
|
@@ -4843,12 +5126,15 @@ recoveryHold = false) {
|
|
|
4843
5126
|
// pointer that a later fetch + push would resurrect.
|
|
4844
5127
|
if ((tombstoneSuppresses || localTombstoneSuppresses) &&
|
|
4845
5128
|
journalEntry?.removedReason !== "local-delete") {
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4851
|
-
|
|
5129
|
+
if (!orphanPlannedKeys.has(remoteFile.key)) {
|
|
5130
|
+
orphanPlannedKeys.add(remoteFile.key);
|
|
5131
|
+
items.push({
|
|
5132
|
+
action: "orphan-cloud-placeholder",
|
|
5133
|
+
remoteFile,
|
|
5134
|
+
localPath,
|
|
5135
|
+
reason: "deleted from the vault by another device or teammate",
|
|
5136
|
+
});
|
|
5137
|
+
}
|
|
4852
5138
|
continue;
|
|
4853
5139
|
}
|
|
4854
5140
|
const baselineEtag = placeholder.remoteEtag ??
|
|
@@ -4858,7 +5144,9 @@ recoveryHold = false) {
|
|
|
4858
5144
|
const remoteChanged = baselineEtag
|
|
4859
5145
|
? normalizeEtag(remoteFile.etag) !== baselineEtag
|
|
4860
5146
|
: remoteFile.size !== placeholder.size;
|
|
4861
|
-
|
|
5147
|
+
// An orphaned placeholder whose key is listed again (same ETag) is
|
|
5148
|
+
// live again: the update executor clears the orphan mark.
|
|
5149
|
+
if (remoteChanged || placeholder.orphaned) {
|
|
4862
5150
|
items.push({
|
|
4863
5151
|
action: "update-cloud-placeholder",
|
|
4864
5152
|
remoteFile,
|
|
@@ -5318,6 +5606,10 @@ recoveryHold = false) {
|
|
|
5318
5606
|
// it to propagate the S3 delete. Skip the re-download without dropping the
|
|
5319
5607
|
// entry. Other tombstone kinds (scope-shrink / FILE_TOMBSTONE) drop it.
|
|
5320
5608
|
if (journalEntry?.removedReason === "local-delete") {
|
|
5609
|
+
// Earlier pulls left these pending for push. Reconsider the whole
|
|
5610
|
+
// burst, unless the server says the object was actually deleted.
|
|
5611
|
+
if (!tombstoneSuppresses)
|
|
5612
|
+
intentionalDeleteCandidates.push({ remoteFile, localPath });
|
|
5321
5613
|
continue;
|
|
5322
5614
|
}
|
|
5323
5615
|
items.push({
|
|
@@ -5342,6 +5634,8 @@ recoveryHold = false) {
|
|
|
5342
5634
|
journalEntry !== undefined &&
|
|
5343
5635
|
!isTombstone(journalEntry) &&
|
|
5344
5636
|
!journalEntry.localDiverges &&
|
|
5637
|
+
// feedback_65ee9e51: an unattended push held this delete; restore it.
|
|
5638
|
+
journalEntry.unattendedHeldBy === undefined &&
|
|
5345
5639
|
(hasCurrentLocalDeleteIntent(journalEntry) ||
|
|
5346
5640
|
propagateDeletePolicy === "currency-gated") &&
|
|
5347
5641
|
normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag) {
|
|
@@ -5376,11 +5670,51 @@ recoveryHold = false) {
|
|
|
5376
5670
|
totalJournalEntries > 0 &&
|
|
5377
5671
|
intentionalDeleteCandidates.length / totalJournalEntries >=
|
|
5378
5672
|
PULL_INTENTIONAL_DELETE_RATIO;
|
|
5673
|
+
// feedback_65089095: the push leg's scattered-delete breaker refuses a
|
|
5674
|
+
// burst of files missing from folders that still exist, even under the
|
|
5675
|
+
// ratio. Apply the same rule here, with the same bypasses, so this pull
|
|
5676
|
+
// downloads those files back instead of stamping `local-delete`
|
|
5677
|
+
// tombstones that would suppress the download and re-trip every push.
|
|
5678
|
+
// Count the same set the push leg counts: under owned-only the push only
|
|
5679
|
+
// considers rows this machine uploaded (direction "up").
|
|
5680
|
+
const scatteredRows = journalRows(journal);
|
|
5681
|
+
const scatteredDeleteRestore = !intentionalDeleteIsBulk && !isMassDeleteGuardOverrideEnabled(propagateDeletePolicy)
|
|
5682
|
+
? detectScatteredDeleteBurst(intentionalDeleteCandidates.flatMap((cand) => propagateDeletePolicy === "owned-only" &&
|
|
5683
|
+
scatteredRows.get(cand.remoteFile.key)?.direction !== "up"
|
|
5684
|
+
? []
|
|
5685
|
+
: [cand.remoteFile.key]), journal, companyRoot, new Set(), { countLocalDeleteTombstones: true })
|
|
5686
|
+
: undefined;
|
|
5687
|
+
const intentionalDeleteIsScattered = scatteredDeleteRestore !== undefined;
|
|
5379
5688
|
for (const cand of intentionalDeleteCandidates) {
|
|
5689
|
+
// Unattended hold (feedback_65ee9e51): only a file the caller created
|
|
5690
|
+
// may become a delete. Another member's file, or one that came down
|
|
5691
|
+
// with no creator, is restored. A file the caller uploaded with no
|
|
5692
|
+
// recorded creator is left for the push leg, which reads `created-by-sub`
|
|
5693
|
+
// from HEAD and deletes or holds it.
|
|
5694
|
+
//
|
|
5380
5695
|
// Recovery hold: a missing local file is not proof of a user delete when
|
|
5381
5696
|
// the journal row may be stale, and a local-delete tombstone would let
|
|
5382
5697
|
// the push leg delete the vault copy once the marker clears. Restore it.
|
|
5383
|
-
|
|
5698
|
+
// The breakers and the recovery hold restore first, so a "defer" can
|
|
5699
|
+
// never leave a candidate they would have restored missing.
|
|
5700
|
+
const forcedRestore = intentionalDeleteIsBulk || intentionalDeleteIsScattered || recoveryHold;
|
|
5701
|
+
const unattendedDecision = unattendedDeleteHold && !forcedRestore
|
|
5702
|
+
? unattendedPullDeleteDecision(unattendedDeleteHold, scatteredRows.get(cand.remoteFile.key) ?? { direction: "down" })
|
|
5703
|
+
: "tombstone";
|
|
5704
|
+
if (unattendedDecision === "defer")
|
|
5705
|
+
continue;
|
|
5706
|
+
if (unattendedDecision === "restore" && unattendedDeleteHold) {
|
|
5707
|
+
const row = scatteredRows.get(cand.remoteFile.key);
|
|
5708
|
+
unattendedDeleteHold.onHold(cand.remoteFile.key, row?.remoteEtag ?? "<not-checked>", normalizeEtag(cand.remoteFile.etag), row?.createdBySub ?? UNKNOWN_CREATOR);
|
|
5709
|
+
}
|
|
5710
|
+
if (forcedRestore || unattendedDecision === "restore") {
|
|
5711
|
+
// Pending local-delete rows are candidates again, so a restore has to
|
|
5712
|
+
// drop the tombstone or the next push still deletes the vault copy.
|
|
5713
|
+
const row = scatteredRows.get(cand.remoteFile.key);
|
|
5714
|
+
if (row?.removedReason === "local-delete") {
|
|
5715
|
+
const { removedAt: _removedAt, removedReason: _removedReason, localDeleteIntent: _intent, ...rest } = row;
|
|
5716
|
+
scatteredRows.set(cand.remoteFile.key, rest);
|
|
5717
|
+
}
|
|
5384
5718
|
items.push({
|
|
5385
5719
|
action: "download",
|
|
5386
5720
|
remoteFile: cand.remoteFile,
|
|
@@ -5393,6 +5727,17 @@ recoveryHold = false) {
|
|
|
5393
5727
|
tombstoneEntry(journal, cand.remoteFile.key, "local-delete");
|
|
5394
5728
|
}
|
|
5395
5729
|
}
|
|
5730
|
+
for (const [key, localPath] of orphanCandidates) {
|
|
5731
|
+
if (orphanPlannedKeys.has(key))
|
|
5732
|
+
continue;
|
|
5733
|
+
orphanPlannedKeys.add(key);
|
|
5734
|
+
items.push({
|
|
5735
|
+
action: "orphan-cloud-placeholder",
|
|
5736
|
+
remoteFile: { key, size: 0, lastModified: new Date(0), etag: "" },
|
|
5737
|
+
localPath,
|
|
5738
|
+
reason: "deleted from the vault by another device or teammate",
|
|
5739
|
+
});
|
|
5740
|
+
}
|
|
5396
5741
|
let filesToDownload = 0;
|
|
5397
5742
|
let bytesToDownload = 0;
|
|
5398
5743
|
let filesToSkip = 0;
|
|
@@ -5617,6 +5962,15 @@ recoveryHold = false) {
|
|
|
5617
5962
|
tombstones,
|
|
5618
5963
|
invalidRemoteKeys,
|
|
5619
5964
|
caseAliasedKeys,
|
|
5965
|
+
...(scatteredDeleteRestore
|
|
5966
|
+
? {
|
|
5967
|
+
scatteredDeleteRestore: {
|
|
5968
|
+
candidates: scatteredDeleteRestore.keys.length,
|
|
5969
|
+
folders: scatteredDeleteRestore.areas,
|
|
5970
|
+
samplePaths: scatteredDeleteRestore.samplePaths,
|
|
5971
|
+
},
|
|
5972
|
+
}
|
|
5973
|
+
: {}),
|
|
5620
5974
|
};
|
|
5621
5975
|
}
|
|
5622
5976
|
/**
|
|
@@ -5631,6 +5985,9 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5631
5985
|
else if (event.type === "identity-switch-push-held" && event.reason === "company-rebind") {
|
|
5632
5986
|
console.warn(` ! this folder was linked to a different company, so ${event.count} local file${event.count === 1 ? " was" : "s were"} not uploaded and nothing was deleted in the cloud. The hold clears after a full \`hq sync\` pull that finishes cleanly: no refused deletes, and every file it sets aside moved into .hq/switch-quarantine (which must be a folder you can write to; fix that first if a pull reported it). HQ_SYNC_DELETE_BULK_OVERRIDE=1 skips only the bulk-delete refusal; files are still moved into quarantine, not deleted.`);
|
|
5633
5987
|
}
|
|
5988
|
+
else if (event.type === "identity-switch-push-held" && event.message) {
|
|
5989
|
+
console.warn(` ! ${event.message}`);
|
|
5990
|
+
}
|
|
5634
5991
|
else if (event.type === "identity-switch-push-held" && event.count === 0) {
|
|
5635
5992
|
console.warn(" ! someone else was signed in on this computer, so uploads from here are held until you review and release them.");
|
|
5636
5993
|
}
|
|
@@ -5767,6 +6124,11 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5767
6124
|
const mb = (event.bytes / (1024 * 1024)).toFixed(1);
|
|
5768
6125
|
console.warn(` ! ${event.path} skipped — ${mb} MB exceeds the 50 MB sync size limit`);
|
|
5769
6126
|
}
|
|
6127
|
+
else if (event.type === "pull-restored-scattered-deletes") {
|
|
6128
|
+
console.warn(` ⚠ scattered-delete guard: downloading ${event.candidates} missing file(s) again instead of deleting them; they vanished from ${event.folders} top-level folders that still exist, which looks like a rollback rather than a cleanup. To delete them on purpose, re-run with HQ_SYNC_DELETE_BULK_OVERRIDE=1.`);
|
|
6129
|
+
for (const samplePath of event.samplePaths.slice(0, 5))
|
|
6130
|
+
console.warn(` · ${samplePath}`);
|
|
6131
|
+
}
|
|
5770
6132
|
else if (event.type === "scope-shrink-refused") {
|
|
5771
6133
|
const why = event.reason === "empty-grants"
|
|
5772
6134
|
? "your sync scope came back with no shared folders, which usually means a server or grant problem"
|