@indigoai-us/hq-cloud 6.18.94 → 6.18.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +3 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +12 -3
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +28 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +33 -2
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/conflict.d.ts +12 -1
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +6 -2
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/reindex.d.ts +24 -0
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +26 -1
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/rescue.d.ts +8 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +43 -2
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/share.d.ts +46 -5
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +187 -23
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +48 -2
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +211 -20
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/tombstones.d.ts +8 -0
- package/dist/cli/tombstones.d.ts.map +1 -1
- package/dist/cli/tombstones.js +9 -0
- package/dist/cli/tombstones.js.map +1 -1
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +61 -79
- package/dist/context.js.map +1 -1
- package/dist/hq-root-recovery.d.ts +37 -6
- package/dist/hq-root-recovery.d.ts.map +1 -1
- package/dist/hq-root-recovery.js +266 -22
- package/dist/hq-root-recovery.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts +68 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +265 -2
- package/dist/journal.js.map +1 -1
- package/dist/lib/cloud-authoritative.d.ts +10 -0
- package/dist/lib/cloud-authoritative.d.ts.map +1 -1
- package/dist/lib/cloud-authoritative.js +23 -0
- package/dist/lib/cloud-authoritative.js.map +1 -1
- package/dist/lib/net-errors.d.ts +14 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +49 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/server-owned-skips.d.ts +64 -0
- package/dist/lib/server-owned-skips.d.ts.map +1 -0
- package/dist/lib/server-owned-skips.js +182 -0
- package/dist/lib/server-owned-skips.js.map +1 -0
- package/dist/manifest-reconcile.d.ts +14 -0
- package/dist/manifest-reconcile.d.ts.map +1 -1
- package/dist/manifest-reconcile.js +20 -2
- package/dist/manifest-reconcile.js.map +1 -1
- package/dist/outposts/client.d.ts +6 -1
- package/dist/outposts/client.d.ts.map +1 -1
- package/dist/outposts/client.js +13 -1
- package/dist/outposts/client.js.map +1 -1
- package/dist/outposts/command.d.ts +1 -1
- package/dist/outposts/command.d.ts.map +1 -1
- package/dist/outposts/command.js +58 -2
- package/dist/outposts/command.js.map +1 -1
- package/dist/outposts/company.d.ts +15 -1
- package/dist/outposts/company.d.ts.map +1 -1
- package/dist/outposts/company.js +36 -1
- package/dist/outposts/company.js.map +1 -1
- package/dist/s3.d.ts +22 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +44 -0
- package/dist/s3.js.map +1 -1
- package/dist/session-log-capture.d.ts.map +1 -1
- package/dist/session-log-capture.js +5 -2
- package/dist/session-log-capture.js.map +1 -1
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +3 -3
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/durable-apply.js +2 -2
- package/dist/sync/durable-apply.js.map +1 -1
- package/dist/sync/per-area-maintenance.js +2 -2
- package/dist/sync/per-area-maintenance.js.map +1 -1
- package/dist/sync/reconcile-cursor.js +2 -2
- package/dist/sync/reconcile-cursor.js.map +1 -1
- package/dist/sync/secret-path-policy.d.ts +37 -11
- package/dist/sync/secret-path-policy.d.ts.map +1 -1
- package/dist/sync/secret-path-policy.js +68 -34
- package/dist/sync/secret-path-policy.js.map +1 -1
- package/dist/sync/snapshot-client.js +2 -2
- package/dist/sync/snapshot-client.js.map +1 -1
- package/dist/sync/state-store.d.ts +26 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +275 -8
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/uploader-finalization.d.ts.map +1 -1
- package/dist/sync/uploader-finalization.js +2 -2
- package/dist/sync/uploader-finalization.js.map +1 -1
- package/dist/test-fixtures/unknown-journal-generation.d.ts +9 -0
- package/dist/test-fixtures/unknown-journal-generation.d.ts.map +1 -0
- package/dist/test-fixtures/unknown-journal-generation.js +37 -0
- package/dist/test-fixtures/unknown-journal-generation.js.map +1 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +3 -3
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/sync.js
CHANGED
|
@@ -27,7 +27,7 @@ import { readPendingConflictNotices, recordConflictParked } from "../conflict-no
|
|
|
27
27
|
import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
|
|
28
28
|
import { SKILLS_KEY_PREFIX } from "./doctor.js";
|
|
29
29
|
import { downloadFile, createDownloadStagingPath, replaceStagedPath, streamRemoteFiles, streamRemoteFilePages, headRemoteFile, FILE_CONTENT_SHA256_META_KEY, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
30
|
-
import { readJournalForPass, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, dropJournalDecodeWindow, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
30
|
+
import { readJournalForPass, bindPersonalVaultJournalToIdentity, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, dropJournalDecodeWindow, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, configureRejectedDeltaRecovery, getJournalRecoveryMarker, clearJournalRecoveryMarker, migratePersonalVaultJournal, } from "../journal.js";
|
|
31
31
|
import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
|
|
32
32
|
import { isPersonalVaultExcluded, isSafePersonalMcpConfigFile } from "../personal-vault-exclusions.js";
|
|
33
33
|
import { resolvePersonalConnectorSettingsSync } from "../lib/personal-connector-settings-sync.js";
|
|
@@ -41,7 +41,7 @@ import { resolveSymlinkTargetRefusalFlag } from "../sync/symlink-target-refusal-
|
|
|
41
41
|
import { coalesceDenseTargetedPullPaths } from "../sync/targeted-pull-prefixes.js";
|
|
42
42
|
import { listRemoteForScope, POST_FILTER_THRESHOLD } from "../remote-pull.js";
|
|
43
43
|
import { createIgnoreFilter, isPathExcludedFromSync, SYNC_MAX_FILE_BYTES, } from "../ignore.js";
|
|
44
|
-
import {
|
|
44
|
+
import { claimNeverSyncedLocalNotices, formatSecretsNotSyncedLine, isNeverSyncedVaultPath, isSecretBearingVaultPath, isSkipSecretPathsEnabled, isWalRejectedDeltaRecoveryEnabled, } from "../sync/secret-path-policy.js";
|
|
45
45
|
import { hasRemoteChanged, isAccessDenied, isNotFound, resolveActiveCompany, resolveTransferConcurrency, readHostLoad, } from "../sync-core.js";
|
|
46
46
|
import { formatIgnoreExcludedWarningLines, isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
47
47
|
import { createHumanSkipLineLog, formatReconcileSkipLine, } from "./reconcile-skip-reason.js";
|
|
@@ -51,7 +51,7 @@ import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/boar
|
|
|
51
51
|
import { createHash } from "node:crypto";
|
|
52
52
|
import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
53
53
|
import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
|
|
54
|
-
import { isCloudAuthoritative } from "../lib/cloud-authoritative.js";
|
|
54
|
+
import { isCloudAuthoritative, isPullReconciledServerOwned, } from "../lib/cloud-authoritative.js";
|
|
55
55
|
import { isMassDeleteGuardOverrideEnabled, MASS_DELETE_ABSOLUTE_CEILING, MASS_DELETE_SAMPLE_CAP, massDeleteGuardReason, resolveMassDeleteGuards, } from "../sync/mass-delete-guards.js";
|
|
56
56
|
import { reindex } from "./reindex.js";
|
|
57
57
|
import { withOperationLock } from "../operation-lock.js";
|
|
@@ -515,6 +515,7 @@ function snapshotCooperativeRowVisits() {
|
|
|
515
515
|
* Do NOT import from `_testing` outside of tests in this package.
|
|
516
516
|
*/
|
|
517
517
|
export const _testing = {
|
|
518
|
+
clearRecoveryMarkerAfterPass,
|
|
518
519
|
enableCooperativeRowVisits,
|
|
519
520
|
disableCooperativeRowVisits,
|
|
520
521
|
cooperativeRowVisits: snapshotCooperativeRowVisits,
|
|
@@ -635,9 +636,13 @@ function safeErrorName(err) {
|
|
|
635
636
|
* runner's all-complete summary carries a stable machine tag alongside the
|
|
636
637
|
* paginated path list.
|
|
637
638
|
*/
|
|
638
|
-
function extractAccessDeniedReason(err) {
|
|
639
|
+
function extractAccessDeniedReason(err, key) {
|
|
639
640
|
if (err instanceof Error) {
|
|
640
|
-
|
|
641
|
+
// object-io messages embed the key ("Access denied for START-HERE.md:
|
|
642
|
+
// ..."), and an upper-case key segment would otherwise be read as the
|
|
643
|
+
// machine tag. Remove the key before matching.
|
|
644
|
+
const raw = err.message ?? "";
|
|
645
|
+
const msg = key ? raw.split(key).join("") : raw;
|
|
641
646
|
// Prefer a bounded SCREAMING_SNAKE_CASE token like FILES_PRESIGN_FORBIDDEN
|
|
642
647
|
// — object-io embeds these inside a parenthesized fragment such as
|
|
643
648
|
// "(server get denied: FILES_PRESIGN_FORBIDDEN)". The prior stricter
|
|
@@ -751,6 +756,7 @@ async function syncWithOperationLockHeld(options) {
|
|
|
751
756
|
// they begin, so their failure cannot turn a successful bulk transfer into
|
|
752
757
|
// a full replay on the next pass.
|
|
753
758
|
flushDownloadedJournalTail(run);
|
|
759
|
+
reportAccessDeniedSkipsToConsole(run);
|
|
754
760
|
if (run.options.cooperativeCheckpoint?.shouldAbort?.() === true) {
|
|
755
761
|
const newFiles = reportableNewFiles(run, plan);
|
|
756
762
|
return {
|
|
@@ -767,6 +773,7 @@ async function syncWithOperationLockHeld(options) {
|
|
|
767
773
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
768
774
|
filesTombstoned: counters.filesTombstoned,
|
|
769
775
|
filesOutOfScope: counters.filesOutOfScope,
|
|
776
|
+
filesAccessDenied: run.accessDeniedSkips.size,
|
|
770
777
|
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
771
778
|
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
772
779
|
changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
|
|
@@ -823,6 +830,7 @@ async function buildPullContext(options) {
|
|
|
823
830
|
},
|
|
824
831
|
});
|
|
825
832
|
const skipSecretPaths = await isSkipSecretPathsEnabled(vaultConfig, ctx.uid);
|
|
833
|
+
configureRejectedDeltaRecovery(await isWalRejectedDeltaRecoveryEnabled(vaultConfig, ctx.uid));
|
|
826
834
|
const ignoreFilter = createIgnoreFilter(hqRoot, {
|
|
827
835
|
allowRootMcpConfig: personalConnectorSettingsSync,
|
|
828
836
|
});
|
|
@@ -839,6 +847,19 @@ async function buildPullContext(options) {
|
|
|
839
847
|
return ignoreFilter(filePath, isDir);
|
|
840
848
|
};
|
|
841
849
|
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
850
|
+
// hq-cloud#1028: bind the personal journal to this identity before any
|
|
851
|
+
// journal handle (quarantine, targeted session, scoped or full read) opens.
|
|
852
|
+
// Migrate the legacy slug first so the copied journal is fenced too.
|
|
853
|
+
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG)
|
|
854
|
+
migratePersonalVaultJournal();
|
|
855
|
+
const personalBind = journalSlug === PERSONAL_VAULT_JOURNAL_SLUG
|
|
856
|
+
? bindPersonalVaultJournalToIdentity({
|
|
857
|
+
uid: ctx.uid,
|
|
858
|
+
bucket: ctx.bucketName,
|
|
859
|
+
region: ctx.region,
|
|
860
|
+
endpoint: vaultConfig?.apiUrl,
|
|
861
|
+
})
|
|
862
|
+
: undefined;
|
|
842
863
|
const quarantinedPaths = readQuarantinedJournalPaths(journalSlug);
|
|
843
864
|
const startedAt = new Date().toISOString();
|
|
844
865
|
// Mirror every progress event into the shared cross-process snapshot so the
|
|
@@ -855,15 +876,15 @@ async function buildPullContext(options) {
|
|
|
855
876
|
baseEmit(event);
|
|
856
877
|
recordProgress(event);
|
|
857
878
|
};
|
|
879
|
+
if (personalBind?.fenced) {
|
|
880
|
+
emit({ type: "personal-journal-identity-fenced", reason: personalBind.fenced });
|
|
881
|
+
}
|
|
858
882
|
if (quarantinedPaths.size > 0) {
|
|
859
883
|
emit({ type: "journal-quarantine-unresolved", count: quarantinedPaths.size });
|
|
860
884
|
}
|
|
861
|
-
const neverSyncedLocalPath
|
|
862
|
-
if (neverSyncedLocalPath) {
|
|
885
|
+
for (const neverSyncedLocalPath of claimNeverSyncedLocalNotices(hqRoot, companyRoot)) {
|
|
863
886
|
emit({ type: "secrets-not-synced", path: neverSyncedLocalPath });
|
|
864
887
|
}
|
|
865
|
-
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG)
|
|
866
|
-
migratePersonalVaultJournal();
|
|
867
888
|
const syncMode = options.syncMode ?? "all";
|
|
868
889
|
const currentPrefixSet = syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
|
|
869
890
|
// Push-only exclusions (US-006). Coalesced independently of `currentPrefixSet`
|
|
@@ -964,6 +985,7 @@ async function buildPullContext(options) {
|
|
|
964
985
|
hqRoot,
|
|
965
986
|
emit,
|
|
966
987
|
humanSkipLines,
|
|
988
|
+
accessDeniedSkips: new Map(),
|
|
967
989
|
ctx,
|
|
968
990
|
companyRoot,
|
|
969
991
|
personalConnectorSettingsSync,
|
|
@@ -1101,7 +1123,7 @@ async function planPull(run) {
|
|
|
1101
1123
|
if (!rememberLinkWarning(run.linkWarnings, warning))
|
|
1102
1124
|
return;
|
|
1103
1125
|
run.emit(brokenLinkSkipEvent(warning));
|
|
1104
|
-
});
|
|
1126
|
+
}, getJournalRecoveryMarker(run.journalSlug) !== undefined);
|
|
1105
1127
|
const localDeleteKeys = Array.from(new Set([
|
|
1106
1128
|
...plan.tombstones.map((tombstone) => tombstone.key),
|
|
1107
1129
|
...plan.items.flatMap((item) => item.action === "tombstone-delete" ? [item.remoteFile.key] : []),
|
|
@@ -1260,8 +1282,15 @@ function createPullCounters() {
|
|
|
1260
1282
|
hasRetryableOutcome: false,
|
|
1261
1283
|
};
|
|
1262
1284
|
}
|
|
1263
|
-
function reportableNewFiles(
|
|
1264
|
-
|
|
1285
|
+
function reportableNewFiles(run, plan) {
|
|
1286
|
+
// A file whose GET was refused never reached this device, so it is not a
|
|
1287
|
+
// "new file" here. Reporting it would also send the created-by enrichment
|
|
1288
|
+
// back to HEAD a key the caller cannot read, one failing request and one
|
|
1289
|
+
// stderr line per denied key (feedback_18ecc0b1: a burst of
|
|
1290
|
+
// "HeadObject failed ...: UnknownError" on a pull that downloaded nothing).
|
|
1291
|
+
if (run.accessDeniedSkips.size === 0)
|
|
1292
|
+
return plan.newFiles;
|
|
1293
|
+
return plan.newFiles.filter((nf) => !run.accessDeniedSkips.has(nf.path));
|
|
1265
1294
|
}
|
|
1266
1295
|
/** Sidecar listing the moves an in-flight scope shrink has not finished. */
|
|
1267
1296
|
function scopeShrinkPendingMovesFile(run) {
|
|
@@ -2336,6 +2365,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
2336
2365
|
remoteBytes: fs.readFileSync(probeAbs),
|
|
2337
2366
|
localMtime,
|
|
2338
2367
|
remoteMtime: remoteFile.lastModified,
|
|
2368
|
+
noJournalBaseline: journalEntryAtConflict === undefined,
|
|
2339
2369
|
});
|
|
2340
2370
|
}
|
|
2341
2371
|
catch (readErr) {
|
|
@@ -2794,6 +2824,9 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
2794
2824
|
? { bypassBandwidthGovernor: true }
|
|
2795
2825
|
: {}),
|
|
2796
2826
|
refuseUnsafeSymlinkTargets: run.symlinkTargetRefusal,
|
|
2827
|
+
// feedback_63da614f: never replay another machine's absolute link
|
|
2828
|
+
// target into the live tree; skip that key, keep the pass going.
|
|
2829
|
+
skipForeignAbsoluteSymlinkTargets: true,
|
|
2797
2830
|
symlinkTargetScope: run.options.personalMode === true
|
|
2798
2831
|
? { kind: "personal" }
|
|
2799
2832
|
: { kind: "company", companySlug: run.ctx.slug },
|
|
@@ -2942,10 +2975,12 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
2942
2975
|
// `filesDone` (parity with the other `skip-*` paths) and the
|
|
2943
2976
|
// runner produce a compact denied-summary on all-complete.
|
|
2944
2977
|
counters.filesSkipped++;
|
|
2978
|
+
const reason = extractAccessDeniedReason(err, remoteFile.key);
|
|
2979
|
+
run.accessDeniedSkips.set(remoteFile.key, reason);
|
|
2945
2980
|
run.emit({
|
|
2946
2981
|
type: "skip-access-denied",
|
|
2947
2982
|
path: remoteFile.key,
|
|
2948
|
-
reason
|
|
2983
|
+
reason,
|
|
2949
2984
|
message: describeError(err),
|
|
2950
2985
|
});
|
|
2951
2986
|
}
|
|
@@ -3100,6 +3135,34 @@ function persistPullJournal(run) {
|
|
|
3100
3135
|
release();
|
|
3101
3136
|
}
|
|
3102
3137
|
}
|
|
3138
|
+
const ACCESS_DENIED_SAMPLE_CAP = 10;
|
|
3139
|
+
/**
|
|
3140
|
+
* Human-mode summary of the files this pull could not read. Each refused GET
|
|
3141
|
+
* already emits `skip-access-denied`, which `--json` callers and the runner
|
|
3142
|
+
* consume; the plain console renderer ignored it, so a pull whose every file
|
|
3143
|
+
* was refused printed "N skipped, 0 error(s)" with no reason
|
|
3144
|
+
* (feedback_18ecc0b1). One summary block, not a line per key, because a
|
|
3145
|
+
* fresh member can be refused thousands of keys.
|
|
3146
|
+
*/
|
|
3147
|
+
function reportAccessDeniedSkipsToConsole(run) {
|
|
3148
|
+
if (run.options.onEvent)
|
|
3149
|
+
return;
|
|
3150
|
+
const denied = run.accessDeniedSkips;
|
|
3151
|
+
if (denied.size === 0)
|
|
3152
|
+
return;
|
|
3153
|
+
const reasons = [...new Set(denied.values())].sort().join(", ");
|
|
3154
|
+
console.warn(` ! ${denied.size} file${denied.size === 1 ? " was" : "s were"} not downloaded — access denied (${reasons}). Ask a company owner or admin to grant you read access, then pull again:`);
|
|
3155
|
+
let shown = 0;
|
|
3156
|
+
for (const key of denied.keys()) {
|
|
3157
|
+
if (shown >= ACCESS_DENIED_SAMPLE_CAP)
|
|
3158
|
+
break;
|
|
3159
|
+
console.warn(` · ${key}`);
|
|
3160
|
+
shown++;
|
|
3161
|
+
}
|
|
3162
|
+
if (denied.size > shown) {
|
|
3163
|
+
console.warn(` ... and ${denied.size - shown} more`);
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3103
3166
|
async function emitAndReportNewFiles(run, plan) {
|
|
3104
3167
|
const enrichedNewFiles = [];
|
|
3105
3168
|
const newFiles = reportableNewFiles(run, plan);
|
|
@@ -3152,6 +3215,11 @@ async function emitAndReportNewFiles(run, plan) {
|
|
|
3152
3215
|
isRetryableIdentityRefusal(headErr)) {
|
|
3153
3216
|
throw headErr;
|
|
3154
3217
|
}
|
|
3218
|
+
// Author enrichment is best-effort. A read refusal is already
|
|
3219
|
+
// reported by the access-denied summary, so do not log it per key.
|
|
3220
|
+
if (isAccessDenied(headErr)) {
|
|
3221
|
+
return { path: nf.path, bytes: nf.bytes, addedBy };
|
|
3222
|
+
}
|
|
3155
3223
|
try {
|
|
3156
3224
|
console.error(`[hq-sync] HeadObject failed for ${nf.path}: ${headErr instanceof Error ? headErr.message : String(headErr)}`);
|
|
3157
3225
|
}
|
|
@@ -3249,6 +3317,16 @@ async function executeJournalTombstoneDeletes(run, plan, counters) {
|
|
|
3249
3317
|
await new Promise((resolve) => setImmediate(resolve));
|
|
3250
3318
|
}
|
|
3251
3319
|
const { key, localPath, localSnapshot } = tombstone;
|
|
3320
|
+
if (tombstone.recoveryHold) {
|
|
3321
|
+
removeEntry(run.journal, key);
|
|
3322
|
+
run.emit({
|
|
3323
|
+
type: "error",
|
|
3324
|
+
diagnostic: true,
|
|
3325
|
+
path: key,
|
|
3326
|
+
message: "journal recovery hold: kept the local file the cloud no longer lists; dropped its journal row instead of deleting it",
|
|
3327
|
+
});
|
|
3328
|
+
continue;
|
|
3329
|
+
}
|
|
3252
3330
|
if (tombstone.journalOnly || plan.caseAliasedKeys?.has(toPosixKey(key))) {
|
|
3253
3331
|
retireCaseAliasJournalRow(run, key);
|
|
3254
3332
|
continue;
|
|
@@ -3315,6 +3393,43 @@ function scopePlanPreviousMode(run) {
|
|
|
3315
3393
|
function scopePlanPreviousPrefixSet(run) {
|
|
3316
3394
|
return [...(lastPullRecord(run.journal, run.ctx.uid)?.prefixSet ?? [])];
|
|
3317
3395
|
}
|
|
3396
|
+
function clearRecoveryMarkerAfterPass(input) {
|
|
3397
|
+
const authoritativeFullListing = input.syncMode === "all" &&
|
|
3398
|
+
input.currentPrefixSet.length === 1 && input.currentPrefixSet[0] === "" &&
|
|
3399
|
+
input.listedPrefixSet.length === 1 && input.listedPrefixSet[0] === "" &&
|
|
3400
|
+
input.pullPrefixes.length === 0;
|
|
3401
|
+
if (!getJournalRecoveryMarker(input.journalSlug) || !authoritativeFullListing || input.hasRetryableOutcome || input.aborted) {
|
|
3402
|
+
return false;
|
|
3403
|
+
}
|
|
3404
|
+
clearJournalRecoveryMarker(input.journalSlug);
|
|
3405
|
+
return true;
|
|
3406
|
+
}
|
|
3407
|
+
/** Most paths the human pull-delete summary names before "... and N more". */
|
|
3408
|
+
export const PULL_DELETE_SUMMARY_SAMPLE_CAP = 10;
|
|
3409
|
+
/**
|
|
3410
|
+
* Human console summary of the local files this pull removed because they
|
|
3411
|
+
* were deleted in the cloud (feedback_97906212). Only printed for the human
|
|
3412
|
+
* CLI (no `onEvent`); machine callers already receive one `progress` event
|
|
3413
|
+
* with `deleted: true` per key and `SyncResult.appliedDeletes`.
|
|
3414
|
+
*/
|
|
3415
|
+
export function formatPullLocalDeleteSummary(appliedDeletes) {
|
|
3416
|
+
if (appliedDeletes.length === 0)
|
|
3417
|
+
return [];
|
|
3418
|
+
const n = appliedDeletes.length;
|
|
3419
|
+
const lines = [
|
|
3420
|
+
`Deleted ${n} local file${n === 1 ? "" : "s"} because ${n === 1 ? "it was" : "they were"} deleted in the cloud:`,
|
|
3421
|
+
];
|
|
3422
|
+
for (const key of appliedDeletes.slice(0, PULL_DELETE_SUMMARY_SAMPLE_CAP))
|
|
3423
|
+
lines.push(` · ${key}`);
|
|
3424
|
+
if (n > PULL_DELETE_SUMMARY_SAMPLE_CAP)
|
|
3425
|
+
lines.push(` ... and ${n - PULL_DELETE_SUMMARY_SAMPLE_CAP} more`);
|
|
3426
|
+
lines.push(" To bring a file back, run hq files versions <path>, then hq files restore <path> --version-id <id>.");
|
|
3427
|
+
return lines;
|
|
3428
|
+
}
|
|
3429
|
+
function printPullLocalDeleteSummary(appliedDeletes) {
|
|
3430
|
+
for (const line of formatPullLocalDeleteSummary(appliedDeletes))
|
|
3431
|
+
console.log(line);
|
|
3432
|
+
}
|
|
3318
3433
|
function finalizePullRun(run, plan, scopeRun, counters) {
|
|
3319
3434
|
appendPullRecord(run.journal, {
|
|
3320
3435
|
pullId: generatePullId(),
|
|
@@ -3340,6 +3455,15 @@ function finalizePullRun(run, plan, scopeRun, counters) {
|
|
|
3340
3455
|
});
|
|
3341
3456
|
run.journal.lastSync = new Date().toISOString();
|
|
3342
3457
|
persistPullJournal(run);
|
|
3458
|
+
clearRecoveryMarkerAfterPass({
|
|
3459
|
+
journalSlug: run.journalSlug,
|
|
3460
|
+
syncMode: run.syncMode,
|
|
3461
|
+
currentPrefixSet: run.currentPrefixSet,
|
|
3462
|
+
listedPrefixSet: run.listedPrefixSet,
|
|
3463
|
+
pullPrefixes: run.options.pullPrefixes ?? [],
|
|
3464
|
+
hasRetryableOutcome: counters.hasRetryableOutcome,
|
|
3465
|
+
aborted: run.options.cooperativeCheckpoint?.shouldAbort?.() === true,
|
|
3466
|
+
});
|
|
3343
3467
|
const changedOnDisk = counters.filesDownloaded > 0 ||
|
|
3344
3468
|
counters.filesTombstoned > 0 ||
|
|
3345
3469
|
scopeRun.scopeOrphansRemoved > 0;
|
|
@@ -3363,6 +3487,8 @@ function finalizePullRun(run, plan, scopeRun, counters) {
|
|
|
3363
3487
|
// best-effort: a post-sync refresh failure never fails the sync
|
|
3364
3488
|
}
|
|
3365
3489
|
}
|
|
3490
|
+
if (run.humanSkipLines)
|
|
3491
|
+
printPullLocalDeleteSummary(counters.appliedDeletes);
|
|
3366
3492
|
const newFiles = reportableNewFiles(run, plan);
|
|
3367
3493
|
const result = {
|
|
3368
3494
|
filesDownloaded: counters.filesDownloaded,
|
|
@@ -3378,6 +3504,7 @@ function finalizePullRun(run, plan, scopeRun, counters) {
|
|
|
3378
3504
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
3379
3505
|
filesTombstoned: counters.filesTombstoned,
|
|
3380
3506
|
filesOutOfScope: counters.filesOutOfScope,
|
|
3507
|
+
filesAccessDenied: run.accessDeniedSkips.size,
|
|
3381
3508
|
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
3382
3509
|
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
3383
3510
|
changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
|
|
@@ -3709,7 +3836,13 @@ fileTombstones = new Map(),
|
|
|
3709
3836
|
excludePrefixes = [],
|
|
3710
3837
|
// The runner resolves this once and shares it with the push leg. Direct
|
|
3711
3838
|
// callers preserve the pre-policy, watcher-intent-only behavior.
|
|
3712
|
-
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size, cooperativeCheckpoint, onCooperativeHandoff, deleteDenominatorKeys, pullContentHashes = {}, ctx, caseFoldingVolume = false, allowRootMcpConfig = false, quarantinedPaths = new Set(), onAuditRemoteFile, onLinkWarning
|
|
3839
|
+
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size, cooperativeCheckpoint, onCooperativeHandoff, deleteDenominatorKeys, pullContentHashes = {}, ctx, caseFoldingVolume = false, allowRootMcpConfig = false, quarantinedPaths = new Set(), onAuditRemoteFile, onLinkWarning,
|
|
3840
|
+
// A journal recovery marker exists (rebuilt from a valid prefix or reset
|
|
3841
|
+
// from an unreadable state store). The journal rows may be stale or missing,
|
|
3842
|
+
// so they are not evidence that a local file is a clean synced copy. While
|
|
3843
|
+
// set, pull never deletes a local file and never replaces an existing local
|
|
3844
|
+
// file without the conflict executor's byte comparison (feedback_b088e391).
|
|
3845
|
+
recoveryHold = false) {
|
|
3713
3846
|
const prefixMatcher = compileScopeMatcher(prefixSet);
|
|
3714
3847
|
const excludeMatcher = compileScopeMatcher(excludePrefixes);
|
|
3715
3848
|
const items = [];
|
|
@@ -4086,6 +4219,11 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4086
4219
|
continue;
|
|
4087
4220
|
}
|
|
4088
4221
|
const journalEntry = getEntry(journal, remoteFile.key);
|
|
4222
|
+
// A server-owned key (sources/, signals/, ontology/ outside entities,
|
|
4223
|
+
// company-brief.md) can never be pushed or deleted from this machine, so
|
|
4224
|
+
// its local state never wins: a divergent local copy is replaced from
|
|
4225
|
+
// the cloud and a missing one is re-downloaded (feedback_5a2609f2).
|
|
4226
|
+
const serverOwnedPullReconciles = isPullReconciledServerOwned(remoteFile.key);
|
|
4089
4227
|
// Reindex and package wiring own symlinks under these `core/<type>`
|
|
4090
4228
|
// surfaces; their canonical sources are elsewhere. A prior pull records
|
|
4091
4229
|
// the remote object as a symlink, then reindex removes the generated link.
|
|
@@ -4161,7 +4299,10 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4161
4299
|
// The currency-gated producer also stamps local-delete without a watcher
|
|
4162
4300
|
// intent. Preserve that decision while the remote ETag is still current;
|
|
4163
4301
|
// otherwise a second pull undoes the first before push can propagate it.
|
|
4164
|
-
|
|
4302
|
+
// A server-owned key is exempt: push never propagates its delete, so a
|
|
4303
|
+
// local-delete tombstone would hide the cloud copy from this machine forever.
|
|
4304
|
+
const localTombstoneSuppresses = !serverOwnedPullReconciles &&
|
|
4305
|
+
journalEntry !== undefined &&
|
|
4165
4306
|
isTombstone(journalEntry) &&
|
|
4166
4307
|
(hasCurrentLocalDeleteIntent(journalEntry) ||
|
|
4167
4308
|
(propagateDeletePolicy === "currency-gated" &&
|
|
@@ -4428,7 +4569,7 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4428
4569
|
// is the stale version → delete it + drop the journal entry so it stays
|
|
4429
4570
|
// gone on this machine.
|
|
4430
4571
|
if (tombstoneSuppresses) {
|
|
4431
|
-
if (localChanged || !journalEntry) {
|
|
4572
|
+
if (localChanged || !journalEntry || recoveryHold) {
|
|
4432
4573
|
items.push({
|
|
4433
4574
|
action: "conflict",
|
|
4434
4575
|
remoteFile,
|
|
@@ -4590,6 +4731,22 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4590
4731
|
continue;
|
|
4591
4732
|
}
|
|
4592
4733
|
if (journalEntry && localChanged && !remoteChanged) {
|
|
4734
|
+
// Push refuses a server-owned key, so "local only" would never
|
|
4735
|
+
// converge. Route it through the conflict executor, which parks the
|
|
4736
|
+
// local body under `.hq/conflict-backups/` and adopts the cloud copy
|
|
4737
|
+
// (cloud-authoritative PULL-WINS).
|
|
4738
|
+
if (serverOwnedPullReconciles) {
|
|
4739
|
+
items.push({
|
|
4740
|
+
action: "conflict",
|
|
4741
|
+
remoteFile,
|
|
4742
|
+
localPath,
|
|
4743
|
+
localHash,
|
|
4744
|
+
localMtime: localLstat.mtime,
|
|
4745
|
+
localSize: isLocalSymlink ? 0 : localLstat.size,
|
|
4746
|
+
localSnapshot: plannedLocalSnapshot,
|
|
4747
|
+
});
|
|
4748
|
+
continue;
|
|
4749
|
+
}
|
|
4593
4750
|
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
4594
4751
|
continue;
|
|
4595
4752
|
}
|
|
@@ -4615,6 +4772,22 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4615
4772
|
continue;
|
|
4616
4773
|
}
|
|
4617
4774
|
if (journalEntry && !localChanged && remoteChanged) {
|
|
4775
|
+
// Recovery hold: the journal baseline this file "matches" may be a
|
|
4776
|
+
// stale generation, so a clean-looking local copy can hold edits the
|
|
4777
|
+
// journal never saw. Compare bytes in the conflict executor instead
|
|
4778
|
+
// of downloading over it; matching bytes still converge silently.
|
|
4779
|
+
if (recoveryHold && !isLocalSymlink) {
|
|
4780
|
+
items.push({
|
|
4781
|
+
action: "conflict",
|
|
4782
|
+
remoteFile,
|
|
4783
|
+
localPath,
|
|
4784
|
+
localHash,
|
|
4785
|
+
localMtime: localLstat.mtime,
|
|
4786
|
+
localSize: localLstat.size,
|
|
4787
|
+
localSnapshot: plannedLocalSnapshot,
|
|
4788
|
+
});
|
|
4789
|
+
continue;
|
|
4790
|
+
}
|
|
4618
4791
|
// ── Anti-rollback guard (silent newer→older overwrite) ───────────
|
|
4619
4792
|
// Remote etag moved but local still matches the journal baseline, so
|
|
4620
4793
|
// the classic 3-way merge calls this a clean `download` and overwrites
|
|
@@ -4726,7 +4899,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4726
4899
|
// remains the whole-set guard for BOTH candidate classes. A stale-etag,
|
|
4727
4900
|
// divergent, tombstoned, or never-journaled missing file falls through to
|
|
4728
4901
|
// the normal download below.
|
|
4729
|
-
if (!
|
|
4902
|
+
if (!serverOwnedPullReconciles &&
|
|
4903
|
+
!localExists &&
|
|
4730
4904
|
journalEntry !== undefined &&
|
|
4731
4905
|
!isTombstone(journalEntry) &&
|
|
4732
4906
|
!journalEntry.localDiverges &&
|
|
@@ -4765,7 +4939,10 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4765
4939
|
intentionalDeleteCandidates.length / totalJournalEntries >=
|
|
4766
4940
|
PULL_INTENTIONAL_DELETE_RATIO;
|
|
4767
4941
|
for (const cand of intentionalDeleteCandidates) {
|
|
4768
|
-
|
|
4942
|
+
// Recovery hold: a missing local file is not proof of a user delete when
|
|
4943
|
+
// the journal row may be stale, and a local-delete tombstone would let
|
|
4944
|
+
// the push leg delete the vault copy once the marker clears. Restore it.
|
|
4945
|
+
if (intentionalDeleteIsBulk || recoveryHold) {
|
|
4769
4946
|
items.push({
|
|
4770
4947
|
action: "download",
|
|
4771
4948
|
remoteFile: cand.remoteFile,
|
|
@@ -4977,6 +5154,13 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
4977
5154
|
const localSnapshot = snapshotLocalState(localPath);
|
|
4978
5155
|
if (localSnapshot === null)
|
|
4979
5156
|
continue;
|
|
5157
|
+
if (recoveryHold && localSnapshot.kind !== "absent") {
|
|
5158
|
+
// Recovery hold: the journal row that says this file was synced may be
|
|
5159
|
+
// stale. Keep the local file and drop only the row, so the file is
|
|
5160
|
+
// treated as untracked local work from here on.
|
|
5161
|
+
tombstones.push({ key, localPath, localSnapshot, journalOnly: true, recoveryHold: true });
|
|
5162
|
+
continue;
|
|
5163
|
+
}
|
|
4980
5164
|
tombstones.push({ key, localPath, localSnapshot });
|
|
4981
5165
|
}
|
|
4982
5166
|
return {
|
|
@@ -5003,7 +5187,10 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
5003
5187
|
* without an `onEvent` see no behavioral change.
|
|
5004
5188
|
*/
|
|
5005
5189
|
function defaultConsoleLogger(event, humanSkipLines) {
|
|
5006
|
-
if (event.type === "journal-
|
|
5190
|
+
if (event.type === "personal-journal-identity-fenced") {
|
|
5191
|
+
console.warn(` ! personal sync record of unproven ownership set aside (${event.reason}); no local files were deleted`);
|
|
5192
|
+
}
|
|
5193
|
+
else if (event.type === "journal-quarantine-unresolved") {
|
|
5007
5194
|
console.warn(` ! ${event.count} journal path${event.count === 1 ? " is" : "s are"} unresolved and held for manual review`);
|
|
5008
5195
|
}
|
|
5009
5196
|
else if (event.type === "journal-quarantine") {
|
|
@@ -5033,7 +5220,11 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5033
5220
|
console.warn(formatPlanLimitNotice(event));
|
|
5034
5221
|
}
|
|
5035
5222
|
else if (event.type === "progress") {
|
|
5036
|
-
if (event.
|
|
5223
|
+
if (event.deleted === true) {
|
|
5224
|
+
// A local delete must never read like a download (feedback_97906212).
|
|
5225
|
+
console.log(` − ${event.path} deleted locally${event.message ? ` — "${event.message}"` : ""}`);
|
|
5226
|
+
}
|
|
5227
|
+
else if (event.message) {
|
|
5037
5228
|
console.log(` ✓ ${event.path} — "${event.message}"`);
|
|
5038
5229
|
}
|
|
5039
5230
|
else {
|