@indigoai-us/hq-cloud 6.16.46-alpha.0 → 6.16.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/rescue-core.d.ts +7 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +36 -22
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +76 -135
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +1 -11
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/s3.d.ts +0 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +2 -20
- package/dist/s3.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/cli/sync.js
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "fs";
|
|
8
8
|
import * as path from "path";
|
|
9
|
-
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
10
9
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
11
10
|
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
12
11
|
import { describeError } from "../lib/describe-error.js";
|
|
@@ -17,7 +16,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
|
17
16
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
18
17
|
import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
|
|
19
18
|
import { SKILLS_KEY_PREFIX } from "./doctor.js";
|
|
20
|
-
import { downloadFile,
|
|
19
|
+
import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
21
20
|
import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
22
21
|
import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
|
|
23
22
|
import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
|
|
@@ -321,17 +320,13 @@ export async function sync(options) {
|
|
|
321
320
|
async function syncWithOperationLockHeld(options) {
|
|
322
321
|
const run = await buildPullContext(options);
|
|
323
322
|
try {
|
|
324
|
-
const plan =
|
|
323
|
+
const plan = planPull(run);
|
|
325
324
|
emitPullPlan(run.emit, plan);
|
|
326
325
|
reportInvalidRemoteKeys(run, plan);
|
|
327
326
|
const scopeRun = run.keyedJournal
|
|
328
327
|
? emptyTargetedScopeRun(run)
|
|
329
328
|
: executeScopeShrink(run, planScopeShrink(run));
|
|
330
329
|
const counters = createPullCounters();
|
|
331
|
-
// A streaming all-mode planner retains only executor-relevant items. Its
|
|
332
|
-
// ordinary skips have already been counted while the page was live.
|
|
333
|
-
counters.filesSkipped = plan.preplannedFilesSkipped;
|
|
334
|
-
counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
|
|
335
330
|
const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
|
|
336
331
|
const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
|
|
337
332
|
if (conflictRun.abortResult) {
|
|
@@ -441,9 +436,8 @@ async function buildPullContext(options) {
|
|
|
441
436
|
? targetedJournalSession(journalSlug)
|
|
442
437
|
: undefined;
|
|
443
438
|
const keyedJournal = keyedJournalLease?.store;
|
|
444
|
-
let remoteFiles;
|
|
445
439
|
try {
|
|
446
|
-
remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
440
|
+
const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
447
441
|
const fileTombstones = options.fileTombstones ??
|
|
448
442
|
await fetchFileTombstones(vaultConfig, options.personalMode === true
|
|
449
443
|
? { personal: true }
|
|
@@ -491,27 +485,43 @@ async function buildPullContext(options) {
|
|
|
491
485
|
catch (error) {
|
|
492
486
|
// `buildPullContext` can fail during LIST/freshness. Do not leave the
|
|
493
487
|
// leased projection marked in use when no PullRunContext is returned.
|
|
494
|
-
if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
|
|
495
|
-
remoteFiles.dispose();
|
|
496
488
|
keyedJournalLease?.release();
|
|
497
489
|
throw error;
|
|
498
490
|
}
|
|
499
491
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
492
|
+
/**
|
|
493
|
+
* List the remote objects the current pull can actually materialize.
|
|
494
|
+
*
|
|
495
|
+
* Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
|
|
496
|
+
* a `shared`/`custom` membership whose effective scope is a handful of
|
|
497
|
+
* prefixes — so every pull materialized the entire remote listing (tens of
|
|
498
|
+
* thousands of RemoteFile objects on large vaults) only for `computePullPlan`
|
|
499
|
+
* to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
|
|
500
|
+
* in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
|
|
501
|
+
* bounded parallel, deduped), which bounds the listing's memory to the scope's
|
|
502
|
+
* actual footprint.
|
|
503
|
+
*
|
|
504
|
+
* `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
|
|
505
|
+
* the single broad listing: fanning `all` out by top-level prefix would need a
|
|
506
|
+
* delimiter LIST to discover the top-level names first and still union the
|
|
507
|
+
* same total object count in one plan, so it bounds nothing today. (A
|
|
508
|
+
* streaming planner is the real fix for `all` and is out of scope here.)
|
|
509
|
+
*
|
|
510
|
+
* An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
|
|
511
|
+
* issue zero LIST calls. Consumers already treat an empty listing correctly:
|
|
512
|
+
* the deletion sweep in `computePullPlan` only considers journal keys covered
|
|
513
|
+
* by the LISTED scope, so keys absent because they were never listed can never
|
|
514
|
+
* be mistaken for remote deletes.
|
|
515
|
+
*/
|
|
506
516
|
async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
|
|
507
517
|
const entries = toScopePrefixEntries(listedPrefixSet);
|
|
508
518
|
if (entries.some((e) => e.prefix === ""))
|
|
509
|
-
return
|
|
519
|
+
return listRemoteFiles(ctx);
|
|
510
520
|
if (entries.length === 0) {
|
|
511
521
|
// `all` mode always carries `[""]`, so an empty set only occurs for a
|
|
512
522
|
// scoped mode (empty grants, or a targeted intersection that excluded
|
|
513
523
|
// everything). Defensive `all` branch: never silently narrow a full pull.
|
|
514
|
-
return syncMode === "all" ?
|
|
524
|
+
return syncMode === "all" ? listRemoteFiles(ctx) : [];
|
|
515
525
|
}
|
|
516
526
|
return listRemoteForScope({
|
|
517
527
|
ctx,
|
|
@@ -532,14 +542,8 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
|
|
|
532
542
|
},
|
|
533
543
|
});
|
|
534
544
|
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
538
|
-
}
|
|
539
|
-
finally {
|
|
540
|
-
if (isStreamingRemoteFileSource(run.remoteFiles))
|
|
541
|
-
run.remoteFiles.dispose();
|
|
542
|
-
}
|
|
545
|
+
function planPull(run) {
|
|
546
|
+
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
543
547
|
}
|
|
544
548
|
/** Build only the rows a targeted remote LIST can observe. */
|
|
545
549
|
function journalForTargetedPull(store, prefixes) {
|
|
@@ -1896,8 +1900,7 @@ class LocalSnapshotChangedError extends Error {
|
|
|
1896
1900
|
* the real outcome of the upcoming Stage-2 execution rather than an
|
|
1897
1901
|
* upper-bound guess.
|
|
1898
1902
|
*
|
|
1899
|
-
*
|
|
1900
|
-
* source performs its paged classification LIST while it is consumed; the
|
|
1903
|
+
* Pure function: no S3 calls, no journal writes, no event emission. The
|
|
1901
1904
|
* caller (`sync()`) is responsible for emitting the resulting plan event
|
|
1902
1905
|
* before iterating `items`.
|
|
1903
1906
|
*/
|
|
@@ -1965,36 +1968,6 @@ excludePrefixes = [],
|
|
|
1965
1968
|
// callers preserve the pre-policy, watcher-intent-only behavior.
|
|
1966
1969
|
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
|
|
1967
1970
|
const items = [];
|
|
1968
|
-
const streaming = isStreamingRemoteFileSource(remoteFiles);
|
|
1969
|
-
const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
|
|
1970
|
-
let preplannedFilesSkipped = 0;
|
|
1971
|
-
let preplannedFilesOutOfScope = 0;
|
|
1972
|
-
let preplannedFilesExcludedByPolicy = 0;
|
|
1973
|
-
if (streaming) {
|
|
1974
|
-
const retain = items.push.bind(items);
|
|
1975
|
-
items.push = (...next) => {
|
|
1976
|
-
for (const item of next) {
|
|
1977
|
-
// These actions still need the executor for an event or a state change.
|
|
1978
|
-
if (item.action === "download" ||
|
|
1979
|
-
item.action === "conflict" ||
|
|
1980
|
-
item.action === "tombstone-delete" ||
|
|
1981
|
-
item.action === "skip-junk-key-spelling" ||
|
|
1982
|
-
item.action === "skip-unreadable-link") {
|
|
1983
|
-
retain(item);
|
|
1984
|
-
}
|
|
1985
|
-
else if (item.action === "skip-out-of-scope") {
|
|
1986
|
-
preplannedFilesOutOfScope++;
|
|
1987
|
-
}
|
|
1988
|
-
else if (item.action === "skip-excluded-policy") {
|
|
1989
|
-
preplannedFilesExcludedByPolicy++;
|
|
1990
|
-
}
|
|
1991
|
-
else {
|
|
1992
|
-
preplannedFilesSkipped++;
|
|
1993
|
-
}
|
|
1994
|
-
}
|
|
1995
|
-
return items.length;
|
|
1996
|
-
};
|
|
1997
|
-
}
|
|
1998
1971
|
// Remote keys the server will never presign — collected so the caller can
|
|
1999
1972
|
// warn + emit telemetry without allowing one poison object to fail a pull.
|
|
2000
1973
|
const invalidRemoteKeys = [];
|
|
@@ -2010,35 +1983,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2010
1983
|
// never gated by this map (see the guard below), so pre-existing families
|
|
2011
1984
|
// keep syncing untouched until the vault doctor (US-003) collapses them.
|
|
2012
1985
|
const journaledKeyByCanonical = new Map();
|
|
2013
|
-
// The delete sweep starts with journaled keys and clears them as the streamed
|
|
2014
|
-
// LIST proves each one present. This retains existing journal-key references
|
|
2015
|
-
// instead of a second, whole-vault population of long remote key strings.
|
|
2016
|
-
const unseenJournalKeys = new Set();
|
|
2017
|
-
// Most journal keys have already been normalized to POSIX and can be cleared
|
|
2018
|
-
// directly. Keep the legacy Windows spelling defense-in-depth without
|
|
2019
|
-
// imposing a second map entry on every current journal row.
|
|
2020
|
-
const legacyJournalKeysByPosix = new Map();
|
|
2021
|
-
// The junk-spelling delete guard needs all journal twins, while
|
|
2022
|
-
// `journaledKeyByCanonical` above deliberately retains only the preferred
|
|
2023
|
-
// representative for classification.
|
|
2024
|
-
const journaledSkillKeysByCanonical = new Map();
|
|
2025
1986
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
2026
1987
|
onHeartbeat?.();
|
|
2027
|
-
const posixJournaledKey = toPosixKey(journaledKey);
|
|
2028
|
-
// A scoped LIST cannot prove anything about keys outside its prefixes.
|
|
2029
|
-
// Do not copy the entire vault's journal into its presence bookkeeping.
|
|
2030
|
-
const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
|
|
2031
|
-
if (inListedScope)
|
|
2032
|
-
unseenJournalKeys.add(journaledKey);
|
|
2033
|
-
if (inListedScope && posixJournaledKey !== journaledKey) {
|
|
2034
|
-
const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
|
|
2035
|
-
if (legacyKeys === undefined) {
|
|
2036
|
-
legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
|
|
2037
|
-
}
|
|
2038
|
-
else {
|
|
2039
|
-
legacyKeys.push(journaledKey);
|
|
2040
|
-
}
|
|
2041
|
-
}
|
|
2042
1988
|
if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
|
|
2043
1989
|
continue;
|
|
2044
1990
|
const canonical = canonicalVaultKeySpelling(journaledKey);
|
|
@@ -2046,15 +1992,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2046
1992
|
if (existing === undefined || journaledKey === canonical) {
|
|
2047
1993
|
journaledKeyByCanonical.set(canonical, journaledKey);
|
|
2048
1994
|
}
|
|
2049
|
-
if (!inListedScope)
|
|
2050
|
-
continue;
|
|
2051
|
-
const skillKeys = journaledSkillKeysByCanonical.get(canonical);
|
|
2052
|
-
if (skillKeys === undefined) {
|
|
2053
|
-
journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
|
|
2054
|
-
}
|
|
2055
|
-
else {
|
|
2056
|
-
skillKeys.push(journaledKey);
|
|
2057
|
-
}
|
|
2058
1995
|
}
|
|
2059
1996
|
// Remote keys that ALSO appear as an ancestor of another remote key — i.e.
|
|
2060
1997
|
// keys that carry child objects under `${key}/…` in this same LIST. The
|
|
@@ -2064,40 +2001,22 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2064
2001
|
// objects beneath it. A REAL directory, by contrast, only ever syncs as
|
|
2065
2002
|
// those child objects — never as a single object AT its own key. So a single
|
|
2066
2003
|
// remote object sitting at a key that also has children is unambiguously a
|
|
2067
|
-
// directory-overlay marker, not a regular file.
|
|
2068
|
-
//
|
|
2069
|
-
//
|
|
2070
|
-
//
|
|
2071
|
-
const remoteKeysWithChildren =
|
|
2072
|
-
|
|
2073
|
-
: collectRemoteAncestors(remoteFiles);
|
|
2074
|
-
// Both sources replay synchronously. for-await on a scoped array introduces
|
|
2075
|
-
// promise/microtask work per object even though no I/O is being awaited.
|
|
2076
|
-
for (const remoteFile of listedFiles) {
|
|
2004
|
+
// directory-overlay marker, not a regular file. Precomputed once (pure path
|
|
2005
|
+
// derivation, no I/O — the planner is synchronous) so the dir-vs-object
|
|
2006
|
+
// collision branch can tell an inert overlay marker apart from a genuine
|
|
2007
|
+
// file-vs-directory structural collision. feedback_d2082110.
|
|
2008
|
+
const remoteKeysWithChildren = new Set();
|
|
2009
|
+
for (const rf of remoteFiles) {
|
|
2077
2010
|
onHeartbeat?.();
|
|
2078
|
-
const
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
if (remoteFile.key === posixRemoteKey) {
|
|
2084
|
-
unseenJournalKeys.delete(remoteFile.key);
|
|
2085
|
-
for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
|
|
2086
|
-
unseenJournalKeys.delete(legacyKey);
|
|
2087
|
-
}
|
|
2088
|
-
}
|
|
2089
|
-
if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2090
|
-
// A remote junk spelling means its canonical journal twin is still
|
|
2091
|
-
// remotely present. Keep that twin out of the delete sweep unless an
|
|
2092
|
-
// exact FILE_TOMBSTONE is authoritative, precisely matching the old
|
|
2093
|
-
// remoteCanonicalSkillKeys protection without retaining remote keys.
|
|
2094
|
-
const canonical = canonicalVaultKeySpelling(remoteFile.key);
|
|
2095
|
-
for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
|
|
2096
|
-
if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
|
|
2097
|
-
unseenJournalKeys.delete(journaledKey);
|
|
2098
|
-
}
|
|
2099
|
-
}
|
|
2011
|
+
const key = toPosixKey(rf.key);
|
|
2012
|
+
let slash = key.indexOf("/");
|
|
2013
|
+
while (slash !== -1) {
|
|
2014
|
+
remoteKeysWithChildren.add(key.slice(0, slash));
|
|
2015
|
+
slash = key.indexOf("/", slash + 1);
|
|
2100
2016
|
}
|
|
2017
|
+
}
|
|
2018
|
+
for (const remoteFile of remoteFiles) {
|
|
2019
|
+
onHeartbeat?.();
|
|
2101
2020
|
const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
|
|
2102
2021
|
if (invalidKey !== null) {
|
|
2103
2022
|
invalidRemoteKeys.push({
|
|
@@ -2750,9 +2669,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2750
2669
|
filesToSkip++;
|
|
2751
2670
|
}
|
|
2752
2671
|
}
|
|
2753
|
-
filesToSkip += preplannedFilesSkipped;
|
|
2754
|
-
filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
|
|
2755
|
-
filesOutOfScope += preplannedFilesOutOfScope;
|
|
2756
2672
|
// Bug #9 — cross-machine delete propagation. The peer's push leg
|
|
2757
2673
|
// already removed the file from S3 (verified in the deep-test
|
|
2758
2674
|
// addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
|
|
@@ -2761,21 +2677,38 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2761
2677
|
// that-was-there-before" — so the file stayed forever on every
|
|
2762
2678
|
// receiver and `filesTombstoned: 0` showed up on every pull.
|
|
2763
2679
|
//
|
|
2764
|
-
// Closing the loop: walk journal
|
|
2765
|
-
// whose path passes the current
|
|
2680
|
+
// Closing the loop: walk the journal, find every entry whose key is
|
|
2681
|
+
// NOT in the remote LIST set, AND whose path passes the current
|
|
2766
2682
|
// ignore filter (paths newly excluded by .hqignore must not trigger
|
|
2767
2683
|
// mass-delete), AND — in personalMode — survives the same companies/*
|
|
2768
2684
|
// gating the download branch applies. The executor will apply each
|
|
2769
2685
|
// as a local delete + journal removal. Symmetric to the push side's
|
|
2770
2686
|
// `propagateDeletes` plan in share.ts.
|
|
2687
|
+
const remoteKeySet = new Set();
|
|
2688
|
+
// Skill-farm twin protection (companion to the junk-spelling pull guard):
|
|
2689
|
+
// when the LIST holds only a junk spelling of a journaled canonical key,
|
|
2690
|
+
// the logical object is still remotely present — a raw has() would class
|
|
2691
|
+
// the canonical journal entry as remote-deleted, unlink the good local
|
|
2692
|
+
// wrapper, and drop the journal entry (after which the next pull would
|
|
2693
|
+
// materialize the junk spelling the guard just refused). Track canonical
|
|
2694
|
+
// forms of listed skill keys and consult them below.
|
|
2695
|
+
const remoteCanonicalSkillKeys = new Set();
|
|
2696
|
+
for (const rf of remoteFiles) {
|
|
2697
|
+
remoteKeySet.add(rf.key);
|
|
2698
|
+
if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2699
|
+
remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
|
|
2700
|
+
}
|
|
2701
|
+
}
|
|
2771
2702
|
const tombstones = [];
|
|
2772
|
-
for (const key of
|
|
2703
|
+
for (const key of journalRows(journal).keys()) {
|
|
2773
2704
|
// Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
|
|
2774
2705
|
// carry backslash separators; the remote LIST is always forward-slash, so a
|
|
2775
2706
|
// raw `has(key)` would miss the match and class a live local file as
|
|
2776
2707
|
// remote-deleted. normalizeJournalKeys() defuses this on load, but the
|
|
2777
2708
|
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
2778
2709
|
const posixKey = toPosixKey(key);
|
|
2710
|
+
if (remoteKeySet.has(posixKey))
|
|
2711
|
+
continue;
|
|
2779
2712
|
// Listing-scope guard (targeted sync / scoped listing). The remote LIST
|
|
2780
2713
|
// now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
|
|
2781
2714
|
// pull narrows further), so "absent from the LIST" is evidence of a remote
|
|
@@ -2787,6 +2720,16 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2787
2720
|
// `all` mode passes `[""]`, which covers every key — no behavior change.
|
|
2788
2721
|
if (!isCoveredByAny(posixKey, prefixSet))
|
|
2789
2722
|
continue;
|
|
2723
|
+
if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
|
|
2724
|
+
remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
|
|
2725
|
+
// An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
|
|
2726
|
+
// delete and wins over the twin protection — a lingering junk
|
|
2727
|
+
// spelling in the LIST must not keep a deliberately-deleted wrapper
|
|
2728
|
+
// alive forever. (The key is absent from the LIST here, so there is
|
|
2729
|
+
// no newer re-create to defer to.)
|
|
2730
|
+
fileTombstones.get(posixKey) === undefined) {
|
|
2731
|
+
continue;
|
|
2732
|
+
}
|
|
2790
2733
|
// Fast-path ignore, BEFORE containment canonicalization. The tombstone
|
|
2791
2734
|
// sweep runs for every journaled key, and a stale journal can carry
|
|
2792
2735
|
// hundreds of thousands of entries under an ignored runtime tree (e.g.
|
|
@@ -2895,8 +2838,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2895
2838
|
}
|
|
2896
2839
|
return {
|
|
2897
2840
|
items,
|
|
2898
|
-
preplannedFilesSkipped,
|
|
2899
|
-
preplannedFilesOutOfScope,
|
|
2900
2841
|
filesToDownload,
|
|
2901
2842
|
bytesToDownload,
|
|
2902
2843
|
filesToSkip,
|