@indigoai-us/hq-cloud 6.16.45 → 6.16.46-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +108 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +9 -1
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +15 -4
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +27 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +211 -2
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +135 -76
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +11 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/s3.d.ts +1 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +20 -2
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/session-log-retention.d.ts +23 -0
- package/dist/session-log-retention.d.ts.map +1 -1
- package/dist/session-log-retention.js +107 -4
- package/dist/session-log-retention.js.map +1 -1
- package/dist/session-log-retention.test.js +124 -1
- package/dist/session-log-retention.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/package.json +2 -1
package/dist/cli/sync.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "fs";
|
|
8
8
|
import * as path from "path";
|
|
9
|
+
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
9
10
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
10
11
|
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
11
12
|
import { describeError } from "../lib/describe-error.js";
|
|
@@ -16,7 +17,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
|
16
17
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
17
18
|
import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
|
|
18
19
|
import { SKILLS_KEY_PREFIX } from "./doctor.js";
|
|
19
|
-
import { downloadFile,
|
|
20
|
+
import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
20
21
|
import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
21
22
|
import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
|
|
22
23
|
import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
|
|
@@ -320,13 +321,17 @@ export async function sync(options) {
|
|
|
320
321
|
async function syncWithOperationLockHeld(options) {
|
|
321
322
|
const run = await buildPullContext(options);
|
|
322
323
|
try {
|
|
323
|
-
const plan = planPull(run);
|
|
324
|
+
const plan = await planPull(run);
|
|
324
325
|
emitPullPlan(run.emit, plan);
|
|
325
326
|
reportInvalidRemoteKeys(run, plan);
|
|
326
327
|
const scopeRun = run.keyedJournal
|
|
327
328
|
? emptyTargetedScopeRun(run)
|
|
328
329
|
: executeScopeShrink(run, planScopeShrink(run));
|
|
329
330
|
const counters = createPullCounters();
|
|
331
|
+
// A streaming all-mode planner retains only executor-relevant items. Its
|
|
332
|
+
// ordinary skips have already been counted while the page was live.
|
|
333
|
+
counters.filesSkipped = plan.preplannedFilesSkipped;
|
|
334
|
+
counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
|
|
330
335
|
const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
|
|
331
336
|
const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
|
|
332
337
|
if (conflictRun.abortResult) {
|
|
@@ -436,8 +441,9 @@ async function buildPullContext(options) {
|
|
|
436
441
|
? targetedJournalSession(journalSlug)
|
|
437
442
|
: undefined;
|
|
438
443
|
const keyedJournal = keyedJournalLease?.store;
|
|
444
|
+
let remoteFiles;
|
|
439
445
|
try {
|
|
440
|
-
|
|
446
|
+
remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
441
447
|
const fileTombstones = options.fileTombstones ??
|
|
442
448
|
await fetchFileTombstones(vaultConfig, options.personalMode === true
|
|
443
449
|
? { personal: true }
|
|
@@ -485,43 +491,27 @@ async function buildPullContext(options) {
|
|
|
485
491
|
catch (error) {
|
|
486
492
|
// `buildPullContext` can fail during LIST/freshness. Do not leave the
|
|
487
493
|
// leased projection marked in use when no PullRunContext is returned.
|
|
494
|
+
if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
|
|
495
|
+
remoteFiles.dispose();
|
|
488
496
|
keyedJournalLease?.release();
|
|
489
497
|
throw error;
|
|
490
498
|
}
|
|
491
499
|
}
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
* thousands of RemoteFile objects on large vaults) only for `computePullPlan`
|
|
499
|
-
* to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
|
|
500
|
-
* in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
|
|
501
|
-
* bounded parallel, deduped), which bounds the listing's memory to the scope's
|
|
502
|
-
* actual footprint.
|
|
503
|
-
*
|
|
504
|
-
* `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
|
|
505
|
-
* the single broad listing: fanning `all` out by top-level prefix would need a
|
|
506
|
-
* delimiter LIST to discover the top-level names first and still union the
|
|
507
|
-
* same total object count in one plan, so it bounds nothing today. (A
|
|
508
|
-
* streaming planner is the real fix for `all` and is out of scope here.)
|
|
509
|
-
*
|
|
510
|
-
* An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
|
|
511
|
-
* issue zero LIST calls. Consumers already treat an empty listing correctly:
|
|
512
|
-
* the deletion sweep in `computePullPlan` only considers journal keys covered
|
|
513
|
-
* by the LISTED scope, so keys absent because they were never listed can never
|
|
514
|
-
* be mistaken for remote deletes.
|
|
515
|
-
*/
|
|
500
|
+
function isStreamingRemoteFileSource(source) {
|
|
501
|
+
return !Array.isArray(source);
|
|
502
|
+
}
|
|
503
|
+
function allModeRemoteFileStream(ctx) {
|
|
504
|
+
return spoolRemoteFiles(streamRemoteFiles(ctx));
|
|
505
|
+
}
|
|
516
506
|
async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
|
|
517
507
|
const entries = toScopePrefixEntries(listedPrefixSet);
|
|
518
508
|
if (entries.some((e) => e.prefix === ""))
|
|
519
|
-
return
|
|
509
|
+
return allModeRemoteFileStream(ctx);
|
|
520
510
|
if (entries.length === 0) {
|
|
521
511
|
// `all` mode always carries `[""]`, so an empty set only occurs for a
|
|
522
512
|
// scoped mode (empty grants, or a targeted intersection that excluded
|
|
523
513
|
// everything). Defensive `all` branch: never silently narrow a full pull.
|
|
524
|
-
return syncMode === "all" ?
|
|
514
|
+
return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
|
|
525
515
|
}
|
|
526
516
|
return listRemoteForScope({
|
|
527
517
|
ctx,
|
|
@@ -542,8 +532,14 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
|
|
|
542
532
|
},
|
|
543
533
|
});
|
|
544
534
|
}
|
|
545
|
-
function planPull(run) {
|
|
546
|
-
|
|
535
|
+
async function planPull(run) {
|
|
536
|
+
try {
|
|
537
|
+
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
538
|
+
}
|
|
539
|
+
finally {
|
|
540
|
+
if (isStreamingRemoteFileSource(run.remoteFiles))
|
|
541
|
+
run.remoteFiles.dispose();
|
|
542
|
+
}
|
|
547
543
|
}
|
|
548
544
|
/** Build only the rows a targeted remote LIST can observe. */
|
|
549
545
|
function journalForTargetedPull(store, prefixes) {
|
|
@@ -1900,7 +1896,8 @@ class LocalSnapshotChangedError extends Error {
|
|
|
1900
1896
|
* the real outcome of the upcoming Stage-2 execution rather than an
|
|
1901
1897
|
* upper-bound guess.
|
|
1902
1898
|
*
|
|
1903
|
-
*
|
|
1899
|
+
* It performs no writes, journal mutations, or event emission. An all-mode
|
|
1900
|
+
* source performs its paged classification LIST while it is consumed; the
|
|
1904
1901
|
* caller (`sync()`) is responsible for emitting the resulting plan event
|
|
1905
1902
|
* before iterating `items`.
|
|
1906
1903
|
*/
|
|
@@ -1968,6 +1965,36 @@ excludePrefixes = [],
|
|
|
1968
1965
|
// callers preserve the pre-policy, watcher-intent-only behavior.
|
|
1969
1966
|
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
|
|
1970
1967
|
const items = [];
|
|
1968
|
+
const streaming = isStreamingRemoteFileSource(remoteFiles);
|
|
1969
|
+
const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
|
|
1970
|
+
let preplannedFilesSkipped = 0;
|
|
1971
|
+
let preplannedFilesOutOfScope = 0;
|
|
1972
|
+
let preplannedFilesExcludedByPolicy = 0;
|
|
1973
|
+
if (streaming) {
|
|
1974
|
+
const retain = items.push.bind(items);
|
|
1975
|
+
items.push = (...next) => {
|
|
1976
|
+
for (const item of next) {
|
|
1977
|
+
// These actions still need the executor for an event or a state change.
|
|
1978
|
+
if (item.action === "download" ||
|
|
1979
|
+
item.action === "conflict" ||
|
|
1980
|
+
item.action === "tombstone-delete" ||
|
|
1981
|
+
item.action === "skip-junk-key-spelling" ||
|
|
1982
|
+
item.action === "skip-unreadable-link") {
|
|
1983
|
+
retain(item);
|
|
1984
|
+
}
|
|
1985
|
+
else if (item.action === "skip-out-of-scope") {
|
|
1986
|
+
preplannedFilesOutOfScope++;
|
|
1987
|
+
}
|
|
1988
|
+
else if (item.action === "skip-excluded-policy") {
|
|
1989
|
+
preplannedFilesExcludedByPolicy++;
|
|
1990
|
+
}
|
|
1991
|
+
else {
|
|
1992
|
+
preplannedFilesSkipped++;
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
return items.length;
|
|
1996
|
+
};
|
|
1997
|
+
}
|
|
1971
1998
|
// Remote keys the server will never presign — collected so the caller can
|
|
1972
1999
|
// warn + emit telemetry without allowing one poison object to fail a pull.
|
|
1973
2000
|
const invalidRemoteKeys = [];
|
|
@@ -1983,8 +2010,35 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
1983
2010
|
// never gated by this map (see the guard below), so pre-existing families
|
|
1984
2011
|
// keep syncing untouched until the vault doctor (US-003) collapses them.
|
|
1985
2012
|
const journaledKeyByCanonical = new Map();
|
|
2013
|
+
// The delete sweep starts with journaled keys and clears them as the streamed
|
|
2014
|
+
// LIST proves each one present. This retains existing journal-key references
|
|
2015
|
+
// instead of a second, whole-vault population of long remote key strings.
|
|
2016
|
+
const unseenJournalKeys = new Set();
|
|
2017
|
+
// Most journal keys have already been normalized to POSIX and can be cleared
|
|
2018
|
+
// directly. Keep the legacy Windows spelling defense-in-depth without
|
|
2019
|
+
// imposing a second map entry on every current journal row.
|
|
2020
|
+
const legacyJournalKeysByPosix = new Map();
|
|
2021
|
+
// The junk-spelling delete guard needs all journal twins, while
|
|
2022
|
+
// `journaledKeyByCanonical` above deliberately retains only the preferred
|
|
2023
|
+
// representative for classification.
|
|
2024
|
+
const journaledSkillKeysByCanonical = new Map();
|
|
1986
2025
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
1987
2026
|
onHeartbeat?.();
|
|
2027
|
+
const posixJournaledKey = toPosixKey(journaledKey);
|
|
2028
|
+
// A scoped LIST cannot prove anything about keys outside its prefixes.
|
|
2029
|
+
// Do not copy the entire vault's journal into its presence bookkeeping.
|
|
2030
|
+
const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
|
|
2031
|
+
if (inListedScope)
|
|
2032
|
+
unseenJournalKeys.add(journaledKey);
|
|
2033
|
+
if (inListedScope && posixJournaledKey !== journaledKey) {
|
|
2034
|
+
const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
|
|
2035
|
+
if (legacyKeys === undefined) {
|
|
2036
|
+
legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
|
|
2037
|
+
}
|
|
2038
|
+
else {
|
|
2039
|
+
legacyKeys.push(journaledKey);
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
1988
2042
|
if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
|
|
1989
2043
|
continue;
|
|
1990
2044
|
const canonical = canonicalVaultKeySpelling(journaledKey);
|
|
@@ -1992,6 +2046,15 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
1992
2046
|
if (existing === undefined || journaledKey === canonical) {
|
|
1993
2047
|
journaledKeyByCanonical.set(canonical, journaledKey);
|
|
1994
2048
|
}
|
|
2049
|
+
if (!inListedScope)
|
|
2050
|
+
continue;
|
|
2051
|
+
const skillKeys = journaledSkillKeysByCanonical.get(canonical);
|
|
2052
|
+
if (skillKeys === undefined) {
|
|
2053
|
+
journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
|
|
2054
|
+
}
|
|
2055
|
+
else {
|
|
2056
|
+
skillKeys.push(journaledKey);
|
|
2057
|
+
}
|
|
1995
2058
|
}
|
|
1996
2059
|
// Remote keys that ALSO appear as an ancestor of another remote key — i.e.
|
|
1997
2060
|
// keys that carry child objects under `${key}/…` in this same LIST. The
|
|
@@ -2001,22 +2064,40 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2001
2064
|
// objects beneath it. A REAL directory, by contrast, only ever syncs as
|
|
2002
2065
|
// those child objects — never as a single object AT its own key. So a single
|
|
2003
2066
|
// remote object sitting at a key that also has children is unambiguously a
|
|
2004
|
-
// directory-overlay marker, not a regular file.
|
|
2005
|
-
//
|
|
2006
|
-
//
|
|
2007
|
-
//
|
|
2008
|
-
const remoteKeysWithChildren =
|
|
2009
|
-
|
|
2067
|
+
// directory-overlay marker, not a regular file. Evidence is complete before
|
|
2068
|
+
// classification for both the spooled all-mode LIST and scoped arrays.
|
|
2069
|
+
// Build scoped evidence incrementally: flatMap would transiently retain an
|
|
2070
|
+
// ancestor string for every row/depth before Set deduplication.
|
|
2071
|
+
const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
|
|
2072
|
+
? remoteFiles.remoteKeysWithChildren
|
|
2073
|
+
: collectRemoteAncestors(remoteFiles);
|
|
2074
|
+
// Both sources replay synchronously. for-await on a scoped array introduces
|
|
2075
|
+
// promise/microtask work per object even though no I/O is being awaited.
|
|
2076
|
+
for (const remoteFile of listedFiles) {
|
|
2010
2077
|
onHeartbeat?.();
|
|
2011
|
-
const
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2078
|
+
const posixRemoteKey = toPosixKey(remoteFile.key);
|
|
2079
|
+
// S3 object keys are POSIX. Match the old delete-sweep membership check:
|
|
2080
|
+
// a legacy backslash journal entry counts as present only when its POSIX
|
|
2081
|
+
// form appears in the LIST, while a malformed backslash remote key does
|
|
2082
|
+
// not accidentally change that behavior.
|
|
2083
|
+
if (remoteFile.key === posixRemoteKey) {
|
|
2084
|
+
unseenJournalKeys.delete(remoteFile.key);
|
|
2085
|
+
for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
|
|
2086
|
+
unseenJournalKeys.delete(legacyKey);
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2090
|
+
// A remote junk spelling means its canonical journal twin is still
|
|
2091
|
+
// remotely present. Keep that twin out of the delete sweep unless an
|
|
2092
|
+
// exact FILE_TOMBSTONE is authoritative, precisely matching the old
|
|
2093
|
+
// remoteCanonicalSkillKeys protection without retaining remote keys.
|
|
2094
|
+
const canonical = canonicalVaultKeySpelling(remoteFile.key);
|
|
2095
|
+
for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
|
|
2096
|
+
if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
|
|
2097
|
+
unseenJournalKeys.delete(journaledKey);
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2016
2100
|
}
|
|
2017
|
-
}
|
|
2018
|
-
for (const remoteFile of remoteFiles) {
|
|
2019
|
-
onHeartbeat?.();
|
|
2020
2101
|
const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
|
|
2021
2102
|
if (invalidKey !== null) {
|
|
2022
2103
|
invalidRemoteKeys.push({
|
|
@@ -2669,6 +2750,9 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2669
2750
|
filesToSkip++;
|
|
2670
2751
|
}
|
|
2671
2752
|
}
|
|
2753
|
+
filesToSkip += preplannedFilesSkipped;
|
|
2754
|
+
filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
|
|
2755
|
+
filesOutOfScope += preplannedFilesOutOfScope;
|
|
2672
2756
|
// Bug #9 — cross-machine delete propagation. The peer's push leg
|
|
2673
2757
|
// already removed the file from S3 (verified in the deep-test
|
|
2674
2758
|
// addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
|
|
@@ -2677,38 +2761,21 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2677
2761
|
// that-was-there-before" — so the file stayed forever on every
|
|
2678
2762
|
// receiver and `filesTombstoned: 0` showed up on every pull.
|
|
2679
2763
|
//
|
|
2680
|
-
// Closing the loop: walk the
|
|
2681
|
-
//
|
|
2764
|
+
// Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
|
|
2765
|
+
// whose path passes the current
|
|
2682
2766
|
// ignore filter (paths newly excluded by .hqignore must not trigger
|
|
2683
2767
|
// mass-delete), AND — in personalMode — survives the same companies/*
|
|
2684
2768
|
// gating the download branch applies. The executor will apply each
|
|
2685
2769
|
// as a local delete + journal removal. Symmetric to the push side's
|
|
2686
2770
|
// `propagateDeletes` plan in share.ts.
|
|
2687
|
-
const remoteKeySet = new Set();
|
|
2688
|
-
// Skill-farm twin protection (companion to the junk-spelling pull guard):
|
|
2689
|
-
// when the LIST holds only a junk spelling of a journaled canonical key,
|
|
2690
|
-
// the logical object is still remotely present — a raw has() would class
|
|
2691
|
-
// the canonical journal entry as remote-deleted, unlink the good local
|
|
2692
|
-
// wrapper, and drop the journal entry (after which the next pull would
|
|
2693
|
-
// materialize the junk spelling the guard just refused). Track canonical
|
|
2694
|
-
// forms of listed skill keys and consult them below.
|
|
2695
|
-
const remoteCanonicalSkillKeys = new Set();
|
|
2696
|
-
for (const rf of remoteFiles) {
|
|
2697
|
-
remoteKeySet.add(rf.key);
|
|
2698
|
-
if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2699
|
-
remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
|
|
2700
|
-
}
|
|
2701
|
-
}
|
|
2702
2771
|
const tombstones = [];
|
|
2703
|
-
for (const key of
|
|
2772
|
+
for (const key of unseenJournalKeys) {
|
|
2704
2773
|
// Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
|
|
2705
2774
|
// carry backslash separators; the remote LIST is always forward-slash, so a
|
|
2706
2775
|
// raw `has(key)` would miss the match and class a live local file as
|
|
2707
2776
|
// remote-deleted. normalizeJournalKeys() defuses this on load, but the
|
|
2708
2777
|
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
2709
2778
|
const posixKey = toPosixKey(key);
|
|
2710
|
-
if (remoteKeySet.has(posixKey))
|
|
2711
|
-
continue;
|
|
2712
2779
|
// Listing-scope guard (targeted sync / scoped listing). The remote LIST
|
|
2713
2780
|
// now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
|
|
2714
2781
|
// pull narrows further), so "absent from the LIST" is evidence of a remote
|
|
@@ -2720,16 +2787,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2720
2787
|
// `all` mode passes `[""]`, which covers every key — no behavior change.
|
|
2721
2788
|
if (!isCoveredByAny(posixKey, prefixSet))
|
|
2722
2789
|
continue;
|
|
2723
|
-
if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
|
|
2724
|
-
remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
|
|
2725
|
-
// An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
|
|
2726
|
-
// delete and wins over the twin protection — a lingering junk
|
|
2727
|
-
// spelling in the LIST must not keep a deliberately-deleted wrapper
|
|
2728
|
-
// alive forever. (The key is absent from the LIST here, so there is
|
|
2729
|
-
// no newer re-create to defer to.)
|
|
2730
|
-
fileTombstones.get(posixKey) === undefined) {
|
|
2731
|
-
continue;
|
|
2732
|
-
}
|
|
2733
2790
|
// Fast-path ignore, BEFORE containment canonicalization. The tombstone
|
|
2734
2791
|
// sweep runs for every journaled key, and a stale journal can carry
|
|
2735
2792
|
// hundreds of thousands of entries under an ignored runtime tree (e.g.
|
|
@@ -2838,6 +2895,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2838
2895
|
}
|
|
2839
2896
|
return {
|
|
2840
2897
|
items,
|
|
2898
|
+
preplannedFilesSkipped,
|
|
2899
|
+
preplannedFilesOutOfScope,
|
|
2841
2900
|
filesToDownload,
|
|
2842
2901
|
bytesToDownload,
|
|
2843
2902
|
filesToSkip,
|