@indigoai-us/hq-cloud 6.16.50 → 6.16.51-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/rescue-runner.js +1 -2
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +6 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +30 -1
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +55 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +108 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +281 -113
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +33 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +34 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +555 -78
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +0 -31
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +63 -596
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-rsync-preflight.test.js +4 -256
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +0 -4
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +1 -20
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +1 -16
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +0 -2
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +57 -37
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +87 -93
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +226 -118
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +215 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +0 -2
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +0 -30
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +0 -175
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-store.test.js +130 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +9 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +30 -21
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +0 -295
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +3 -18
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +24 -47
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/s3.test.js +1 -27
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync-core.d.ts +4 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +4 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +1 -25
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/cli/rescue-breaker.test.d.ts +0 -2
- package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
- package/dist/cli/rescue-breaker.test.js +0 -334
- package/dist/cli/rescue-breaker.test.js.map +0 -1
- package/dist/cli/rescue-realpath.test.d.ts +0 -2
- package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
- package/dist/cli/rescue-realpath.test.js +0 -109
- package/dist/cli/rescue-realpath.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.js +0 -260
- package/dist/cli/rescue-snapshot-recovery.test.js.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.js +0 -210
- package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
package/dist/cli/sync.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "fs";
|
|
8
8
|
import * as path from "path";
|
|
9
|
+
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
9
10
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
10
11
|
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
11
12
|
import { describeError } from "../lib/describe-error.js";
|
|
@@ -16,7 +17,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
|
16
17
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
17
18
|
import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
|
|
18
19
|
import { SKILLS_KEY_PREFIX } from "./doctor.js";
|
|
19
|
-
import { downloadFile,
|
|
20
|
+
import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
20
21
|
import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
21
22
|
import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
|
|
22
23
|
import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
|
|
@@ -185,6 +186,11 @@ export function targetedJournalSessionCensusSizes() {
|
|
|
185
186
|
* dirty entries through a run-scoped journal writer.
|
|
186
187
|
*/
|
|
187
188
|
const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
|
|
189
|
+
/**
|
|
190
|
+
* A full-pull hand-off unit. Every member has finished its transfer and the
|
|
191
|
+
* resulting journal delta is durable before realtime can acquire the writer.
|
|
192
|
+
*/
|
|
193
|
+
export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
188
194
|
/**
|
|
189
195
|
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
190
196
|
* an oversized batch wholesale, so the client MUST split a large report into
|
|
@@ -318,36 +324,47 @@ export async function sync(options) {
|
|
|
318
324
|
return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
|
|
319
325
|
}
|
|
320
326
|
async function syncWithOperationLockHeld(options) {
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
327
|
+
for (;;) {
|
|
328
|
+
const run = await buildPullContext(options);
|
|
329
|
+
try {
|
|
330
|
+
const plan = await planPull(run);
|
|
331
|
+
emitPullPlan(run.emit, plan);
|
|
332
|
+
reportInvalidRemoteKeys(run, plan);
|
|
333
|
+
const scopeRun = run.keyedJournal
|
|
334
|
+
? emptyTargetedScopeRun(run)
|
|
335
|
+
: executeScopeShrink(run, planScopeShrink(run));
|
|
336
|
+
const counters = createPullCounters();
|
|
337
|
+
// A streaming all-mode planner retains only executor-relevant items. Its
|
|
338
|
+
// ordinary skips have already been counted while the page was live.
|
|
339
|
+
counters.filesSkipped = plan.preplannedFilesSkipped;
|
|
340
|
+
counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
|
|
341
|
+
const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
|
|
342
|
+
const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
|
|
343
|
+
if (conflictRun.abortResult) {
|
|
344
|
+
return conflictRun.abortResult;
|
|
345
|
+
}
|
|
346
|
+
await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
|
|
347
|
+
// The following added-by and tombstone phases may still make authenticated
|
|
348
|
+
// vault calls. Make a completed sub-threshold download tail durable before
|
|
349
|
+
// they begin, so their failure cannot turn a successful bulk transfer into
|
|
350
|
+
// a full replay on the next pass.
|
|
351
|
+
flushDownloadedJournalTail(run);
|
|
352
|
+
await emitAndReportNewFiles(run, plan);
|
|
353
|
+
await verifyPlannedJournalTombstones(run, plan);
|
|
354
|
+
executeJournalTombstoneDeletes(run, plan, counters);
|
|
355
|
+
emitScopeMaterializationGap(run, plan);
|
|
356
|
+
return finalizePullRun(run, plan, scopeRun, counters);
|
|
357
|
+
}
|
|
358
|
+
catch (error) {
|
|
359
|
+
if (error instanceof CooperativePullHandoff)
|
|
360
|
+
continue;
|
|
361
|
+
throw error;
|
|
362
|
+
}
|
|
363
|
+
finally {
|
|
364
|
+
// Release a pass-local lease even on conflict, transfer, or auth failure.
|
|
365
|
+
// The one bounded hot lease remains available only for its same vault.
|
|
366
|
+
run.releaseKeyedJournal?.();
|
|
367
|
+
}
|
|
351
368
|
}
|
|
352
369
|
}
|
|
353
370
|
async function buildPullContext(options) {
|
|
@@ -436,8 +453,9 @@ async function buildPullContext(options) {
|
|
|
436
453
|
? targetedJournalSession(journalSlug)
|
|
437
454
|
: undefined;
|
|
438
455
|
const keyedJournal = keyedJournalLease?.store;
|
|
456
|
+
let remoteFiles;
|
|
439
457
|
try {
|
|
440
|
-
|
|
458
|
+
remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
441
459
|
const fileTombstones = options.fileTombstones ??
|
|
442
460
|
await fetchFileTombstones(vaultConfig, options.personalMode === true
|
|
443
461
|
? { personal: true }
|
|
@@ -485,43 +503,27 @@ async function buildPullContext(options) {
|
|
|
485
503
|
catch (error) {
|
|
486
504
|
// `buildPullContext` can fail during LIST/freshness. Do not leave the
|
|
487
505
|
// leased projection marked in use when no PullRunContext is returned.
|
|
506
|
+
if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
|
|
507
|
+
remoteFiles.dispose();
|
|
488
508
|
keyedJournalLease?.release();
|
|
489
509
|
throw error;
|
|
490
510
|
}
|
|
491
511
|
}
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
* thousands of RemoteFile objects on large vaults) only for `computePullPlan`
|
|
499
|
-
* to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
|
|
500
|
-
* in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
|
|
501
|
-
* bounded parallel, deduped), which bounds the listing's memory to the scope's
|
|
502
|
-
* actual footprint.
|
|
503
|
-
*
|
|
504
|
-
* `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
|
|
505
|
-
* the single broad listing: fanning `all` out by top-level prefix would need a
|
|
506
|
-
* delimiter LIST to discover the top-level names first and still union the
|
|
507
|
-
* same total object count in one plan, so it bounds nothing today. (A
|
|
508
|
-
* streaming planner is the real fix for `all` and is out of scope here.)
|
|
509
|
-
*
|
|
510
|
-
* An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
|
|
511
|
-
* issue zero LIST calls. Consumers already treat an empty listing correctly:
|
|
512
|
-
* the deletion sweep in `computePullPlan` only considers journal keys covered
|
|
513
|
-
* by the LISTED scope, so keys absent because they were never listed can never
|
|
514
|
-
* be mistaken for remote deletes.
|
|
515
|
-
*/
|
|
512
|
+
function isStreamingRemoteFileSource(source) {
|
|
513
|
+
return !Array.isArray(source);
|
|
514
|
+
}
|
|
515
|
+
function allModeRemoteFileStream(ctx) {
|
|
516
|
+
return spoolRemoteFiles(streamRemoteFiles(ctx));
|
|
517
|
+
}
|
|
516
518
|
async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
|
|
517
519
|
const entries = toScopePrefixEntries(listedPrefixSet);
|
|
518
520
|
if (entries.some((e) => e.prefix === ""))
|
|
519
|
-
return
|
|
521
|
+
return allModeRemoteFileStream(ctx);
|
|
520
522
|
if (entries.length === 0) {
|
|
521
523
|
// `all` mode always carries `[""]`, so an empty set only occurs for a
|
|
522
524
|
// scoped mode (empty grants, or a targeted intersection that excluded
|
|
523
525
|
// everything). Defensive `all` branch: never silently narrow a full pull.
|
|
524
|
-
return syncMode === "all" ?
|
|
526
|
+
return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
|
|
525
527
|
}
|
|
526
528
|
return listRemoteForScope({
|
|
527
529
|
ctx,
|
|
@@ -542,8 +544,14 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
|
|
|
542
544
|
},
|
|
543
545
|
});
|
|
544
546
|
}
|
|
545
|
-
function planPull(run) {
|
|
546
|
-
|
|
547
|
+
async function planPull(run) {
|
|
548
|
+
try {
|
|
549
|
+
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
550
|
+
}
|
|
551
|
+
finally {
|
|
552
|
+
if (isStreamingRemoteFileSource(run.remoteFiles))
|
|
553
|
+
run.remoteFiles.dispose();
|
|
554
|
+
}
|
|
547
555
|
}
|
|
548
556
|
/** Build only the rows a targeted remote LIST can observe. */
|
|
549
557
|
function journalForTargetedPull(store, prefixes) {
|
|
@@ -1253,10 +1261,18 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1253
1261
|
discardProbe();
|
|
1254
1262
|
}
|
|
1255
1263
|
}
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1264
|
+
/**
|
|
1265
|
+
* The old plan may have tombstone evidence that predates a realtime hand-off.
|
|
1266
|
+
* Rebuilding from disk after a hand-off makes the continuation observe the
|
|
1267
|
+
* realtime journal and a fresh LIST before it can perform another mutation.
|
|
1268
|
+
*/
|
|
1269
|
+
class CooperativePullHandoff extends Error {
|
|
1270
|
+
constructor() {
|
|
1271
|
+
super("full pull handed off to realtime work");
|
|
1272
|
+
this.name = "CooperativePullHandoff";
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
|
|
1260
1276
|
const queue = [...downloadItems];
|
|
1261
1277
|
const inFlight = new Set();
|
|
1262
1278
|
const workerErrors = [];
|
|
@@ -1271,7 +1287,7 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
|
|
|
1271
1287
|
inFlight.size < transferConcurrency &&
|
|
1272
1288
|
queue.length > 0) {
|
|
1273
1289
|
const downloadItem = queue.shift();
|
|
1274
|
-
const download = () => downloadOne(run, downloadItem, counters);
|
|
1290
|
+
const download = () => downloadOne(run, downloadItem, counters, deferJournalCheckpoint);
|
|
1275
1291
|
const p = (run.options.transferSemaphore
|
|
1276
1292
|
? run.options.transferSemaphore.withPermit(download)
|
|
1277
1293
|
: download())
|
|
@@ -1295,16 +1311,44 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
|
|
|
1295
1311
|
break;
|
|
1296
1312
|
}
|
|
1297
1313
|
}
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1314
|
+
return workerErrors;
|
|
1315
|
+
}
|
|
1316
|
+
function throwDownloadWorkerErrors(run, workerErrors) {
|
|
1317
|
+
persistPullJournal(run);
|
|
1318
|
+
const first = workerErrors[0];
|
|
1319
|
+
if (workerErrors.length > 1) {
|
|
1320
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
1321
|
+
}
|
|
1322
|
+
throw first;
|
|
1323
|
+
}
|
|
1324
|
+
async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
|
|
1325
|
+
if (downloadItems.length === 0)
|
|
1326
|
+
return;
|
|
1327
|
+
await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
|
|
1328
|
+
// Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
|
|
1329
|
+
// Only a watch chore that supplied the cooperative seam pays for drained
|
|
1330
|
+
// units and can return its root lock to queued realtime work.
|
|
1331
|
+
if (!run.options.cooperativeCheckpoint) {
|
|
1332
|
+
const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
|
|
1333
|
+
if (workerErrors.length > 0)
|
|
1334
|
+
throwDownloadWorkerErrors(run, workerErrors);
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
|
|
1338
|
+
const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
|
|
1339
|
+
const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
|
|
1340
|
+
// `executeDownloadQueue` does not return until every member of this chunk
|
|
1341
|
+
// has settled. Persist before the callback so the suspended chore owns no
|
|
1342
|
+
// mutable pass state while realtime uses the root lock.
|
|
1343
|
+
flushDownloadedJournalTail(run);
|
|
1344
|
+
if (workerErrors.length > 0)
|
|
1345
|
+
throwDownloadWorkerErrors(run, workerErrors);
|
|
1346
|
+
if (await run.options.cooperativeCheckpoint() === true) {
|
|
1347
|
+
throw new CooperativePullHandoff();
|
|
1303
1348
|
}
|
|
1304
|
-
throw first;
|
|
1305
1349
|
}
|
|
1306
1350
|
}
|
|
1307
|
-
async function downloadOne(run, downloadItem, counters) {
|
|
1351
|
+
async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint = false) {
|
|
1308
1352
|
const { remoteFile, localPath } = downloadItem;
|
|
1309
1353
|
await refreshRunContextIfExpiring(run);
|
|
1310
1354
|
if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
|
|
@@ -1461,7 +1505,8 @@ async function downloadOne(run, downloadItem, counters) {
|
|
|
1461
1505
|
return;
|
|
1462
1506
|
}
|
|
1463
1507
|
run.downloadsSinceJournalCheckpoint++;
|
|
1464
|
-
if (
|
|
1508
|
+
if (!deferJournalCheckpoint &&
|
|
1509
|
+
run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
|
|
1465
1510
|
flushDownloadedJournalTail(run);
|
|
1466
1511
|
}
|
|
1467
1512
|
}
|
|
@@ -1900,7 +1945,8 @@ class LocalSnapshotChangedError extends Error {
|
|
|
1900
1945
|
* the real outcome of the upcoming Stage-2 execution rather than an
|
|
1901
1946
|
* upper-bound guess.
|
|
1902
1947
|
*
|
|
1903
|
-
*
|
|
1948
|
+
* It performs no writes, journal mutations, or event emission. An all-mode
|
|
1949
|
+
* source performs its paged classification LIST while it is consumed; the
|
|
1904
1950
|
* caller (`sync()`) is responsible for emitting the resulting plan event
|
|
1905
1951
|
* before iterating `items`.
|
|
1906
1952
|
*/
|
|
@@ -1968,6 +2014,36 @@ excludePrefixes = [],
|
|
|
1968
2014
|
// callers preserve the pre-policy, watcher-intent-only behavior.
|
|
1969
2015
|
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
|
|
1970
2016
|
const items = [];
|
|
2017
|
+
const streaming = isStreamingRemoteFileSource(remoteFiles);
|
|
2018
|
+
const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
|
|
2019
|
+
let preplannedFilesSkipped = 0;
|
|
2020
|
+
let preplannedFilesOutOfScope = 0;
|
|
2021
|
+
let preplannedFilesExcludedByPolicy = 0;
|
|
2022
|
+
if (streaming) {
|
|
2023
|
+
const retain = items.push.bind(items);
|
|
2024
|
+
items.push = (...next) => {
|
|
2025
|
+
for (const item of next) {
|
|
2026
|
+
// These actions still need the executor for an event or a state change.
|
|
2027
|
+
if (item.action === "download" ||
|
|
2028
|
+
item.action === "conflict" ||
|
|
2029
|
+
item.action === "tombstone-delete" ||
|
|
2030
|
+
item.action === "skip-junk-key-spelling" ||
|
|
2031
|
+
item.action === "skip-unreadable-link") {
|
|
2032
|
+
retain(item);
|
|
2033
|
+
}
|
|
2034
|
+
else if (item.action === "skip-out-of-scope") {
|
|
2035
|
+
preplannedFilesOutOfScope++;
|
|
2036
|
+
}
|
|
2037
|
+
else if (item.action === "skip-excluded-policy") {
|
|
2038
|
+
preplannedFilesExcludedByPolicy++;
|
|
2039
|
+
}
|
|
2040
|
+
else {
|
|
2041
|
+
preplannedFilesSkipped++;
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
return items.length;
|
|
2045
|
+
};
|
|
2046
|
+
}
|
|
1971
2047
|
// Remote keys the server will never presign — collected so the caller can
|
|
1972
2048
|
// warn + emit telemetry without allowing one poison object to fail a pull.
|
|
1973
2049
|
const invalidRemoteKeys = [];
|
|
@@ -1983,8 +2059,35 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
1983
2059
|
// never gated by this map (see the guard below), so pre-existing families
|
|
1984
2060
|
// keep syncing untouched until the vault doctor (US-003) collapses them.
|
|
1985
2061
|
const journaledKeyByCanonical = new Map();
|
|
2062
|
+
// The delete sweep starts with journaled keys and clears them as the streamed
|
|
2063
|
+
// LIST proves each one present. This retains existing journal-key references
|
|
2064
|
+
// instead of a second, whole-vault population of long remote key strings.
|
|
2065
|
+
const unseenJournalKeys = new Set();
|
|
2066
|
+
// Most journal keys have already been normalized to POSIX and can be cleared
|
|
2067
|
+
// directly. Keep the legacy Windows spelling defense-in-depth without
|
|
2068
|
+
// imposing a second map entry on every current journal row.
|
|
2069
|
+
const legacyJournalKeysByPosix = new Map();
|
|
2070
|
+
// The junk-spelling delete guard needs all journal twins, while
|
|
2071
|
+
// `journaledKeyByCanonical` above deliberately retains only the preferred
|
|
2072
|
+
// representative for classification.
|
|
2073
|
+
const journaledSkillKeysByCanonical = new Map();
|
|
1986
2074
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
1987
2075
|
onHeartbeat?.();
|
|
2076
|
+
const posixJournaledKey = toPosixKey(journaledKey);
|
|
2077
|
+
// A scoped LIST cannot prove anything about keys outside its prefixes.
|
|
2078
|
+
// Do not copy the entire vault's journal into its presence bookkeeping.
|
|
2079
|
+
const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
|
|
2080
|
+
if (inListedScope)
|
|
2081
|
+
unseenJournalKeys.add(journaledKey);
|
|
2082
|
+
if (inListedScope && posixJournaledKey !== journaledKey) {
|
|
2083
|
+
const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
|
|
2084
|
+
if (legacyKeys === undefined) {
|
|
2085
|
+
legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
|
|
2086
|
+
}
|
|
2087
|
+
else {
|
|
2088
|
+
legacyKeys.push(journaledKey);
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
1988
2091
|
if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
|
|
1989
2092
|
continue;
|
|
1990
2093
|
const canonical = canonicalVaultKeySpelling(journaledKey);
|
|
@@ -1992,6 +2095,15 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
1992
2095
|
if (existing === undefined || journaledKey === canonical) {
|
|
1993
2096
|
journaledKeyByCanonical.set(canonical, journaledKey);
|
|
1994
2097
|
}
|
|
2098
|
+
if (!inListedScope)
|
|
2099
|
+
continue;
|
|
2100
|
+
const skillKeys = journaledSkillKeysByCanonical.get(canonical);
|
|
2101
|
+
if (skillKeys === undefined) {
|
|
2102
|
+
journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
|
|
2103
|
+
}
|
|
2104
|
+
else {
|
|
2105
|
+
skillKeys.push(journaledKey);
|
|
2106
|
+
}
|
|
1995
2107
|
}
|
|
1996
2108
|
// Remote keys that ALSO appear as an ancestor of another remote key — i.e.
|
|
1997
2109
|
// keys that carry child objects under `${key}/…` in this same LIST. The
|
|
@@ -2001,22 +2113,40 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2001
2113
|
// objects beneath it. A REAL directory, by contrast, only ever syncs as
|
|
2002
2114
|
// those child objects — never as a single object AT its own key. So a single
|
|
2003
2115
|
// remote object sitting at a key that also has children is unambiguously a
|
|
2004
|
-
// directory-overlay marker, not a regular file.
|
|
2005
|
-
//
|
|
2006
|
-
//
|
|
2007
|
-
//
|
|
2008
|
-
const remoteKeysWithChildren =
|
|
2009
|
-
|
|
2116
|
+
// directory-overlay marker, not a regular file. Evidence is complete before
|
|
2117
|
+
// classification for both the spooled all-mode LIST and scoped arrays.
|
|
2118
|
+
// Build scoped evidence incrementally: flatMap would transiently retain an
|
|
2119
|
+
// ancestor string for every row/depth before Set deduplication.
|
|
2120
|
+
const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
|
|
2121
|
+
? remoteFiles.remoteKeysWithChildren
|
|
2122
|
+
: collectRemoteAncestors(remoteFiles);
|
|
2123
|
+
// Both sources replay synchronously. for-await on a scoped array introduces
|
|
2124
|
+
// promise/microtask work per object even though no I/O is being awaited.
|
|
2125
|
+
for (const remoteFile of listedFiles) {
|
|
2010
2126
|
onHeartbeat?.();
|
|
2011
|
-
const
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2127
|
+
const posixRemoteKey = toPosixKey(remoteFile.key);
|
|
2128
|
+
// S3 object keys are POSIX. Match the old delete-sweep membership check:
|
|
2129
|
+
// a legacy backslash journal entry counts as present only when its POSIX
|
|
2130
|
+
// form appears in the LIST, while a malformed backslash remote key does
|
|
2131
|
+
// not accidentally change that behavior.
|
|
2132
|
+
if (remoteFile.key === posixRemoteKey) {
|
|
2133
|
+
unseenJournalKeys.delete(remoteFile.key);
|
|
2134
|
+
for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
|
|
2135
|
+
unseenJournalKeys.delete(legacyKey);
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2139
|
+
// A remote junk spelling means its canonical journal twin is still
|
|
2140
|
+
// remotely present. Keep that twin out of the delete sweep unless an
|
|
2141
|
+
// exact FILE_TOMBSTONE is authoritative, precisely matching the old
|
|
2142
|
+
// remoteCanonicalSkillKeys protection without retaining remote keys.
|
|
2143
|
+
const canonical = canonicalVaultKeySpelling(remoteFile.key);
|
|
2144
|
+
for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
|
|
2145
|
+
if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
|
|
2146
|
+
unseenJournalKeys.delete(journaledKey);
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2016
2149
|
}
|
|
2017
|
-
}
|
|
2018
|
-
for (const remoteFile of remoteFiles) {
|
|
2019
|
-
onHeartbeat?.();
|
|
2020
2150
|
const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
|
|
2021
2151
|
if (invalidKey !== null) {
|
|
2022
2152
|
invalidRemoteKeys.push({
|
|
@@ -2669,6 +2799,9 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2669
2799
|
filesToSkip++;
|
|
2670
2800
|
}
|
|
2671
2801
|
}
|
|
2802
|
+
filesToSkip += preplannedFilesSkipped;
|
|
2803
|
+
filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
|
|
2804
|
+
filesOutOfScope += preplannedFilesOutOfScope;
|
|
2672
2805
|
// Bug #9 — cross-machine delete propagation. The peer's push leg
|
|
2673
2806
|
// already removed the file from S3 (verified in the deep-test
|
|
2674
2807
|
// addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
|
|
@@ -2677,38 +2810,21 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2677
2810
|
// that-was-there-before" — so the file stayed forever on every
|
|
2678
2811
|
// receiver and `filesTombstoned: 0` showed up on every pull.
|
|
2679
2812
|
//
|
|
2680
|
-
// Closing the loop: walk the
|
|
2681
|
-
//
|
|
2813
|
+
// Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
|
|
2814
|
+
// whose path passes the current
|
|
2682
2815
|
// ignore filter (paths newly excluded by .hqignore must not trigger
|
|
2683
2816
|
// mass-delete), AND — in personalMode — survives the same companies/*
|
|
2684
2817
|
// gating the download branch applies. The executor will apply each
|
|
2685
2818
|
// as a local delete + journal removal. Symmetric to the push side's
|
|
2686
2819
|
// `propagateDeletes` plan in share.ts.
|
|
2687
|
-
const remoteKeySet = new Set();
|
|
2688
|
-
// Skill-farm twin protection (companion to the junk-spelling pull guard):
|
|
2689
|
-
// when the LIST holds only a junk spelling of a journaled canonical key,
|
|
2690
|
-
// the logical object is still remotely present — a raw has() would class
|
|
2691
|
-
// the canonical journal entry as remote-deleted, unlink the good local
|
|
2692
|
-
// wrapper, and drop the journal entry (after which the next pull would
|
|
2693
|
-
// materialize the junk spelling the guard just refused). Track canonical
|
|
2694
|
-
// forms of listed skill keys and consult them below.
|
|
2695
|
-
const remoteCanonicalSkillKeys = new Set();
|
|
2696
|
-
for (const rf of remoteFiles) {
|
|
2697
|
-
remoteKeySet.add(rf.key);
|
|
2698
|
-
if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2699
|
-
remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
|
|
2700
|
-
}
|
|
2701
|
-
}
|
|
2702
2820
|
const tombstones = [];
|
|
2703
|
-
for (const key of
|
|
2821
|
+
for (const key of unseenJournalKeys) {
|
|
2704
2822
|
// Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
|
|
2705
2823
|
// carry backslash separators; the remote LIST is always forward-slash, so a
|
|
2706
2824
|
// raw `has(key)` would miss the match and class a live local file as
|
|
2707
2825
|
// remote-deleted. normalizeJournalKeys() defuses this on load, but the
|
|
2708
2826
|
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
2709
2827
|
const posixKey = toPosixKey(key);
|
|
2710
|
-
if (remoteKeySet.has(posixKey))
|
|
2711
|
-
continue;
|
|
2712
2828
|
// Listing-scope guard (targeted sync / scoped listing). The remote LIST
|
|
2713
2829
|
// now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
|
|
2714
2830
|
// pull narrows further), so "absent from the LIST" is evidence of a remote
|
|
@@ -2720,16 +2836,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2720
2836
|
// `all` mode passes `[""]`, which covers every key — no behavior change.
|
|
2721
2837
|
if (!isCoveredByAny(posixKey, prefixSet))
|
|
2722
2838
|
continue;
|
|
2723
|
-
if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
|
|
2724
|
-
remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
|
|
2725
|
-
// An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
|
|
2726
|
-
// delete and wins over the twin protection — a lingering junk
|
|
2727
|
-
// spelling in the LIST must not keep a deliberately-deleted wrapper
|
|
2728
|
-
// alive forever. (The key is absent from the LIST here, so there is
|
|
2729
|
-
// no newer re-create to defer to.)
|
|
2730
|
-
fileTombstones.get(posixKey) === undefined) {
|
|
2731
|
-
continue;
|
|
2732
|
-
}
|
|
2733
2839
|
// Fast-path ignore, BEFORE containment canonicalization. The tombstone
|
|
2734
2840
|
// sweep runs for every journaled key, and a stale journal can carry
|
|
2735
2841
|
// hundreds of thousands of entries under an ignored runtime tree (e.g.
|
|
@@ -2838,6 +2944,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2838
2944
|
}
|
|
2839
2945
|
return {
|
|
2840
2946
|
items,
|
|
2947
|
+
preplannedFilesSkipped,
|
|
2948
|
+
preplannedFilesOutOfScope,
|
|
2841
2949
|
filesToDownload,
|
|
2842
2950
|
bytesToDownload,
|
|
2843
2951
|
filesToSkip,
|