@indigoai-us/hq-cloud 6.16.55-alpha.0 → 6.16.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +23 -1
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +92 -8
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +36 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -14
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +29 -46
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +247 -57
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -108
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -51
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +164 -403
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +4 -14
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -59
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -48
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +221 -1520
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +40 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +1454 -162
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +256 -4
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +627 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +467 -150
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +608 -95
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +15 -13
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +118 -226
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -215
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +30 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +175 -0
- package/dist/cognito-auth.js.map +1 -1
- package/dist/context.d.ts +5 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +92 -12
- package/dist/context.js.map +1 -1
- package/dist/context.test.js +77 -2
- package/dist/context.test.js.map +1 -1
- package/dist/ignore.d.ts +10 -0
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +47 -5
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +30 -3
- package/dist/ignore.test.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -40
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -240
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +0 -54
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -259
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +15 -43
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +79 -297
- package/dist/journal.js.map +1 -1
- package/dist/lib/describe-error.d.ts.map +1 -1
- package/dist/lib/describe-error.js +7 -0
- package/dist/lib/describe-error.js.map +1 -1
- package/dist/lib/describe-error.test.js +11 -0
- package/dist/lib/describe-error.test.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/lib/net-errors.d.ts +12 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +35 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/net-errors.test.js +19 -1
- package/dist/lib/net-errors.test.js.map +1 -1
- package/dist/machine-auth.test.js +295 -0
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/manifest/build-manifest-resident-parity.test.js +3 -3
- package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +18 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +47 -24
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +2 -2
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +34 -66
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +5 -90
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +0 -6
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +2 -14
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +33 -5
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +69 -21
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +65 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +0 -12
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +0 -131
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +0 -63
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/cli/sync.js
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "fs";
|
|
8
8
|
import * as path from "path";
|
|
9
|
-
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
10
9
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
11
10
|
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
12
11
|
import { describeError } from "../lib/describe-error.js";
|
|
@@ -17,7 +16,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
|
17
16
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
18
17
|
import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
|
|
19
18
|
import { SKILLS_KEY_PREFIX } from "./doctor.js";
|
|
20
|
-
import { downloadFile,
|
|
19
|
+
import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
21
20
|
import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
22
21
|
import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
|
|
23
22
|
import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
|
|
@@ -186,11 +185,6 @@ export function targetedJournalSessionCensusSizes() {
|
|
|
186
185
|
* dirty entries through a run-scoped journal writer.
|
|
187
186
|
*/
|
|
188
187
|
const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
|
|
189
|
-
/**
|
|
190
|
-
* A full-pull hand-off unit. Every member has finished its transfer and the
|
|
191
|
-
* resulting journal delta is durable before realtime can acquire the writer.
|
|
192
|
-
*/
|
|
193
|
-
export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
194
188
|
/**
|
|
195
189
|
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
196
190
|
* an oversized batch wholesale, so the client MUST split a large report into
|
|
@@ -324,47 +318,36 @@ export async function sync(options) {
|
|
|
324
318
|
return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
|
|
325
319
|
}
|
|
326
320
|
async function syncWithOperationLockHeld(options) {
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
}
|
|
358
|
-
catch (error) {
|
|
359
|
-
if (error instanceof CooperativePullHandoff)
|
|
360
|
-
continue;
|
|
361
|
-
throw error;
|
|
362
|
-
}
|
|
363
|
-
finally {
|
|
364
|
-
// Release a pass-local lease even on conflict, transfer, or auth failure.
|
|
365
|
-
// The one bounded hot lease remains available only for its same vault.
|
|
366
|
-
run.releaseKeyedJournal?.();
|
|
367
|
-
}
|
|
321
|
+
const run = await buildPullContext(options);
|
|
322
|
+
try {
|
|
323
|
+
const plan = planPull(run);
|
|
324
|
+
emitPullPlan(run.emit, plan);
|
|
325
|
+
reportInvalidRemoteKeys(run, plan);
|
|
326
|
+
const scopeRun = run.keyedJournal
|
|
327
|
+
? emptyTargetedScopeRun(run)
|
|
328
|
+
: executeScopeShrink(run, planScopeShrink(run));
|
|
329
|
+
const counters = createPullCounters();
|
|
330
|
+
const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
|
|
331
|
+
const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
|
|
332
|
+
if (conflictRun.abortResult) {
|
|
333
|
+
return conflictRun.abortResult;
|
|
334
|
+
}
|
|
335
|
+
await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
|
|
336
|
+
// The following added-by and tombstone phases may still make authenticated
|
|
337
|
+
// vault calls. Make a completed sub-threshold download tail durable before
|
|
338
|
+
// they begin, so their failure cannot turn a successful bulk transfer into
|
|
339
|
+
// a full replay on the next pass.
|
|
340
|
+
flushDownloadedJournalTail(run);
|
|
341
|
+
await emitAndReportNewFiles(run, plan);
|
|
342
|
+
await verifyPlannedJournalTombstones(run, plan);
|
|
343
|
+
executeJournalTombstoneDeletes(run, plan, counters);
|
|
344
|
+
emitScopeMaterializationGap(run, plan);
|
|
345
|
+
return finalizePullRun(run, plan, scopeRun, counters);
|
|
346
|
+
}
|
|
347
|
+
finally {
|
|
348
|
+
// Release a pass-local lease even on conflict, transfer, or auth failure.
|
|
349
|
+
// The one bounded hot lease remains available only for its same vault.
|
|
350
|
+
run.releaseKeyedJournal?.();
|
|
368
351
|
}
|
|
369
352
|
}
|
|
370
353
|
async function buildPullContext(options) {
|
|
@@ -453,9 +436,8 @@ async function buildPullContext(options) {
|
|
|
453
436
|
? targetedJournalSession(journalSlug)
|
|
454
437
|
: undefined;
|
|
455
438
|
const keyedJournal = keyedJournalLease?.store;
|
|
456
|
-
let remoteFiles;
|
|
457
439
|
try {
|
|
458
|
-
remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
440
|
+
const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
459
441
|
const fileTombstones = options.fileTombstones ??
|
|
460
442
|
await fetchFileTombstones(vaultConfig, options.personalMode === true
|
|
461
443
|
? { personal: true }
|
|
@@ -503,27 +485,43 @@ async function buildPullContext(options) {
|
|
|
503
485
|
catch (error) {
|
|
504
486
|
// `buildPullContext` can fail during LIST/freshness. Do not leave the
|
|
505
487
|
// leased projection marked in use when no PullRunContext is returned.
|
|
506
|
-
if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
|
|
507
|
-
remoteFiles.dispose();
|
|
508
488
|
keyedJournalLease?.release();
|
|
509
489
|
throw error;
|
|
510
490
|
}
|
|
511
491
|
}
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
492
|
+
/**
|
|
493
|
+
* List the remote objects the current pull can actually materialize.
|
|
494
|
+
*
|
|
495
|
+
* Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
|
|
496
|
+
* a `shared`/`custom` membership whose effective scope is a handful of
|
|
497
|
+
* prefixes — so every pull materialized the entire remote listing (tens of
|
|
498
|
+
* thousands of RemoteFile objects on large vaults) only for `computePullPlan`
|
|
499
|
+
* to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
|
|
500
|
+
* in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
|
|
501
|
+
* bounded parallel, deduped), which bounds the listing's memory to the scope's
|
|
502
|
+
* actual footprint.
|
|
503
|
+
*
|
|
504
|
+
* `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
|
|
505
|
+
* the single broad listing: fanning `all` out by top-level prefix would need a
|
|
506
|
+
* delimiter LIST to discover the top-level names first and still union the
|
|
507
|
+
* same total object count in one plan, so it bounds nothing today. (A
|
|
508
|
+
* streaming planner is the real fix for `all` and is out of scope here.)
|
|
509
|
+
*
|
|
510
|
+
* An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
|
|
511
|
+
* issue zero LIST calls. Consumers already treat an empty listing correctly:
|
|
512
|
+
* the deletion sweep in `computePullPlan` only considers journal keys covered
|
|
513
|
+
* by the LISTED scope, so keys absent because they were never listed can never
|
|
514
|
+
* be mistaken for remote deletes.
|
|
515
|
+
*/
|
|
518
516
|
async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
|
|
519
517
|
const entries = toScopePrefixEntries(listedPrefixSet);
|
|
520
518
|
if (entries.some((e) => e.prefix === ""))
|
|
521
|
-
return
|
|
519
|
+
return listRemoteFiles(ctx);
|
|
522
520
|
if (entries.length === 0) {
|
|
523
521
|
// `all` mode always carries `[""]`, so an empty set only occurs for a
|
|
524
522
|
// scoped mode (empty grants, or a targeted intersection that excluded
|
|
525
523
|
// everything). Defensive `all` branch: never silently narrow a full pull.
|
|
526
|
-
return syncMode === "all" ?
|
|
524
|
+
return syncMode === "all" ? listRemoteFiles(ctx) : [];
|
|
527
525
|
}
|
|
528
526
|
return listRemoteForScope({
|
|
529
527
|
ctx,
|
|
@@ -544,14 +542,8 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
|
|
|
544
542
|
},
|
|
545
543
|
});
|
|
546
544
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
550
|
-
}
|
|
551
|
-
finally {
|
|
552
|
-
if (isStreamingRemoteFileSource(run.remoteFiles))
|
|
553
|
-
run.remoteFiles.dispose();
|
|
554
|
-
}
|
|
545
|
+
function planPull(run) {
|
|
546
|
+
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
555
547
|
}
|
|
556
548
|
/** Build only the rows a targeted remote LIST can observe. */
|
|
557
549
|
function journalForTargetedPull(store, prefixes) {
|
|
@@ -1261,18 +1253,10 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1261
1253
|
discardProbe();
|
|
1262
1254
|
}
|
|
1263
1255
|
}
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
*/
|
|
1269
|
-
class CooperativePullHandoff extends Error {
|
|
1270
|
-
constructor() {
|
|
1271
|
-
super("full pull handed off to realtime work");
|
|
1272
|
-
this.name = "CooperativePullHandoff";
|
|
1273
|
-
}
|
|
1274
|
-
}
|
|
1275
|
-
async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
|
|
1256
|
+
async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
|
|
1257
|
+
if (downloadItems.length === 0)
|
|
1258
|
+
return;
|
|
1259
|
+
await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
|
|
1276
1260
|
const queue = [...downloadItems];
|
|
1277
1261
|
const inFlight = new Set();
|
|
1278
1262
|
const workerErrors = [];
|
|
@@ -1287,7 +1271,7 @@ async function executeDownloadQueue(run, downloadItems, transferConcurrency, cou
|
|
|
1287
1271
|
inFlight.size < transferConcurrency &&
|
|
1288
1272
|
queue.length > 0) {
|
|
1289
1273
|
const downloadItem = queue.shift();
|
|
1290
|
-
const download = () => downloadOne(run, downloadItem, counters
|
|
1274
|
+
const download = () => downloadOne(run, downloadItem, counters);
|
|
1291
1275
|
const p = (run.options.transferSemaphore
|
|
1292
1276
|
? run.options.transferSemaphore.withPermit(download)
|
|
1293
1277
|
: download())
|
|
@@ -1311,44 +1295,16 @@ async function executeDownloadQueue(run, downloadItems, transferConcurrency, cou
|
|
|
1311
1295
|
break;
|
|
1312
1296
|
}
|
|
1313
1297
|
}
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
if (workerErrors.length > 1) {
|
|
1320
|
-
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
1321
|
-
}
|
|
1322
|
-
throw first;
|
|
1323
|
-
}
|
|
1324
|
-
async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
|
|
1325
|
-
if (downloadItems.length === 0)
|
|
1326
|
-
return;
|
|
1327
|
-
await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
|
|
1328
|
-
// Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
|
|
1329
|
-
// Only a watch chore that supplied the cooperative seam pays for drained
|
|
1330
|
-
// units and can return its root lock to queued realtime work.
|
|
1331
|
-
if (!run.options.cooperativeCheckpoint) {
|
|
1332
|
-
const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
|
|
1333
|
-
if (workerErrors.length > 0)
|
|
1334
|
-
throwDownloadWorkerErrors(run, workerErrors);
|
|
1335
|
-
return;
|
|
1336
|
-
}
|
|
1337
|
-
for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
|
|
1338
|
-
const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
|
|
1339
|
-
const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
|
|
1340
|
-
// `executeDownloadQueue` does not return until every member of this chunk
|
|
1341
|
-
// has settled. Persist before the callback so the suspended chore owns no
|
|
1342
|
-
// mutable pass state while realtime uses the root lock.
|
|
1343
|
-
flushDownloadedJournalTail(run);
|
|
1344
|
-
if (workerErrors.length > 0)
|
|
1345
|
-
throwDownloadWorkerErrors(run, workerErrors);
|
|
1346
|
-
if (await run.options.cooperativeCheckpoint() === true) {
|
|
1347
|
-
throw new CooperativePullHandoff();
|
|
1298
|
+
if (workerErrors.length > 0) {
|
|
1299
|
+
persistPullJournal(run);
|
|
1300
|
+
const first = workerErrors[0];
|
|
1301
|
+
if (workerErrors.length > 1) {
|
|
1302
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
1348
1303
|
}
|
|
1304
|
+
throw first;
|
|
1349
1305
|
}
|
|
1350
1306
|
}
|
|
1351
|
-
async function downloadOne(run, downloadItem, counters
|
|
1307
|
+
async function downloadOne(run, downloadItem, counters) {
|
|
1352
1308
|
const { remoteFile, localPath } = downloadItem;
|
|
1353
1309
|
await refreshRunContextIfExpiring(run);
|
|
1354
1310
|
if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
|
|
@@ -1505,8 +1461,7 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
|
|
|
1505
1461
|
return;
|
|
1506
1462
|
}
|
|
1507
1463
|
run.downloadsSinceJournalCheckpoint++;
|
|
1508
|
-
if (
|
|
1509
|
-
run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
|
|
1464
|
+
if (run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
|
|
1510
1465
|
flushDownloadedJournalTail(run);
|
|
1511
1466
|
}
|
|
1512
1467
|
}
|
|
@@ -1945,8 +1900,7 @@ class LocalSnapshotChangedError extends Error {
|
|
|
1945
1900
|
* the real outcome of the upcoming Stage-2 execution rather than an
|
|
1946
1901
|
* upper-bound guess.
|
|
1947
1902
|
*
|
|
1948
|
-
*
|
|
1949
|
-
* source performs its paged classification LIST while it is consumed; the
|
|
1903
|
+
* Pure function: no S3 calls, no journal writes, no event emission. The
|
|
1950
1904
|
* caller (`sync()`) is responsible for emitting the resulting plan event
|
|
1951
1905
|
* before iterating `items`.
|
|
1952
1906
|
*/
|
|
@@ -2014,36 +1968,6 @@ excludePrefixes = [],
|
|
|
2014
1968
|
// callers preserve the pre-policy, watcher-intent-only behavior.
|
|
2015
1969
|
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
|
|
2016
1970
|
const items = [];
|
|
2017
|
-
const streaming = isStreamingRemoteFileSource(remoteFiles);
|
|
2018
|
-
const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
|
|
2019
|
-
let preplannedFilesSkipped = 0;
|
|
2020
|
-
let preplannedFilesOutOfScope = 0;
|
|
2021
|
-
let preplannedFilesExcludedByPolicy = 0;
|
|
2022
|
-
if (streaming) {
|
|
2023
|
-
const retain = items.push.bind(items);
|
|
2024
|
-
items.push = (...next) => {
|
|
2025
|
-
for (const item of next) {
|
|
2026
|
-
// These actions still need the executor for an event or a state change.
|
|
2027
|
-
if (item.action === "download" ||
|
|
2028
|
-
item.action === "conflict" ||
|
|
2029
|
-
item.action === "tombstone-delete" ||
|
|
2030
|
-
item.action === "skip-junk-key-spelling" ||
|
|
2031
|
-
item.action === "skip-unreadable-link") {
|
|
2032
|
-
retain(item);
|
|
2033
|
-
}
|
|
2034
|
-
else if (item.action === "skip-out-of-scope") {
|
|
2035
|
-
preplannedFilesOutOfScope++;
|
|
2036
|
-
}
|
|
2037
|
-
else if (item.action === "skip-excluded-policy") {
|
|
2038
|
-
preplannedFilesExcludedByPolicy++;
|
|
2039
|
-
}
|
|
2040
|
-
else {
|
|
2041
|
-
preplannedFilesSkipped++;
|
|
2042
|
-
}
|
|
2043
|
-
}
|
|
2044
|
-
return items.length;
|
|
2045
|
-
};
|
|
2046
|
-
}
|
|
2047
1971
|
// Remote keys the server will never presign — collected so the caller can
|
|
2048
1972
|
// warn + emit telemetry without allowing one poison object to fail a pull.
|
|
2049
1973
|
const invalidRemoteKeys = [];
|
|
@@ -2059,35 +1983,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2059
1983
|
// never gated by this map (see the guard below), so pre-existing families
|
|
2060
1984
|
// keep syncing untouched until the vault doctor (US-003) collapses them.
|
|
2061
1985
|
const journaledKeyByCanonical = new Map();
|
|
2062
|
-
// The delete sweep starts with journaled keys and clears them as the streamed
|
|
2063
|
-
// LIST proves each one present. This retains existing journal-key references
|
|
2064
|
-
// instead of a second, whole-vault population of long remote key strings.
|
|
2065
|
-
const unseenJournalKeys = new Set();
|
|
2066
|
-
// Most journal keys have already been normalized to POSIX and can be cleared
|
|
2067
|
-
// directly. Keep the legacy Windows spelling defense-in-depth without
|
|
2068
|
-
// imposing a second map entry on every current journal row.
|
|
2069
|
-
const legacyJournalKeysByPosix = new Map();
|
|
2070
|
-
// The junk-spelling delete guard needs all journal twins, while
|
|
2071
|
-
// `journaledKeyByCanonical` above deliberately retains only the preferred
|
|
2072
|
-
// representative for classification.
|
|
2073
|
-
const journaledSkillKeysByCanonical = new Map();
|
|
2074
1986
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
2075
1987
|
onHeartbeat?.();
|
|
2076
|
-
const posixJournaledKey = toPosixKey(journaledKey);
|
|
2077
|
-
// A scoped LIST cannot prove anything about keys outside its prefixes.
|
|
2078
|
-
// Do not copy the entire vault's journal into its presence bookkeeping.
|
|
2079
|
-
const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
|
|
2080
|
-
if (inListedScope)
|
|
2081
|
-
unseenJournalKeys.add(journaledKey);
|
|
2082
|
-
if (inListedScope && posixJournaledKey !== journaledKey) {
|
|
2083
|
-
const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
|
|
2084
|
-
if (legacyKeys === undefined) {
|
|
2085
|
-
legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
|
|
2086
|
-
}
|
|
2087
|
-
else {
|
|
2088
|
-
legacyKeys.push(journaledKey);
|
|
2089
|
-
}
|
|
2090
|
-
}
|
|
2091
1988
|
if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
|
|
2092
1989
|
continue;
|
|
2093
1990
|
const canonical = canonicalVaultKeySpelling(journaledKey);
|
|
@@ -2095,15 +1992,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2095
1992
|
if (existing === undefined || journaledKey === canonical) {
|
|
2096
1993
|
journaledKeyByCanonical.set(canonical, journaledKey);
|
|
2097
1994
|
}
|
|
2098
|
-
if (!inListedScope)
|
|
2099
|
-
continue;
|
|
2100
|
-
const skillKeys = journaledSkillKeysByCanonical.get(canonical);
|
|
2101
|
-
if (skillKeys === undefined) {
|
|
2102
|
-
journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
|
|
2103
|
-
}
|
|
2104
|
-
else {
|
|
2105
|
-
skillKeys.push(journaledKey);
|
|
2106
|
-
}
|
|
2107
1995
|
}
|
|
2108
1996
|
// Remote keys that ALSO appear as an ancestor of another remote key — i.e.
|
|
2109
1997
|
// keys that carry child objects under `${key}/…` in this same LIST. The
|
|
@@ -2113,40 +2001,22 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2113
2001
|
// objects beneath it. A REAL directory, by contrast, only ever syncs as
|
|
2114
2002
|
// those child objects — never as a single object AT its own key. So a single
|
|
2115
2003
|
// remote object sitting at a key that also has children is unambiguously a
|
|
2116
|
-
// directory-overlay marker, not a regular file.
|
|
2117
|
-
//
|
|
2118
|
-
//
|
|
2119
|
-
//
|
|
2120
|
-
const remoteKeysWithChildren =
|
|
2121
|
-
|
|
2122
|
-
: collectRemoteAncestors(remoteFiles);
|
|
2123
|
-
// Both sources replay synchronously. for-await on a scoped array introduces
|
|
2124
|
-
// promise/microtask work per object even though no I/O is being awaited.
|
|
2125
|
-
for (const remoteFile of listedFiles) {
|
|
2004
|
+
// directory-overlay marker, not a regular file. Precomputed once (pure path
|
|
2005
|
+
// derivation, no I/O — the planner is synchronous) so the dir-vs-object
|
|
2006
|
+
// collision branch can tell an inert overlay marker apart from a genuine
|
|
2007
|
+
// file-vs-directory structural collision. feedback_d2082110.
|
|
2008
|
+
const remoteKeysWithChildren = new Set();
|
|
2009
|
+
for (const rf of remoteFiles) {
|
|
2126
2010
|
onHeartbeat?.();
|
|
2127
|
-
const
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
if (remoteFile.key === posixRemoteKey) {
|
|
2133
|
-
unseenJournalKeys.delete(remoteFile.key);
|
|
2134
|
-
for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
|
|
2135
|
-
unseenJournalKeys.delete(legacyKey);
|
|
2136
|
-
}
|
|
2137
|
-
}
|
|
2138
|
-
if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2139
|
-
// A remote junk spelling means its canonical journal twin is still
|
|
2140
|
-
// remotely present. Keep that twin out of the delete sweep unless an
|
|
2141
|
-
// exact FILE_TOMBSTONE is authoritative, precisely matching the old
|
|
2142
|
-
// remoteCanonicalSkillKeys protection without retaining remote keys.
|
|
2143
|
-
const canonical = canonicalVaultKeySpelling(remoteFile.key);
|
|
2144
|
-
for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
|
|
2145
|
-
if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
|
|
2146
|
-
unseenJournalKeys.delete(journaledKey);
|
|
2147
|
-
}
|
|
2148
|
-
}
|
|
2011
|
+
const key = toPosixKey(rf.key);
|
|
2012
|
+
let slash = key.indexOf("/");
|
|
2013
|
+
while (slash !== -1) {
|
|
2014
|
+
remoteKeysWithChildren.add(key.slice(0, slash));
|
|
2015
|
+
slash = key.indexOf("/", slash + 1);
|
|
2149
2016
|
}
|
|
2017
|
+
}
|
|
2018
|
+
for (const remoteFile of remoteFiles) {
|
|
2019
|
+
onHeartbeat?.();
|
|
2150
2020
|
const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
|
|
2151
2021
|
if (invalidKey !== null) {
|
|
2152
2022
|
invalidRemoteKeys.push({
|
|
@@ -2799,9 +2669,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2799
2669
|
filesToSkip++;
|
|
2800
2670
|
}
|
|
2801
2671
|
}
|
|
2802
|
-
filesToSkip += preplannedFilesSkipped;
|
|
2803
|
-
filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
|
|
2804
|
-
filesOutOfScope += preplannedFilesOutOfScope;
|
|
2805
2672
|
// Bug #9 — cross-machine delete propagation. The peer's push leg
|
|
2806
2673
|
// already removed the file from S3 (verified in the deep-test
|
|
2807
2674
|
// addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
|
|
@@ -2810,21 +2677,38 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2810
2677
|
// that-was-there-before" — so the file stayed forever on every
|
|
2811
2678
|
// receiver and `filesTombstoned: 0` showed up on every pull.
|
|
2812
2679
|
//
|
|
2813
|
-
// Closing the loop: walk journal
|
|
2814
|
-
// whose path passes the current
|
|
2680
|
+
// Closing the loop: walk the journal, find every entry whose key is
|
|
2681
|
+
// NOT in the remote LIST set, AND whose path passes the current
|
|
2815
2682
|
// ignore filter (paths newly excluded by .hqignore must not trigger
|
|
2816
2683
|
// mass-delete), AND — in personalMode — survives the same companies/*
|
|
2817
2684
|
// gating the download branch applies. The executor will apply each
|
|
2818
2685
|
// as a local delete + journal removal. Symmetric to the push side's
|
|
2819
2686
|
// `propagateDeletes` plan in share.ts.
|
|
2687
|
+
const remoteKeySet = new Set();
|
|
2688
|
+
// Skill-farm twin protection (companion to the junk-spelling pull guard):
|
|
2689
|
+
// when the LIST holds only a junk spelling of a journaled canonical key,
|
|
2690
|
+
// the logical object is still remotely present — a raw has() would class
|
|
2691
|
+
// the canonical journal entry as remote-deleted, unlink the good local
|
|
2692
|
+
// wrapper, and drop the journal entry (after which the next pull would
|
|
2693
|
+
// materialize the junk spelling the guard just refused). Track canonical
|
|
2694
|
+
// forms of listed skill keys and consult them below.
|
|
2695
|
+
const remoteCanonicalSkillKeys = new Set();
|
|
2696
|
+
for (const rf of remoteFiles) {
|
|
2697
|
+
remoteKeySet.add(rf.key);
|
|
2698
|
+
if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2699
|
+
remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
|
|
2700
|
+
}
|
|
2701
|
+
}
|
|
2820
2702
|
const tombstones = [];
|
|
2821
|
-
for (const key of
|
|
2703
|
+
for (const key of journalRows(journal).keys()) {
|
|
2822
2704
|
// Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
|
|
2823
2705
|
// carry backslash separators; the remote LIST is always forward-slash, so a
|
|
2824
2706
|
// raw `has(key)` would miss the match and class a live local file as
|
|
2825
2707
|
// remote-deleted. normalizeJournalKeys() defuses this on load, but the
|
|
2826
2708
|
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
2827
2709
|
const posixKey = toPosixKey(key);
|
|
2710
|
+
if (remoteKeySet.has(posixKey))
|
|
2711
|
+
continue;
|
|
2828
2712
|
// Listing-scope guard (targeted sync / scoped listing). The remote LIST
|
|
2829
2713
|
// now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
|
|
2830
2714
|
// pull narrows further), so "absent from the LIST" is evidence of a remote
|
|
@@ -2836,6 +2720,16 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2836
2720
|
// `all` mode passes `[""]`, which covers every key — no behavior change.
|
|
2837
2721
|
if (!isCoveredByAny(posixKey, prefixSet))
|
|
2838
2722
|
continue;
|
|
2723
|
+
if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
|
|
2724
|
+
remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
|
|
2725
|
+
// An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
|
|
2726
|
+
// delete and wins over the twin protection — a lingering junk
|
|
2727
|
+
// spelling in the LIST must not keep a deliberately-deleted wrapper
|
|
2728
|
+
// alive forever. (The key is absent from the LIST here, so there is
|
|
2729
|
+
// no newer re-create to defer to.)
|
|
2730
|
+
fileTombstones.get(posixKey) === undefined) {
|
|
2731
|
+
continue;
|
|
2732
|
+
}
|
|
2839
2733
|
// Fast-path ignore, BEFORE containment canonicalization. The tombstone
|
|
2840
2734
|
// sweep runs for every journaled key, and a stale journal can carry
|
|
2841
2735
|
// hundreds of thousands of entries under an ignored runtime tree (e.g.
|
|
@@ -2944,8 +2838,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2944
2838
|
}
|
|
2945
2839
|
return {
|
|
2946
2840
|
items,
|
|
2947
|
-
preplannedFilesSkipped,
|
|
2948
|
-
preplannedFilesOutOfScope,
|
|
2949
2841
|
filesToDownload,
|
|
2950
2842
|
bytesToDownload,
|
|
2951
2843
|
filesToSkip,
|