@indigoai-us/hq-cloud 6.18.11-alpha.1 → 6.18.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +1 -1
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +2 -2
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -31
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +51 -212
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +20 -72
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +1 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +15 -7
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +16 -15
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +1 -69
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +10 -50
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +40 -121
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +0 -2
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +0 -76
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +8 -11
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +23 -92
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +182 -228
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +0 -3
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +0 -8
- package/dist/ignore.test.js.map +1 -1
- package/dist/journal.d.ts +0 -15
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +2 -66
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +1 -16
- package/dist/journal.test.js.map +1 -1
- package/dist/manifest/build-manifest-resident-parity.test.js +3 -5
- package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +1 -58
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +0 -46
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/logger.test.js +0 -3
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +0 -3
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -52
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +50 -416
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +40 -673
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/targeted-upsert-exact-list.d.ts +33 -0
- package/dist/sync/targeted-upsert-exact-list.d.ts.map +1 -0
- package/dist/sync/targeted-upsert-exact-list.js +137 -0
- package/dist/sync/targeted-upsert-exact-list.js.map +1 -0
- package/dist/sync/targeted-upsert-exact-list.test.d.ts +2 -0
- package/dist/sync/targeted-upsert-exact-list.test.d.ts.map +1 -0
- package/dist/sync/targeted-upsert-exact-list.test.js +151 -0
- package/dist/sync/targeted-upsert-exact-list.test.js.map +1 -0
- package/dist/watcher.d.ts +0 -18
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +13 -78
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +0 -72
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/bin/sync-runner-args.test.d.ts +0 -2
- package/dist/bin/sync-runner-args.test.d.ts.map +0 -1
- package/dist/bin/sync-runner-args.test.js +0 -82
- package/dist/bin/sync-runner-args.test.js.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts +0 -2
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js +0 -366
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts +0 -2
- package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot.bench.test.js +0 -317
- package/dist/bin/sync-runner-delete-snapshot.bench.test.js.map +0 -1
- package/dist/sync/content-hash.d.ts +0 -4
- package/dist/sync/content-hash.d.ts.map +0 -1
- package/dist/sync/content-hash.js +0 -11
- package/dist/sync/content-hash.js.map +0 -1
- package/dist/sync/content-hash.test.d.ts +0 -2
- package/dist/sync/content-hash.test.d.ts.map +0 -1
- package/dist/sync/content-hash.test.js +0 -16
- package/dist/sync/content-hash.test.js.map +0 -1
- package/dist/sync/unchanged-push-events.d.ts +0 -16
- package/dist/sync/unchanged-push-events.d.ts.map +0 -1
- package/dist/sync/unchanged-push-events.js +0 -51
- package/dist/sync/unchanged-push-events.js.map +0 -1
- package/dist/sync/unchanged-push-events.test.d.ts +0 -2
- package/dist/sync/unchanged-push-events.test.d.ts.map +0 -1
- package/dist/sync/unchanged-push-events.test.js +0 -76
- package/dist/sync/unchanged-push-events.test.js.map +0 -1
|
@@ -17,8 +17,7 @@ import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watche
|
|
|
17
17
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
18
18
|
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
19
19
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
20
|
-
import { NoopPushReceiver,
|
|
21
|
-
import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
|
|
20
|
+
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
22
21
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
23
22
|
import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
|
|
24
23
|
import { resolveAreaLayoutAuthority } from "../sync/area-ledger-migration.js";
|
|
@@ -27,11 +26,12 @@ import { emitCloudTelemetry } from "../telemetry-events.js";
|
|
|
27
26
|
import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPressureCloudProperties, heapPressureStderrLine, heapRecycleCloudProperties, heapRecycleStderrLine, performHeapHardExit, nodeHeapReaders, nodeMajorGcObserver, resolveDeclaredHeapCeilingBytes, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveHeapFootprintRecycleEnabled, resolveHeapFootprintRecycleBytes, resolveHeapFootprintHardExitBytes, HEAP_SAMPLE_INTERVAL_ENV, HEAP_PRESSURE_FRACTION_ENV, HEAP_RECYCLE_ENABLED_ENV, HEAP_RECYCLE_LIVE_FRACTION_ENV, HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV, HEAP_RECYCLE_EMERGENCY_FRACTION_ENV, HEAP_RECYCLE_MIN_UPTIME_MS_ENV, HEAP_CHECKPOINT_MS_ENV, HEAP_PINNED_FRACTION_ENV, HEAP_PINNED_SUSTAIN_MS_ENV, HEAP_HARD_EXIT_FRACTION_ENV, HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV, HEAP_FOOTPRINT_RECYCLE_ENABLED_ENV, HEAP_FOOTPRINT_RECYCLE_MB_ENV, HEAP_FOOTPRINT_HARD_EXIT_MB_ENV, } from "./sync-runner-heap-telemetry.js";
|
|
28
27
|
import { setRunnerEmitObserver } from "./sync-runner-events.js";
|
|
29
28
|
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
29
|
+
import { exactUpsertPathsForCoordinatorSlice } from "../sync/targeted-upsert-exact-list.js";
|
|
30
30
|
import { createHqFlagsCapSource, FullPassCapController, } from "../sync/full-pass-caps.js";
|
|
31
31
|
import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
|
|
32
32
|
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
|
|
33
33
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
34
|
-
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey,
|
|
34
|
+
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
35
35
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
36
36
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
37
37
|
/** Maximum time shutdown waits for event-sync startup before abandoning it. */
|
|
@@ -858,6 +858,21 @@ function emitFullReconcileDecision({ mode, reasons, triggers, pollTick, configur
|
|
|
858
858
|
*/
|
|
859
859
|
export const DEFAULT_EVENT_BATCH_LIMIT = 10_000;
|
|
860
860
|
export const EVENT_BATCH_LIMIT_ENV = "HQ_SYNC_EVENT_BATCH_LIMIT";
|
|
861
|
+
/** Remove transient watcher scopes from argv reused for scheduled reconciles. */
|
|
862
|
+
export function stripTransientWatchScopeArgs(argv) {
|
|
863
|
+
const valueFlags = new Set([
|
|
864
|
+
"--poll-remote-ms",
|
|
865
|
+
"--scope-path",
|
|
866
|
+
"--scope-upsert-path",
|
|
867
|
+
"--delete-scope-path",
|
|
868
|
+
]);
|
|
869
|
+
const standaloneFlags = new Set(["--watch", "--event-push"]);
|
|
870
|
+
return argv.filter((arg, index) => {
|
|
871
|
+
if (standaloneFlags.has(arg) || valueFlags.has(arg))
|
|
872
|
+
return false;
|
|
873
|
+
return index === 0 || !valueFlags.has(argv[index - 1]);
|
|
874
|
+
});
|
|
875
|
+
}
|
|
861
876
|
/**
|
|
862
877
|
* Fast-lane admission is deliberately conservative: a normal editor save is
|
|
863
878
|
* a handful of files and well below 4 MiB, whereas session-log catch-up is
|
|
@@ -1200,33 +1215,6 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
|
|
|
1200
1215
|
nextDueAt = now + intervalMs;
|
|
1201
1216
|
return { nextDueAt, delayMs: intervalMs };
|
|
1202
1217
|
}
|
|
1203
|
-
async function withScopeContentHashesFile(route, events, scopedPaths, run) {
|
|
1204
|
-
const targetPaths = new Set(scopedPaths
|
|
1205
|
-
.map((relativePath) => scopePathForRoute(route, relativePath))
|
|
1206
|
-
.filter((relativePath) => relativePath !== null));
|
|
1207
|
-
const contentHashes = Object.create(null);
|
|
1208
|
-
for (const event of events) {
|
|
1209
|
-
if (event.kind !== "upsert" ||
|
|
1210
|
-
!event.contentHash ||
|
|
1211
|
-
!CONTENT_HASH_PATTERN.test(event.contentHash))
|
|
1212
|
-
continue;
|
|
1213
|
-
const scopePath = scopePathForRoute(route, event.relativePath);
|
|
1214
|
-
if (scopePath !== null && targetPaths.has(scopePath)) {
|
|
1215
|
-
contentHashes[scopePath] = event.contentHash;
|
|
1216
|
-
}
|
|
1217
|
-
}
|
|
1218
|
-
if (Object.keys(contentHashes).length === 0)
|
|
1219
|
-
return run();
|
|
1220
|
-
const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "hq-sync-pull-content-hashes-"));
|
|
1221
|
-
const contentHashesFile = path.join(temporaryDirectory, "hashes.json");
|
|
1222
|
-
try {
|
|
1223
|
-
fs.writeFileSync(contentHashesFile, JSON.stringify(contentHashes), { mode: 0o600 });
|
|
1224
|
-
return await run(contentHashesFile);
|
|
1225
|
-
}
|
|
1226
|
-
finally {
|
|
1227
|
-
fs.rmSync(temporaryDirectory, { recursive: true, force: true });
|
|
1228
|
-
}
|
|
1229
|
-
}
|
|
1230
1218
|
/**
|
|
1231
1219
|
* Select journal entries covered by one watcher delete. Exact file unlinks
|
|
1232
1220
|
* deliberately avoid enumeration; only directory unlinks expand descendants.
|
|
@@ -1296,22 +1284,6 @@ function isBulkIntentOverride() {
|
|
|
1296
1284
|
function passExitCode(outcome) {
|
|
1297
1285
|
return typeof outcome === "number" ? outcome : outcome.exitCode;
|
|
1298
1286
|
}
|
|
1299
|
-
function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode) {
|
|
1300
|
-
const exitCode = passExitCode(outcome);
|
|
1301
|
-
const result = typeof outcome === "number" ? undefined : outcome.result;
|
|
1302
|
-
const diagnostics = [
|
|
1303
|
-
...(result?.errors ?? []).map(({ company, message }) => `${company}: ${message}`),
|
|
1304
|
-
...(result?.transient ?? []).map(({ company, message }) => `${company}: ${message}`),
|
|
1305
|
-
...(result?.companyFailures ?? []).map(({ company, message }) => `${company}: ${message}`),
|
|
1306
|
-
];
|
|
1307
|
-
const failureReason = `targeted pull failed with exit code ${exitCode}, accepted=${accepted}` +
|
|
1308
|
-
(diagnostics.length === 0 ? "" : `: ${[...new Set(diagnostics)].join("; ")}`);
|
|
1309
|
-
const isTransportFailure = exitCode === authRequiredExitCode ||
|
|
1310
|
-
exitCode === TRANSIENT_NETWORK_EXIT ||
|
|
1311
|
-
(result?.transient?.length ?? 0) > 0 ||
|
|
1312
|
-
isReceiverTransportFailure(failureReason);
|
|
1313
|
-
return new ReceiverDispatchError(isTransportFailure ? "transport" : "path-specific", failureReason);
|
|
1314
|
-
}
|
|
1315
1287
|
/**
|
|
1316
1288
|
* Exit code for one scoped/catch-up route. A 404 company route returns
|
|
1317
1289
|
* UNRESOLVED_COMPANY_EXIT from the runner; that must not poison the rest of
|
|
@@ -1404,25 +1376,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1404
1376
|
} });
|
|
1405
1377
|
process.stderr.write(`hq-sync-runner: backup retention dry-run — ${plan.candidates.length} candidate(s), ${plan.protected.length} protected\n`);
|
|
1406
1378
|
});
|
|
1407
|
-
const passArgv = argv
|
|
1408
|
-
if (a === "--watch")
|
|
1409
|
-
return false;
|
|
1410
|
-
if (a === "--poll-remote-ms")
|
|
1411
|
-
return false;
|
|
1412
|
-
if (a === "--event-push")
|
|
1413
|
-
return false;
|
|
1414
|
-
if (a === "--scope-path")
|
|
1415
|
-
return false;
|
|
1416
|
-
if (a === "--delete-scope-path")
|
|
1417
|
-
return false;
|
|
1418
|
-
if (i > 0 && argv[i - 1] === "--poll-remote-ms")
|
|
1419
|
-
return false;
|
|
1420
|
-
if (i > 0 && argv[i - 1] === "--scope-path")
|
|
1421
|
-
return false;
|
|
1422
|
-
if (i > 0 && argv[i - 1] === "--delete-scope-path")
|
|
1423
|
-
return false;
|
|
1424
|
-
return true;
|
|
1425
|
-
});
|
|
1379
|
+
const passArgv = stripTransientWatchScopeArgs(argv);
|
|
1426
1380
|
// Authoritative direction from parseArgs — never re-derive from argv (that
|
|
1427
1381
|
// mis-defaults an omitted --direction to "both" and picks the wrong
|
|
1428
1382
|
// occurrence on duplicates). Fall back to the parser's own default ("pull").
|
|
@@ -2692,137 +2646,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2692
2646
|
}
|
|
2693
2647
|
return snapshots;
|
|
2694
2648
|
};
|
|
2695
|
-
/**
|
|
2696
|
-
* Fold file unlinks in each watcher-turn run into one durable delta per
|
|
2697
|
-
* journal. Directory deletes are ordered barriers: flush the preceding file
|
|
2698
|
-
* run, apply the directory event through its existing path, then continue.
|
|
2699
|
-
* That preserves quarantine and descendant-revocation semantics without
|
|
2700
|
-
* sending every file in a mixed burst through a per-item append.
|
|
2701
|
-
*/
|
|
2702
|
-
const captureQueuedLocalDeleteSnapshotsBatch = async (captures) => {
|
|
2703
|
-
const results = captures.map(() => []);
|
|
2704
|
-
if (captures.length === 0)
|
|
2705
|
-
return results;
|
|
2706
|
-
const captureFileUnlinkGroup = async (scoped) => {
|
|
2707
|
-
if (scoped.length === 0)
|
|
2708
|
-
return;
|
|
2709
|
-
const slug = scoped[0].coordinates.slug;
|
|
2710
|
-
const session = await watcherJournalStoreAsync(slug);
|
|
2711
|
-
await session.refreshAsync();
|
|
2712
|
-
const changed = new Map();
|
|
2713
|
-
const owners = new Map();
|
|
2714
|
-
const snapshots = new Map();
|
|
2715
|
-
const own = (key, index) => {
|
|
2716
|
-
const indexes = owners.get(key) ?? new Set();
|
|
2717
|
-
indexes.add(index);
|
|
2718
|
-
owners.set(key, indexes);
|
|
2719
|
-
};
|
|
2720
|
-
for (const item of scoped) {
|
|
2721
|
-
if (!item.capture.isCurrent())
|
|
2722
|
-
continue;
|
|
2723
|
-
const key = item.coordinates.key;
|
|
2724
|
-
const entry = changed.get(key) ?? session.get(key);
|
|
2725
|
-
if (!entry)
|
|
2726
|
-
continue;
|
|
2727
|
-
if (quarantineRootFor(item.coordinates.slug, key)) {
|
|
2728
|
-
const next = structuredClone(entry);
|
|
2729
|
-
const journal = {
|
|
2730
|
-
version: "2", lastSync: "", files: { [key]: next }, pulls: [],
|
|
2731
|
-
};
|
|
2732
|
-
if (clearLocalDeleteIntent(journal, key)) {
|
|
2733
|
-
changed.set(key, journalRows(journal).get(key));
|
|
2734
|
-
own(key, item.index);
|
|
2735
|
-
}
|
|
2736
|
-
continue;
|
|
2737
|
-
}
|
|
2738
|
-
if (!entry.remoteEtag || !entry.kind || entry.removedAt !== undefined)
|
|
2739
|
-
continue;
|
|
2740
|
-
const snapshot = {
|
|
2741
|
-
journalSlug: item.coordinates.slug,
|
|
2742
|
-
journalPath: key,
|
|
2743
|
-
deleteScopeRoot: item.coordinates.key,
|
|
2744
|
-
absolutePath: item.route.kind === "company"
|
|
2745
|
-
? path.join(hqRoot, "companies", item.route.slug, key)
|
|
2746
|
-
: path.join(hqRoot, key),
|
|
2747
|
-
remoteEtag: entry.remoteEtag,
|
|
2748
|
-
localHash: entry.hash,
|
|
2749
|
-
localKind: entry.kind,
|
|
2750
|
-
};
|
|
2751
|
-
const next = structuredClone(entry);
|
|
2752
|
-
const journal = {
|
|
2753
|
-
version: "2", lastSync: "", files: { [key]: next }, pulls: [],
|
|
2754
|
-
};
|
|
2755
|
-
if (!markLocalDeleteIntent(journal, key, snapshot.localHash, snapshot.localKind, snapshot.remoteEtag))
|
|
2756
|
-
continue;
|
|
2757
|
-
changed.set(key, journalRows(journal).get(key));
|
|
2758
|
-
own(key, item.index);
|
|
2759
|
-
const perCapture = snapshots.get(item.index) ?? [];
|
|
2760
|
-
perCapture.push(snapshot);
|
|
2761
|
-
snapshots.set(item.index, perCapture);
|
|
2762
|
-
}
|
|
2763
|
-
const persistedKeys = new Set();
|
|
2764
|
-
const changedEntries = [...changed];
|
|
2765
|
-
const maxDeltaRows = 1_000;
|
|
2766
|
-
for (let offset = 0; offset < changedEntries.length; offset += maxDeltaRows) {
|
|
2767
|
-
const chunk = changedEntries.slice(offset, offset + maxDeltaRows);
|
|
2768
|
-
let chunkPersistedKeys = new Set();
|
|
2769
|
-
const persisted = await session.applyDeltaGeneratedAsync(() => {
|
|
2770
|
-
const accepted = chunk.filter(([key]) => [...(owners.get(key) ?? [])].some((index) => captures[index]?.isCurrent()));
|
|
2771
|
-
chunkPersistedKeys = new Set(accepted.map(([key]) => key));
|
|
2772
|
-
if (accepted.length === 0)
|
|
2773
|
-
return undefined;
|
|
2774
|
-
return { upserts: Object.fromEntries(accepted) };
|
|
2775
|
-
});
|
|
2776
|
-
if (persisted) {
|
|
2777
|
-
for (const key of chunkPersistedKeys)
|
|
2778
|
-
persistedKeys.add(key);
|
|
2779
|
-
}
|
|
2780
|
-
if (offset + maxDeltaRows < changedEntries.length) {
|
|
2781
|
-
await new Promise((resolve) => setImmediate(resolve));
|
|
2782
|
-
}
|
|
2783
|
-
}
|
|
2784
|
-
for (const [index, values] of snapshots) {
|
|
2785
|
-
if (!captures[index]?.isCurrent())
|
|
2786
|
-
continue;
|
|
2787
|
-
results[index] = values.filter((snapshot) => persistedKeys.has(snapshot.journalPath));
|
|
2788
|
-
}
|
|
2789
|
-
};
|
|
2790
|
-
const fileRun = [];
|
|
2791
|
-
const flushFileRun = async () => {
|
|
2792
|
-
if (fileRun.length === 0)
|
|
2793
|
-
return;
|
|
2794
|
-
const groupsByJournal = new Map();
|
|
2795
|
-
for (const item of fileRun) {
|
|
2796
|
-
const group = groupsByJournal.get(item.coordinates.slug) ?? [];
|
|
2797
|
-
group.push(item);
|
|
2798
|
-
groupsByJournal.set(item.coordinates.slug, group);
|
|
2799
|
-
}
|
|
2800
|
-
fileRun.length = 0;
|
|
2801
|
-
// Map iteration preserves the first-seen journal order within this run;
|
|
2802
|
-
// rows inside each journal retain their original watcher order.
|
|
2803
|
-
for (const group of groupsByJournal.values()) {
|
|
2804
|
-
await captureFileUnlinkGroup(group);
|
|
2805
|
-
}
|
|
2806
|
-
};
|
|
2807
|
-
for (let index = 0; index < captures.length; index += 1) {
|
|
2808
|
-
const capture = captures[index];
|
|
2809
|
-
if (capture.kind === "unlinkDir") {
|
|
2810
|
-
await flushFileRun();
|
|
2811
|
-
results[index] = await captureQueuedLocalDeleteSnapshots(capture.relativePath, capture.kind, capture.isCurrent);
|
|
2812
|
-
continue;
|
|
2813
|
-
}
|
|
2814
|
-
const coordinates = journalCoordinates(capture.relativePath, capture.kind);
|
|
2815
|
-
const route = routeChangeToTarget(capture.relativePath, capture.kind);
|
|
2816
|
-
if (!coordinates || !route)
|
|
2817
|
-
continue;
|
|
2818
|
-
if (route.kind === "company" &&
|
|
2819
|
-
resolveSkipCompanies().has(route.slug.toLowerCase()))
|
|
2820
|
-
continue;
|
|
2821
|
-
fileRun.push({ index, capture, coordinates, route });
|
|
2822
|
-
}
|
|
2823
|
-
await flushFileRun();
|
|
2824
|
-
return results;
|
|
2825
|
-
};
|
|
2826
2649
|
const warmupJournalPaths = function* () {
|
|
2827
2650
|
const journals = listJournals();
|
|
2828
2651
|
const vaultJournalPresent = journals.some(({ slug }) => slug === PERSONAL_VAULT_JOURNAL_SLUG);
|
|
@@ -3172,7 +2995,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3172
2995
|
// Paused workspaces sync in NO direction while Off.
|
|
3173
2996
|
if (isPausedRoute(group.route))
|
|
3174
2997
|
continue;
|
|
3175
|
-
const
|
|
2998
|
+
const exactFileUpserts = [...group.paths.entries()].flatMap(([absolutePath, relativePath]) => {
|
|
2999
|
+
const kind = group.changes.get(absolutePath)?.kind;
|
|
3000
|
+
// Only create/change events are file upserts. Keep unlinks on the
|
|
3001
|
+
// legacy exact-plus-descendant scope so delete reconciliation is
|
|
3002
|
+
// not narrowed by this listing optimization.
|
|
3003
|
+
return kind === "add" || kind === "change" ? [relativePath] : [];
|
|
3004
|
+
});
|
|
3005
|
+
const targetedPullArgv = buildTargetedPullArgv(group.route, passArgv, group.relPaths, exactFileUpserts);
|
|
3176
3006
|
const pullArgv = group.route.kind === "company" ||
|
|
3177
3007
|
targetedPullArgv.includes("--scope-path")
|
|
3178
3008
|
? targetedPullArgv
|
|
@@ -3186,7 +3016,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3186
3016
|
? "fast"
|
|
3187
3017
|
: "slow";
|
|
3188
3018
|
const execution = parallelCoordinatorEnabled
|
|
3189
|
-
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) =>
|
|
3019
|
+
? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
|
|
3020
|
+
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
3021
|
+
return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
|
|
3022
|
+
})
|
|
3190
3023
|
: { result: await runGuarded(pullArgv, undefined, lane, "realtime"), accepted: true };
|
|
3191
3024
|
const exit = scopedRouteDrainExit(execution);
|
|
3192
3025
|
if (exit !== 0 && worstExit === 0)
|
|
@@ -3337,7 +3170,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3337
3170
|
onDegraded: opts.onDegraded,
|
|
3338
3171
|
captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
|
|
3339
3172
|
queueLocalDeleteSnapshots: opts.queueLocalDeleteSnapshots,
|
|
3340
|
-
queueLocalDeleteSnapshotsBatch: opts.queueLocalDeleteSnapshotsBatch,
|
|
3341
3173
|
isUntrackedDelete: opts.isUntrackedDelete,
|
|
3342
3174
|
telemetryClient: watcherTelemetryClient,
|
|
3343
3175
|
telemetryClaims: claims,
|
|
@@ -3366,7 +3198,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3366
3198
|
},
|
|
3367
3199
|
captureLocalDeleteSnapshots,
|
|
3368
3200
|
queueLocalDeleteSnapshots: captureQueuedLocalDeleteSnapshots,
|
|
3369
|
-
queueLocalDeleteSnapshotsBatch: captureQueuedLocalDeleteSnapshotsBatch,
|
|
3370
3201
|
isUntrackedDelete,
|
|
3371
3202
|
});
|
|
3372
3203
|
activeWatcherGeneration = ++watcherGeneration;
|
|
@@ -3542,12 +3373,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3542
3373
|
// Submit every route in this lane before awaiting any result. In
|
|
3543
3374
|
// particular, a later route's fast receipt must never wait for an
|
|
3544
3375
|
// earlier route's slow pull to finish.
|
|
3545
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => {
|
|
3546
|
-
|
|
3547
|
-
|
|
3376
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
|
|
3377
|
+
const slicePaths = slice.paths.map((path) => path.canonicalPath);
|
|
3378
|
+
const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
|
|
3379
|
+
return runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced);
|
|
3380
|
+
})));
|
|
3548
3381
|
for (const execution of completed) {
|
|
3549
3382
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
3550
|
-
throw
|
|
3383
|
+
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
3551
3384
|
}
|
|
3552
3385
|
}
|
|
3553
3386
|
};
|
|
@@ -3582,7 +3415,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3582
3415
|
}
|
|
3583
3416
|
const lanes = partitionByKnownSizes(events, receiverEventSize);
|
|
3584
3417
|
const drainLane = async (lane, laneEvents) => {
|
|
3585
|
-
const
|
|
3418
|
+
const pathsByRoute = new Map();
|
|
3586
3419
|
for (const event of laneEvents) {
|
|
3587
3420
|
const route = routeChangeToTarget(event.relativePath);
|
|
3588
3421
|
// The receiver is an independent ingress from the local watcher.
|
|
@@ -3592,15 +3425,21 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3592
3425
|
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
3593
3426
|
continue;
|
|
3594
3427
|
const key = routeKey(route);
|
|
3595
|
-
const group =
|
|
3596
|
-
|
|
3597
|
-
|
|
3428
|
+
const group = pathsByRoute.get(key) ?? {
|
|
3429
|
+
route,
|
|
3430
|
+
paths: [],
|
|
3431
|
+
upsertPaths: [],
|
|
3432
|
+
};
|
|
3433
|
+
group.paths.push(event.relativePath);
|
|
3434
|
+
if (event.kind === "upsert")
|
|
3435
|
+
group.upsertPaths.push(event.relativePath);
|
|
3436
|
+
pathsByRoute.set(key, group);
|
|
3598
3437
|
}
|
|
3599
|
-
for (const { route,
|
|
3600
|
-
const
|
|
3601
|
-
const result = await
|
|
3438
|
+
for (const { route, paths, upsertPaths } of pathsByRoute.values()) {
|
|
3439
|
+
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths, upsertPaths);
|
|
3440
|
+
const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
|
|
3602
3441
|
if (passExitCode(result) !== 0) {
|
|
3603
|
-
throw
|
|
3442
|
+
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
3604
3443
|
}
|
|
3605
3444
|
}
|
|
3606
3445
|
};
|