@indigoai-us/hq-cloud 6.16.49 → 6.16.51-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/rescue-runner.js +1 -1
- package/dist/bin/sync-runner-company.d.ts +6 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +30 -1
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +55 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +108 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +281 -113
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +33 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +34 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +555 -78
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +0 -29
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +39 -236
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-rsync-preflight.test.js +4 -256
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +0 -6
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +0 -12
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +0 -6
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +0 -2
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +57 -37
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +87 -93
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +226 -118
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +215 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +0 -2
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +0 -30
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +0 -175
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-store.test.js +130 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +9 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +30 -21
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +0 -295
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +3 -18
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +24 -47
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/s3.test.js +1 -27
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync-core.d.ts +4 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +4 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +1 -25
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/cli/rescue-breaker.test.d.ts +0 -2
- package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
- package/dist/cli/rescue-breaker.test.js +0 -314
- package/dist/cli/rescue-breaker.test.js.map +0 -1
- package/dist/cli/rescue-realpath.test.d.ts +0 -2
- package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
- package/dist/cli/rescue-realpath.test.js +0 -109
- package/dist/cli/rescue-realpath.test.js.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.js +0 -210
- package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
package/dist/cli/share.js
CHANGED
|
@@ -11,7 +11,7 @@ import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../c
|
|
|
11
11
|
import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
12
12
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
13
13
|
import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
|
|
14
|
-
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads,
|
|
14
|
+
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
15
15
|
import * as crypto from "crypto";
|
|
16
16
|
import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, timestampsTooCoarseToTrust, updateEntry, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
|
|
17
17
|
import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
@@ -628,10 +628,33 @@ export function wrapFilterWithIgnoreVisibility(underlying, hqRoot, onIgnoreExclu
|
|
|
628
628
|
return false;
|
|
629
629
|
};
|
|
630
630
|
}
|
|
631
|
+
class CooperativePushHandoff extends Error {
|
|
632
|
+
constructor() {
|
|
633
|
+
super("Realtime work ran at a settled full-push boundary");
|
|
634
|
+
this.name = "CooperativePushHandoff";
|
|
635
|
+
}
|
|
636
|
+
}
|
|
631
637
|
/**
|
|
632
638
|
* Share local file(s) to the entity vault.
|
|
639
|
+
*
|
|
640
|
+
* A cooperative hand-off discards the old plan instead of continuing it after
|
|
641
|
+
* another writer has run. Journal delta computation treats missing keys as
|
|
642
|
+
* deletes, so reusing a pre-handoff plan would be unsafe even though the
|
|
643
|
+
* hand-off itself only happens at a durable unit boundary.
|
|
633
644
|
*/
|
|
634
645
|
export async function share(options) {
|
|
646
|
+
for (;;) {
|
|
647
|
+
try {
|
|
648
|
+
return await shareOnce(options);
|
|
649
|
+
}
|
|
650
|
+
catch (error) {
|
|
651
|
+
if (error instanceof CooperativePushHandoff)
|
|
652
|
+
continue;
|
|
653
|
+
throw error;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
async function shareOnce(options) {
|
|
635
658
|
const run = await createPushRunContext(options);
|
|
636
659
|
const counters = createShareCounters();
|
|
637
660
|
const filesRefusedStalePaths = [];
|
|
@@ -1333,7 +1356,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1333
1356
|
key: it.relativePath,
|
|
1334
1357
|
localPath: it.absolutePath,
|
|
1335
1358
|
isSymlink: it.kind === "symlink",
|
|
1336
|
-
...(it.kind === "file" ? { contentHash: it.localHash } : {}),
|
|
1337
1359
|
author: run.options.author,
|
|
1338
1360
|
})));
|
|
1339
1361
|
// Warm the GET presigns the per-item conflict HEAD (remoteMeta) reuses, so a
|
|
@@ -1505,39 +1527,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1505
1527
|
}
|
|
1506
1528
|
if (remoteMeta) {
|
|
1507
1529
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
1508
|
-
const
|
|
1509
|
-
const remoteChanged = !!journalEntry &&
|
|
1510
|
-
hasRemoteChanged({ ...remoteMeta, contentHash: remoteContentHash }, journalEntry);
|
|
1511
|
-
// A matching published digest is a content no-op even when SSE-KMS gave
|
|
1512
|
-
// this object a fresh ETag. Reconcile the journal to that version without
|
|
1513
|
-
// a needless PUT or a phantom two-sided conflict.
|
|
1514
|
-
if (remoteContentHash === localHash) {
|
|
1515
|
-
const lstat = fs.lstatSync(absolutePath);
|
|
1516
|
-
updateEntry(run.journal, relativePath, localHash, lstat.size, journalEntry?.direction ?? "up", absolutePath, {
|
|
1517
|
-
remoteEtag: remoteMeta.etag,
|
|
1518
|
-
mtimeMs: lstat.mtimeMs,
|
|
1519
|
-
ctimeMs: ctimeToStamp(item.kind === "file" ? item.statAtHash : undefined, lstat),
|
|
1520
|
-
kind: "file",
|
|
1521
|
-
});
|
|
1522
|
-
checkpointShareJournal(run);
|
|
1523
|
-
run.emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
1524
|
-
counters.filesSkipped++;
|
|
1525
|
-
return;
|
|
1526
|
-
}
|
|
1527
|
-
// This machine's journal baseline is a superseded content generation.
|
|
1528
|
-
// Replaying that unchanged baseline is a stale revert, not an edit. A
|
|
1529
|
-
// user who deliberately restores some other historic content has a hash
|
|
1530
|
-
// different from the journal baseline and still reaches normal conflict
|
|
1531
|
-
// resolution below.
|
|
1532
|
-
if (journalEntry !== undefined &&
|
|
1533
|
-
remoteContentHash !== undefined &&
|
|
1534
|
-
journalEntry.hash === localHash &&
|
|
1535
|
-
remoteContentHash !== localHash) {
|
|
1536
|
-
conflictPaths.push(relativePath);
|
|
1537
|
-
run.emit({ type: "conflict", path: relativePath, direction: "push", resolution: "skip" });
|
|
1538
|
-
counters.filesSkipped++;
|
|
1539
|
-
return;
|
|
1540
|
-
}
|
|
1530
|
+
const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
|
|
1541
1531
|
// Every path below that needs the remote BODY — the no-row equality
|
|
1542
1532
|
// probe, and a `keep` that adopts the remote — is impossible against a
|
|
1543
1533
|
// DEEP_ARCHIVE session transcript: each GET fails with InvalidObjectState.
|
|
@@ -1770,8 +1760,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1770
1760
|
const skillContentAtPut = !isSymlinkUpload && CANONICAL_COMPANY_SKILL_RE.test(relativePath)
|
|
1771
1761
|
? fs.readFileSync(absolutePath, "utf8")
|
|
1772
1762
|
: undefined;
|
|
1773
|
-
if (!isSymlinkUpload)
|
|
1774
|
-
setUploadContentHash(absolutePath, localHash);
|
|
1775
1763
|
const { etag } = isSymlinkUpload
|
|
1776
1764
|
? await uploadSymlink(run.ctx, item.target, relativePath, run.options.author, pc)
|
|
1777
1765
|
: await uploadFile(run.ctx, absolutePath, relativePath, run.options.author, pc);
|
|
@@ -2015,9 +2003,16 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2015
2003
|
// this flag only halts admission, so the refusal still surfaces via
|
|
2016
2004
|
// throwUploadWorkerErrors below. In-flight legs still drain.
|
|
2017
2005
|
let sessionAuthAborted = false;
|
|
2006
|
+
// Cooperative work is admitted in bounded units, then fully drained before
|
|
2007
|
+
// the root lock can change owners. Skipped rows are not durable mutations
|
|
2008
|
+
// and therefore do not count toward the unit.
|
|
2009
|
+
let admittedSinceCooperativeCheckpoint = 0;
|
|
2018
2010
|
while (queue.length > 0 || inFlight.size > 0) {
|
|
2019
2011
|
while (!aborted &&
|
|
2020
2012
|
!sessionAuthAborted &&
|
|
2013
|
+
(!run.options.cooperativeCheckpoint ||
|
|
2014
|
+
admittedSinceCooperativeCheckpoint <
|
|
2015
|
+
FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE) &&
|
|
2021
2016
|
inFlight.size < TRANSFER_CONCURRENCY &&
|
|
2022
2017
|
queue.length > 0) {
|
|
2023
2018
|
const previewItem = queue[0];
|
|
@@ -2044,6 +2039,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2044
2039
|
if (inFlight.size > 0 && inFlightBytes + nextBytes > byteBudget)
|
|
2045
2040
|
break;
|
|
2046
2041
|
queue.shift();
|
|
2042
|
+
admittedSinceCooperativeCheckpoint++;
|
|
2047
2043
|
inFlightBytes += nextBytes;
|
|
2048
2044
|
const upload = () => processUploadItem(item);
|
|
2049
2045
|
const p = (run.options.transferSemaphore
|
|
@@ -2071,6 +2067,24 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2071
2067
|
if (inFlight.size > 0) {
|
|
2072
2068
|
await Promise.race(Array.from(inFlight));
|
|
2073
2069
|
}
|
|
2070
|
+
else if (run.options.cooperativeCheckpoint &&
|
|
2071
|
+
!aborted &&
|
|
2072
|
+
!sessionAuthAborted &&
|
|
2073
|
+
!hadUploadFailure &&
|
|
2074
|
+
admittedSinceCooperativeCheckpoint > 0 &&
|
|
2075
|
+
(admittedSinceCooperativeCheckpoint >=
|
|
2076
|
+
FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE ||
|
|
2077
|
+
queue.length === 0)) {
|
|
2078
|
+
// Every admitted upload has settled, so `processUploadItem` has
|
|
2079
|
+
// finished its mutable journal updates. Persist this durable unit
|
|
2080
|
+
// before yielding. A real hand-off restarts share() above and thereby
|
|
2081
|
+
// discards its now-stale upload/delete plan.
|
|
2082
|
+
flushShareJournal(run);
|
|
2083
|
+
admittedSinceCooperativeCheckpoint = 0;
|
|
2084
|
+
if ((await run.options.cooperativeCheckpoint()) === true) {
|
|
2085
|
+
throw new CooperativePushHandoff();
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2074
2088
|
else {
|
|
2075
2089
|
break;
|
|
2076
2090
|
}
|
|
@@ -2585,6 +2599,12 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
|
|
|
2585
2599
|
*/
|
|
2586
2600
|
const CHECKPOINT_MIN_APPLIED = 250;
|
|
2587
2601
|
const CHECKPOINT_MIN_INTERVAL_MS = 5_000;
|
|
2602
|
+
/**
|
|
2603
|
+
* Maximum completed upload mutations between cooperative full-push hand-offs.
|
|
2604
|
+
* The executor drains a unit before yielding, so realtime never overlaps an
|
|
2605
|
+
* upload body or the push's mutable journal state.
|
|
2606
|
+
*/
|
|
2607
|
+
export const FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
2588
2608
|
/**
|
|
2589
2609
|
* Fraction of the journal's current size that must be re-applied before another
|
|
2590
2610
|
* checkpoint is worth its write. `writeJournal` publishes a whole compact
|