@indigoai-us/hq-cloud 6.16.68-alpha.1 → 6.16.68-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  2. package/dist/bin/sync-runner-watch-loop.js +24 -1
  3. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  4. package/dist/cli/share.d.ts +3 -6
  5. package/dist/cli/share.d.ts.map +1 -1
  6. package/dist/cli/share.js +33 -11
  7. package/dist/cli/share.js.map +1 -1
  8. package/dist/cli/share.test.js +45 -0
  9. package/dist/cli/share.test.js.map +1 -1
  10. package/dist/cli/sync.d.ts +7 -0
  11. package/dist/cli/sync.d.ts.map +1 -1
  12. package/dist/cli/sync.js +42 -6
  13. package/dist/cli/sync.js.map +1 -1
  14. package/dist/cli/sync.test.js +96 -2
  15. package/dist/cli/sync.test.js.map +1 -1
  16. package/dist/journal-row-cache.d.ts +102 -0
  17. package/dist/journal-row-cache.d.ts.map +1 -0
  18. package/dist/journal-row-cache.heap.test.d.ts +2 -0
  19. package/dist/journal-row-cache.heap.test.d.ts.map +1 -0
  20. package/dist/journal-row-cache.heap.test.js +102 -0
  21. package/dist/journal-row-cache.heap.test.js.map +1 -0
  22. package/dist/journal-row-cache.js +549 -0
  23. package/dist/journal-row-cache.js.map +1 -0
  24. package/dist/journal-row-cache.test.d.ts +2 -0
  25. package/dist/journal-row-cache.test.d.ts.map +1 -0
  26. package/dist/journal-row-cache.test.js +155 -0
  27. package/dist/journal-row-cache.test.js.map +1 -0
  28. package/dist/journal-row-store.d.ts +14 -0
  29. package/dist/journal-row-store.d.ts.map +1 -1
  30. package/dist/journal-row-store.js +19 -0
  31. package/dist/journal-row-store.js.map +1 -1
  32. package/dist/journal-snapshot-rows.d.ts +4 -0
  33. package/dist/journal-snapshot-rows.d.ts.map +1 -1
  34. package/dist/journal-snapshot-rows.js +12 -0
  35. package/dist/journal-snapshot-rows.js.map +1 -1
  36. package/dist/journal.d.ts +11 -0
  37. package/dist/journal.d.ts.map +1 -1
  38. package/dist/journal.js +59 -3
  39. package/dist/journal.js.map +1 -1
  40. package/dist/journal.test.js +24 -1
  41. package/dist/journal.test.js.map +1 -1
  42. package/dist/lib/net-errors.d.ts +6 -0
  43. package/dist/lib/net-errors.d.ts.map +1 -1
  44. package/dist/lib/net-errors.js +10 -0
  45. package/dist/lib/net-errors.js.map +1 -1
  46. package/dist/lib/net-errors.test.js +4 -0
  47. package/dist/lib/net-errors.test.js.map +1 -1
  48. package/dist/object-io.d.ts +8 -0
  49. package/dist/object-io.d.ts.map +1 -1
  50. package/dist/object-io.js +41 -12
  51. package/dist/object-io.js.map +1 -1
  52. package/dist/object-io.test.js +44 -1
  53. package/dist/object-io.test.js.map +1 -1
  54. package/dist/s3.d.ts +8 -1
  55. package/dist/s3.d.ts.map +1 -1
  56. package/dist/s3.js +5 -9
  57. package/dist/s3.js.map +1 -1
  58. package/dist/vault-client.d.ts.map +1 -1
  59. package/dist/vault-client.js +12 -10
  60. package/dist/vault-client.js.map +1 -1
  61. package/package.json +1 -1
package/dist/cli/share.js CHANGED
@@ -14,7 +14,7 @@ import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.
14
14
  import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
15
15
  import * as crypto from "crypto";
16
16
  import { RateLimitedError } from "../object-io.js";
17
- import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
17
+ import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
18
18
  import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
19
19
  import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
20
20
  import { isGeneratedCoreMirrorKey, isPersonalVaultDeletionExcluded, isSessionLogDeletionKey, purgeExcludedPersonalVaultJournalEntries, } from "../personal-vault.js";
@@ -303,6 +303,10 @@ function ctimeToStamp(hashedStat, lstat) {
303
303
  * True when `absolutePath` provably has not been written since its journal
304
304
  * entry was stamped, judged from stat alone. See `journalEntryMatchesStat`.
305
305
  */
306
+ function skipFingerprint(journal, relativePath) {
307
+ const rows = journalRows(journal);
308
+ return rows.statFingerprint?.(relativePath) ?? rows.get(relativePath);
309
+ }
306
310
  function statMatchesJournal(absolutePath, entry) {
307
311
  if (!entry)
308
312
  return false;
@@ -315,6 +319,10 @@ function statMatchesJournal(absolutePath, entry) {
315
319
  }
316
320
  // The entry checks and the ctime argument live in `journalEntryMatchesStat`
317
321
  // (journal.ts) so the pull planner applies the identical gate.
322
+ if (journalFingerprintMatchesStat(entry, lstat))
323
+ return true;
324
+ if (!("syncedAt" in entry))
325
+ return false;
318
326
  return journalEntryMatchesStat(entry, lstat);
319
327
  }
320
328
  /**
@@ -342,6 +350,7 @@ async function yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, force =
342
350
  }
343
351
  function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
344
352
  const items = [];
353
+ let omittedSkips = 0;
345
354
  for (const entry of filesToShare) {
346
355
  onHeartbeat?.();
347
356
  const { absolutePath, relativePath } = entry;
@@ -364,12 +373,12 @@ function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
364
373
  if (entry.kind === "symlink") {
365
374
  const localHash = hashSymlinkTarget(entry.target);
366
375
  if (skipUnchanged) {
367
- const existing = journalRows(journal).get(relativePath);
376
+ const existing = skipFingerprint(journal, relativePath);
368
377
  // `!localDiverges`: see the regular-file gate below — a divergent entry
369
378
  // stores the current LOCAL hash, so hash equality here proves only that
370
379
  // local has not moved since the pull-side hold, never that it matches remote.
371
380
  if (existing && existing.hash === localHash && !existing.localDiverges) {
372
- items.push({ action: "skip-unchanged", absolutePath, relativePath });
381
+ omittedSkips += 1;
373
382
  continue;
374
383
  }
375
384
  }
@@ -406,8 +415,8 @@ function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
406
415
  // by the restamp below so the next pass is free;
407
416
  // - `localDiverges` entries — see the hash gate below for why equality
408
417
  // proves nothing about them.
409
- if (skipUnchanged && statMatchesJournal(absolutePath, journalRows(journal).get(relativePath))) {
410
- items.push({ action: "skip-unchanged", absolutePath, relativePath });
418
+ if (skipUnchanged && statMatchesJournal(absolutePath, skipFingerprint(journal, relativePath))) {
419
+ omittedSkips += 1;
411
420
  continue;
412
421
  }
413
422
  // Snapshot stat either side of the read. If the two differ the file was
@@ -459,7 +468,10 @@ function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
459
468
  existing.size !== statAtHash.size)) {
460
469
  restamp = statAtHash;
461
470
  }
462
- items.push({ action: "skip-unchanged", absolutePath, relativePath, restamp });
471
+ if (restamp)
472
+ items.push({ action: "skip-unchanged", absolutePath, relativePath, restamp });
473
+ else
474
+ omittedSkips += 1;
463
475
  continue;
464
476
  }
465
477
  }
@@ -486,7 +498,8 @@ function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
486
498
  filesToSkip++;
487
499
  }
488
500
  }
489
- return { items, filesToUpload, bytesToUpload, filesToSkip };
501
+ filesToSkip += omittedSkips;
502
+ return { items, filesToUpload, bytesToUpload, filesToSkip, preplannedFilesSkipped: omittedSkips };
490
503
  }
491
504
  /**
492
505
  * Build a full-pass plan in bounded row batches. A scoped replan deliberately
@@ -498,6 +511,7 @@ async function computePushPlanCooperatively(filesToShare, journal, skipUnchanged
498
511
  let filesToUpload = 0;
499
512
  let bytesToUpload = 0;
500
513
  let filesToSkip = 0;
514
+ let preplannedFilesSkipped = 0;
501
515
  const cooperativeWork = { rowsSinceYield: 0 };
502
516
  for (let start = 0; start < filesToShare.length; start += FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
503
517
  const slice = filesToShare.slice(start, start + FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
@@ -506,12 +520,13 @@ async function computePushPlanCooperatively(filesToShare, journal, skipUnchanged
506
520
  filesToUpload += plan.filesToUpload;
507
521
  bytesToUpload += plan.bytesToUpload;
508
522
  filesToSkip += plan.filesToSkip;
523
+ preplannedFilesSkipped += plan.preplannedFilesSkipped;
509
524
  for (const entry of slice) {
510
525
  noteCooperativeRowVisit("plan", entry.relativePath);
511
526
  }
512
527
  await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, start + slice.length);
513
528
  }
514
- return { items, filesToUpload, bytesToUpload, filesToSkip };
529
+ return { items, filesToUpload, bytesToUpload, filesToSkip, preplannedFilesSkipped };
515
530
  }
516
531
  /** Re-read one collected path at the execution seam. */
517
532
  function recollectPushEntry(item) {
@@ -1712,6 +1727,7 @@ function refreshPushJournalAfterWalk(run, collectedKeys = [], deleteScopeRoots =
1712
1727
  }
1713
1728
  }
1714
1729
  async function executeUploads(run, pushPlan, counters, conflictPaths) {
1730
+ counters.filesSkipped += pushPlan.preplannedFilesSkipped;
1715
1731
  const TRANSFER_CONCURRENCY = resolveTransferConcurrency(run.hostLoadAtStart);
1716
1732
  let conflictPromptChain = Promise.resolve();
1717
1733
  const resolveConflictSerialized = (info) => {
@@ -2467,7 +2483,8 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2467
2483
  // scheduler until the full pass ends.
2468
2484
  let admittedSinceCooperativeCheckpoint = 0;
2469
2485
  let skippedSinceCooperativeCheckpoint = 0;
2470
- while (queue.length > 0 || inFlight.size > 0) {
2486
+ let remainingPreplannedSkips = pushPlan.preplannedFilesSkipped;
2487
+ while (queue.length > 0 || inFlight.size > 0 || remainingPreplannedSkips > 0) {
2471
2488
  while (!aborted &&
2472
2489
  !sessionAuthAborted &&
2473
2490
  (!run.options.cooperativeCheckpoint ||
@@ -2476,7 +2493,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2476
2493
  skippedSinceCooperativeCheckpoint <
2477
2494
  FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE)) &&
2478
2495
  inFlight.size < TRANSFER_CONCURRENCY &&
2479
- queue.length > 0) {
2496
+ (queue.length > 0 || remainingPreplannedSkips > 0)) {
2497
+ if (remainingPreplannedSkips > 0) {
2498
+ remainingPreplannedSkips -= 1;
2499
+ skippedSinceCooperativeCheckpoint += 1;
2500
+ continue;
2501
+ }
2480
2502
  const previewItem = queue[0];
2481
2503
  const item = replanPushItemAtExecution(previewItem, run.journal, run.skipUnchanged === true, run.options.onHeartbeat);
2482
2504
  if (item === null) {
@@ -2541,7 +2563,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2541
2563
  FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE ||
2542
2564
  skippedSinceCooperativeCheckpoint >=
2543
2565
  FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE ||
2544
- queue.length === 0)) {
2566
+ (queue.length === 0 && remainingPreplannedSkips === 0))) {
2545
2567
  // Every admitted upload has settled, so `processUploadItem` has
2546
2568
  // finished its mutable journal updates. Persist that durable unit before
2547
2569
  // yielding. Skip-only units have no new durable state; resume the