@indigoai-us/hq-cloud 6.16.68 → 6.16.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  2. package/dist/bin/sync-runner-company.js +14 -0
  3. package/dist/bin/sync-runner-company.js.map +1 -1
  4. package/dist/bin/sync-runner-company.test.js +64 -0
  5. package/dist/bin/sync-runner-company.test.js.map +1 -1
  6. package/dist/bin/sync-runner-events.test.js +19 -0
  7. package/dist/bin/sync-runner-events.test.js.map +1 -1
  8. package/dist/bin/sync-runner.d.ts +5 -0
  9. package/dist/bin/sync-runner.d.ts.map +1 -1
  10. package/dist/bin/sync-runner.js.map +1 -1
  11. package/dist/cli/share.d.ts +18 -1
  12. package/dist/cli/share.d.ts.map +1 -1
  13. package/dist/cli/share.js +135 -120
  14. package/dist/cli/share.js.map +1 -1
  15. package/dist/cli/sync.d.ts +181 -1
  16. package/dist/cli/sync.d.ts.map +1 -1
  17. package/dist/cli/sync.js +63 -27
  18. package/dist/cli/sync.js.map +1 -1
  19. package/dist/cli/sync.test.js +105 -0
  20. package/dist/cli/sync.test.js.map +1 -1
  21. package/dist/journal-decode-window.d.ts +42 -0
  22. package/dist/journal-decode-window.d.ts.map +1 -0
  23. package/dist/journal-decode-window.js +60 -0
  24. package/dist/journal-decode-window.js.map +1 -0
  25. package/dist/journal-decode-window.test.d.ts +2 -0
  26. package/dist/journal-decode-window.test.d.ts.map +1 -0
  27. package/dist/journal-decode-window.test.js +99 -0
  28. package/dist/journal-decode-window.test.js.map +1 -0
  29. package/dist/journal-row-store.d.ts +4 -0
  30. package/dist/journal-row-store.d.ts.map +1 -1
  31. package/dist/journal-row-store.js +64 -2
  32. package/dist/journal-row-store.js.map +1 -1
  33. package/dist/journal-streaming.heap.test.d.ts +2 -0
  34. package/dist/journal-streaming.heap.test.d.ts.map +1 -0
  35. package/dist/journal-streaming.heap.test.js +190 -0
  36. package/dist/journal-streaming.heap.test.js.map +1 -0
  37. package/dist/journal.d.ts +1 -11
  38. package/dist/journal.d.ts.map +1 -1
  39. package/dist/journal.js +4 -53
  40. package/dist/journal.js.map +1 -1
  41. package/dist/journal.test.js +13 -7
  42. package/dist/journal.test.js.map +1 -1
  43. package/dist/lib/net-errors.d.ts +12 -0
  44. package/dist/lib/net-errors.d.ts.map +1 -1
  45. package/dist/lib/net-errors.js +22 -0
  46. package/dist/lib/net-errors.js.map +1 -1
  47. package/dist/lib/net-errors.test.js +35 -1
  48. package/dist/lib/net-errors.test.js.map +1 -1
  49. package/dist/lib/s3-content-checksum.d.ts +16 -0
  50. package/dist/lib/s3-content-checksum.d.ts.map +1 -0
  51. package/dist/lib/s3-content-checksum.js +91 -0
  52. package/dist/lib/s3-content-checksum.js.map +1 -0
  53. package/dist/lib/s3-content-checksum.test.d.ts +2 -0
  54. package/dist/lib/s3-content-checksum.test.d.ts.map +1 -0
  55. package/dist/lib/s3-content-checksum.test.js +26 -0
  56. package/dist/lib/s3-content-checksum.test.js.map +1 -0
  57. package/dist/object-io.d.ts +2 -0
  58. package/dist/object-io.d.ts.map +1 -1
  59. package/dist/object-io.js +37 -19
  60. package/dist/object-io.js.map +1 -1
  61. package/dist/object-io.test.js +89 -2
  62. package/dist/object-io.test.js.map +1 -1
  63. package/dist/sync/candidate-uploader.d.ts.map +1 -1
  64. package/dist/sync/candidate-uploader.js +6 -1
  65. package/dist/sync/candidate-uploader.js.map +1 -1
  66. package/dist/sync/multipart-uploader.d.ts.map +1 -1
  67. package/dist/sync/multipart-uploader.js +2 -1
  68. package/dist/sync/multipart-uploader.js.map +1 -1
  69. package/package.json +1 -1
  70. package/dist/journal-row-cache.d.ts +0 -102
  71. package/dist/journal-row-cache.d.ts.map +0 -1
  72. package/dist/journal-row-cache.heap.test.d.ts +0 -2
  73. package/dist/journal-row-cache.heap.test.d.ts.map +0 -1
  74. package/dist/journal-row-cache.heap.test.js +0 -102
  75. package/dist/journal-row-cache.heap.test.js.map +0 -1
  76. package/dist/journal-row-cache.js +0 -549
  77. package/dist/journal-row-cache.js.map +0 -1
  78. package/dist/journal-row-cache.test.d.ts +0 -2
  79. package/dist/journal-row-cache.test.d.ts.map +0 -1
  80. package/dist/journal-row-cache.test.js +0 -155
  81. package/dist/journal-row-cache.test.js.map +0 -1
package/dist/cli/share.js CHANGED
@@ -14,7 +14,7 @@ import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.
14
14
  import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
15
15
  import * as crypto from "crypto";
16
16
  import { RateLimitedError } from "../object-io.js";
17
- import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
17
+ import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, JOURNAL_DECODE_WINDOW, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
18
18
  import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
19
19
  import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
20
20
  import { isGeneratedCoreMirrorKey, isPersonalVaultDeletionExcluded, isSessionLogDeletionKey, purgeExcludedPersonalVaultJournalEntries, } from "../personal-vault.js";
@@ -260,6 +260,8 @@ export const _testing = {
260
260
  enableCooperativeRowVisits,
261
261
  disableCooperativeRowVisits,
262
262
  cooperativeRowVisits: snapshotCooperativeRowVisits,
263
+ computePushPlan,
264
+ computePushPlanCooperatively,
263
265
  };
264
266
  /** Snapshot of `absolutePath`, or null when it is missing or not a plain file. */
265
267
  function statSnapshot(absolutePath) {
@@ -351,141 +353,150 @@ async function yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, force =
351
353
  function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
352
354
  const items = [];
353
355
  let omittedSkips = 0;
356
+ let plannedRows = 0;
354
357
  for (const entry of filesToShare) {
355
358
  onHeartbeat?.();
356
- const { absolutePath, relativePath } = entry;
357
- // Server-generated paths are read-only to normal client sync. Classify
358
- // them before hashing or skip-unchanged so local-only files and files with
359
- // a divergent remote baseline both surface the same honest refusal. The
360
- // board remains the documented push-side exception.
361
- if (relativePath !== "board.json" && isCloudAuthoritative(relativePath)) {
362
- items.push({ action: "refuse-server-owned", absolutePath, relativePath });
363
- continue;
364
- }
365
- // Symlinks bypass the size-limit gate (the wire body is small,
366
- // bounded by target length) and hash the target through the
367
- // symlink-namespaced hash so a symlink to "real.md" can never
368
- // collide with a regular file containing the bytes "real.md" in
369
- // the skip-unchanged gate. The target is what we're actually
370
- // uploading, so its hash is what should drive change detection —
371
- // a target rewrite must re-fire an upload even when the link
372
- // itself "looks" identical.
373
- if (entry.kind === "symlink") {
374
- const localHash = hashSymlinkTarget(entry.target);
359
+ plannedRows += 1;
360
+ try {
361
+ const { absolutePath, relativePath } = entry;
362
+ // Server-generated paths are read-only to normal client sync. Classify
363
+ // them before hashing or skip-unchanged so local-only files and files with
364
+ // a divergent remote baseline both surface the same honest refusal. The
365
+ // board remains the documented push-side exception.
366
+ if (relativePath !== "board.json" && isCloudAuthoritative(relativePath)) {
367
+ items.push({ action: "refuse-server-owned", absolutePath, relativePath });
368
+ continue;
369
+ }
370
+ // Symlinks bypass the size-limit gate (the wire body is small,
371
+ // bounded by target length) and hash the target through the
372
+ // symlink-namespaced hash so a symlink to "real.md" can never
373
+ // collide with a regular file containing the bytes "real.md" in
374
+ // the skip-unchanged gate. The target is what we're actually
375
+ // uploading, so its hash is what should drive change detection —
376
+ // a target rewrite must re-fire an upload even when the link
377
+ // itself "looks" identical.
378
+ if (entry.kind === "symlink") {
379
+ const localHash = hashSymlinkTarget(entry.target);
380
+ if (skipUnchanged) {
381
+ const existing = skipFingerprint(journal, relativePath);
382
+ // `!localDiverges`: see the regular-file gate below — a divergent entry
383
+ // stores the current LOCAL hash, so hash equality here proves only that
384
+ // local has not moved since the pull-side hold, never that it matches remote.
385
+ if (existing && existing.hash === localHash && !existing.localDiverges) {
386
+ omittedSkips += 1;
387
+ continue;
388
+ }
389
+ }
390
+ items.push({
391
+ action: "upload",
392
+ kind: "symlink",
393
+ absolutePath,
394
+ relativePath,
395
+ target: entry.target,
396
+ localHash,
397
+ size: 0,
398
+ });
399
+ continue;
400
+ }
401
+ if (!isWithinSizeLimit(absolutePath)) {
402
+ items.push({ action: "skip-size-limit", absolutePath, relativePath });
403
+ continue;
404
+ }
405
+ // Stat fast path. Reading and hashing every regular file every pass costs
406
+ // one full read of the vault — hundreds of thousands of files on a real
407
+ // install — to answer a question stat can usually answer for free.
408
+ //
409
+ // The gate is size + mtime + ctime, and ctime is what makes it safe. mtime
410
+ // is forgeable: `cp -p`, `touch -r` and `rsync --times` put an old mtime
411
+ // back on new bytes, so a size+mtime check would skip such an edit forever
412
+ // — the failure this code previously hashed everything to avoid. ctime is
413
+ // the inode-change stamp and userspace cannot set it; the very utimes call
414
+ // that restores mtime moves ctime to now. A file that clears all three has
415
+ // not been written since we hashed it.
416
+ //
417
+ // Deliberately NOT skipped here:
418
+ // - entries with no recorded mtime/ctime (written before stat tracking,
419
+ // or by a downloader that did not stamp) — hashed once, then stamped
420
+ // by the restamp below so the next pass is free;
421
+ // - `localDiverges` entries — see the hash gate below for why equality
422
+ // proves nothing about them.
423
+ if (skipUnchanged && statMatchesJournal(absolutePath, skipFingerprint(journal, relativePath))) {
424
+ omittedSkips += 1;
425
+ continue;
426
+ }
427
+ // Snapshot stat either side of the read. If the two differ the file was
428
+ // rewritten while we hashed it, so `localHash` may describe bytes that are
429
+ // already gone — and nothing downstream may stamp a timestamp against it.
430
+ const statBeforeHash = statSnapshot(absolutePath);
431
+ let localHash;
432
+ let statAtHash;
433
+ if (entry.knownHash !== undefined &&
434
+ statBeforeHash !== null &&
435
+ sameStatSnapshot(entry.knownHash.stat, statBeforeHash) &&
436
+ !timestampsTooCoarseToTrust(statBeforeHash.mtimeMs, statBeforeHash.ctimeMs)) {
437
+ // The execution seam re-plans every upload candidate up to three times
438
+ // (admission, dispatch, after HEAD). Each re-plan hashed the body again,
439
+ // so a candidate that never changes — on a real install ~3.5k archived
440
+ // transcripts, 6.3 GB, skipped after HEAD — was read four times per pass.
441
+ // size + mtime + ctime unchanged since the earlier hash proves the bytes
442
+ // are the same (ctime cannot be set from userspace), so reuse it.
443
+ localHash = entry.knownHash.hash;
444
+ statAtHash = statBeforeHash;
445
+ }
446
+ else {
447
+ localHash = hashFile(absolutePath);
448
+ const statAfterHash = statSnapshot(absolutePath);
449
+ statAtHash = sameStatSnapshot(statBeforeHash, statAfterHash)
450
+ ? statAfterHash ?? undefined
451
+ : undefined;
452
+ }
375
453
  if (skipUnchanged) {
376
- const existing = skipFingerprint(journal, relativePath);
377
- // `!localDiverges`: see the regular-file gate below — a divergent entry
378
- // stores the current LOCAL hash, so hash equality here proves only that
379
- // local has not moved since the pull-side hold, never that it matches remote.
454
+ const existing = journalRows(journal).get(relativePath);
455
+ // A `localDiverges` entry records the hash of the KEPT LOCAL body, so
456
+ // `existing.hash === localHash` is true for every such file by
457
+ // construction. Skipping on that equality asserts "local matches remote",
458
+ // which is precisely the claim the flag exists to deny — and it would
459
+ // strand the file: the runner's push leg always passes skipUnchanged,
460
+ // so the conflict path that could publish it never runs, and the pull leg
461
+ // sees an unchanged remote and re-arms the flag. Route these to the HEAD
462
+ // + conflict path instead and let the resolution decide.
380
463
  if (existing && existing.hash === localHash && !existing.localDiverges) {
381
- omittedSkips += 1;
464
+ // The hash proved the bytes are unchanged — a no-op skip. Capture the
465
+ // current (mtimeMs, size) so the journal metadata stays accurate. Only
466
+ // attach when the stored stat actually differs (avoid a needless
467
+ // journal write on an already-current entry). Stat failure → no
468
+ // restamp; the skip still stands.
469
+ let restamp;
470
+ if (statAtHash &&
471
+ (existing.mtimeMs !== statAtHash.mtimeMs ||
472
+ existing.ctimeMs !== statAtHash.ctimeMs ||
473
+ existing.size !== statAtHash.size)) {
474
+ restamp = statAtHash;
475
+ }
476
+ if (restamp)
477
+ items.push({ action: "skip-unchanged", absolutePath, relativePath, restamp });
478
+ else
479
+ omittedSkips += 1;
382
480
  continue;
383
481
  }
384
482
  }
483
+ const size = fs.statSync(absolutePath).size;
385
484
  items.push({
386
485
  action: "upload",
387
- kind: "symlink",
486
+ kind: "file",
388
487
  absolutePath,
389
488
  relativePath,
390
- target: entry.target,
391
489
  localHash,
392
- size: 0,
490
+ size,
491
+ ...(statAtHash ? { statAtHash } : {}),
393
492
  });
394
- continue;
395
- }
396
- if (!isWithinSizeLimit(absolutePath)) {
397
- items.push({ action: "skip-size-limit", absolutePath, relativePath });
398
- continue;
399
- }
400
- // Stat fast path. Reading and hashing every regular file every pass costs
401
- // one full read of the vault — hundreds of thousands of files on a real
402
- // install — to answer a question stat can usually answer for free.
403
- //
404
- // The gate is size + mtime + ctime, and ctime is what makes it safe. mtime
405
- // is forgeable: `cp -p`, `touch -r` and `rsync --times` put an old mtime
406
- // back on new bytes, so a size+mtime check would skip such an edit forever
407
- // — the failure this code previously hashed everything to avoid. ctime is
408
- // the inode-change stamp and userspace cannot set it; the very utimes call
409
- // that restores mtime moves ctime to now. A file that clears all three has
410
- // not been written since we hashed it.
411
- //
412
- // Deliberately NOT skipped here:
413
- // - entries with no recorded mtime/ctime (written before stat tracking,
414
- // or by a downloader that did not stamp) — hashed once, then stamped
415
- // by the restamp below so the next pass is free;
416
- // - `localDiverges` entries — see the hash gate below for why equality
417
- // proves nothing about them.
418
- if (skipUnchanged && statMatchesJournal(absolutePath, skipFingerprint(journal, relativePath))) {
419
- omittedSkips += 1;
420
- continue;
421
493
  }
422
- // Snapshot stat either side of the read. If the two differ the file was
423
- // rewritten while we hashed it, so `localHash` may describe bytes that are
424
- // already gone — and nothing downstream may stamp a timestamp against it.
425
- const statBeforeHash = statSnapshot(absolutePath);
426
- let localHash;
427
- let statAtHash;
428
- if (entry.knownHash !== undefined &&
429
- statBeforeHash !== null &&
430
- sameStatSnapshot(entry.knownHash.stat, statBeforeHash) &&
431
- !timestampsTooCoarseToTrust(statBeforeHash.mtimeMs, statBeforeHash.ctimeMs)) {
432
- // The execution seam re-plans every upload candidate up to three times
433
- // (admission, dispatch, after HEAD). Each re-plan hashed the body again,
434
- // so a candidate that never changes — on a real install ~3.5k archived
435
- // transcripts, 6.3 GB, skipped after HEAD — was read four times per pass.
436
- // size + mtime + ctime unchanged since the earlier hash proves the bytes
437
- // are the same (ctime cannot be set from userspace), so reuse it.
438
- localHash = entry.knownHash.hash;
439
- statAtHash = statBeforeHash;
440
- }
441
- else {
442
- localHash = hashFile(absolutePath);
443
- const statAfterHash = statSnapshot(absolutePath);
444
- statAtHash = sameStatSnapshot(statBeforeHash, statAfterHash)
445
- ? statAfterHash ?? undefined
446
- : undefined;
447
- }
448
- if (skipUnchanged) {
449
- const existing = journalRows(journal).get(relativePath);
450
- // A `localDiverges` entry records the hash of the KEPT LOCAL body, so
451
- // `existing.hash === localHash` is true for every such file by
452
- // construction. Skipping on that equality asserts "local matches remote",
453
- // which is precisely the claim the flag exists to deny — and it would
454
- // strand the file: the runner's push leg always passes skipUnchanged,
455
- // so the conflict path that could publish it never runs, and the pull leg
456
- // sees an unchanged remote and re-arms the flag. Route these to the HEAD
457
- // + conflict path instead and let the resolution decide.
458
- if (existing && existing.hash === localHash && !existing.localDiverges) {
459
- // The hash proved the bytes are unchanged — a no-op skip. Capture the
460
- // current (mtimeMs, size) so the journal metadata stays accurate. Only
461
- // attach when the stored stat actually differs (avoid a needless
462
- // journal write on an already-current entry). Stat failure → no
463
- // restamp; the skip still stands.
464
- let restamp;
465
- if (statAtHash &&
466
- (existing.mtimeMs !== statAtHash.mtimeMs ||
467
- existing.ctimeMs !== statAtHash.ctimeMs ||
468
- existing.size !== statAtHash.size)) {
469
- restamp = statAtHash;
470
- }
471
- if (restamp)
472
- items.push({ action: "skip-unchanged", absolutePath, relativePath, restamp });
473
- else
474
- omittedSkips += 1;
475
- continue;
476
- }
494
+ finally {
495
+ if (plannedRows % JOURNAL_DECODE_WINDOW === 0)
496
+ dropJournalDecodeWindow();
477
497
  }
478
- const size = fs.statSync(absolutePath).size;
479
- items.push({
480
- action: "upload",
481
- kind: "file",
482
- absolutePath,
483
- relativePath,
484
- localHash,
485
- size,
486
- ...(statAtHash ? { statAtHash } : {}),
487
- });
488
498
  }
499
+ dropJournalDecodeWindow();
489
500
  let filesToUpload = 0;
490
501
  let bytesToUpload = 0;
491
502
  let filesToSkip = 0;
@@ -525,6 +536,7 @@ async function computePushPlanCooperatively(filesToShare, journal, skipUnchanged
525
536
  noteCooperativeRowVisit("plan", entry.relativePath);
526
537
  }
527
538
  await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, start + slice.length);
539
+ dropJournalDecodeWindow();
528
540
  }
529
541
  return { items, filesToUpload, bytesToUpload, filesToSkip, preplannedFilesSkipped };
530
542
  }
@@ -3513,6 +3525,9 @@ function defaultConsoleLogger(event) {
3513
3525
  else if (event.type === "skip-rename-blocked") {
3514
3526
  console.warn(` ! ${event.path} skipped — another program on this PC is holding the file open, so HQ could not replace it yet. ${event.remedy}`);
3515
3527
  }
3528
+ else if (event.type === "skip-vanished-remote") {
3529
+ console.warn(` ! ${event.path} skipped — the vault object disappeared before it could be downloaded (${event.message})`);
3530
+ }
3516
3531
  else if (event.type === "skip-archived-session-log") {
3517
3532
  console.log(` - ${event.path} skipped — the cloud copy of this session log is archived`);
3518
3533
  }