@indigoai-us/hq-cloud 6.16.1 → 6.16.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/bin/area-ledger-migration.test.js +3 -1
  2. package/dist/bin/area-ledger-migration.test.js.map +1 -1
  3. package/dist/bin/sync-runner-watch-loop.d.ts +11 -0
  4. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  5. package/dist/bin/sync-runner-watch-loop.js +60 -24
  6. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  7. package/dist/bin/sync-runner-watch-loop.test.js +84 -2
  8. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  9. package/dist/cli/reindex.d.ts.map +1 -1
  10. package/dist/cli/reindex.js +2 -253
  11. package/dist/cli/reindex.js.map +1 -1
  12. package/dist/cli/sync.d.ts +2 -0
  13. package/dist/cli/sync.d.ts.map +1 -1
  14. package/dist/cli/sync.js +135 -75
  15. package/dist/cli/sync.js.map +1 -1
  16. package/dist/cli/sync.test.js +107 -2
  17. package/dist/cli/sync.test.js.map +1 -1
  18. package/dist/ignore.d.ts +26 -8
  19. package/dist/ignore.d.ts.map +1 -1
  20. package/dist/ignore.js +66 -18
  21. package/dist/ignore.js.map +1 -1
  22. package/dist/ignore.test.js +77 -5
  23. package/dist/ignore.test.js.map +1 -1
  24. package/dist/journal-store.test.js +17 -0
  25. package/dist/journal-store.test.js.map +1 -1
  26. package/dist/journal.d.ts +14 -2
  27. package/dist/journal.d.ts.map +1 -1
  28. package/dist/journal.js +42 -1
  29. package/dist/journal.js.map +1 -1
  30. package/dist/object-io.d.ts +2 -0
  31. package/dist/object-io.d.ts.map +1 -1
  32. package/dist/object-io.js +2 -0
  33. package/dist/object-io.js.map +1 -1
  34. package/dist/object-io.test.js +5 -1
  35. package/dist/object-io.test.js.map +1 -1
  36. package/dist/personal-vault.d.ts.map +1 -1
  37. package/dist/personal-vault.js +3 -1
  38. package/dist/personal-vault.js.map +1 -1
  39. package/dist/personal-vault.test.js +6 -0
  40. package/dist/personal-vault.test.js.map +1 -1
  41. package/dist/s3.d.ts +2 -0
  42. package/dist/s3.d.ts.map +1 -1
  43. package/dist/s3.js +1 -0
  44. package/dist/s3.js.map +1 -1
  45. package/dist/s3.test.js +22 -0
  46. package/dist/s3.test.js.map +1 -1
  47. package/dist/session-log-capture.d.ts +20 -0
  48. package/dist/session-log-capture.d.ts.map +1 -0
  49. package/dist/session-log-capture.js +648 -0
  50. package/dist/session-log-capture.js.map +1 -0
  51. package/dist/session-log-capture.test.d.ts +2 -0
  52. package/dist/session-log-capture.test.d.ts.map +1 -0
  53. package/dist/session-log-capture.test.js +276 -0
  54. package/dist/session-log-capture.test.js.map +1 -0
  55. package/dist/sync/area-ledger-migration.d.ts +8 -1
  56. package/dist/sync/area-ledger-migration.d.ts.map +1 -1
  57. package/dist/sync/area-ledger-migration.js +81 -50
  58. package/dist/sync/area-ledger-migration.js.map +1 -1
  59. package/dist/sync/area-ledger-migration.test.js +1 -0
  60. package/dist/sync/area-ledger-migration.test.js.map +1 -1
  61. package/dist/sync/event-sync.d.ts +3 -1
  62. package/dist/sync/event-sync.d.ts.map +1 -1
  63. package/dist/sync/event-sync.js +1 -0
  64. package/dist/sync/event-sync.js.map +1 -1
  65. package/dist/sync/state-store.js +5 -1
  66. package/dist/sync/state-store.js.map +1 -1
  67. package/dist/vault-client.d.ts +5 -0
  68. package/dist/vault-client.d.ts.map +1 -1
  69. package/dist/vault-client.js +1 -0
  70. package/dist/vault-client.js.map +1 -1
  71. package/dist/vault-client.test.js +15 -0
  72. package/dist/vault-client.test.js.map +1 -1
  73. package/dist/watcher.d.ts +13 -4
  74. package/dist/watcher.d.ts.map +1 -1
  75. package/dist/watcher.js +18 -3
  76. package/dist/watcher.js.map +1 -1
  77. package/dist/watcher.test.js +28 -0
  78. package/dist/watcher.test.js.map +1 -1
  79. package/package.json +1 -1
package/dist/cli/sync.js CHANGED
@@ -16,7 +16,7 @@ import { createSyncProgressRecorder } from "../sync-progress.js";
16
16
  import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
17
17
  import { SKILLS_KEY_PREFIX } from "./doctor.js";
18
18
  import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
19
- import { readJournal, writeJournal, openJournalStoreSession, hashFile, hashSymlinkTarget, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
19
+ import { readJournal, writeJournal, getStateDir, openJournalStoreSession, hashFile, hashSymlinkTarget, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
20
20
  import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
21
21
  import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
22
22
  import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
@@ -134,18 +134,48 @@ export function resolveAutoPruneCap() {
134
134
  }
135
135
  /** Max time to wait on the best-effort new-files notification POST. */
136
136
  const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
137
- // A watcher invokes `sync()` repeatedly in one daemon. Keep the keyed v3
138
- // projection per journal shard for those targeted receive passes; `refresh()`
139
- // below remains the authoritative cross-process freshness boundary.
140
- const targetedJournalSessions = new Map();
137
+ // Keep the existing hot targeted-pull performance contract without retaining
138
+ // one full journal per vault/shard. One current runner can have exactly one
139
+ // idle projection; a different or concurrent target opens a pass-local store.
140
+ const TARGETED_JOURNAL_SESSION_LIMIT = 1;
141
+ let cachedTargetedJournalSession;
141
142
  function targetedJournalSession(slug) {
142
- const key = `${process.env.HQ_STATE_DIR ?? "default"}\0${slug}`;
143
- let session = targetedJournalSessions.get(key);
144
- if (!session) {
145
- session = openJournalStoreSession(slug);
146
- targetedJournalSessions.set(key, session);
143
+ const cacheKey = `${getStateDir()}\u0000${slug}`;
144
+ const cached = cachedTargetedJournalSession;
145
+ if (cached?.cacheKey === cacheKey && cached.leases === 0) {
146
+ cached.leases = 1;
147
+ return {
148
+ store: cached.store,
149
+ release: () => { cached.leases -= 1; },
150
+ };
147
151
  }
148
- return session;
152
+ if ((!cached || cached.leases === 0) && TARGETED_JOURNAL_SESSION_LIMIT === 1) {
153
+ // Keep the current idle lease intact until the replacement opens. If the
154
+ // open fails, the same-key caller can still reuse this healthy session.
155
+ const store = openJournalStoreSession(slug);
156
+ const replacement = {
157
+ cacheKey,
158
+ store,
159
+ leases: 1,
160
+ };
161
+ cached?.store.dispose();
162
+ cachedTargetedJournalSession = replacement;
163
+ return {
164
+ store: replacement.store,
165
+ release: () => { replacement.leases -= 1; },
166
+ };
167
+ }
168
+ // A concurrent or alternate-vault pass must not mutate the hot projection.
169
+ // Its store is pass-local and released on every completion/error path.
170
+ const transient = openJournalStoreSession(slug);
171
+ return { store: transient, release: () => transient.dispose() };
172
+ }
173
+ /** Fixed-token census for the single retained targeted-pull projection. */
174
+ export function targetedJournalSessionCensusSizes() {
175
+ return {
176
+ targetedJournalSessions: cachedTargetedJournalSession ? 1 : 0,
177
+ targetedJournalRows: cachedTargetedJournalSession?.store.rowCount() ?? 0,
178
+ };
149
179
  }
150
180
  /**
151
181
  * Bound crash recovery while avoiding full-journal checkpoint amplification on
@@ -287,29 +317,36 @@ export async function sync(options) {
287
317
  }
288
318
  async function syncWithOperationLockHeld(options) {
289
319
  const run = await buildPullContext(options);
290
- const plan = planPull(run);
291
- emitPullPlan(run.emit, plan);
292
- reportInvalidRemoteKeys(run, plan);
293
- const scopeRun = run.keyedJournal
294
- ? emptyTargetedScopeRun(run)
295
- : executeScopeShrink(run, planScopeShrink(run));
296
- const counters = createPullCounters();
297
- const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
298
- const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
299
- if (conflictRun.abortResult) {
300
- return conflictRun.abortResult;
301
- }
302
- await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
303
- // The following added-by and tombstone phases may still make authenticated
304
- // vault calls. Make a completed sub-threshold download tail durable before
305
- // they begin, so their failure cannot turn a successful bulk transfer into
306
- // a full replay on the next pass.
307
- flushDownloadedJournalTail(run);
308
- await emitAndReportNewFiles(run, plan);
309
- await verifyPlannedJournalTombstones(run, plan);
310
- executeJournalTombstoneDeletes(run, plan, counters);
311
- emitScopeMaterializationGap(run, plan);
312
- return finalizePullRun(run, plan, scopeRun, counters);
320
+ try {
321
+ const plan = planPull(run);
322
+ emitPullPlan(run.emit, plan);
323
+ reportInvalidRemoteKeys(run, plan);
324
+ const scopeRun = run.keyedJournal
325
+ ? emptyTargetedScopeRun(run)
326
+ : executeScopeShrink(run, planScopeShrink(run));
327
+ const counters = createPullCounters();
328
+ const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
329
+ const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
330
+ if (conflictRun.abortResult) {
331
+ return conflictRun.abortResult;
332
+ }
333
+ await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
334
+ // The following added-by and tombstone phases may still make authenticated
335
+ // vault calls. Make a completed sub-threshold download tail durable before
336
+ // they begin, so their failure cannot turn a successful bulk transfer into
337
+ // a full replay on the next pass.
338
+ flushDownloadedJournalTail(run);
339
+ await emitAndReportNewFiles(run, plan);
340
+ await verifyPlannedJournalTombstones(run, plan);
341
+ executeJournalTombstoneDeletes(run, plan, counters);
342
+ emitScopeMaterializationGap(run, plan);
343
+ return finalizePullRun(run, plan, scopeRun, counters);
344
+ }
345
+ finally {
346
+ // Release a pass-local lease even on conflict, transfer, or auth failure.
347
+ // The one bounded hot lease remains available only for its same vault.
348
+ run.releaseKeyedJournal?.();
349
+ }
313
350
  }
314
351
  async function buildPullContext(options) {
315
352
  const { company, vaultConfig, hqRoot } = options;
@@ -381,49 +418,62 @@ async function buildPullContext(options) {
381
418
  // A targeted receive already knows the only remote prefixes it will inspect.
382
419
  // Keep one keyed projection alive across the LIST/planning/write seams rather
383
420
  // than reopening and materialising the full v3 journal for this pass.
384
- const keyedJournal = requestedPullPrefixes.length > 0
421
+ const keyedJournalLease = requestedPullPrefixes.length > 0
385
422
  ? targetedJournalSession(journalSlug)
386
423
  : undefined;
387
- const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
388
- const fileTombstones = options.fileTombstones ??
389
- await fetchFileTombstones(vaultConfig, options.personalMode === true
390
- ? { personal: true }
391
- : { companyUid: ctx.uid });
392
- // LIST can take minutes on a large personal vault. Reload after that window
393
- // so a watcher intent committed while it ran participates in classification
394
- // instead of allowing this pass to re-download the user's deleted object.
395
- // PR #385 freshness boundary: LIST may take minutes. A pass-start projection
396
- // is deliberately never used for classification after that window.
397
- keyedJournal?.refresh();
398
- const journal = keyedJournal
399
- ? journalForTargetedPull(keyedJournal, listedPrefixSet)
400
- : migrateToV2(readJournal(journalSlug));
401
- gcTombstones(journal, Date.now());
402
- return {
403
- options,
404
- hostLoadAtStart,
405
- companyRef,
406
- vaultConfig,
407
- hqRoot,
408
- emit,
409
- ctx,
410
- companyRoot,
411
- shouldSync,
412
- journalSlug,
413
- startedAt,
414
- journal,
415
- ...(keyedJournal
416
- ? { keyedJournal, keyedBaseline: structuredClone(journal) }
417
- : {}),
418
- journalRowCount: keyedJournal?.rowCount() ?? Object.keys(journal.files).length,
419
- remoteFiles,
420
- syncMode,
421
- currentPrefixSet,
422
- listedPrefixSet,
423
- currentExcludeSet,
424
- fileTombstones,
425
- downloadsSinceJournalCheckpoint: 0,
426
- };
424
+ const keyedJournal = keyedJournalLease?.store;
425
+ try {
426
+ const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
427
+ const fileTombstones = options.fileTombstones ??
428
+ await fetchFileTombstones(vaultConfig, options.personalMode === true
429
+ ? { personal: true }
430
+ : { companyUid: ctx.uid });
431
+ // LIST can take minutes on a large personal vault. Reload after that window
432
+ // so a watcher intent committed while it ran participates in classification
433
+ // instead of allowing this pass to re-download the user's deleted object.
434
+ // PR #385 freshness boundary: LIST may take minutes. A pass-start projection
435
+ // is deliberately never used for classification after that window.
436
+ keyedJournal?.refresh();
437
+ const journal = keyedJournal
438
+ ? journalForTargetedPull(keyedJournal, listedPrefixSet)
439
+ : migrateToV2(readJournal(journalSlug));
440
+ gcTombstones(journal, Date.now());
441
+ return {
442
+ options,
443
+ hostLoadAtStart,
444
+ companyRef,
445
+ vaultConfig,
446
+ hqRoot,
447
+ emit,
448
+ ctx,
449
+ companyRoot,
450
+ shouldSync,
451
+ journalSlug,
452
+ startedAt,
453
+ journal,
454
+ ...(keyedJournal
455
+ ? {
456
+ keyedJournal,
457
+ keyedBaseline: structuredClone(journal),
458
+ releaseKeyedJournal: keyedJournalLease.release,
459
+ }
460
+ : {}),
461
+ journalRowCount: keyedJournal?.rowCount() ?? Object.keys(journal.files).length,
462
+ remoteFiles,
463
+ syncMode,
464
+ currentPrefixSet,
465
+ listedPrefixSet,
466
+ currentExcludeSet,
467
+ fileTombstones,
468
+ downloadsSinceJournalCheckpoint: 0,
469
+ };
470
+ }
471
+ catch (error) {
472
+ // `buildPullContext` can fail during LIST/freshness. Do not leave the
473
+ // leased projection marked in use when no PullRunContext is returned.
474
+ keyedJournalLease?.release();
475
+ throw error;
476
+ }
427
477
  }
428
478
  /**
429
479
  * List the remote objects the current pull can actually materialize.
@@ -692,6 +742,7 @@ async function executeConflictExecutor(run, plan, scopeRun, counters) {
692
742
  const downloadItems = [];
693
743
  for (const item of plan.items) {
694
744
  if (item.action === "skip-ignored" ||
745
+ item.action === "skip-archived-session-log" ||
695
746
  item.action === "skip-personal-mode" ||
696
747
  item.action === "skip-unchanged" ||
697
748
  item.action === "skip-local-only" ||
@@ -1941,6 +1992,15 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = Object.keys
1941
1992
  items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
1942
1993
  continue;
1943
1994
  }
1995
+ // This is intentionally exact-prefix and storage-class-specific. Other
1996
+ // archived objects retain their established sync behavior until a separate
1997
+ // policy explicitly defines it. LIST retains the object in remoteKeys, so
1998
+ // an existing journal row is neither tombstoned nor deleted.
1999
+ if (remoteFile.key.startsWith("workspace/.session-logs/") &&
2000
+ remoteFile.storageClass === "DEEP_ARCHIVE") {
2001
+ items.push({ action: "skip-archived-session-log", remoteFile, localPath });
2002
+ continue;
2003
+ }
1944
2004
  // Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
1945
2005
  // in the 5.33.0 deep-test: the push side has refused to upload conflict
1946
2006
  // mirrors since 5.33.0, but the pull side downloaded them freely from