@indigoai-us/hq-cloud 6.11.12 → 6.11.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/.github/workflows/ci.yml +17 -26
  2. package/dist/bin/sync-runner-company.d.ts +35 -0
  3. package/dist/bin/sync-runner-company.d.ts.map +1 -0
  4. package/dist/bin/sync-runner-company.js +290 -0
  5. package/dist/bin/sync-runner-company.js.map +1 -0
  6. package/dist/bin/sync-runner-events.d.ts +12 -0
  7. package/dist/bin/sync-runner-events.d.ts.map +1 -0
  8. package/dist/bin/sync-runner-events.js +12 -0
  9. package/dist/bin/sync-runner-events.js.map +1 -0
  10. package/dist/bin/sync-runner-planning.d.ts +53 -0
  11. package/dist/bin/sync-runner-planning.d.ts.map +1 -0
  12. package/dist/bin/sync-runner-planning.js +59 -0
  13. package/dist/bin/sync-runner-planning.js.map +1 -0
  14. package/dist/bin/sync-runner-rollup.d.ts +24 -0
  15. package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
  16. package/dist/bin/sync-runner-rollup.js +46 -0
  17. package/dist/bin/sync-runner-rollup.js.map +1 -0
  18. package/dist/bin/sync-runner-telemetry.d.ts +5 -0
  19. package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
  20. package/dist/bin/sync-runner-telemetry.js +5 -0
  21. package/dist/bin/sync-runner-telemetry.js.map +1 -0
  22. package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
  23. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
  24. package/dist/bin/sync-runner-watch-loop.js +372 -0
  25. package/dist/bin/sync-runner-watch-loop.js.map +1 -0
  26. package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
  27. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
  28. package/dist/bin/sync-runner-watch-routes.js +74 -0
  29. package/dist/bin/sync-runner-watch-routes.js.map +1 -0
  30. package/dist/bin/sync-runner.d.ts +3 -54
  31. package/dist/bin/sync-runner.d.ts.map +1 -1
  32. package/dist/bin/sync-runner.js +73 -1154
  33. package/dist/bin/sync-runner.js.map +1 -1
  34. package/dist/cli/reindex.d.ts.map +1 -1
  35. package/dist/cli/reindex.js +34 -17
  36. package/dist/cli/reindex.js.map +1 -1
  37. package/dist/cli/reindex.test.js +39 -5
  38. package/dist/cli/reindex.test.js.map +1 -1
  39. package/dist/cli/rescue-classify-ordering.test.js +17 -0
  40. package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
  41. package/dist/cli/rescue-core.d.ts +45 -0
  42. package/dist/cli/rescue-core.d.ts.map +1 -1
  43. package/dist/cli/rescue-core.js +197 -170
  44. package/dist/cli/rescue-core.js.map +1 -1
  45. package/dist/cli/share.d.ts.map +1 -1
  46. package/dist/cli/share.js +235 -677
  47. package/dist/cli/share.js.map +1 -1
  48. package/dist/cli/sync.d.ts.map +1 -1
  49. package/dist/cli/sync.js +412 -727
  50. package/dist/cli/sync.js.map +1 -1
  51. package/dist/cli/sync.test.js +20 -0
  52. package/dist/cli/sync.test.js.map +1 -1
  53. package/dist/daemon-worker.d.ts +2 -2
  54. package/dist/daemon-worker.js +3 -3
  55. package/dist/daemon-worker.js.map +1 -1
  56. package/dist/index.d.ts +2 -0
  57. package/dist/index.d.ts.map +1 -1
  58. package/dist/index.js +3 -0
  59. package/dist/index.js.map +1 -1
  60. package/dist/object-io.js +1 -1
  61. package/dist/object-io.js.map +1 -1
  62. package/dist/remote-pull.d.ts +2 -2
  63. package/dist/remote-pull.d.ts.map +1 -1
  64. package/dist/remote-pull.js +23 -3
  65. package/dist/remote-pull.js.map +1 -1
  66. package/dist/remote-pull.test.js +24 -2
  67. package/dist/remote-pull.test.js.map +1 -1
  68. package/dist/sync/push-receiver.d.ts +6 -0
  69. package/dist/sync/push-receiver.d.ts.map +1 -1
  70. package/dist/sync/push-receiver.js +32 -2
  71. package/dist/sync/push-receiver.js.map +1 -1
  72. package/dist/sync/push-receiver.test.js +31 -0
  73. package/dist/sync/push-receiver.test.js.map +1 -1
  74. package/dist/sync-core.d.ts +27 -0
  75. package/dist/sync-core.d.ts.map +1 -0
  76. package/dist/sync-core.js +54 -0
  77. package/dist/sync-core.js.map +1 -0
  78. package/dist/sync-progress.d.ts +37 -0
  79. package/dist/sync-progress.d.ts.map +1 -0
  80. package/dist/sync-progress.js +104 -0
  81. package/dist/sync-progress.js.map +1 -0
  82. package/dist/sync-progress.test.d.ts +2 -0
  83. package/dist/sync-progress.test.d.ts.map +1 -0
  84. package/dist/sync-progress.test.js +70 -0
  85. package/dist/sync-progress.test.js.map +1 -0
  86. package/dist/vault-client.d.ts.map +1 -1
  87. package/dist/vault-client.js +284 -36
  88. package/dist/vault-client.js.map +1 -1
  89. package/dist/vault-client.test.js +59 -0
  90. package/dist/vault-client.test.js.map +1 -1
  91. package/dist/watcher.d.ts +2 -20
  92. package/dist/watcher.d.ts.map +1 -1
  93. package/dist/watcher.js +3 -113
  94. package/dist/watcher.js.map +1 -1
  95. package/package.json +1 -1
  96. package/src/bin/sync-runner-company.ts +350 -0
  97. package/src/bin/sync-runner-events.ts +25 -0
  98. package/src/bin/sync-runner-planning.ts +121 -0
  99. package/src/bin/sync-runner-rollup.ts +72 -0
  100. package/src/bin/sync-runner-telemetry.ts +8 -0
  101. package/src/bin/sync-runner-watch-loop.ts +443 -0
  102. package/src/bin/sync-runner-watch-routes.ts +86 -0
  103. package/src/bin/sync-runner.ts +96 -1253
  104. package/src/cli/reindex.test.ts +41 -3
  105. package/src/cli/reindex.ts +35 -19
  106. package/src/cli/rescue-classify-ordering.test.ts +20 -0
  107. package/src/cli/rescue-core.ts +252 -176
  108. package/src/cli/share.ts +374 -706
  109. package/src/cli/sync.test.ts +25 -0
  110. package/src/cli/sync.ts +625 -803
  111. package/src/daemon-worker.ts +3 -3
  112. package/src/index.ts +9 -0
  113. package/src/object-io.ts +1 -1
  114. package/src/remote-pull.test.ts +30 -1
  115. package/src/remote-pull.ts +29 -4
  116. package/src/sync/push-receiver.test.ts +35 -0
  117. package/src/sync/push-receiver.ts +41 -2
  118. package/src/sync-core.ts +58 -0
  119. package/src/sync-progress.test.ts +81 -0
  120. package/src/sync-progress.ts +133 -0
  121. package/src/vault-client.test.ts +74 -0
  122. package/src/vault-client.ts +395 -43
  123. package/src/watcher.ts +6 -141
@@ -4,11 +4,11 @@
4
4
  *
5
5
  * Day 1: not invoked by CLI surface; retained for future automatic-sync milestone.
6
6
  * When re-enabled, this worker will need to resolve an EntityContext before
7
- * constructing the SyncWatcher. The process argv will need to include company
8
- * context (slug or UID) and vault-service config.
7
+ * constructing the active watcher/runner path. The process argv will need to
8
+ * include company context (slug or UID) and vault-service config.
9
9
  */
10
10
 
11
- // Day 1: SyncWatcher now requires an EntityContext.
11
+ // Day 1: this worker still requires EntityContext-aware startup wiring.
12
12
  // This file is retained for the automatic-sync milestone but is not functional
13
13
  // until the daemon startup path is updated to resolve entity context.
14
14
 
package/src/index.ts CHANGED
@@ -57,6 +57,15 @@ export {
57
57
 
58
58
  export type { JournalSummary } from "./journal.js";
59
59
 
60
+ // Cross-process sync progress — every sync path writes one shared snapshot so
61
+ // a watcher (the hq-sync menubar) can show live progress for ANY sync.
62
+ export {
63
+ createSyncProgressRecorder,
64
+ readSyncProgress,
65
+ syncProgressPath,
66
+ } from "./sync-progress.js";
67
+ export type { SyncProgressSnapshot } from "./sync-progress.js";
68
+
60
69
  // Prefix coalescing helper (US-005)
61
70
  export {
62
71
  coalescePrefixes,
package/src/object-io.ts CHANGED
@@ -540,7 +540,7 @@ export class PresignObjectIO implements ObjectIO {
540
540
  ) {}
541
541
 
542
542
  private cacheKey(op: PresignOp, key: string): string {
543
- return `${op}${key}`;
543
+ return `${op}\u0000${key}`;
544
544
  }
545
545
 
546
546
  hasPrimedPut(key: string): boolean {
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Failing-test seed for the Auto-sync (Beta) remote-pull loop.
3
3
  *
4
- * Background: SyncWatcher (watcher.ts) pushes local edits to S3 in seconds,
4
+ * Background: the watcher push path ships local edits to S3 in seconds,
5
5
  * but pulls happen only on a manual sync today. Auto-sync adds a periodic
6
6
  * (every 10 min) remote-pull pass per company. The decision of *which* keys
7
7
  * to download / delete locally / skip is pure given a remote listing, the
@@ -25,6 +25,7 @@ import {
25
25
  POST_FILTER_THRESHOLD,
26
26
  pullCompany,
27
27
  resolveCompanyScope,
28
+ VEND_FANOUT_CONCURRENCY,
28
29
  VEND_PATH_CAP,
29
30
  } from "./remote-pull.js";
30
31
  import type { RemoteFile } from "./s3.js";
@@ -579,6 +580,34 @@ describe("listRemoteForScope", () => {
579
580
  expect(calls.size).toBe(VEND_PATH_CAP + 3); // every prefix listed
580
581
  });
581
582
 
583
+ it("F28: bounds concurrent per-prefix listings across vend-fanout batches", async () => {
584
+ const prefixes = Array.from(
585
+ { length: POST_FILTER_THRESHOLD },
586
+ (_, i) => `companies/indigo/p${i}/`,
587
+ );
588
+ let active = 0;
589
+ let maxActive = 0;
590
+
591
+ await listRemoteForScope({
592
+ ctx: makeCtx(),
593
+ scope: {
594
+ companyUid: "cmp_indigo",
595
+ syncMode: "shared",
596
+ prefixSet: prefixes,
597
+ strategy: "vend-fanout",
598
+ },
599
+ listFn: async () => {
600
+ active += 1;
601
+ maxActive = Math.max(maxActive, active);
602
+ await new Promise((resolve) => setTimeout(resolve, 1));
603
+ active -= 1;
604
+ return [];
605
+ },
606
+ });
607
+
608
+ expect(maxActive).toBeLessThanOrEqual(VEND_FANOUT_CONCURRENCY);
609
+ });
610
+
582
611
  it("strategy=vend-fanout uses vendForBatchFn to narrow credentials per batch", async () => {
583
612
  const vendCalls: Array<{ paths: string[] }> = [];
584
613
  await listRemoteForScope({
@@ -8,7 +8,7 @@
8
8
  * sync-runner.ts trivial: list S3 → call `decideRemotePulls` → drive S3 +
9
9
  * filesystem from the result.
10
10
  *
11
- * Pairs with `SyncWatcher` (push-side) — together they implement the
11
+ * Pairs with the TreeWatcher push path — together they implement the
12
12
  * bidirectional auto-sync the Settings toggle exposes.
13
13
  */
14
14
  import type { RemoteFile } from "./s3.js";
@@ -160,7 +160,7 @@ export const VEND_PATH_CAP = 10;
160
160
  */
161
161
  export const POST_FILTER_THRESHOLD = 50;
162
162
 
163
- /** Bounded parallelism for vend fan-out (5 concurrent STS+ListObjectsV2 calls). */
163
+ /** Bounded parallelism for vend fan-out (5 concurrent vends/list paginators). */
164
164
  export const VEND_FANOUT_CONCURRENCY = 5;
165
165
 
166
166
  /**
@@ -291,6 +291,27 @@ async function mapWithConcurrency<T, R>(
291
291
  return results;
292
292
  }
293
293
 
294
+ function createConcurrencyLimiter(
295
+ concurrency: number,
296
+ ): <R>(fn: () => Promise<R>) => Promise<R> {
297
+ const limit = Math.max(1, concurrency);
298
+ let active = 0;
299
+ const waiters: Array<() => void> = [];
300
+
301
+ return async function runLimited<R>(fn: () => Promise<R>): Promise<R> {
302
+ if (active >= limit) {
303
+ await new Promise<void>((resolve) => waiters.push(resolve));
304
+ }
305
+ active += 1;
306
+ try {
307
+ return await fn();
308
+ } finally {
309
+ active -= 1;
310
+ waiters.shift()?.();
311
+ }
312
+ };
313
+ }
314
+
294
315
  export interface ListRemoteForScopeInput {
295
316
  ctx: EntityContext;
296
317
  scope: CompanyScope;
@@ -347,6 +368,7 @@ export async function listRemoteForScope(
347
368
  const scopeEntries = toScopePrefixEntries(scope.prefixSet);
348
369
  const listPrefixes = scopeEntries.map((entry) => entry.prefix);
349
370
  const batches = batchPrefixesForVend(listPrefixes);
371
+ const listWithLimit = createConcurrencyLimiter(VEND_FANOUT_CONCURRENCY);
350
372
  const perBatch = await mapWithConcurrency(
351
373
  batches,
352
374
  VEND_FANOUT_CONCURRENCY,
@@ -357,8 +379,11 @@ export async function listRemoteForScope(
357
379
  // For a coalesced batch we issue one ListObjectsV2 per prefix in the
358
380
  // batch. We can't issue one ListObjectsV2 across N prefixes (the API
359
381
  // takes a single Prefix); the per-batch grouping exists for the STS
360
- // session policy ceiling, not the list call itself.
361
- const lists = await Promise.all(paths.map((p) => list(batchCtx, p)));
382
+ // session policy ceiling, not the list call itself. The shared limiter
383
+ // keeps the total active prefix paginators bounded across all batches.
384
+ const lists = await Promise.all(
385
+ paths.map((p) => listWithLimit(() => list(batchCtx, p))),
386
+ );
362
387
  return lists.flat();
363
388
  },
364
389
  );
@@ -396,6 +396,41 @@ describe("US-009: SqsPushReceiver — reconnect / catch-up replay", () => {
396
396
  expect(sqs.deleted).not.toContain("rh-first-attempt");
397
397
  expect(sqs.deleted).toContain("rh-redelivery");
398
398
  });
399
+
400
+ it("F29: bounds dedupe state by evicting least-recently-used paths", async () => {
401
+ const sqs = new FakeSqs();
402
+ const processed: string[] = [];
403
+ const syncFn: SyncEngineFn = async (ctx) => {
404
+ processed.push(ctx.event.relativePath);
405
+ };
406
+
407
+ sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/a.md", sequenceNumber: 1 }), "rh-a1")]);
408
+ sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/b.md", sequenceNumber: 1 }), "rh-b1")]);
409
+ sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/c.md", sequenceNumber: 1 }), "rh-c1")]);
410
+ sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/a.md", sequenceNumber: 1 }), "rh-a-redelivered")]);
411
+
412
+ const receiver = new SqsPushReceiver({
413
+ tenantId: TENANT,
414
+ queueUrl: QUEUE_URL,
415
+ sqs,
416
+ syncFn,
417
+ enabled: true,
418
+ sleep: fastSleep,
419
+ dedupeMaxPaths: 2,
420
+ });
421
+ await receiver.start();
422
+ await waitFor(() => receiver.processedCount === 4);
423
+ await receiver.dispose();
424
+
425
+ expect(processed).toEqual([
426
+ "companies/indigo/a.md",
427
+ "companies/indigo/b.md",
428
+ "companies/indigo/c.md",
429
+ "companies/indigo/a.md",
430
+ ]);
431
+ expect(receiver.dedupedCount).toBe(0);
432
+ expect(sqs.deleted).toContain("rh-a-redelivered");
433
+ });
399
434
  });
400
435
 
401
436
  describe("US-009: SqsPushReceiver — flag gating", () => {
@@ -125,6 +125,39 @@ export const DEFAULT_MAX_MESSAGES = 10;
125
125
  export const DEFAULT_RECONNECT_INITIAL_MS = 250;
126
126
  export const DEFAULT_RECONNECT_MAX_MS = 30_000;
127
127
 
128
+ /** Maximum distinct paths retained in receiver dedupe state. */
129
+ export const DEFAULT_RECEIVER_DEDUPE_MAX_PATHS = 50_000;
130
+
131
+ class BoundedSequenceDedupe {
132
+ private readonly maxPaths: number;
133
+ private readonly seen = new Map<string, number>();
134
+
135
+ constructor(maxPaths: number | undefined) {
136
+ this.maxPaths = Math.max(
137
+ 1,
138
+ Math.floor(maxPaths ?? DEFAULT_RECEIVER_DEDUPE_MAX_PATHS),
139
+ );
140
+ }
141
+
142
+ get(relativePath: string): number | undefined {
143
+ const sequenceNumber = this.seen.get(relativePath);
144
+ if (sequenceNumber === undefined) return undefined;
145
+ this.seen.delete(relativePath);
146
+ this.seen.set(relativePath, sequenceNumber);
147
+ return sequenceNumber;
148
+ }
149
+
150
+ set(relativePath: string, sequenceNumber: number): void {
151
+ if (this.seen.has(relativePath)) this.seen.delete(relativePath);
152
+ this.seen.set(relativePath, sequenceNumber);
153
+ while (this.seen.size > this.maxPaths) {
154
+ const oldest = this.seen.keys().next().value;
155
+ if (oldest === undefined) return;
156
+ this.seen.delete(oldest);
157
+ }
158
+ }
159
+ }
160
+
128
161
  // ─── Narrow SQS surface (the injectable transport seam) ──────────────────────
129
162
 
130
163
  /**
@@ -321,6 +354,8 @@ export interface SqsPushReceiverOptions {
321
354
  maxMessages?: number;
322
355
  /** Max time `dispose()` waits for an in-flight syncFn after abort. */
323
356
  disposeDrainMs?: number;
357
+ /** Maximum distinct paths retained in dedupe state. */
358
+ dedupeMaxPaths?: number;
324
359
  /** Reconnect backoff config. */
325
360
  reconnect?: {
326
361
  initialMs?: number;
@@ -384,7 +419,7 @@ export class SqsPushReceiver implements PushReceiver {
384
419
  private inFlightSync: Promise<void> | null = null;
385
420
 
386
421
  /** Per-path highest sequence number already PROCESSED by syncFn. */
387
- private readonly seenSequencePerPath = new Map<string, number>();
422
+ private readonly seenSequencePerPath: BoundedSequenceDedupe;
388
423
 
389
424
  private _processedCount = 0;
390
425
  private _dedupedCount = 0;
@@ -413,6 +448,7 @@ export class SqsPushReceiver implements PushReceiver {
413
448
  this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
414
449
  this.disposeDrainMs =
415
450
  opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
451
+ this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
416
452
  this.reconnectInitialMs =
417
453
  opts.reconnect?.initialMs ?? DEFAULT_RECONNECT_INITIAL_MS;
418
454
  this.reconnectMaxMs = opts.reconnect?.maxMs ?? DEFAULT_RECONNECT_MAX_MS;
@@ -821,6 +857,8 @@ export interface InMemoryPushReceiverOptions {
821
857
  flagProvider?: EventDrivenPushFlagProvider;
822
858
  env?: Record<string, string | undefined>;
823
859
  disposeDrainMs?: number;
860
+ /** Maximum distinct paths retained in dedupe state. */
861
+ dedupeMaxPaths?: number;
824
862
  }
825
863
 
826
864
  /**
@@ -846,7 +884,7 @@ export class InMemoryPushReceiver implements PushReceiver {
846
884
 
847
885
  private disconnectedFlag = false;
848
886
  private readonly pendingDuringDisconnect: PushEvent[] = [];
849
- private readonly seenSequencePerPath = new Map<string, number>();
887
+ private readonly seenSequencePerPath: BoundedSequenceDedupe;
850
888
 
851
889
  private inFlightAbort: AbortController | null = null;
852
890
  private inFlightSync: Promise<void> | null = null;
@@ -862,6 +900,7 @@ export class InMemoryPushReceiver implements PushReceiver {
862
900
  this.logger = opts.logger ?? NOOP_LOGGER;
863
901
  this.disposeDrainMs =
864
902
  opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
903
+ this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
865
904
  this.enabled = resolveEnabled({
866
905
  explicit: opts.enabled,
867
906
  flagProvider: opts.flagProvider,
@@ -0,0 +1,58 @@
1
+ import * as fs from "fs";
2
+ import * as path from "path";
3
+ import { normalizeEtag } from "./journal.js";
4
+
5
+ /**
6
+ * Is this error the S3/STS "access denied" class? Expected when a scoped
7
+ * member/guest credential touches a key outside its granted ACL prefixes
8
+ * (the server's `SCOPE_EXCEEDS_PARENT` surfaces as a 403 AccessDenied /
9
+ * Forbidden).
10
+ */
11
+ export function isAccessDenied(err: unknown): boolean {
12
+ if (err && typeof err === "object" && "name" in err) {
13
+ const name = (err as { name?: unknown }).name;
14
+ return name === "AccessDenied" || name === "Forbidden";
15
+ }
16
+ return false;
17
+ }
18
+
19
+ export function resolveTransferConcurrency(): number {
20
+ const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
21
+ if (raw === undefined || raw === "") return 16;
22
+ const parsed = Number.parseInt(raw, 10);
23
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
24
+ }
25
+
26
+ /**
27
+ * Resolve active company from .hq/config.json.
28
+ */
29
+ export function resolveActiveCompany(hqRoot: string): string | undefined {
30
+ const configPath = path.join(hqRoot, ".hq", "config.json");
31
+ if (fs.existsSync(configPath)) {
32
+ try {
33
+ const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
34
+ return config.activeCompany ?? config.companySlug;
35
+ } catch {
36
+ // Ignore parse errors
37
+ }
38
+ }
39
+ return undefined;
40
+ }
41
+
42
+ /**
43
+ * Returns true when the remote object appears to have moved since the
44
+ * journal entry's last-recorded sync. Prefers ETag equality; falls back to
45
+ * `lastModified > syncedAt` for legacy entries written before remoteEtag
46
+ * was tracked. Conservative on tie (`<=` skews "remote unchanged") so an
47
+ * S3-side mtime that exactly equals our syncedAt is not treated as drift.
48
+ */
49
+ export function hasRemoteChanged(
50
+ remote: { lastModified: Date; etag: string },
51
+ entry: { syncedAt: string; remoteEtag?: string },
52
+ ): boolean {
53
+ if (entry.remoteEtag) {
54
+ return normalizeEtag(remote.etag) !== entry.remoteEtag;
55
+ }
56
+ const syncedAt = new Date(entry.syncedAt).getTime();
57
+ return remote.lastModified.getTime() > syncedAt;
58
+ }
@@ -0,0 +1,81 @@
1
+ import { describe, it, expect, beforeEach, afterEach } from "vitest";
2
+ import * as fs from "fs";
3
+ import * as os from "os";
4
+ import * as path from "path";
5
+ import type { SyncProgressEvent } from "./cli/sync.js";
6
+ import {
7
+ createSyncProgressRecorder,
8
+ readSyncProgress,
9
+ } from "./sync-progress.js";
10
+
11
+ const planEvent = (download: number, upload: number, del: number): SyncProgressEvent =>
12
+ ({
13
+ type: "plan",
14
+ filesToDownload: download,
15
+ bytesToDownload: 0,
16
+ filesToUpload: upload,
17
+ bytesToUpload: 0,
18
+ filesToSkip: 0,
19
+ filesToConflict: 0,
20
+ filesToDelete: del,
21
+ }) as SyncProgressEvent;
22
+
23
+ describe("sync-progress", () => {
24
+ let stateDir: string;
25
+ beforeEach(() => {
26
+ stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-progress-test-"));
27
+ process.env.HQ_STATE_DIR = stateDir;
28
+ });
29
+ afterEach(() => {
30
+ delete process.env.HQ_STATE_DIR;
31
+ fs.rmSync(stateDir, { recursive: true, force: true });
32
+ });
33
+
34
+ it("returns null when no sync has written a snapshot", () => {
35
+ expect(readSyncProgress()).toBeNull();
36
+ });
37
+
38
+ it("accumulates plan totals, progress, and conflicts into the shared snapshot", () => {
39
+ const record = createSyncProgressRecorder({ company: "acme", phase: "pull" });
40
+ record(planEvent(3, 0, 1)); // 3 to download + 1 to delete = 4 total
41
+ record({ type: "progress", path: "a.md", bytes: 10 } as SyncProgressEvent);
42
+ record({ type: "progress", path: "b.md", bytes: 10 } as SyncProgressEvent);
43
+ record({
44
+ type: "conflict",
45
+ path: "c.md",
46
+ direction: "pull",
47
+ resolution: "keep",
48
+ } as SyncProgressEvent);
49
+
50
+ const snap = readSyncProgress();
51
+ expect(snap).not.toBeNull();
52
+ expect(snap!.company).toBe("acme");
53
+ expect(snap!.phase).toBe("pull");
54
+ expect(snap!.filesTotal).toBe(4);
55
+ expect(snap!.filesDone).toBe(2);
56
+ expect(snap!.conflicts).toBe(1);
57
+ expect(snap!.currentFile).toBe("b.md");
58
+ expect(snap!.status).toBe("syncing");
59
+ expect(snap!.pid).toBe(process.pid);
60
+ expect(typeof snap!.updatedAt).toBe("string");
61
+ });
62
+
63
+ it("records the push leg + personal scope (company null)", () => {
64
+ const record = createSyncProgressRecorder({ company: null, phase: "push" });
65
+ record(planEvent(0, 5, 0));
66
+ const snap = readSyncProgress();
67
+ expect(snap!.phase).toBe("push");
68
+ expect(snap!.company).toBeNull();
69
+ expect(snap!.filesTotal).toBe(5);
70
+ });
71
+
72
+ it("ignores non-progress events (no churn from new-files/reconciled)", () => {
73
+ const record = createSyncProgressRecorder({ company: "acme", phase: "pull" });
74
+ record(planEvent(1, 0, 0));
75
+ record({ type: "new-files", files: [] } as SyncProgressEvent);
76
+ record({ type: "reconciled", path: "x.md", direction: "pull" } as SyncProgressEvent);
77
+ const snap = readSyncProgress();
78
+ expect(snap!.filesDone).toBe(0);
79
+ expect(snap!.filesTotal).toBe(1);
80
+ });
81
+ });
@@ -0,0 +1,133 @@
1
+ /**
2
+ * Cross-process sync progress.
3
+ *
4
+ * Every sync path — the menubar watch daemon (auto-sync), "Sync Now", and a
5
+ * CLI `hq sync` — runs through the same engine (`sync()` pull + `share()`
6
+ * push). Each writes one shared status snapshot here so a watcher (the menubar)
7
+ * can show LIVE progress for ANY sync, not just the one it spawned over stdout.
8
+ *
9
+ * Because syncs are serialized (the op-lock; the watch daemon coordinates with
10
+ * one-shots), there is at most one active sync, so this single file is
11
+ * unambiguous. Writes are atomic (temp + rename) — a torn read at worst shows
12
+ * slightly stale progress. There is intentionally no explicit "done": the
13
+ * engine simply stops writing when a sync ends, so a consumer treats the file
14
+ * as idle once `updatedAt` is older than a few seconds. Progress reporting must
15
+ * never break a sync, so every fs error here is swallowed.
16
+ */
17
+ import * as fs from "fs";
18
+ import * as os from "os";
19
+ import * as path from "path";
20
+ import type { SyncProgressEvent } from "./cli/sync.js";
21
+
22
+ /** Mirror operation-lock's stateDir so the file lands next to the journal/lock. */
23
+ function stateDir(): string {
24
+ return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
25
+ }
26
+
27
+ /** Absolute path of the shared progress snapshot. Exported for the watcher + tests. */
28
+ export function syncProgressPath(): string {
29
+ return path.join(stateDir(), "sync-progress.json");
30
+ }
31
+
32
+ export interface SyncProgressSnapshot {
33
+ schema: 1;
34
+ /** PID of the writing sync — lets a watcher detect a dead/stale holder. */
35
+ pid: number;
36
+ /** Company slug being synced, or null for the personal vault. */
37
+ company: string | null;
38
+ /** Which leg is running. */
39
+ phase: "pull" | "push";
40
+ filesTotal: number;
41
+ filesDone: number;
42
+ conflicts: number;
43
+ /** Path of the file currently transferring, or null before the first one. */
44
+ currentFile: string | null;
45
+ startedAt: string;
46
+ updatedAt: string;
47
+ status: "syncing";
48
+ }
49
+
50
+ export interface SyncProgressRecorderOptions {
51
+ company?: string | null;
52
+ phase: "pull" | "push";
53
+ }
54
+
55
+ /**
56
+ * Build an event sink that accumulates a sync's progress and atomic-writes the
57
+ * shared snapshot on each meaningful event. Wrap the engine's `onEvent` with
58
+ * it: `const emit = (e) => { baseEmit(e); record(e); }`.
59
+ */
60
+ export function createSyncProgressRecorder(
61
+ opts: SyncProgressRecorderOptions,
62
+ ): (event: SyncProgressEvent) => void {
63
+ let filesTotal = 0;
64
+ let filesDone = 0;
65
+ let conflicts = 0;
66
+ let currentFile: string | null = null;
67
+ const startedAt = new Date().toISOString();
68
+ const file = syncProgressPath();
69
+
70
+ const flush = (): void => {
71
+ const snap: SyncProgressSnapshot = {
72
+ schema: 1,
73
+ pid: process.pid,
74
+ company: opts.company ?? null,
75
+ phase: opts.phase,
76
+ filesTotal,
77
+ filesDone,
78
+ conflicts,
79
+ currentFile,
80
+ startedAt,
81
+ updatedAt: new Date().toISOString(),
82
+ status: "syncing",
83
+ };
84
+ try {
85
+ fs.mkdirSync(path.dirname(file), { recursive: true });
86
+ const tmp = `${file}.${process.pid}.tmp`;
87
+ fs.writeFileSync(tmp, JSON.stringify(snap));
88
+ fs.renameSync(tmp, file);
89
+ } catch {
90
+ // Progress reporting is best-effort and must never break a sync.
91
+ }
92
+ };
93
+
94
+ return (event: SyncProgressEvent): void => {
95
+ switch (event.type) {
96
+ case "plan":
97
+ // Sum the work this leg will do. `plan` arrives once before any
98
+ // progress events for the leg.
99
+ filesTotal +=
100
+ event.filesToDownload + event.filesToUpload + event.filesToDelete;
101
+ flush();
102
+ break;
103
+ case "progress":
104
+ filesDone += 1;
105
+ currentFile = event.path;
106
+ flush();
107
+ break;
108
+ case "conflict":
109
+ conflicts += 1;
110
+ flush();
111
+ break;
112
+ default:
113
+ // error / new-files / reconciled / delete-refused etc. don't move the
114
+ // bar — skip the write to keep the file churn-free.
115
+ break;
116
+ }
117
+ };
118
+ }
119
+
120
+ /**
121
+ * Read the current snapshot, or null if absent/unreadable. Consumers should
122
+ * additionally treat a snapshot whose `updatedAt` is older than a few seconds
123
+ * (or whose `pid` is dead) as idle.
124
+ */
125
+ export function readSyncProgress(): SyncProgressSnapshot | null {
126
+ try {
127
+ const raw = fs.readFileSync(syncProgressPath(), "utf8");
128
+ const parsed = JSON.parse(raw) as SyncProgressSnapshot;
129
+ return parsed && parsed.schema === 1 ? parsed : null;
130
+ } catch {
131
+ return null;
132
+ }
133
+ }
@@ -440,6 +440,80 @@ describe("API surface", () => {
440
440
  });
441
441
  });
442
442
 
443
+ describe("response validation (F26)", () => {
444
+ it("F26: validates a valid files/list response and returns typed data", async () => {
445
+ fetchSpy.mockResolvedValueOnce(
446
+ jsonResponse(200, {
447
+ objects: [
448
+ {
449
+ key: "shared/docs/a.txt",
450
+ size: 123,
451
+ lastModified: "2026-05-20T12:00:00.000Z",
452
+ etag: "abc123",
453
+ permission: "read",
454
+ },
455
+ ],
456
+ cursor: null,
457
+ truncated: false,
458
+ }),
459
+ );
460
+
461
+ const page = await client.listFiles("cmp_abc", "shared/docs/");
462
+
463
+ expect(page.objects[0]).toEqual({
464
+ key: "shared/docs/a.txt",
465
+ size: 123,
466
+ lastModified: "2026-05-20T12:00:00.000Z",
467
+ etag: "abc123",
468
+ permission: "read",
469
+ });
470
+ expect(page.cursor).toBeNull();
471
+ expect(page.truncated).toBe(false);
472
+ });
473
+
474
+ it("F26: defaults omitted files/list paging fields", async () => {
475
+ fetchSpy.mockResolvedValueOnce(jsonResponse(200, {}));
476
+
477
+ const page = await client.listFiles("cmp_abc", "shared/docs/");
478
+
479
+ expect(page).toEqual({
480
+ objects: [],
481
+ cursor: null,
482
+ truncated: false,
483
+ });
484
+ });
485
+
486
+ it("F26: defaults omitted telemetry opt-in fields to disabled", async () => {
487
+ fetchSpy.mockResolvedValueOnce(jsonResponse(200, {}));
488
+
489
+ const optIn = await client.getTelemetryOptIn();
490
+
491
+ expect(optIn).toEqual({ enabled: false, updatedAt: null });
492
+ });
493
+
494
+ it("F26: maps a malformed files/list response to VaultClientError", async () => {
495
+ fetchSpy.mockResolvedValueOnce(
496
+ jsonResponse(200, {
497
+ objects: [
498
+ {
499
+ key: "shared/docs/a.txt",
500
+ size: 123,
501
+ lastModified: "2026-05-20T12:00:00.000Z",
502
+ permission: "read",
503
+ },
504
+ ],
505
+ cursor: null,
506
+ truncated: false,
507
+ }),
508
+ );
509
+
510
+ await expect(client.listFiles("cmp_abc")).rejects.toMatchObject({
511
+ name: "VaultClientError",
512
+ statusCode: 502,
513
+ });
514
+ });
515
+ });
516
+
443
517
  describe("VaultClient identity bootstrap", () => {
444
518
  let client: VaultClient;
445
519
  let fetchSpy: MockInstance<typeof fetch>;