@indigoai-us/hq-cloud 5.26.0 → 5.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/.github/workflows/ci.yml +34 -0
  2. package/dist/bin/sync-runner.d.ts +38 -0
  3. package/dist/bin/sync-runner.d.ts.map +1 -1
  4. package/dist/bin/sync-runner.js +75 -1
  5. package/dist/bin/sync-runner.js.map +1 -1
  6. package/dist/index.d.ts +4 -2
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +4 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/sync/feature-flags.d.ts +136 -0
  11. package/dist/sync/feature-flags.d.ts.map +1 -0
  12. package/dist/sync/feature-flags.js +160 -0
  13. package/dist/sync/feature-flags.js.map +1 -0
  14. package/dist/sync/feature-flags.test.d.ts +24 -0
  15. package/dist/sync/feature-flags.test.d.ts.map +1 -0
  16. package/dist/sync/feature-flags.test.js +330 -0
  17. package/dist/sync/feature-flags.test.js.map +1 -0
  18. package/dist/sync/index.d.ts +10 -2
  19. package/dist/sync/index.d.ts.map +1 -1
  20. package/dist/sync/index.js +5 -1
  21. package/dist/sync/index.js.map +1 -1
  22. package/dist/sync/logger.d.ts +61 -0
  23. package/dist/sync/logger.d.ts.map +1 -0
  24. package/dist/sync/logger.js +51 -0
  25. package/dist/sync/logger.js.map +1 -0
  26. package/dist/sync/logger.test.d.ts +19 -0
  27. package/dist/sync/logger.test.d.ts.map +1 -0
  28. package/dist/sync/logger.test.js +199 -0
  29. package/dist/sync/logger.test.js.map +1 -0
  30. package/dist/sync/metrics.d.ts +89 -0
  31. package/dist/sync/metrics.d.ts.map +1 -0
  32. package/dist/sync/metrics.js +105 -0
  33. package/dist/sync/metrics.js.map +1 -0
  34. package/dist/sync/metrics.test.d.ts +19 -0
  35. package/dist/sync/metrics.test.d.ts.map +1 -0
  36. package/dist/sync/metrics.test.js +280 -0
  37. package/dist/sync/metrics.test.js.map +1 -0
  38. package/dist/sync/push-receiver.d.ts +442 -0
  39. package/dist/sync/push-receiver.d.ts.map +1 -0
  40. package/dist/sync/push-receiver.js +782 -0
  41. package/dist/sync/push-receiver.js.map +1 -0
  42. package/dist/sync/push-receiver.test.d.ts +25 -0
  43. package/dist/sync/push-receiver.test.d.ts.map +1 -0
  44. package/dist/sync/push-receiver.test.js +477 -0
  45. package/dist/sync/push-receiver.test.js.map +1 -0
  46. package/dist/sync/push-transport.d.ts +84 -1
  47. package/dist/sync/push-transport.d.ts.map +1 -1
  48. package/dist/sync/push-transport.js +84 -0
  49. package/dist/sync/push-transport.js.map +1 -1
  50. package/dist/watcher.d.ts +113 -2
  51. package/dist/watcher.d.ts.map +1 -1
  52. package/dist/watcher.js +204 -25
  53. package/dist/watcher.js.map +1 -1
  54. package/package.json +9 -5
  55. package/src/bin/sync-runner.ts +102 -1
  56. package/src/index.ts +21 -0
  57. package/src/sync/feature-flags.test.ts +392 -0
  58. package/src/sync/feature-flags.ts +229 -0
  59. package/src/sync/index.ts +57 -2
  60. package/src/sync/logger.test.ts +241 -0
  61. package/src/sync/logger.ts +79 -0
  62. package/src/sync/metrics.test.ts +380 -0
  63. package/src/sync/metrics.ts +158 -0
  64. package/src/sync/push-receiver.test.ts +545 -0
  65. package/src/sync/push-receiver.ts +1077 -0
  66. package/src/sync/push-transport.ts +148 -1
  67. package/src/watcher.ts +299 -17
  68. package/test/e2e/sync/cross-tenant-isolation.test.ts +502 -0
  69. package/test/e2e/watcher-real-chokidar.test.ts +105 -0
@@ -0,0 +1,782 @@
1
+ /**
2
+ * PushReceiver — inbound subscription seam for the hq-cloud watcher daemon
3
+ * (project event-driven-sync-menubar US-009).
4
+ *
5
+ * Mirrors {@link PushTransport} (`./push-transport.ts`) but for the opposite
6
+ * direction of travel: where the transport SHIPS local file changes out to the
7
+ * cloud, the receiver SUBSCRIBES to the tenant fanout and triggers an
8
+ * immediate, TARGETED local pull the moment a peer device of the same tenant
9
+ * publishes a change. Together they form the event-driven primary path; the
10
+ * existing `--poll-remote-ms` poll in `runRunnerWithLoop` is the safety net
11
+ * behind both.
12
+ *
13
+ * Transport: SNS → per-client SQS (US-000 decision)
14
+ * ─────────────────────────────────────────────────
15
+ * Per the US-000 transport investigation (companies/indigo/projects/
16
+ * event-driven-sync-menubar/references.md): reuse PR #112's SNS publish +
17
+ * DynamoDB catch-up log, and build the client RECEIVE side as a per-client
18
+ * SQS queue subscribed to `sync-push-{tenantId}`. The receiver long-polls its
19
+ * own queue, decodes each message body as a {@link PushEvent}, dedupes by
20
+ * `sequenceNumber` per `relativePath`, and bridges into the existing sync
21
+ * engine via an injected {@link SyncEngineFn} (→ targeted `runRunner` pull).
22
+ *
23
+ * The live queue is NOT provisioned yet (the server SQS-provisioning Lambda is
24
+ * an unbuilt follow-up — see references.md "Open items handed to the plan").
25
+ * So this module ships:
26
+ * - {@link SqsClientLike} — the narrow SQS surface the receiver depends on
27
+ * (`receiveMessage` / `deleteMessage`). Production callers pass an
28
+ * `@aws-sdk/client-sqs` `SQSClient` adapted to this shape; unit tests inject
29
+ * a fake. NO real AWS is required to exercise the receiver.
30
+ * - {@link SqsPushReceiver} — the real receiver. Long-polls the queue,
31
+ * dispatches each event through the shared dedupe path, deletes the message
32
+ * on successful handoff, and reconnects on transient `receiveMessage`
33
+ * failures with exponential backoff. SQS's own 14-day retention buffers
34
+ * messages while the device is offline → reconnect-replay is "free": on
35
+ * reconnect the poll loop simply resumes and the retained messages are
36
+ * redelivered, then dedupe skips anything already processed.
37
+ * - {@link NoopPushReceiver} — the dormant default. Flips `connected` on
38
+ * start, opens no subscription. Wired when the daemon runs without a real
39
+ * queue (or when the feature flag is OFF).
40
+ * - {@link createPushReceiver} — factory the daemon uses; returns the noop
41
+ * unless an SQS client + queue URL are supplied.
42
+ *
43
+ * Lifecycle (mirrors PushTransport)
44
+ * ─────────────────────────────────
45
+ * - `start()` opens the subscription (begins the long-poll loop). Awaited
46
+ * AFTER the watcher starts so a synthetic event can't race a half-built
47
+ * daemon. When the feature flag is OFF, `start()` is a no-op and `connected`
48
+ * stays false — NO queue is polled (dormant; AC#4).
49
+ * - On each received message: validate with {@link decodePushEvent} (defense
50
+ * in depth at the wire boundary), dedupe by `relativePath` against the
51
+ * highest `sequenceNumber` seen for that path, then call the injected
52
+ * {@link SyncEngineFn}. The sync engine is an injected seam — this story
53
+ * does NOT re-implement download logic; it bridges to `runRunner` pull.
54
+ * - `dispose()` aborts in-flight via AbortController, stops the poll loop,
55
+ * awaits the in-flight sync up to a drain deadline, then disconnects.
56
+ *
57
+ * Dedupe (AC#3)
58
+ * ─────────────
59
+ * A per-`relativePath` map of the highest `sequenceNumber` already passed to
60
+ * `syncFn`. An incoming event with `sequenceNumber <= seen` is skipped. SQS
61
+ * at-least-once delivery + reconnect-replay means the SAME event can arrive
62
+ * twice; dedupe makes that idempotent.
63
+ *
64
+ * Disconnect / reconnect with catch-up replay (AC#3/#4)
65
+ * ─────────────────────────────────────────────────────
66
+ * `receiveMessage` failures (network blip, throttling) are caught; the loop
67
+ * backs off (exponential + jitter, capped) and resumes. Because the per-client
68
+ * SQS queue retains undelivered messages for 14 days, anything published while
69
+ * the device was offline/disconnected is redelivered when the poll resumes —
70
+ * catch-up replay with no server round-trip. Redelivered-but-already-processed
71
+ * events are absorbed by the dedupe path. The in-memory fake's
72
+ * `simulateDisconnect()` / `simulateReconnect()` model exactly this buffering.
73
+ *
74
+ * Feature flag (AC#4)
75
+ * ───────────────────
76
+ * Gated by the per-tenant {@link EventDrivenPushFlagProvider} (US-008's
77
+ * feature-flags.ts) — honors `HQ_SYNC_EVENT_DRIVEN_PUSH_ENABLED_TENANTS`
78
+ * (per-tenant) + the legacy global `HQ_SYNC_EVENT_DRIVEN_PUSH_ENABLED=true`.
79
+ * Default DISABLED. Resolution precedence: explicit `enabled` > injected
80
+ * `flagProvider.isEnabled(tenantId)` > default env-driven provider.
81
+ *
82
+ * Cross-tenant isolation (US-010)
83
+ * ───────────────────────────────
84
+ * Each receiver instance binds exactly ONE `tenantId` and polls exactly ONE
85
+ * queue URL (its own tenant's per-client queue). Isolation is enforced at the
86
+ * subscription boundary — the receiver never reads another tenant's queue, and
87
+ * never filters cross-tenant data post-hoc.
88
+ *
89
+ * @see ./push-transport.ts (the outbound seam this mirrors)
90
+ * @see ./feature-flags.ts (the per-tenant flag provider — US-008)
91
+ * @see ../bin/sync-runner.ts (the wiring site — runRunnerWithLoop)
92
+ * @see companies/indigo/projects/event-driven-sync-menubar/references.md (US-000)
93
+ *
94
+ * Adapted from indigoai-us/hq-pro PR #112 (src/sync/push-receiver.ts) into
95
+ * @indigoai-us/hq-cloud (Path B). The hq-pro source shipped only Noop +
96
+ * InMemory receivers (the production SQS path was deferred there); this module
97
+ * builds the real SQS receiver behind the same lifecycle/dedupe/flag seam.
98
+ */
99
+ import { decodePushEvent } from "./push-event.js";
100
+ import { defaultFlagProvider, } from "./feature-flags.js";
101
+ import { publishSyncLatencyMetric, } from "./metrics.js";
102
+ // ─── Constants ─────────────────────────────────────────────────────────────
103
+ /**
104
+ * How long `dispose()` awaits an in-flight `syncFn` after aborting its signal,
105
+ * before abandoning it (the poll/cadence safety net re-pulls on the next tick).
106
+ */
107
+ export const DEFAULT_RECEIVER_DISPOSE_DRAIN_MS = 5_000;
108
+ /** Default SQS long-poll wait (seconds). 20 is the SQS max — true long-poll. */
109
+ export const DEFAULT_WAIT_TIME_SECONDS = 20;
110
+ /** Default max messages pulled per `receiveMessage` call (SQS max is 10). */
111
+ export const DEFAULT_MAX_MESSAGES = 10;
112
+ /** Reconnect backoff defaults. */
113
+ export const DEFAULT_RECONNECT_INITIAL_MS = 250;
114
+ export const DEFAULT_RECONNECT_MAX_MS = 30_000;
115
+ const NOOP_LOGGER = {
116
+ info: () => undefined,
117
+ warn: () => undefined,
118
+ error: () => undefined,
119
+ debug: () => undefined,
120
+ };
121
+ // ─── Noop default ─────────────────────────────────────────────────────────
122
+ /**
123
+ * Default `PushReceiver` used when no real queue is wired (or the flag is OFF
124
+ * at the factory). `start()` flips `connected` true; `dispose()` flips it
125
+ * false. No subscription work, no events. Mirrors {@link NoopPushTransport}.
126
+ */
127
+ export class NoopPushReceiver {
128
+ _connected = false;
129
+ get connected() {
130
+ return this._connected;
131
+ }
132
+ async start() {
133
+ this._connected = true;
134
+ }
135
+ async dispose() {
136
+ this._connected = false;
137
+ }
138
+ }
139
+ // ─── Feature-flag resolution ─────────────────────────────────────────────────
140
+ /**
141
+ * Resolve the enabled boolean from layered config. Precedence (highest first):
142
+ * 1. `explicit` — `enabled: true|false` wins outright (test contract).
143
+ * 2. `flagProvider.isEnabled(tenantId)` — injected per-tenant provider.
144
+ * 3. default {@link EnvTenantListFlagProvider} from `env ?? process.env`.
145
+ */
146
+ function resolveEnabled(args) {
147
+ if (args.explicit !== undefined)
148
+ return args.explicit;
149
+ if (args.flagProvider !== undefined) {
150
+ return args.flagProvider.isEnabled(args.tenantId);
151
+ }
152
+ return defaultFlagProvider(args.env ?? process.env).isEnabled(args.tenantId);
153
+ }
154
+ /**
155
+ * Real client `PushReceiver` backed by a per-tenant SQS queue.
156
+ *
157
+ * Poll loop: long-poll `receiveMessage` → for each message, decode + dedupe +
158
+ * dispatch through `syncFn`, then `deleteMessage` on successful handoff. A
159
+ * `receiveMessage` rejection is treated as a transient disconnect: log, back
160
+ * off, resume. SQS retention covers offline catch-up; dedupe covers redelivery.
161
+ */
162
+ export class SqsPushReceiver {
163
+ tenantId;
164
+ queueUrl;
165
+ sqs;
166
+ syncFn;
167
+ logger;
168
+ enabled;
169
+ waitTimeSeconds;
170
+ maxMessages;
171
+ disposeDrainMs;
172
+ reconnectInitialMs;
173
+ reconnectMaxMs;
174
+ reconnectJitter;
175
+ sleep;
176
+ publishMetric;
177
+ now;
178
+ _connected = false;
179
+ disposed = false;
180
+ disposing = false;
181
+ disposePromise = null;
182
+ /** Abort signal shared by the poll loop + in-flight sync; fired on dispose. */
183
+ loopAbort = null;
184
+ /** The running poll loop promise; awaited (best-effort) during dispose. */
185
+ loopPromise = null;
186
+ /** AbortController for the in-flight syncFn; refreshed each dispatch. */
187
+ inFlightAbort = null;
188
+ inFlightSync = null;
189
+ /** Per-path highest sequence number already PROCESSED by syncFn. */
190
+ seenSequencePerPath = new Map();
191
+ _processedCount = 0;
192
+ _dedupedCount = 0;
193
+ _decodeFailureCount = 0;
194
+ _receiveErrorCount = 0;
195
+ constructor(opts) {
196
+ if (!opts.tenantId || opts.tenantId.trim() === "") {
197
+ throw new Error("SqsPushReceiver: tenantId is required");
198
+ }
199
+ if (!opts.queueUrl || opts.queueUrl.trim() === "") {
200
+ throw new Error("SqsPushReceiver: queueUrl is required");
201
+ }
202
+ this.tenantId = opts.tenantId;
203
+ this.queueUrl = opts.queueUrl;
204
+ this.sqs = opts.sqs;
205
+ this.syncFn = opts.syncFn;
206
+ this.logger = opts.logger ?? NOOP_LOGGER;
207
+ this.enabled = resolveEnabled({
208
+ explicit: opts.enabled,
209
+ flagProvider: opts.flagProvider,
210
+ tenantId: opts.tenantId,
211
+ env: opts.env,
212
+ });
213
+ this.waitTimeSeconds = opts.waitTimeSeconds ?? DEFAULT_WAIT_TIME_SECONDS;
214
+ this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
215
+ this.disposeDrainMs =
216
+ opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
217
+ this.reconnectInitialMs =
218
+ opts.reconnect?.initialMs ?? DEFAULT_RECONNECT_INITIAL_MS;
219
+ this.reconnectMaxMs = opts.reconnect?.maxMs ?? DEFAULT_RECONNECT_MAX_MS;
220
+ this.reconnectJitter = opts.reconnect?.jitter ?? true;
221
+ this.sleep = opts.sleep ?? defaultSleep;
222
+ this.publishMetric =
223
+ opts.publishMetric ?? ((m) => publishSyncLatencyMetric(m, { logger: undefined }));
224
+ this.now = opts.now ?? (() => Date.now());
225
+ }
226
+ // ─── PushReceiver surface ────────────────────────────────────────────────
227
+ get connected() {
228
+ return this._connected;
229
+ }
230
+ async start() {
231
+ if (this.disposed)
232
+ return;
233
+ if (!this.enabled) {
234
+ this.logger.info({
235
+ event: "receiver.start.disabled",
236
+ tenantId: this.tenantId,
237
+ reason: "feature flag off",
238
+ }, "push receiver disabled by feature flag");
239
+ return;
240
+ }
241
+ if (this.loopAbort !== null) {
242
+ // Double-start is a no-op — matches PushTransport idempotency posture.
243
+ this.logger.debug({ event: "receiver.start.noop", tenantId: this.tenantId }, "push receiver already started");
244
+ return;
245
+ }
246
+ this.loopAbort = new AbortController();
247
+ this._connected = true;
248
+ this.logger.info({ event: "receiver.start", tenantId: this.tenantId, queueUrl: this.queueUrl }, "push receiver subscribed (sqs long-poll)");
249
+ // Kick the loop off; do NOT await — start() returns once subscribed.
250
+ this.loopPromise = this.pollLoop(this.loopAbort.signal);
251
+ }
252
+ async dispose() {
253
+ if (this.disposed)
254
+ return;
255
+ if (this.disposePromise !== null)
256
+ return this.disposePromise;
257
+ this.disposing = true;
258
+ this.disposePromise = (async () => {
259
+ // Stop the poll loop + abort any in-flight long-poll / syncFn.
260
+ try {
261
+ this.loopAbort?.abort();
262
+ }
263
+ catch {
264
+ /* AbortController.abort never throws on Node; defensive. */
265
+ }
266
+ try {
267
+ this.inFlightAbort?.abort();
268
+ }
269
+ catch {
270
+ /* defensive */
271
+ }
272
+ if (this.inFlightSync !== null) {
273
+ const drainDeadline = new Promise((resolve) => {
274
+ const t = setTimeout(resolve, this.disposeDrainMs);
275
+ // Unref so a hung syncFn can't keep the loop alive past process exit.
276
+ t.unref?.();
277
+ });
278
+ await Promise.race([
279
+ this.inFlightSync.catch(() => undefined),
280
+ drainDeadline,
281
+ ]);
282
+ }
283
+ // Best-effort: let the poll loop observe the abort and exit.
284
+ if (this.loopPromise !== null) {
285
+ await this.loopPromise.catch(() => undefined);
286
+ }
287
+ this._connected = false;
288
+ this.disposed = true;
289
+ this.logger.info({
290
+ event: "receiver.stop",
291
+ tenantId: this.tenantId,
292
+ processed: this._processedCount,
293
+ deduped: this._dedupedCount,
294
+ }, "push receiver stopped");
295
+ })();
296
+ return this.disposePromise;
297
+ }
298
+ // ─── Observability ─────────────────────────────────────────────────────────
299
+ /** Events that passed dedupe AND completed `syncFn` successfully. */
300
+ get processedCount() {
301
+ return this._processedCount;
302
+ }
303
+ /** Events skipped by dedupe. */
304
+ get dedupedCount() {
305
+ return this._dedupedCount;
306
+ }
307
+ /** Events dropped at the wire-boundary decode step. */
308
+ get decodeFailureCount() {
309
+ return this._decodeFailureCount;
310
+ }
311
+ /** `receiveMessage` failures (transient disconnects) the loop recovered from. */
312
+ get receiveErrorCount() {
313
+ return this._receiveErrorCount;
314
+ }
315
+ // ─── Internals ───────────────────────────────────────────────────────────
316
+ /**
317
+ * The long-poll loop. Runs until the loop abort signal fires (dispose). A
318
+ * `receiveMessage` rejection is a transient disconnect — log, back off,
319
+ * resume. Because the SQS queue retains messages, resuming after a blip
320
+ * replays the gap (catch-up). The loop never throws past this method; it
321
+ * is fire-and-forgotten by `start()` and awaited best-effort by `dispose()`.
322
+ */
323
+ async pollLoop(signal) {
324
+ let attempt = 0;
325
+ while (!signal.aborted) {
326
+ let received;
327
+ try {
328
+ received = await this.sqs.receiveMessage({
329
+ queueUrl: this.queueUrl,
330
+ maxMessages: this.maxMessages,
331
+ waitTimeSeconds: this.waitTimeSeconds,
332
+ signal,
333
+ });
334
+ attempt = 0; // success → reset backoff
335
+ if (received.messages.length > 0) {
336
+ this._connected = true;
337
+ }
338
+ }
339
+ catch (err) {
340
+ if (signal.aborted)
341
+ return; // dispose-driven abort — clean exit
342
+ this._receiveErrorCount += 1;
343
+ this._connected = false;
344
+ const backoff = this.computeBackoff(attempt);
345
+ attempt += 1;
346
+ this.logger.warn({
347
+ event: "receiver.receive.failed",
348
+ tenantId: this.tenantId,
349
+ attempt,
350
+ backoffMs: backoff,
351
+ err: { message: err?.message },
352
+ }, "push receiver receiveMessage failed; backing off (catch-up replay on resume)");
353
+ await this.sleep(backoff, signal);
354
+ continue;
355
+ }
356
+ for (const msg of received.messages) {
357
+ if (signal.aborted)
358
+ return;
359
+ await this.handleMessage(msg);
360
+ }
361
+ }
362
+ }
363
+ /**
364
+ * Decode → dedupe → dispatch → delete a single SQS message. Decode failures
365
+ * and syncFn throws are logged + absorbed (never crash the loop). The message
366
+ * is deleted only after a successful handoff so an unprocessed message stays
367
+ * on the queue for redelivery (the dedupe path makes redelivery idempotent).
368
+ */
369
+ async handleMessage(msg) {
370
+ if (this.disposing || this.disposed)
371
+ return;
372
+ let validated;
373
+ try {
374
+ validated = decodePushEvent(msg.Body ?? "");
375
+ }
376
+ catch (err) {
377
+ this._decodeFailureCount += 1;
378
+ this.logger.warn({
379
+ event: "receiver.decode.failed",
380
+ tenantId: this.tenantId,
381
+ messageId: msg.MessageId,
382
+ err: { message: err.message },
383
+ }, "push receiver dropped event: decode failed");
384
+ // A poison message we can't decode is deleted so it doesn't redeliver
385
+ // forever — it carries no actionable path. (Defense in depth.)
386
+ await this.safeDelete(msg);
387
+ return;
388
+ }
389
+ const handled = await this.dispatch(validated);
390
+ if (handled) {
391
+ // Delete only after the handoff (success OR dedupe-skip — both mean we
392
+ // don't need this message again). A syncFn throw still counts as handled
393
+ // for delete purposes: the seen-counter advanced, so a redelivery would
394
+ // dedupe; the poll/cadence safety net is the recovery path for the throw.
395
+ await this.safeDelete(msg);
396
+ }
397
+ }
398
+ /**
399
+ * Dedupe + invoke `syncFn`. Returns true once the event has been accounted
400
+ * for (deduped, or syncFn settled) so the caller can delete the message.
401
+ * Stores the in-flight promise so `dispose()` can drain it.
402
+ */
403
+ async dispatch(event) {
404
+ if (this.disposing || this.disposed)
405
+ return false;
406
+ const seen = this.seenSequencePerPath.get(event.relativePath);
407
+ if (seen !== undefined && event.sequenceNumber <= seen) {
408
+ this._dedupedCount += 1;
409
+ this.logger.debug({
410
+ event: "receiver.event.deduped",
411
+ tenantId: this.tenantId,
412
+ relativePath: event.relativePath,
413
+ sequenceNumber: event.sequenceNumber,
414
+ seen,
415
+ }, "push receiver deduped event");
416
+ return true;
417
+ }
418
+ // Record BEFORE invoking syncFn — a back-to-back event for the same path
419
+ // must see this sequence in its dedupe check. The trade-off: a syncFn that
420
+ // throws still advances the counter ("latest we KNOW about"); the safety
421
+ // net poll is the recovery path for the throw.
422
+ this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
423
+ const controller = new AbortController();
424
+ this.inFlightAbort = controller;
425
+ const ctx = { event, signal: controller.signal };
426
+ const holder = { p: null };
427
+ const startMs = this.now();
428
+ const p = (async () => {
429
+ try {
430
+ this.logger.debug({
431
+ event: "receiver.sync.start",
432
+ tenantId: this.tenantId,
433
+ relativePath: event.relativePath,
434
+ sequenceNumber: event.sequenceNumber,
435
+ }, "push receiver invoking sync engine");
436
+ await this.syncFn(ctx);
437
+ this._processedCount += 1;
438
+ this.logger.debug({
439
+ event: "receiver.sync.completed",
440
+ tenantId: this.tenantId,
441
+ relativePath: event.relativePath,
442
+ sequenceNumber: event.sequenceNumber,
443
+ }, "push receiver sync completed");
444
+ // ── US-011: 3-log chain (3rd link) + p95 latency metric ──────────
445
+ // Latency = save-on-A (event.eventTimestamp) → visible-on-B (now),
446
+ // falling back to the local syncFn duration if the event timestamp is
447
+ // unparseable. Emitted ONLY on the success path so failed syncs don't
448
+ // skew p95 toward infinity.
449
+ const endMs = this.now();
450
+ const savedAtMs = Date.parse(event.eventTimestamp);
451
+ const latencyMs = Number.isFinite(savedAtMs)
452
+ ? Math.max(0, endMs - savedAtMs)
453
+ : Math.max(0, endMs - startMs);
454
+ const latencySeconds = latencyMs / 1000;
455
+ // The 3rd correlated log line — shares `sequenceNumber` with
456
+ // watcher.emit (client push) and push.receive (server).
457
+ this.logger.info({
458
+ event: "fanout.receive",
459
+ tenantId: this.tenantId,
460
+ relativePath: event.relativePath,
461
+ sequenceNumber: event.sequenceNumber,
462
+ latencySeconds,
463
+ }, "push receiver fanout-receive (round-trip complete)");
464
+ // Best-effort metric. Fire-and-forget so observability never sits on
465
+ // the dispatch critical path (which gates message deletion) — a slow or
466
+ // hung CloudWatch call must not delay the delete/next-poll. Errors are
467
+ // swallowed; publishMetric is itself best-effort.
468
+ void this.emitLatencyMetric({
469
+ tenantId: this.tenantId,
470
+ relativePath: event.relativePath,
471
+ sequenceNumber: event.sequenceNumber,
472
+ latencySeconds,
473
+ timestamp: new Date(endMs),
474
+ });
475
+ }
476
+ catch (err) {
477
+ // Critical: catch, log, return. A misbehaving sync engine must never
478
+ // crash the receiver loop. The safety-net poll handles eventual
479
+ // consistency for failed pulls.
480
+ const e = err;
481
+ this.logger.error({
482
+ event: "receiver.sync.failed",
483
+ tenantId: this.tenantId,
484
+ relativePath: event.relativePath,
485
+ sequenceNumber: event.sequenceNumber,
486
+ err: { message: e?.message, code: e?.code },
487
+ }, "push receiver sync engine threw");
488
+ }
489
+ finally {
490
+ if (this.inFlightAbort === controller)
491
+ this.inFlightAbort = null;
492
+ if (this.inFlightSync === holder.p)
493
+ this.inFlightSync = null;
494
+ }
495
+ })();
496
+ holder.p = p;
497
+ this.inFlightSync = p;
498
+ await p;
499
+ return true;
500
+ }
501
+ /** Delete a message, swallowing transport errors (redelivery is harmless). */
502
+ async safeDelete(msg) {
503
+ if (!msg.ReceiptHandle)
504
+ return;
505
+ try {
506
+ await this.sqs.deleteMessage({
507
+ queueUrl: this.queueUrl,
508
+ receiptHandle: msg.ReceiptHandle,
509
+ });
510
+ }
511
+ catch (err) {
512
+ this.logger.warn({
513
+ event: "receiver.delete.failed",
514
+ tenantId: this.tenantId,
515
+ messageId: msg.MessageId,
516
+ err: { message: err?.message },
517
+ }, "push receiver failed to delete message (will redeliver; dedupe absorbs)");
518
+ }
519
+ }
520
+ /**
521
+ * Publish one best-effort latency datum (US-011). Awaits `publishMetric` and
522
+ * swallows any rejection so a metric-backend outage can never reach the poll
523
+ * loop. Called fire-and-forget (`void`) off the dispatch critical path.
524
+ */
525
+ async emitLatencyMetric(metric) {
526
+ try {
527
+ await this.publishMetric(metric);
528
+ }
529
+ catch (metricErr) {
530
+ this.logger.warn({
531
+ event: "receiver.metric.failed",
532
+ tenantId: metric.tenantId,
533
+ sequenceNumber: metric.sequenceNumber,
534
+ err: { message: metricErr?.message },
535
+ }, "push receiver failed to publish latency metric (ignored)");
536
+ }
537
+ }
538
+ /** Exponential backoff (capped) with optional full-jitter. */
539
+ computeBackoff(attempt) {
540
+ const exp = Math.min(this.reconnectMaxMs, this.reconnectInitialMs * 2 ** attempt);
541
+ if (!this.reconnectJitter)
542
+ return exp;
543
+ return Math.floor(Math.random() * exp);
544
+ }
545
+ }
546
+ // ─── In-memory receiver (unit-test transport analogue) ───────────────────────
547
+ /**
548
+ * A tiny in-process fanout the {@link InMemoryPushReceiver} subscribes against.
549
+ * Models SNS publish + the per-client SQS queue's disconnect buffering so unit
550
+ * tests can drive the receive path without AWS. `publish` raw strings (the
551
+ * wire form) so decode-failure paths are testable too.
552
+ */
553
+ export class InMemoryFanout {
554
+ subscribers = new Set();
555
+ subscribe(handler) {
556
+ this.subscribers.add(handler);
557
+ return () => this.subscribers.delete(handler);
558
+ }
559
+ /** Publish a raw (already-encoded) message body to all subscribers. */
560
+ publish(raw) {
561
+ for (const h of [...this.subscribers])
562
+ h(raw);
563
+ }
564
+ }
565
+ /**
566
+ * In-memory receiver paired with {@link InMemoryFanout}. Powers the unit
567
+ * tests for dedupe, reconnect-replay, flag gating, and dispose-drain WITHOUT
568
+ * any AWS SDK. The dedupe / dispatch / dispose semantics are identical to
569
+ * {@link SqsPushReceiver} (shared design); the disconnect buffer is the
570
+ * in-process analogue of the per-client SQS queue's 14-day retention.
571
+ */
572
+ export class InMemoryPushReceiver {
573
+ tenantId;
574
+ fanout;
575
+ syncFn;
576
+ logger;
577
+ enabled;
578
+ disposeDrainMs;
579
+ _connected = false;
580
+ disposed = false;
581
+ disposing = false;
582
+ disposePromise = null;
583
+ unsubscribe = null;
584
+ disconnectedFlag = false;
585
+ pendingDuringDisconnect = [];
586
+ seenSequencePerPath = new Map();
587
+ inFlightAbort = null;
588
+ inFlightSync = null;
589
+ _processedCount = 0;
590
+ _dedupedCount = 0;
591
+ _decodeFailureCount = 0;
592
+ constructor(opts) {
593
+ this.tenantId = opts.tenantId;
594
+ this.fanout = opts.fanout;
595
+ this.syncFn = opts.syncFn;
596
+ this.logger = opts.logger ?? NOOP_LOGGER;
597
+ this.disposeDrainMs =
598
+ opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
599
+ this.enabled = resolveEnabled({
600
+ explicit: opts.enabled,
601
+ flagProvider: opts.flagProvider,
602
+ tenantId: opts.tenantId,
603
+ env: opts.env,
604
+ });
605
+ }
606
+ get connected() {
607
+ return this._connected;
608
+ }
609
+ async start() {
610
+ if (this.disposed)
611
+ return;
612
+ if (!this.enabled) {
613
+ this.logger.info({ event: "receiver.start.disabled", tenantId: this.tenantId }, "push receiver disabled by feature flag");
614
+ return;
615
+ }
616
+ if (this.unsubscribe !== null)
617
+ return; // double-start no-op
618
+ this.unsubscribe = this.fanout.subscribe((raw) => {
619
+ let validated;
620
+ try {
621
+ validated = decodePushEvent(raw);
622
+ }
623
+ catch (err) {
624
+ this._decodeFailureCount += 1;
625
+ this.logger.warn({
626
+ event: "receiver.decode.failed",
627
+ tenantId: this.tenantId,
628
+ err: { message: err.message },
629
+ }, "push receiver dropped event: decode failed");
630
+ return;
631
+ }
632
+ if (this.disconnectedFlag) {
633
+ this.pendingDuringDisconnect.push(validated);
634
+ return;
635
+ }
636
+ void this.dispatch(validated);
637
+ });
638
+ this._connected = true;
639
+ this.logger.info({ event: "receiver.start", tenantId: this.tenantId }, "push receiver subscribed (in-memory)");
640
+ }
641
+ async dispose() {
642
+ if (this.disposed)
643
+ return;
644
+ if (this.disposePromise !== null)
645
+ return this.disposePromise;
646
+ this.disposing = true;
647
+ this.disposePromise = (async () => {
648
+ try {
649
+ this.inFlightAbort?.abort();
650
+ }
651
+ catch {
652
+ /* defensive */
653
+ }
654
+ if (this.inFlightSync !== null) {
655
+ const drainDeadline = new Promise((resolve) => {
656
+ const t = setTimeout(resolve, this.disposeDrainMs);
657
+ t.unref?.();
658
+ });
659
+ await Promise.race([
660
+ this.inFlightSync.catch(() => undefined),
661
+ drainDeadline,
662
+ ]);
663
+ }
664
+ if (this.unsubscribe !== null) {
665
+ try {
666
+ this.unsubscribe();
667
+ }
668
+ catch {
669
+ /* defensive */
670
+ }
671
+ this.unsubscribe = null;
672
+ }
673
+ this._connected = false;
674
+ this.disposed = true;
675
+ this.logger.info({
676
+ event: "receiver.stop",
677
+ tenantId: this.tenantId,
678
+ processed: this._processedCount,
679
+ deduped: this._dedupedCount,
680
+ }, "push receiver stopped");
681
+ })();
682
+ return this.disposePromise;
683
+ }
684
+ // ─── Test hooks (model the SQS retention buffer) ────────────────────────────
685
+ get processedCount() {
686
+ return this._processedCount;
687
+ }
688
+ get dedupedCount() {
689
+ return this._dedupedCount;
690
+ }
691
+ get decodeFailureCount() {
692
+ return this._decodeFailureCount;
693
+ }
694
+ get bufferedCount() {
695
+ return this.pendingDuringDisconnect.length;
696
+ }
697
+ /** Emulate a network blip — events buffer instead of dispatching. */
698
+ simulateDisconnect() {
699
+ if (!this.enabled || this.unsubscribe === null)
700
+ return;
701
+ this.disconnectedFlag = true;
702
+ this._connected = false;
703
+ }
704
+ /** Emulate reconnect — drain the buffer through the same dedupe path. */
705
+ simulateReconnect() {
706
+ if (!this.disconnectedFlag)
707
+ return;
708
+ this.disconnectedFlag = false;
709
+ this._connected = true;
710
+ const queued = this.pendingDuringDisconnect.splice(0);
711
+ for (const evt of queued)
712
+ void this.dispatch(evt);
713
+ }
714
+ dispatch(event) {
715
+ if (this.disposing || this.disposed)
716
+ return Promise.resolve();
717
+ const seen = this.seenSequencePerPath.get(event.relativePath);
718
+ if (seen !== undefined && event.sequenceNumber <= seen) {
719
+ this._dedupedCount += 1;
720
+ return Promise.resolve();
721
+ }
722
+ this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
723
+ const controller = new AbortController();
724
+ this.inFlightAbort = controller;
725
+ const ctx = { event, signal: controller.signal };
726
+ const holder = { p: null };
727
+ const p = (async () => {
728
+ try {
729
+ await this.syncFn(ctx);
730
+ this._processedCount += 1;
731
+ }
732
+ catch (err) {
733
+ this.logger.error({
734
+ event: "receiver.sync.failed",
735
+ tenantId: this.tenantId,
736
+ relativePath: event.relativePath,
737
+ err: { message: err?.message },
738
+ }, "push receiver sync engine threw");
739
+ }
740
+ finally {
741
+ if (this.inFlightAbort === controller)
742
+ this.inFlightAbort = null;
743
+ if (this.inFlightSync === holder.p)
744
+ this.inFlightSync = null;
745
+ }
746
+ })();
747
+ holder.p = p;
748
+ this.inFlightSync = p;
749
+ return p;
750
+ }
751
+ }
752
+ export function createPushReceiver(opts) {
753
+ if ("kind" in opts && opts.kind === "noop") {
754
+ return new NoopPushReceiver();
755
+ }
756
+ if ("queueUrl" in opts && "sqs" in opts) {
757
+ return new SqsPushReceiver(opts);
758
+ }
759
+ return new NoopPushReceiver();
760
+ }
761
+ // ─── Helpers ───────────────────────────────────────────────────────────────
762
+ /**
763
+ * Default sleep that resolves after `ms` OR promptly on abort (so a dispose
764
+ * during a backoff wait doesn't block). Never rejects.
765
+ */
766
+ function defaultSleep(ms, signal) {
767
+ return new Promise((resolve) => {
768
+ if (signal.aborted)
769
+ return resolve();
770
+ const t = setTimeout(() => {
771
+ signal.removeEventListener("abort", onAbort);
772
+ resolve();
773
+ }, ms);
774
+ t.unref?.();
775
+ const onAbort = () => {
776
+ clearTimeout(t);
777
+ resolve();
778
+ };
779
+ signal.addEventListener("abort", onAbort, { once: true });
780
+ });
781
+ }
782
+ //# sourceMappingURL=push-receiver.js.map