@indigoai-us/hq-cloud 5.26.0 → 5.28.0

package/src/sync/feature-flags.ts

@@ -0,0 +1,229 @@
+/**
+ * Per-tenant feature-flag module for the event-driven push pipeline.
+ *
+ * Ported from indigoai-us/hq-pro PR #112 (src/sync/feature-flags.ts) into
+ * @indigoai-us/hq-cloud (Path B) per project event-driven-sync-menubar US-008.
+ *
+ * Why this exists
+ * ───────────────
+ * The event-driven push pipeline must default OFF for all tenants and be
+ * enabled per-tenant ("OFF for all tenants by default, ON for Indigo only").
+ * This module provides the read-only seam the watcher (and a later receiver)
+ * consult at start time to decide whether to emit/ship PushEvents at all:
+ *  - {@link EventDrivenPushFlagProvider} — the small read-only seam every
+ *    caller consults. Returns a boolean for a given tenantId.
+ *  - {@link EnvTenantListFlagProvider} — production implementation. Reads
+ *    a comma-separated allow-list from {@link FEATURE_FLAG_TENANTS_ENV_VAR}
+ *    AND honors the legacy global `HQ_SYNC_EVENT_DRIVEN_PUSH_ENABLED=true`
+ *    as a backwards-compat global override. The legacy global wins when set
+ *    — it enables all tenants.
+ *  - {@link StaticFlagProvider} — small test helper. Constructor takes an
+ *    iterable of enabled tenant IDs. Used by watcher + feature-flag tests.
+ *  - {@link defaultFlagProvider} — factory callers default to when no
+ *    provider is explicitly supplied.
+ *
+ * Validation posture
+ * ──────────────────
+ * Tenant IDs from the env are trimmed, empty entries are dropped, and
+ * entries containing internal whitespace are warn-and-skipped (NOT thrown).
+ * The watcher runs inside a long-lived daemon — a malformed env var must
+ * not crash daemon startup; it must surface in logs and fall through to the
+ * default-OFF behavior for the misconfigured entry.
+ *
+ * @see src/watcher.ts — the TreeWatcher consults this seam at start time to
+ *   decide whether to emit + ship PushEvents (dormant when OFF).
+ */
+
+// ─── Constants ─────────────────────────────────────────────────────────────
+
+/**
+ * Env var holding a comma-separated allow-list of tenant IDs for which the
+ * event-driven push pipeline is enabled. Empty/unset means no tenants are
+ * enabled (default OFF for all). Example: `indigo,acme`.
+ *
+ * Tenant matching is exact and case-sensitive after trimming. Whitespace
+ * around commas is tolerated; internal whitespace within a tenant ID is
+ * not (warned + skipped — see `parseTenantList` below).
+ */
+export const FEATURE_FLAG_TENANTS_ENV_VAR =
+  "HQ_SYNC_EVENT_DRIVEN_PUSH_ENABLED_TENANTS";
+
+/**
+ * Legacy process-wide env var. When set to the literal string `"true"`, the
+ * pipeline is unconditionally enabled. Preserved as a "global override that
+ * enables ALL tenants" so a single-flag rollout (or local dev) keeps working.
+ *
+ * Operators flipping rollout via deploy-time config should prefer the
+ * per-tenant {@link FEATURE_FLAG_TENANTS_ENV_VAR} instead.
+ */
+export const FEATURE_FLAG_LEGACY_GLOBAL_ENV_VAR =
+  "HQ_SYNC_EVENT_DRIVEN_PUSH_ENABLED";
+
+// ─── Public types ──────────────────────────────────────────────────────────
+
+/**
+ * Read-only seam for the per-tenant event-driven push feature flag.
+ *
+ * Implementations are pure / synchronous. Callers MUST treat the result as
+ * the truth at the moment of the call — a deploy-time env-var change does
+ * NOT propagate to running daemons until they restart.
+ */
+export interface EventDrivenPushFlagProvider {
+  /**
+   * Returns true iff the event-driven push pipeline is enabled for
+   * `tenantId`. Defaults to false. Case-sensitive exact match against the
+   * configured allow-list (or the legacy global override).
+   */
+  isEnabled(tenantId: string): boolean;
+}
+
+/**
+ * Optional injection seam for warnings emitted while parsing the env var
+ * (invalid entries). Defaults to `console.warn`.
+ */
+export type FlagProviderWarnFn = (
+  message: string,
+  context: Record<string, unknown>,
+) => void;
+
+// ─── Env-driven provider ───────────────────────────────────────────────────
+
+/**
+ * Options for {@link EnvTenantListFlagProvider}.
+ */
+export interface EnvTenantListFlagProviderOptions {
+  /**
+   * Env snapshot. Defaults to `process.env`. Tests inject `{}` or a
+   * synthetic object so they don't mutate real env state.
+   */
+  env?: Record<string, string | undefined>;
+  /**
+   * Where to send "invalid entry, skipping" warnings. Defaults to
+   * `console.warn`.
+   */
+  warn?: FlagProviderWarnFn;
+}
+
+/**
+ * Production flag provider. Reads {@link FEATURE_FLAG_TENANTS_ENV_VAR} at
+ * construction time and caches the parsed set — `isEnabled()` is O(1).
+ *
+ * The legacy {@link FEATURE_FLAG_LEGACY_GLOBAL_ENV_VAR} (set to literal
+ * `"true"`) acts as a global override: when set, `isEnabled()` returns
+ * true for every tenant.
+ *
+ * Parse rules:
+ *  - Empty / unset {@link FEATURE_FLAG_TENANTS_ENV_VAR} → no tenants enabled.
+ *  - Whitespace around commas is tolerated and trimmed.
+ *  - Empty entries (`",,foo,"`) are silently dropped.
+ *  - Entries with INTERNAL whitespace (`"indigo team"`) are warn-and-skipped.
+ *  - Matching is exact + case-sensitive.
+ */
+export class EnvTenantListFlagProvider implements EventDrivenPushFlagProvider {
+  private readonly tenants: ReadonlySet<string>;
+  private readonly legacyGlobal: boolean;
+
+  constructor(opts: EnvTenantListFlagProviderOptions = {}) {
+    const env = opts.env ?? process.env;
+    const warn = opts.warn ?? defaultWarn;
+    this.tenants = parseTenantList(env[FEATURE_FLAG_TENANTS_ENV_VAR], warn);
+    this.legacyGlobal = env[FEATURE_FLAG_LEGACY_GLOBAL_ENV_VAR] === "true";
+  }
+
+  isEnabled(tenantId: string): boolean {
+    if (this.legacyGlobal) return true;
+    return this.tenants.has(tenantId);
+  }
+
+  /**
+   * Debug-only accessor — returns a defensive copy of the cached allow-list.
+   * Used by tests and the daemon's startup log line.
+   */
+  get enabledTenants(): ReadonlySet<string> {
+    return new Set(this.tenants);
+  }
+
+  /** Debug-only accessor — true iff the legacy global override is in effect. */
+  get globalOverride(): boolean {
+    return this.legacyGlobal;
+  }
+}
+
+// ─── Static provider (tests) ───────────────────────────────────────────────
+
+/**
+ * In-memory provider for unit tests. The constructor takes the set of
+ * enabled tenant IDs; `isEnabled()` is a plain `Set.has` check.
+ *
+ * Production callers should NEVER construct this — use
+ * {@link EnvTenantListFlagProvider} via {@link defaultFlagProvider}.
+ */
+export class StaticFlagProvider implements EventDrivenPushFlagProvider {
+  private readonly tenants: ReadonlySet<string>;
+
+  constructor(enabledTenants: Iterable<string>) {
+    this.tenants = new Set(enabledTenants);
+  }
+
+  isEnabled(tenantId: string): boolean {
+    return this.tenants.has(tenantId);
+  }
+}
+
+// ─── Factory ───────────────────────────────────────────────────────────────
+
+/**
+ * Build the default {@link EventDrivenPushFlagProvider} from an env
+ * snapshot. Production callers use this with no args — `process.env` is
+ * the implicit input.
+ */
+export function defaultFlagProvider(
+  env: Record<string, string | undefined> = process.env,
+): EventDrivenPushFlagProvider {
+  return new EnvTenantListFlagProvider({ env });
+}
+
+// ─── Internals ─────────────────────────────────────────────────────────────
+
+/**
+ * Parse a raw env value into a Set of valid tenant IDs.
+ *
+ * Kept as a free function so unit tests can call it via the class accessor
+ * with a synthetic warn spy without exporting an internal symbol.
+ */
+function parseTenantList(
+  raw: string | undefined,
+  warn: FlagProviderWarnFn,
+): ReadonlySet<string> {
+  if (raw === undefined || raw === "") return new Set<string>();
+  const out = new Set<string>();
+  for (const piece of raw.split(",")) {
+    const trimmed = piece.trim();
+    if (trimmed === "") {
+      // Empty slot (`",,foo,"` or trailing commas) — silently skipped.
+      continue;
+    }
+    if (/\s/.test(trimmed)) {
+      // Internal whitespace — operator misconfiguration. Warn but don't
+      // throw: the daemon must keep starting, the misconfigured tenant
+      // just stays OFF.
+      warn("feature-flag: skipping tenant entry with internal whitespace", {
+        envVar: FEATURE_FLAG_TENANTS_ENV_VAR,
+        rawEntry: trimmed,
+      });
+      continue;
+    }
+    out.add(trimmed);
+  }
+  return out;
+}
+
+/**
+ * Default warn destination — plain `console.warn`.
+ */
+function defaultWarn(
+  message: string,
+  context: Record<string, unknown>,
+): void {
+  console.warn(message, context);
+}

package/src/sync/index.ts

@@ -15,5 +15,60 @@ export {
 } from "./push-event.js";
 export type { PushEvent, PushEventDecodeIssue } from "./push-event.js";
 
-export { NoopPushTransport } from "./push-transport.js";
-export type { PushTransport } from "./push-transport.js";
+export { NoopPushTransport, HttpPushTransport } from "./push-transport.js";
+export type {
+  PushTransport,
+  HttpPushTransportOptions,
+  AuthTokenSource,
+  FetchLike,
+} from "./push-transport.js";
+
+export {
+  FEATURE_FLAG_TENANTS_ENV_VAR,
+  FEATURE_FLAG_LEGACY_GLOBAL_ENV_VAR,
+  EnvTenantListFlagProvider,
+  StaticFlagProvider,
+  defaultFlagProvider,
+} from "./feature-flags.js";
+export type {
+  EventDrivenPushFlagProvider,
+  FlagProviderWarnFn,
+  EnvTenantListFlagProviderOptions,
+} from "./feature-flags.js";
+
+export {
+  NoopPushReceiver,
+  SqsPushReceiver,
+  InMemoryFanout,
+  InMemoryPushReceiver,
+  createPushReceiver,
+  DEFAULT_RECEIVER_DISPOSE_DRAIN_MS,
+  DEFAULT_WAIT_TIME_SECONDS,
+  DEFAULT_MAX_MESSAGES,
+} from "./push-receiver.js";
+export type {
+  PushReceiver,
+  PushReceiverContext,
+  SyncEngineFn,
+  PublishMetricFn,
+  ReceiverLogger,
+  SqsClientLike,
+  SqsMessageLike,
+  SqsPushReceiverOptions,
+  InMemoryPushReceiverOptions,
+  CreatePushReceiverOptions,
+} from "./push-receiver.js";
+
+export {
+  SYNC_METRIC_NAMESPACE,
+  SYNC_LATENCY_METRIC_NAME,
+  _setSyncCloudWatchClient,
+  publishSyncLatencyMetric,
+} from "./metrics.js";
+export type {
+  SyncLatencyMetric,
+  PublishSyncLatencyMetricOptions,
+} from "./metrics.js";
+
+export { createLogger } from "./logger.js";
+export type { CreateLoggerOptions, Logger } from "./logger.js";

package/src/sync/logger.test.ts

@@ -0,0 +1,241 @@
+/**
+ * US-011 — unit tests for the pino logger factory + the 3-log diagnostic
+ * chain's CLIENT-side correlated fields.
+ *
+ * The PRD requires three correlated logs sharing the same `sequenceNumber`:
+ *   1. `watcher.emit`    — client push (US-008 PushEventEmitter)   [client]
+ *   2. `push.receive`    — server (hq-pro)                          [server]
+ *   3. `fanout.receive`  — client fanout-receive (US-009 receiver)  [client]
+ *
+ * This package owns the two CLIENT-side links. These tests prove:
+ *   - `createLogger` stamps the `component` tag on every line and respects
+ *     level + injected destination.
+ *   - the watcher emits `event=watcher.emit` carrying `sequenceNumber`.
+ *   - the receiver emits `event=fanout.receive` carrying the SAME
+ *     `sequenceNumber` → an operator can join the two client links (and the
+ *     server `push.receive` line, which carries the same key) end-to-end.
+ */
+
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Writable } from "node:stream";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { createLogger } from "./logger.js";
+import { encodePushEvent, type PushEvent } from "./push-event.js";
+import { NoopPushTransport } from "./push-transport.js";
+import { StaticFlagProvider } from "./feature-flags.js";
+import {
+  SqsPushReceiver,
+  type SqsClientLike,
+  type SqsMessageLike,
+} from "./push-receiver.js";
+import { PushEventEmitter, type TreeChangeBatch } from "../watcher.js";
+
+const TENANT = "tenant-indigo";
+
+// ── Capture stream + JSON line reader ─────────────────────────────────────────
+
+function captureStream(): { stream: Writable; lines: () => Array<Record<string, unknown>> } {
+  const chunks: string[] = [];
+  const stream = new Writable({
+    write(chunk, _enc, cb) {
+      chunks.push(
+        typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8"),
+      );
+      cb();
+    },
+  });
+  return {
+    stream,
+    lines: () =>
+      chunks
+        .join("")
+        .split("\n")
+        .filter((s) => s.length > 0)
+        .map((s) => JSON.parse(s) as Record<string, unknown>),
+  };
+}
+
+async function until(predicate: () => boolean, timeoutMs = 1000): Promise<void> {
+  const start = Date.now();
+  while (!predicate()) {
+    if (Date.now() - start > timeoutMs) throw new Error("until() timed out");
+    await new Promise((r) => setTimeout(r, 5));
+  }
+}
+
+function makeEvent(overrides: Partial<PushEvent> = {}): PushEvent {
+  return {
+    relativePath: "companies/indigo/notes.md",
+    contentHash:
+      "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    mtime: "2026-05-21T12:34:56.000Z",
+    originDeviceId: "device-A",
+    originTenantId: TENANT,
+    sequenceNumber: 1,
+    eventTimestamp: "2026-05-21T12:34:56.000Z",
+    ...overrides,
+  };
+}
+
+class OneBatchSqs implements SqsClientLike {
+  private batch: SqsMessageLike[] | null;
+  constructor(messages: SqsMessageLike[]) {
+    this.batch = messages;
+  }
+  async receiveMessage(args: {
+    queueUrl: string;
+    maxMessages: number;
+    waitTimeSeconds: number;
+    signal: AbortSignal;
+  }): Promise<{ messages: SqsMessageLike[] }> {
+    if (this.batch) {
+      const b = this.batch;
+      this.batch = null;
+      return { messages: b };
+    }
+    return new Promise((resolve) => {
+      if (args.signal.aborted) return resolve({ messages: [] });
+      const t = setTimeout(() => resolve({ messages: [] }), 5);
+      (t as { unref?: () => void }).unref?.();
+      args.signal.addEventListener(
+        "abort",
+        () => {
+          clearTimeout(t);
+          resolve({ messages: [] });
+        },
+        { once: true },
+      );
+    });
+  }
+  async deleteMessage(): Promise<void> {
+    /* no-op */
+  }
+}
+
+// ── createLogger ──────────────────────────────────────────────────────────────
+
+describe("createLogger", () => {
+  it("stamps the `component` tag on every line", () => {
+    const { stream, lines } = captureStream();
+    const log = createLogger({
+      component: "sync-watcher",
+      destination: stream,
+      level: "info",
+    });
+    log.info({ event: "x" }, "hello");
+    log.warn({ event: "y" }, "world");
+    const out = lines();
+    expect(out).toHaveLength(2);
+    expect(out.every((l) => l.component === "sync-watcher")).toBe(true);
+  });
+
+  it("respects the injected level (debug below info is dropped at info)", () => {
+    const { stream, lines } = captureStream();
+    const log = createLogger({
+      component: "c",
+      destination: stream,
+      level: "info",
+    });
+    log.debug({ event: "d" }, "dropped");
+    log.info({ event: "i" }, "kept");
+    const out = lines();
+    expect(out).toHaveLength(1);
+    expect(out[0]!.event).toBe("i");
+  });
+});
+
+// ── 3-log chain: client-side correlated fields share sequenceNumber ───────────
+
+describe("3-log diagnostic chain (client links share sequenceNumber)", () => {
+  let tmp: string | undefined;
+  let receiver: SqsPushReceiver | undefined;
+
+  afterEach(async () => {
+    if (receiver) {
+      await receiver.dispose();
+      receiver = undefined;
+    }
+    if (tmp) {
+      await rm(tmp, { recursive: true, force: true });
+      tmp = undefined;
+    }
+  });
+
+  it("watcher.emit and fanout.receive carry the SAME sequenceNumber", async () => {
+    const SEQ = 4242;
+
+    // ── Link 1: watcher.emit (client push side) ──────────────────────────
+    tmp = await mkdtemp(join(tmpdir(), "us011-logger-"));
+    const absPath = join(tmp, "notes.md");
+    await writeFile(absPath, "hello world", "utf8");
+
+    const watcherCap = captureStream();
+    const watcherLog = createLogger({
+      component: "sync-watcher",
+      destination: watcherCap.stream,
+      level: "info",
+    });
+
+    const emitter = new PushEventEmitter({
+      originTenantId: TENANT,
+      originDeviceId: "device-A",
+      transport: new NoopPushTransport(),
+      flagProvider: new StaticFlagProvider([TENANT]),
+      getSequenceNumber: () => SEQ,
+      logger: watcherLog,
+    });
+
+    const batch: TreeChangeBatch = {
+      paths: new Map([[absPath, "companies/indigo/notes.md"]]),
+    };
+    await emitter.emitForBatch(batch);
+
+    const emitLine = watcherCap
+      .lines()
+      .find((l) => l.event === "watcher.emit");
+    expect(emitLine).toBeDefined();
+    expect(emitLine!.sequenceNumber).toBe(SEQ);
+    expect(emitLine!.component).toBe("sync-watcher");
+
+    // ── Link 3: fanout.receive (client receive side) ─────────────────────
+    const receiverCap = captureStream();
+    const receiverLog = createLogger({
+      component: "sync-receiver",
+      destination: receiverCap.stream,
+      level: "info",
+    });
+    const event = makeEvent({ sequenceNumber: SEQ });
+
+    receiver = new SqsPushReceiver({
+      tenantId: TENANT,
+      queueUrl: "https://sqs.local/q",
+      sqs: new OneBatchSqs([{ Body: encodePushEvent(event), ReceiptHandle: "rh" }]),
+      syncFn: async () => {
+        /* success */
+      },
+      enabled: true,
+      logger: receiverLog,
+      // Inject a no-op metric so no real AWS is touched.
+      publishMetric: async () => {},
+    });
+
+    await receiver.start();
+    await until(() =>
+      receiverCap.lines().some((l) => l.event === "fanout.receive"),
+    );
+
+    const recvLine = receiverCap
+      .lines()
+      .find((l) => l.event === "fanout.receive");
+    expect(recvLine).toBeDefined();
+    expect(recvLine!.sequenceNumber).toBe(SEQ);
+    expect(recvLine!.component).toBe("sync-receiver");
+
+    // ── The join key matches across both client links ────────────────────
+    expect(emitLine!.sequenceNumber).toBe(recvLine!.sequenceNumber);
+  });
+});

package/src/sync/logger.ts

@@ -0,0 +1,79 @@
+/**
+ * Component-tagged logger factory for the hq-cloud sync subsystem
+ * (project event-driven-sync-menubar US-011).
+ *
+ * Every log line emitted from a sync module SHOULD carry a `component` field
+ * so operators can grep / route by subsystem in the aggregated JSON stream.
+ * `createLogger({ component: "sync-watcher" })` is the canonical entry point —
+ * modules call this once at module-load time and use the returned logger for
+ * the lifetime of the process.
+ *
+ * The 3-log diagnostic chain (US-011)
+ * ───────────────────────────────────
+ * Three correlated log lines share a single `sequenceNumber` join key so an
+ * operator can walk one event end-to-end:
+ *   1. `event=watcher.emit`    — client push side (US-008 PushEventEmitter)
+ *   2. `event=push.receive`    — server side (hq-pro; context only here)
+ *   3. `event=fanout.receive`  — client fanout-receive side (US-009 receiver)
+ * The watcher.emit + fanout.receive halves live in this client package; the
+ * push.receive half lives server-side in hq-pro. All three stamp the same
+ * `sequenceNumber` for log-chain correlation.
+ *
+ * Output format: pino's default newline-delimited JSON. No transports, no
+ * pretty-printing — the daemon consumes the raw JSON stream. (Operators who
+ * want pretty output pipe through `pino-pretty` themselves.)
+ *
+ * Destination injection: tests pass a `destination` stream so they can capture
+ * log lines and assert on them. Production callers omit it and pino defaults
+ * to `process.stdout`.
+ *
+ * Adapted from indigoai-us/hq-pro PR #112 (src/sync/logger.ts) into
+ * @indigoai-us/hq-cloud (Path B).
+ */
+
+import {
+  pino,
+  type DestinationStream,
+  type Level,
+  type Logger,
+  type LoggerOptions,
+} from "pino";
+
+export interface CreateLoggerOptions {
+  /**
+   * Component tag stamped on every log line as `"component": <value>`.
+   * Required so daemon / watcher / receiver lines never appear in the stream
+   * untagged.
+   */
+  component: string;
+  /**
+   * Optional pino level. Default: pino's own default (`info`). Set via
+   * `LOG_LEVEL` env var, command-line flag, or test injection.
+   */
+  level?: Level;
+  /**
+   * Optional pino destination. Default: `process.stdout`. Tests inject a
+   * memory stream here to capture lines for assertion.
+   */
+  destination?: DestinationStream;
+}
+
+/**
+ * Build a pino logger pre-bound to a `component` tag.
+ *
+ * Use this — not a bare `pino()` call — so every sync module's log lines carry
+ * the tag uniformly. Adding more bound fields (e.g. `deviceId`, `tenantId`) is
+ * a `logger.child({ ... })` call away.
+ */
+export function createLogger(opts: CreateLoggerOptions): Logger {
+  const { component, level, destination } = opts;
+  const pinoOpts: LoggerOptions = {
+    base: { component },
+    ...(level === undefined ? {} : { level }),
+  };
+  return destination === undefined
+    ? pino(pinoOpts)
+    : pino(pinoOpts, destination);
+}
+
+export type { Logger } from "pino";