@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.3.0

package/src/ui/sse-enhancer.js

@@ -0,0 +1,149 @@
+/**
+ * SSE Enhancer
+ *
+ * Provides a factory createSSEEnhancer({ readJobFn, sseRegistry, debounceMs })
+ * and a singleton export sseEnhancer created with default dependencies.
+ *
+ * Behavior:
+ *  - handleJobChange({ jobId, category, filePath })
+ *    - debounce per jobId (default 200ms)
+ *    - after debounce, call readJobFn(jobId) to obtain latest detail
+ *    - if read succeeds (ok), broadcast { type: "job:updated", data: detail }
+ *    - if read fails, do not broadcast
+ *  - getPendingCount() returns number of pending timers
+ *  - cleanup() clears timers
+ */
+
+import { detectJobChange } from "./job-change-detector.js";
+import { transformJobStatus } from "./transformers/status-transformer.js";
+import { transformJobListForAPI } from "./transformers/list-transformer.js";
+
+export function createSSEEnhancer({
+  readJobFn,
+  sseRegistry,
+  debounceMs = 200,
+} = {}) {
+  if (!readJobFn) {
+    throw new Error("readJobFn is required");
+  }
+  if (!sseRegistry) {
+    throw new Error("sseRegistry is required");
+  }
+
+  const pending = new Map(); // jobId -> timeoutId
+  // Track jobIds we've already emitted a creation event for so we can
+  // emit "job:created" on first successful read, then "job:updated" thereafter.
+  const seen = new Set();
+
+  async function runJobUpdate(jobId) {
+    try {
+      const res = await readJobFn(jobId);
+      if (!res || !res.ok) {
+        return;
+      }
+
+      // First transform to detailed job schema, then to list schema for SSE
+      const detailedJob = transformJobStatus(
+        res.data || {},
+        jobId,
+        res.location
+      );
+
+      if (!detailedJob) {
+        console.warn(
+          `[SSEEnhancer] Failed to transform job ${jobId} for broadcast`
+        );
+        return;
+      }
+
+      // Transform to canonical list schema to match /api/jobs item shape
+      const listJob = transformJobListForAPI([detailedJob], {
+        includePipelineMetadata: true,
+      });
+      const canonicalJob = listJob.length > 0 ? listJob[0] : null;
+
+      if (!canonicalJob) {
+        console.warn(
+          `[SSEEnhancer] Failed to transform job ${jobId} to list schema for broadcast`
+        );
+        return;
+      }
+
+      // If this is the first successful read for this jobId, emit a
+      // "job:created" event so clients can observe new jobs immediately.
+      // Subsequent successful reads will emit "job:updated".
+      try {
+        const alreadySeen = seen.has(jobId);
+        const type = alreadySeen ? "job:updated" : "job:created";
+        sseRegistry.broadcast({ type, data: canonicalJob });
+        if (!alreadySeen) {
+          seen.add(jobId);
+        }
+      } catch (broadcastErr) {
+        // If broadcasting fails for any reason, swallow to avoid crashing the enhancer
+      }
+    } catch (err) {
+      // swallow errors - do not broadcast
+      return;
+    } finally {
+      pending.delete(jobId);
+    }
+  }
+
+  function handleJobChange(change) {
+    if (!change || !change.jobId) return;
+
+    const jobId = change.jobId;
+
+    // debounce/coalesce per jobId
+    if (pending.has(jobId)) {
+      clearTimeout(pending.get(jobId));
+    }
+
+    const t = setTimeout(() => {
+      pending.delete(jobId);
+      // fire async update
+      void runJobUpdate(jobId);
+    }, debounceMs);
+
+    pending.set(jobId, t);
+  }
+
+  function getPendingCount() {
+    return pending.size;
+  }
+
+  function cleanup() {
+    for (const [_, t] of pending) {
+      clearTimeout(t);
+    }
+    pending.clear();
+  }
+
+  return {
+    handleJobChange,
+    getPendingCount,
+    cleanup,
+  };
+}
+
+// Singleton using default dependencies if available
+// Try to import default sseRegistry and readJobFn lazily to avoid cycles
+let sseEnhancer = null;
+try {
+  // eslint-disable-next-line import/no-mutable-exports
+  const { sseRegistry } = await import("./sse.js");
+  const { readJob } = await import("./job-reader.js");
+  sseEnhancer = createSSEEnhancer({
+    readJobFn: readJob,
+    sseRegistry,
+    debounceMs: 200,
+  });
+} catch (err) {
+  // In test environments, consumers will create their own using the factory
+  // Leave sseEnhancer as null if dependencies not available
+  // eslint-disable-next-line no-console
+  console.warn("sseEnhancer singleton not initialized:", err?.message || err);
+}
+
+export { sseEnhancer };

package/src/ui/sse.js

@@ -0,0 +1,204 @@
+/**
+ * Server-Sent Events (SSE) registry for broadcasting to connected clients.
+ * Compatibility-focused: tolerant of mock clients, supports typed & untyped
+ * broadcasts, dead-client cleanup, optional heartbeats, and optional initial ping.
+ */
+
+import {
+  setInterval as nodeSetInterval,
+  clearInterval as nodeClearInterval,
+} from "node:timers";
+
+/**
+ * Create an SSE registry.
+ * @param {{ heartbeatMs?: number, sendInitialPing?: boolean }} [opts]
+ *   - heartbeatMs: send periodic keep-alive comments (default 15000)
+ *   - sendInitialPing: write ': connected\n\n' on addClient (default false)
+ */
+export function createSSERegistry({
+  heartbeatMs = 15000,
+  sendInitialPing = false,
+} = {}) {
+  const clients = new Set(); // Set<{res: http.ServerResponse | {write:Function, end?:Function, on?:Function}, jobId?: string}>
+  let heartbeatTimer = null;
+
+  function _startHeartbeat() {
+    if (!heartbeatMs || heartbeatTimer) return;
+    heartbeatTimer = nodeSetInterval(() => {
+      for (const client of clients) {
+        const res = client.res || client;
+        try {
+          if (typeof res.write === "function") {
+            // Comment line per SSE spec; keeps proxies from buffering/closing.
+            res.write(`: keep-alive\n\n`);
+          }
+        } catch {
+          // Will be cleaned on next broadcast or below
+          try {
+            typeof res.end === "function" && res.end();
+          } catch {}
+          clients.delete(client);
+        }
+      }
+    }, heartbeatMs);
+  }
+
+  /**
+   * Add a client response to the registry and send headers if possible.
+   * Accepts real http.ServerResponse or a test mock {write(), [writeHead], [end], [on]}.
+   * @param {any} res
+   * @param {Object} [metadata] - Optional metadata for the client (e.g., { jobId })
+   */
+  function addClient(res, metadata = {}) {
+    const client = { res, ...metadata };
+
+    try {
+      if (typeof res.writeHead === "function") {
+        res.writeHead(200, {
+          "Content-Type": "text/event-stream",
+          "Cache-Control": "no-cache",
+          Connection: "keep-alive",
+          "X-Accel-Buffering": "no", // helps with nginx buffering
+        });
+      }
+      if (sendInitialPing && typeof res.write === "function") {
+        // Initial ping so EventSource 'open' resolves quickly (server mode)
+        res.write(`: connected\n\n`);
+      }
+    } catch {
+      // If headers or initial write fail, avoid crashing tests—still register client
+    }
+
+    clients.add(client);
+    _startHeartbeat();
+
+    if (res && typeof res.on === "function") {
+      res.on("close", () => {
+        clients.delete(client);
+        if (clients.size === 0 && heartbeatTimer) {
+          nodeClearInterval(heartbeatTimer);
+          heartbeatTimer = null;
+        }
+      });
+    }
+  }
+
+  /**
+   * Remove a client (and end its response if possible).
+   * @param {any} res
+   */
+  function removeClient(res) {
+    // Find client by response object (handle both old and new structure)
+    let clientToRemove = null;
+    for (const client of clients) {
+      const clientRes = client.res || client;
+      if (clientRes === res) {
+        clientToRemove = client;
+        break;
+      }
+    }
+
+    if (!clientToRemove) return;
+
+    const clientRes = clientToRemove.res || clientToRemove;
+    try {
+      typeof clientRes.end === "function" && clientRes.end();
+    } catch {}
+    clients.delete(clientToRemove);
+    if (clients.size === 0 && heartbeatTimer) {
+      nodeClearInterval(heartbeatTimer);
+      heartbeatTimer = null;
+    }
+  }
+
+  /**
+   * Broadcast SSE. Supports:
+   *  - broadcast({ type, data })
+   *  - broadcast("eventName", data)
+   *  - broadcast(data)  // untyped 'message' event (data-only)
+   */
+  function broadcast(arg1, arg2) {
+    /** @type {string | undefined} */
+    let type;
+    /** @type {any} */
+    let data;
+
+    if (typeof arg1 === "string") {
+      type = arg1;
+      data = arg2;
+    } else if (
+      arg1 &&
+      typeof arg1 === "object" &&
+      ("type" in arg1 || "data" in arg1)
+    ) {
+      type = arg1.type;
+      data = arg1.data;
+    } else {
+      type = undefined;
+      data = arg1;
+    }
+
+    const payload =
+      typeof data === "string" ? data : JSON.stringify(data ?? {});
+    const dead = [];
+
+    for (const client of clients) {
+      const res = client.res || client;
+
+      // Apply jobId filtering: if data has a jobId and client has a jobId, only send if they match
+      if (data && data.jobId && client.jobId) {
+        if (data.jobId !== client.jobId) {
+          continue; // Skip this client - event is for a different job
+        }
+      }
+
+      try {
+        if (typeof res.write !== "function") {
+          dead.push(client);
+          continue;
+        }
+        if (type) {
+          res.write(`event: ${type}\n`);
+        }
+        res.write(`data: ${payload}\n\n`);
+      } catch {
+        dead.push(client);
+      }
+    }
+
+    // Clean up dead clients
+    for (const client of dead) {
+      const clientRes = client.res || client;
+      try {
+        typeof clientRes.end === "function" && clientRes.end();
+      } catch {}
+      clients.delete(client);
+    }
+  }
+
+  function getClientCount() {
+    return clients.size;
+  }
+
+  function closeAll() {
+    for (const client of clients) {
+      const res = client.res || client;
+      try {
+        typeof res.end === "function" && res.end();
+      } catch {}
+    }
+    clients.clear();
+    if (heartbeatTimer) {
+      nodeClearInterval(heartbeatTimer);
+      heartbeatTimer = null;
+    }
+  }
+
+  return { addClient, removeClient, broadcast, getClientCount, closeAll };
+}
+
+// Export a singleton used by the server: keep initial ping enabled for real EventSource clients
+export const sseRegistry = createSSERegistry({
+  heartbeatMs: 15000,
+  sendInitialPing: true,
+});

package/src/ui/state-snapshot.js

@@ -0,0 +1,252 @@
+/**
+ * composeStateSnapshot
+ *
+ * Pure function that composes a minimal snapshot object for client bootstrap.
+ *
+ * Signature:
+ *   composeStateSnapshot(options?)
+ *
+ * options:
+ *   - jobs: Array of job-like objects (optional)
+ *   - meta: Object with metadata (optional)
+ *   - transformJob: optional function(job) -> normalizedJob to customize normalization
+ *
+ * Returns:
+ *   {
+ *     jobs: [{ id: string, status: string|null, summary: string|null, updatedAt: string|null }, ...],
+ *     meta: { version: string|number, lastUpdated: string }
+ *   }
+ *
+ * Notes:
+ * - This function is pure and does not perform I/O or mutate inputs.
+ * - It is defensive about input shapes and provides sensible defaults.
+ */
+
+export function composeStateSnapshot(options = {}) {
+  const { jobs = [], meta, transformJob } = options || {};
+
+  // Ensure we don't mutate input arrays/objects; work on copies.
+  const inputJobs = Array.isArray(jobs) ? jobs.slice() : [];
+
+  const normalizedJobs = inputJobs.map((j) => {
+    // If caller provided a transformJob, prefer its output but still normalize missing fields.
+    if (typeof transformJob === "function") {
+      const t = transformJob(j) || {};
+      return {
+        jobId: t.jobId != null ? String(t.jobId) : null,
+        status: t.status ?? null,
+        title: t.title ?? null,
+        updatedAt: t.updatedAt ?? t.lastUpdated ?? null,
+      };
+    }
+
+    // Best-effort normalization for common fields seen in this repo.
+    // Prefer canonical fields, fallback to legacy fields.
+    const rawId = j?.jobId ?? j?.id ?? j?.uid ?? j?.job_id ?? j?.jobID ?? null;
+    const rawStatus = j?.status ?? j?.state ?? j?.s ?? null;
+    const rawTitle = j?.title ?? j?.name ?? j?.summary ?? null;
+    const rawUpdated = j?.updatedAt ?? j?.lastUpdated ?? j?.updated_at ?? null;
+
+    return {
+      jobId: rawId != null ? String(rawId) : null,
+      status: rawStatus ?? null,
+      title: rawTitle ?? null,
+      updatedAt: rawUpdated ?? null,
+    };
+  });
+
+  const resultMeta = {
+    version: meta?.version ?? meta ?? "1",
+    lastUpdated: meta?.lastUpdated ?? new Date().toISOString(),
+  };
+
+  return {
+    jobs: normalizedJobs,
+    meta: resultMeta,
+  };
+}
+
+/**
+ * Build a minimal snapshot composed from files on disk.
+ *
+ * deps (optional) - injected dependencies for testability:
+ *  - listAllJobs() -> { current: [], complete: [] }
+ *  - readJob(jobId, location) -> { ok:true, data, location, path } or error envelope
+ *  - transformMultipleJobs(readResults) -> Array<job>
+ *  - now() -> () => new Date()
+ *  - paths -> optional resolved PATHS object
+ *
+ * Behavior:
+ *  - Reads job ids from current then complete
+ *  - Reads each job (attach jobId & location to the read result)
+ *  - Transforms reads, dedupes (prefer current), sorts, maps to minimal job fields
+ *  - Returns { jobs: [...], meta: { version: "1", lastUpdated } }
+ */
+export async function buildSnapshotFromFilesystem(deps = {}) {
+  // Prefer injected deps; fall back to local modules for convenience.
+  const { listAllJobs, readJob, transformMultipleJobs, now, paths } = deps;
+
+  // Lazy-import fallbacks when deps not provided.
+  // These imports are intentionally dynamic to avoid circular/boot-time issues in tests.
+  const jobScanner = listAllJobs
+    ? null
+    : await import("./job-scanner.js").then((m) => m).catch(() => null);
+  const jobReader = readJob
+    ? null
+    : await import("./job-reader.js").then((m) => m).catch(() => null);
+  const statusTransformer = transformMultipleJobs
+    ? null
+    : await import("./transformers/status-transformer.js")
+        .then((m) => m)
+        .catch(() => null);
+  const configBridge = await import("./config-bridge.js")
+    .then((m) => m)
+    .catch(() => null);
+
+  const _listAllJobs = listAllJobs || (jobScanner && jobScanner.listAllJobs);
+  const _readJob = readJob || (jobReader && jobReader.readJob);
+  const _transformMultipleJobs =
+    transformMultipleJobs ||
+    (statusTransformer && statusTransformer.transformMultipleJobs);
+  const _now = typeof now === "function" ? now : () => new Date();
+  const _paths = paths || (configBridge && configBridge.PATHS) || null;
+
+  if (typeof _listAllJobs !== "function") {
+    throw new Error("Missing dependency: listAllJobs");
+  }
+  if (typeof _readJob !== "function") {
+    throw new Error("Missing dependency: readJob");
+  }
+  if (typeof _transformMultipleJobs !== "function") {
+    throw new Error("Missing dependency: transformMultipleJobs");
+  }
+
+  // 1) Enumerate jobs
+  const all = (await _listAllJobs()) || {};
+  const currentIds = Array.isArray(all.current) ? all.current : [];
+  const completeIds = Array.isArray(all.complete) ? all.complete : [];
+
+  // Build read order: current first, then complete
+  const toRead = [
+    ...currentIds.map((id) => ({ id, location: "current" })),
+    ...completeIds.map((id) => ({ id, location: "complete" })),
+  ];
+
+  // If no jobs, return empty snapshot
+  if (toRead.length === 0) {
+    const meta = { version: "1", lastUpdated: _now().toISOString() };
+    return { jobs: [], meta };
+  }
+
+  // 2) Read jobs concurrently (Promise.all is acceptable for demo-scale)
+  const readPromises = toRead.map(async ({ id, location }) => {
+    try {
+      // Call readJob with (id, location) - extra arg is ignored by implementations that don't accept it.
+      const res = await _readJob(id, location);
+      // Ensure we attach jobId and location for downstream transformers
+      if (res && typeof res === "object") {
+        return { ...res, jobId: id, location };
+      }
+      // If readJob returns non-object, wrap as error
+      return {
+        ok: false,
+        code: "read_error",
+        message: "Invalid read result",
+        jobId: id,
+        location,
+      };
+    } catch (err) {
+      console.warn(
+        `Error reading job ${id} in ${location}: ${err?.message || String(err)}`
+      );
+      return {
+        ok: false,
+        code: "read_exception",
+        message: err?.message || String(err),
+        jobId: id,
+        location,
+      };
+    }
+  });
+
+  const readResults = await Promise.all(readPromises);
+
+  // 3) Transform reads into canonical job objects
+  const transformed = _transformMultipleJobs(readResults || []);
+
+  // 4) Dedupe by id, preferring earlier entries (current before complete)
+  const seen = new Set();
+  const deduped = [];
+  for (const j of transformed || []) {
+    if (!j || !j.id) continue;
+    if (seen.has(j.id)) continue;
+    seen.add(j.id);
+    deduped.push(j);
+  }
+
+  // 5) Sorting
+  // Location weight: current=0, complete=1
+  const locWeight = (loc) => (loc === "current" ? 0 : 1);
+
+  // Status priority from Constants if available
+  const statusOrder = (configBridge &&
+    configBridge.Constants &&
+    configBridge.Constants.STATUS_ORDER) || [
+    "error",
+    "running",
+    "complete",
+    "pending",
+  ];
+
+  const statusPriority = (s) => {
+    const idx = statusOrder.indexOf(s);
+    return idx === -1 ? statusOrder.length : idx;
+  };
+
+  const getTime = (j) => {
+    const ua =
+      j && j.updatedAt ? j.updatedAt : j && j.createdAt ? j.createdAt : null;
+    if (!ua) return null;
+    // Ensure we compare ISO strings or timestamps consistently
+    const t = Date.parse(ua);
+    return Number.isNaN(t) ? null : t;
+  };
+
+  deduped.sort((a, b) => {
+    // 1) location
+    const lw = locWeight(a.location) - locWeight(b.location);
+    if (lw !== 0) return lw;
+
+    // 2) status priority (lower is higher priority)
+    const sp = statusPriority(a.status) - statusPriority(b.status);
+    if (sp !== 0) return sp;
+
+    // 3) updatedAt descending (newer first)
+    const ta = getTime(a);
+    const tb = getTime(b);
+    if (ta !== null && tb !== null && ta !== tb) return tb - ta;
+    if (ta !== null && tb === null) return -1;
+    if (ta === null && tb !== null) return 1;
+
+    // 4) id ascending
+    return String(a.id).localeCompare(String(b.id));
+  });
+
+  // 6) Map to minimal snapshot fields (canonical schema)
+  const snapshotJobs = deduped.map((j) => ({
+    jobId: j.jobId || j.id,
+    title: j.title || j.name || "Unnamed Job",
+    status: j.status || "pending",
+    progress:
+      typeof j.progress === "number" && Number.isFinite(j.progress)
+        ? j.progress
+        : 0,
+    createdAt: j.createdAt || null,
+    updatedAt: j.updatedAt || j.createdAt || null,
+    location: j.location || "current",
+  }));
+
+  const meta = { version: "1", lastUpdated: _now().toISOString() };
+
+  return { jobs: snapshotJobs, meta };
+}