@ryanfw/prompt-orchestration-pipeline 0.11.0 → 0.13.0

package/src/ui/endpoints/schema-file-endpoint.js

@@ -0,0 +1,105 @@
+/**
+ * Schema file endpoint
+ *
+ * Exports:
+ *  - handleSchemaFileRequest(req, res) -> HTTP handler function
+ *
+ * Serves schema and sample JSON files for pipeline tasks.
+ */
+
+import { getPipelineConfig } from "../../core/config.js";
+import { sendJson } from "../utils/http-utils.js";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+
+/**
+ * HTTP handler for schema file requests.
+ *
+ * @param {http.IncomingMessage} req - HTTP request object
+ * @param {http.ServerResponse} res - HTTP response object
+ */
+export async function handleSchemaFileRequest(req, res) {
+  const { slug, fileName } = req.params;
+  const { type } = req.query;
+
+  // Validate slug parameter
+  if (!slug || typeof slug !== "string" || !/^[A-Za-z0-9_-]+$/.test(slug)) {
+    return sendJson(res, 400, {
+      ok: false,
+      code: "invalid_params",
+      message: "Invalid slug parameter",
+    });
+  }
+
+  // Validate fileName parameter (no path traversal)
+  if (
+    !fileName ||
+    typeof fileName !== "string" ||
+    !/^[A-Za-z0-9_.-]+$/.test(fileName)
+  ) {
+    return sendJson(res, 400, {
+      ok: false,
+      code: "invalid_params",
+      message: "Invalid fileName parameter",
+    });
+  }
+
+  // Validate type parameter
+  if (type !== "schema" && type !== "sample") {
+    return sendJson(res, 400, {
+      ok: false,
+      code: "invalid_params",
+      message: "Invalid type parameter - must be 'schema' or 'sample'",
+    });
+  }
+
+  try {
+    // Get pipeline configuration
+    let pipelineConfig;
+    try {
+      pipelineConfig = getPipelineConfig(slug);
+    } catch (error) {
+      return sendJson(res, 404, {
+        ok: false,
+        code: "not_found",
+        message: `Pipeline '${slug}' not found in registry`,
+      });
+    }
+
+    const pipelineDir = path.dirname(pipelineConfig.pipelineJsonPath);
+    const baseName = path.parse(fileName).name;
+    const schemaFilePath = path.join(
+      pipelineDir,
+      "schemas",
+      `${baseName}.${type}.json`
+    );
+
+    // Read schema file
+    let fileContents;
+    try {
+      fileContents = await fs.readFile(schemaFilePath, "utf8");
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return sendJson(res, 404, {
+          ok: false,
+          code: "not_found",
+          message: "Schema file not found",
+        });
+      }
+      throw error;
+    }
+
+    // Return raw file contents (not parsed)
+    return sendJson(res, 200, {
+      ok: true,
+      data: fileContents,
+    });
+  } catch (error) {
+    console.error("handleSchemaFileRequest unexpected error:", error);
+    return sendJson(res, 500, {
+      ok: false,
+      code: "internal_error",
+      message: "Internal server error",
+    });
+  }
+}

package/src/ui/endpoints/sse-endpoints.js

@@ -0,0 +1,223 @@
+/**
+ * SSE (Server-Sent Events) and state management endpoints
+ */
+
+import { sseRegistry } from "../sse.js";
+import { sendJson } from "../utils/http-utils.js";
+
+/**
+ * Decorate a change object with jobId and lifecycle information
+ */
+function decorateChangeWithJobId(change) {
+  if (!change || typeof change !== "object") return change;
+  const normalizedPath = String(change.path || "").replace(/\\/g, "/");
+  const match = normalizedPath.match(
+    /pipeline-data\/(current|complete|pending|rejected)\/([^/]+)/
+  );
+  if (!match) {
+    return change;
+  }
+  return {
+    ...change,
+    lifecycle: match[1],
+    jobId: match[2],
+  };
+}
+
+/**
+ * Prioritize job status changes from a list of changes
+ */
+function prioritizeJobStatusChange(changes = []) {
+  const normalized = changes.map((change) => decorateChangeWithJobId(change));
+  const statusChange = normalized.find(
+    (change) =>
+      typeof change?.path === "string" &&
+      /tasks-status\.json$/.test(change.path)
+  );
+  return statusChange || normalized[0] || null;
+}
+
+/**
+ * Broadcast state update to all SSE clients
+ *
+ * NOTE: Per plan, SSE should emit compact, incremental events rather than
+ * streaming full application state. Use /api/state for full snapshot
+ * retrieval on client bootstrap. This function will emit only the most
+ * recent change when available (type: "state:change") and fall back to a
+ * lightweight summary event if no recent change is present.
+ */
+function broadcastStateUpdate(currentState) {
+  try {
+    const recentChanges = (currentState && currentState.recentChanges) || [];
+    const latest = prioritizeJobStatusChange(recentChanges);
+    console.debug("[Server] Broadcasting state update:", {
+      latest,
+      currentState,
+    });
+    if (latest) {
+      // Emit only the most recent change as a compact, typed event
+      const eventData = { type: "state:change", data: latest };
+      console.debug("[Server] Broadcasting event:", eventData);
+      sseRegistry.broadcast(eventData);
+    } else {
+      // Fallback: emit a minimal summary so clients can observe a state "tick"
+      const eventData = {
+        type: "state:summary",
+        data: {
+          changeCount:
+            currentState && currentState.changeCount
+              ? currentState.changeCount
+              : 0,
+        },
+      };
+      console.debug("[Server] Broadcasting summary event:", eventData);
+      sseRegistry.broadcast(eventData);
+    }
+  } catch (err) {
+    // Defensive: if something unexpected happens, fall back to a lightweight notification
+    try {
+      console.error("[Server] Error in broadcastStateUpdate:", err);
+      sseRegistry.broadcast({
+        type: "state:summary",
+        data: {
+          changeCount:
+            currentState && currentState.changeCount
+              ? currentState.changeCount
+              : 0,
+        },
+      });
+    } catch (fallbackErr) {
+      // Log error to aid debugging; this should never happen unless sseRegistry.broadcast is broken
+      console.error(
+        "Failed to broadcast fallback state summary in broadcastStateUpdate:",
+        fallbackErr
+      );
+    }
+  }
+}
+
+/**
+ * Handle SSE events endpoint (/api/events)
+ */
+function handleSseEvents(req, res, searchParams) {
+  // Parse jobId from query parameters for filtering
+  const jobId = searchParams.get("jobId");
+
+  // Set SSE headers
+  res.writeHead(200, {
+    "Content-Type": "text/event-stream",
+    "Cache-Control": "no-cache",
+    Connection: "keep-alive",
+    "Access-Control-Allow-Origin": "*",
+  });
+
+  // Flush headers immediately
+  res.flushHeaders();
+
+  // Initial full-state is no longer sent over the SSE stream.
+  // Clients should fetch the snapshot from GET /api/state during bootstrap
+  // and then rely on SSE incremental events (state:change/state:summary).
+  // Keep headers flushed; sseRegistry.addClient will optionally send an initial ping.
+  // (Previously sent full state here; removed to reduce SSE payloads.)
+
+  // Add to SSE registry with jobId metadata for filtering
+  sseRegistry.addClient(res, { jobId });
+
+  // Start heartbeat for this connection
+  const heartbeatInterval = setInterval(() => {
+    try {
+      res.write(
+        `event: heartbeat\ndata: ${JSON.stringify({ timestamp: Date.now() })}\n\n`
+      );
+    } catch (err) {
+      // Client disconnected, stop heartbeat
+      clearInterval(heartbeatInterval);
+    }
+  }, 30000);
+
+  // Remove client on disconnect
+  req.on("close", () => {
+    clearInterval(heartbeatInterval);
+    sseRegistry.removeClient(res);
+  });
+}
+
+/**
+ * Handle API state endpoint (/api/state)
+ */
+async function handleApiState(req, res) {
+  if (req.method !== "GET") {
+    sendJson(res, 200, {
+      success: false,
+      error: "Method not allowed",
+      allowed: ["GET"],
+    });
+    return;
+  }
+
+  // Prefer returning in-memory state when available (tests and runtime rely on state.getState()).
+  // If in-memory state is available, return it directly; otherwise fall back to
+  // building a filesystem-backed snapshot for client bootstrap.
+  try {
+    // Dynamically import state to avoid circular dependencies
+    const state = await import("../state.js");
+
+    try {
+      if (state && typeof state.getState === "function") {
+        const inMemory = state.getState();
+        if (inMemory) {
+          sendJson(res, 200, inMemory);
+          return;
+        }
+      }
+    } catch (innerErr) {
+      // If reading in-memory state throws for some reason, fall back to snapshot
+      console.warn("Warning: failed to retrieve in-memory state:", innerErr);
+    }
+
+    // Build a filesystem-backed snapshot for client bootstrap.
+    // Dynamically import the composer and dependencies to avoid circular import issues.
+    const [
+      { buildSnapshotFromFilesystem },
+      jobScannerModule,
+      jobReaderModule,
+      statusTransformerModule,
+      configBridgeModule,
+    ] = await Promise.all([
+      import("../state-snapshot.js"),
+      import("../job-scanner.js").catch(() => null),
+      import("../job-reader.js").catch(() => null),
+      import("../transformers/status-transformer.js").catch(() => null),
+      import("../config-bridge.js").catch(() => null),
+    ]);
+
+    const snapshot = await buildSnapshotFromFilesystem({
+      listAllJobs:
+        jobScannerModule && jobScannerModule.listAllJobs
+          ? jobScannerModule.listAllJobs
+          : undefined,
+      readJob:
+        jobReaderModule && jobReaderModule.readJob
+          ? jobReaderModule.readJob
+          : undefined,
+      transformMultipleJobs:
+        statusTransformerModule && statusTransformerModule.transformMultipleJobs
+          ? statusTransformerModule.transformMultipleJobs
+          : undefined,
+      now: () => new Date(),
+      paths: (configBridgeModule && configBridgeModule.PATHS) || undefined,
+    });
+
+    sendJson(res, 200, snapshot);
+  } catch (err) {
+    console.error("Failed to build /api/state snapshot:", err);
+    sendJson(res, 500, {
+      ok: false,
+      code: "snapshot_error",
+      message: "Failed to build state snapshot",
+      details: err && err.message ? err.message : String(err),
+    });
+  }
+}
+
+export { handleSseEvents, handleApiState, broadcastStateUpdate };

package/src/ui/endpoints/state-endpoint.js

@@ -0,0 +1,85 @@
+/**
+ * Handle GET /api/state endpoint
+ */
+import * as state from "../state.js";
+import { sendJson } from "../utils/http-utils.js";
+
+export async function handleApiState(req, res) {
+  if (req.method !== "GET") {
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(
+      JSON.stringify({
+        success: false,
+        error: "Method not allowed",
+        allowed: ["GET"],
+      })
+    );
+    return;
+  }
+
+  // Prefer returning in-memory state when available (tests and runtime rely on state.getState()).
+  // If in-memory state is available, return it directly; otherwise fall back to
+  // building a filesystem-backed snapshot for client bootstrap.
+  try {
+    try {
+      if (state && typeof state.getState === "function") {
+        const inMemory = state.getState();
+        if (inMemory) {
+          res.writeHead(200, { "Content-Type": "application/json" });
+          res.end(JSON.stringify(inMemory));
+          return;
+        }
+      }
+    } catch (innerErr) {
+      // If reading in-memory state throws for some reason, fall back to snapshot
+      console.warn("Warning: failed to retrieve in-memory state:", innerErr);
+    }
+
+    // Build a filesystem-backed snapshot for client bootstrap.
+    // Dynamically import the composer and dependencies to avoid circular import issues.
+    const [
+      { buildSnapshotFromFilesystem },
+      jobScannerModule,
+      jobReaderModule,
+      statusTransformerModule,
+      configBridgeModule,
+    ] = await Promise.all([
+      import("../state-snapshot.js"),
+      import("../job-scanner.js").catch(() => null),
+      import("../job-reader.js").catch(() => null),
+      import("../transformers/status-transformer.js").catch(() => null),
+      import("../config-bridge.js").catch(() => null),
+    ]);
+
+    const snapshot = await buildSnapshotFromFilesystem({
+      listAllJobs:
+        jobScannerModule && jobScannerModule.listAllJobs
+          ? jobScannerModule.listAllJobs
+          : undefined,
+      readJob:
+        jobReaderModule && jobReaderModule.readJob
+          ? jobReaderModule.readJob
+          : undefined,
+      transformMultipleJobs:
+        statusTransformerModule && statusTransformerModule.transformMultipleJobs
+          ? statusTransformerModule.transformMultipleJobs
+          : undefined,
+      now: () => new Date(),
+      paths: (configBridgeModule && configBridgeModule.PATHS) || undefined,
+    });
+
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(snapshot));
+  } catch (err) {
+    console.error("Failed to build /api/state snapshot:", err);
+    res.writeHead(500, { "Content-Type": "application/json" });
+    res.end(
+      JSON.stringify({
+        ok: false,
+        code: "snapshot_error",
+        message: "Failed to build state snapshot",
+        details: err && err.message ? err.message : String(err),
+      })
+    );
+  }
+}

package/src/ui/endpoints/task-analysis-endpoint.js

@@ -0,0 +1,104 @@
+/**
+ * Task analysis endpoint
+ *
+ * Exports:
+ *  - handleTaskAnalysisRequest(req, res) -> HTTP handler function
+ *
+ * Returns task analysis data if available, or null if no analysis file exists.
+ */
+
+import { getPipelineConfig } from "../../core/config.js";
+import { sendJson } from "../utils/http-utils.js";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+
+/**
+ * HTTP handler for task analysis requests.
+ *
+ * @param {http.IncomingMessage} req - HTTP request object
+ * @param {http.ServerResponse} res - HTTP response object
+ */
+export async function handleTaskAnalysisRequest(req, res) {
+  const { slug, taskId } = req.params;
+
+  // Validate slug parameter
+  if (!slug || typeof slug !== "string" || !/^[A-Za-z0-9_-]+$/.test(slug)) {
+    return sendJson(res, 400, {
+      ok: false,
+      code: "invalid_params",
+      message: "Invalid slug parameter",
+    });
+  }
+
+  // Validate taskId parameter
+  if (
+    !taskId ||
+    typeof taskId !== "string" ||
+    !/^[A-Za-z0-9_-]+$/.test(taskId)
+  ) {
+    return sendJson(res, 400, {
+      ok: false,
+      code: "invalid_params",
+      message: "Invalid taskId parameter",
+    });
+  }
+
+  try {
+    // Get pipeline configuration
+    let pipelineConfig;
+    try {
+      pipelineConfig = getPipelineConfig(slug);
+    } catch (error) {
+      return sendJson(res, 404, {
+        ok: false,
+        code: "not_found",
+        message: `Pipeline '${slug}' not found in registry`,
+      });
+    }
+
+    const pipelineDir = path.dirname(pipelineConfig.pipelineJsonPath);
+    const analysisPath = path.join(
+      pipelineDir,
+      "analysis",
+      `${taskId}.analysis.json`
+    );
+
+    // Attempt to read and parse analysis file
+    let analysisData;
+    try {
+      const contents = await fs.readFile(analysisPath, "utf8");
+      analysisData = JSON.parse(contents);
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        // Analysis file doesn't exist - this is not an error
+        return sendJson(res, 200, {
+          ok: true,
+          data: null,
+        });
+      }
+
+      if (error instanceof SyntaxError) {
+        return sendJson(res, 500, {
+          ok: false,
+          code: "invalid_json",
+          message: "Invalid JSON in analysis file",
+        });
+      }
+
+      throw error;
+    }
+
+    // Return analysis data
+    return sendJson(res, 200, {
+      ok: true,
+      data: analysisData,
+    });
+  } catch (error) {
+    console.error("handleTaskAnalysisRequest unexpected error:", error);
+    return sendJson(res, 500, {
+      ok: false,
+      code: "internal_error",
+      message: "Internal server error",
+    });
+  }
+}

package/src/ui/endpoints/task-creation-endpoint.js

@@ -0,0 +1,114 @@
+import fs from "node:fs";
+import { streamSSE } from "../lib/sse.js";
+import { createHighLevelLLM } from "../../llm/index.js";
+
+export async function handleTaskPlan(req, res) {
+  console.log("[task-creation-endpoint] Request received");
+
+  const { messages, pipelineSlug } = req.body;
+
+  console.log("[task-creation-endpoint] Request details:", {
+    hasMessages: !!messages,
+    messageCount: Array.isArray(messages) ? messages.length : 0,
+    pipelineSlug,
+    bodyKeys: Object.keys(req.body),
+  });
+
+  // Validate input
+  if (!Array.isArray(messages)) {
+    console.error(
+      "[task-creation-endpoint] Validation failed: messages is not an array"
+    );
+    res.status(400).json({ error: "messages must be an array" });
+    return;
+  }
+
+  console.log(
+    "[task-creation-endpoint] Loading guidelines from docs/pipeline-task-guidelines.md..."
+  );
+
+  // Load guidelines - let it throw if missing
+  const guidelinesPath = "docs/pipeline-task-guidelines.md";
+  const guidelines = fs.readFileSync(guidelinesPath, "utf-8");
+
+  console.log(
+    "[task-creation-endpoint] Guidelines loaded, length:",
+    guidelines.length
+  );
+
+  // Build LLM messages array
+  const systemPrompt = `You are a pipeline task assistant. Help users create task definitions following these guidelines:
+
+${guidelines}
+
+Provide complete, working code. Use markdown code blocks.
+
+When you have completed a task definition that the user wants to create, wrap it in this format:
+[TASK_PROPOSAL]
+FILENAME: <filename.js>
+TASKNAME: <task-name>
+CODE:
+\`\`\`javascript
+<the complete task code here>
+\`\`\`
+[/TASK_PROPOSAL]`;
+
+  const llmMessages = [{ role: "system", content: systemPrompt }, ...messages];
+
+  console.log("[task-creation-endpoint] LLM messages array created:", {
+    totalMessages: llmMessages.length,
+    systemPromptLength: systemPrompt.length,
+  });
+
+  // Create SSE stream
+  console.log("[task-creation-endpoint] Creating SSE stream...");
+  const sse = streamSSE(res);
+
+  try {
+    console.log("[task-creation-endpoint] Creating LLM instance...");
+    // Get LLM instance (uses default provider from config)
+    const llm = createHighLevelLLM();
+
+    console.log("[task-creation-endpoint] Calling LLM chat with streaming...");
+    // Call LLM with streaming enabled
+    const response = await llm.chat({
+      messages: llmMessages,
+      responseFormat: "text",
+      stream: true,
+    });
+
+    console.log("[task-creation-endpoint] LLM response received:", {
+      isStream: typeof response[Symbol.asyncIterator] !== "undefined",
+    });
+
+    // Stream is an async generator
+    let chunkCount = 0;
+    for await (const chunk of response) {
+      if (chunk?.content) {
+        sse.send("chunk", { content: chunk.content });
+        chunkCount++;
+      }
+    }
+
+    console.log("[task-creation-endpoint] Sent", chunkCount, "chunks via SSE");
+
+    // Send done event
+    console.log("[task-creation-endpoint] Sending 'done' event...");
+    sse.send("done", {});
+    console.log("[task-creation-endpoint] Ending SSE stream...");
+    sse.end();
+    console.log("[task-creation-endpoint] Request completed successfully");
+  } catch (error) {
+    console.error("[task-creation-endpoint] Error occurred:", {
+      message: error.message,
+      stack: error.stack,
+      name: error.name,
+    });
+    // Send error event
+    sse.send("error", { message: error.message });
+    console.log(
+      "[task-creation-endpoint] Error sent via SSE, ending stream..."
+    );
+    sse.end();
+  }
+}