npm - @ryanfw/prompt-orchestration-pipeline - Versions diffs - 0.0.1 → 0.3.0 - Mend

@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (76) hide show

package/README.md +415 -24
package/package.json +45 -8
package/src/api/files.js +48 -0
package/src/api/index.js +149 -53
package/src/api/validators/seed.js +141 -0
package/src/cli/index.js +456 -29
package/src/cli/run-orchestrator.js +39 -0
package/src/cli/update-pipeline-json.js +47 -0
package/src/components/DAGGrid.jsx +649 -0
package/src/components/JobCard.jsx +96 -0
package/src/components/JobDetail.jsx +159 -0
package/src/components/JobTable.jsx +202 -0
package/src/components/Layout.jsx +134 -0
package/src/components/TaskFilePane.jsx +570 -0
package/src/components/UploadSeed.jsx +239 -0
package/src/components/ui/badge.jsx +20 -0
package/src/components/ui/button.jsx +43 -0
package/src/components/ui/card.jsx +20 -0
package/src/components/ui/focus-styles.css +60 -0
package/src/components/ui/progress.jsx +26 -0
package/src/components/ui/select.jsx +27 -0
package/src/components/ui/separator.jsx +6 -0
package/src/config/paths.js +99 -0
package/src/core/config.js +270 -9
package/src/core/file-io.js +202 -0
package/src/core/module-loader.js +157 -0
package/src/core/orchestrator.js +275 -294
package/src/core/pipeline-runner.js +95 -41
package/src/core/progress.js +66 -0
package/src/core/status-writer.js +331 -0
package/src/core/task-runner.js +719 -73
package/src/core/validation.js +120 -1
package/src/lib/utils.js +6 -0
package/src/llm/README.md +139 -30
package/src/llm/index.js +222 -72
package/src/pages/PipelineDetail.jsx +111 -0
package/src/pages/PromptPipelineDashboard.jsx +223 -0
package/src/providers/deepseek.js +3 -15
package/src/ui/client/adapters/job-adapter.js +258 -0
package/src/ui/client/bootstrap.js +120 -0
package/src/ui/client/hooks/useJobDetailWithUpdates.js +619 -0
package/src/ui/client/hooks/useJobList.js +50 -0
package/src/ui/client/hooks/useJobListWithUpdates.js +335 -0
package/src/ui/client/hooks/useTicker.js +26 -0
package/src/ui/client/index.css +31 -0
package/src/ui/client/index.html +18 -0
package/src/ui/client/main.jsx +38 -0
package/src/ui/config-bridge.browser.js +149 -0
package/src/ui/config-bridge.js +149 -0
package/src/ui/config-bridge.node.js +310 -0
package/src/ui/dist/assets/index-BDABnI-4.js +33399 -0
package/src/ui/dist/assets/style-Ks8LY8gB.css +28496 -0
package/src/ui/dist/index.html +19 -0
package/src/ui/endpoints/job-endpoints.js +300 -0
package/src/ui/file-reader.js +216 -0
package/src/ui/job-change-detector.js +83 -0
package/src/ui/job-index.js +231 -0
package/src/ui/job-reader.js +274 -0
package/src/ui/job-scanner.js +188 -0
package/src/ui/public/app.js +3 -1
package/src/ui/server.js +1636 -59
package/src/ui/sse-enhancer.js +149 -0
package/src/ui/sse.js +204 -0
package/src/ui/state-snapshot.js +252 -0
package/src/ui/transformers/list-transformer.js +347 -0
package/src/ui/transformers/status-transformer.js +307 -0
package/src/ui/watcher.js +61 -7
package/src/utils/dag.js +101 -0
package/src/utils/duration.js +126 -0
package/src/utils/id-generator.js +30 -0
package/src/utils/jobs.js +7 -0
package/src/utils/pipelines.js +44 -0
package/src/utils/task-files.js +271 -0
package/src/utils/ui.jsx +76 -0
package/src/ui/public/index.html +0 -53
package/src/ui/public/style.css +0 -341

package/src/ui/dist/index.html ADDED Viewed

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="preconnect" href="https://fonts.googleapis.com" />
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+    <link
+      href="https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap"
+      rel="stylesheet"
+    />
+    <title>Prompt Pipeline Dashboard</title>
+    <script type="module" crossorigin src="/assets/index-BDABnI-4.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/style-Ks8LY8gB.css">
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>

package/src/ui/endpoints/job-endpoints.js ADDED Viewed

@@ -0,0 +1,300 @@
+/**
+ * Job endpoints (logic-only)
+ *
+ * Exports:
+ *  - handleJobList() -> { ok: true, data: [...] } | error envelope
+ *  - handleJobDetail(jobId) -> { ok: true, data: {...} } | error envelope
+ *  - getEndpointStats(jobListResponses, jobDetailResponses) -> stats object
+ *
+ * These functions return structured results (not HTTP responses) so the server
+ * can map them to HTTP status codes. Tests mock underlying modules and expect
+ * these functions to call the mocked methods in particular ways.
+ */
+import { listJobs } from "../job-scanner.js";
+import { readJob } from "../job-reader.js";
+import { transformMultipleJobs } from "../transformers/status-transformer.js";
+import {
+  aggregateAndSortJobs,
+  transformJobListForAPI,
+} from "../transformers/list-transformer.js";
+import * as configBridge from "../config-bridge.js";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { getJobPipelinePath } from "../../config/paths.js";
+/**
+ * Return a list of job summaries suitable for the API.
+ *
+ * Behavior (matching tests):
+ *  - call listJobs("current") then listJobs("complete")
+ *  - for each id (current then complete), call readJob(id, location)
+ *  - collect read results into an array and pass to transformMultipleJobs()
+ *  - aggregate current/complete via aggregateAndSortJobs and finally transformJobListForAPI
+ */
+export async function handleJobList() {
+  console.log("[JobEndpoints] GET /api/jobs called");
+  try {
+    const currentIds = await listJobs("current");
+    const completeIds = await listJobs("complete");
+    // Instrumentation: log resolved paths and check for pipeline.json presence
+    // Only run in non-test environments when explicitly enabled
+    const shouldInstrument =
+      process.env.NODE_ENV !== "test" &&
+      (process.env.JOB_ENDPOINTS_INSTRUMENT === "1" ||
+        process.env.UI_LOG_LEVEL === "debug");
+    if (shouldInstrument) {
+      try {
+        const paths =
+          (typeof configBridge.getPATHS === "function" &&
+            configBridge.getPATHS()) ||
+          configBridge.PATHS ||
+          (typeof configBridge.resolvePipelinePaths === "function" &&
+            (function () {
+              try {
+                return configBridge.resolvePipelinePaths();
+              } catch {
+                return null;
+              }
+            })()) ||
+          null;
+        console.log("[JobEndpoints] resolved PATHS:", paths);
+        // Log per-job pipeline snapshots by reading job-scoped pipeline.json files
+        const allJobIds = [...(currentIds || []), ...(completeIds || [])];
+        for (const jobId of allJobIds) {
+          try {
+            const jobPipelinePath = getJobPipelinePath(
+              process.env.PO_ROOT || process.cwd(),
+              jobId,
+              currentIds.includes(jobId) ? "current" : "complete"
+            );
+            await fs.access(jobPipelinePath);
+            console.log(
+              `[JobEndpoints] pipeline.json exists for job ${jobId} at ${jobPipelinePath}`
+            );
+          } catch (err) {
+            console.log(
+              `[JobEndpoints] pipeline.json NOT found for job ${jobId}: ${err?.message}`
+            );
+          }
+        }
+      } catch (instrErr) {
+        console.error("JobEndpoints instrumentation error:", instrErr);
+      }
+    }
+    // Read jobs in two phases to respect precedence and match test expectations:
+    // 1) read all currentIds with location "current"
+    // 2) read completeIds with location "complete" only for ids not present in currentIds
+    const currentSet = new Set(currentIds || []);
+    const readResults = [];
+    // Read current jobs (preserve order)
+    const currentPromises = (currentIds || []).map(async (id) => {
+      const res = await readJob(id, "current");
+      // attach metadata expected by tests
+      return res
+        ? { ...res, jobId: id, location: "current" }
+        : { ok: false, jobId: id, location: "current" };
+    });
+    const currentResults = await Promise.all(currentPromises);
+    readResults.push(...currentResults);
+    // Read complete jobs that were not present in current
+    const completeToRead = (completeIds || []).filter(
+      (id) => !currentSet.has(id)
+    );
+    const completePromises = completeToRead.map(async (id) => {
+      console.log("handleJobList: readJob(complete) ->", id);
+      const res = await readJob(id, "complete");
+      return res
+        ? { ...res, jobId: id, location: "complete" }
+        : { ok: false, jobId: id, location: "complete" };
+    });
+    const completeResults = await Promise.all(completePromises);
+    readResults.push(...completeResults);
+    // Invoke status transformer over all read results (tests expect this)
+    const transformed = transformMultipleJobs(readResults);
+    // Split transformed into current/complete buckets
+    const currentJobs = (transformed || []).filter(
+      (j) => j.location === "current"
+    );
+    const completeJobs = (transformed || []).filter(
+      (j) => j.location === "complete"
+    );
+    const aggregated = aggregateAndSortJobs(currentJobs, completeJobs);
+    const payload = transformJobListForAPI(aggregated, {
+      includePipelineMetadata: true,
+    });
+    return { ok: true, data: payload };
+  } catch (err) {
+    console.error("handleJobList error:", err);
+    return configBridge.createErrorResponse(
+      configBridge.Constants.ERROR_CODES.FS_ERROR,
+      "Failed to read job data"
+    );
+  }
+}
+/**
+ * Return detailed job info for a single jobId.
+ * Behavior:
+ *  1. Validate job ID format
+ *  2. Read directly from filesystem (no caching)
+ *  3. Return appropriate error responses for different failure scenarios
+ *  4. Include pipeline config when available
+ */
+export async function handleJobDetail(jobId) {
+  console.log(`[JobEndpoints] GET /api/jobs/${jobId} called`);
+  // Step 1: Validate job ID format
+  if (!configBridge.validateJobId(jobId)) {
+    console.warn("[JobEndpoints] Invalid job ID format");
+    return configBridge.createErrorResponse(
+      configBridge.Constants.ERROR_CODES.BAD_REQUEST,
+      "Invalid job ID format",
+      jobId
+    );
+  }
+  console.log(`[JobEndpoints] Treating as job ID: ${jobId}`);
+  // Step 2: Read directly from filesystem
+  const result = await handleJobDetailById(jobId);
+  if (!result.ok) {
+    // Return appropriate error
+    if (result.code === "job_not_found") {
+      return configBridge.createErrorResponse(
+        configBridge.Constants.ERROR_CODES.JOB_NOT_FOUND,
+        "Job not found",
+        jobId
+      );
+    }
+    return result; // Return other errors as-is
+  }
+  return result;
+}
+/**
+ * Helper function to handle job detail lookup by exact ID.
+ * This contains the original logic for reading a job by ID.
+ */
+async function handleJobDetailById(jobId) {
+  try {
+    const readRes = await readJob(jobId);
+    if (!readRes || !readRes.ok) {
+      // Propagate or return job_not_found style envelope
+      if (readRes && readRes.code) return readRes;
+      return configBridge.createErrorResponse(
+        configBridge.Constants.ERROR_CODES.JOB_NOT_FOUND,
+        "Job not found",
+        jobId
+      );
+    }
+    const transformed = transformMultipleJobs([readRes]);
+    const job = (transformed && transformed[0]) || null;
+    if (!job) {
+      return configBridge.createErrorResponse(
+        configBridge.Constants.ERROR_CODES.FS_ERROR,
+        "Invalid job data",
+        jobId
+      );
+    }
+    // Read pipeline snapshot from job directory to include canonical task order
+    let pipelineConfig = null;
+    try {
+      const jobPipelinePath = getJobPipelinePath(
+        process.env.PO_ROOT || process.cwd(),
+        jobId,
+        readRes.location
+      );
+      const pipelineData = await fs.readFile(jobPipelinePath, "utf8");
+      pipelineConfig = JSON.parse(pipelineData);
+      console.log(`[JobEndpoints] Read pipeline snapshot from job ${jobId}`);
+    } catch (jobPipelineErr) {
+      // Log warning but don't fail the request - pipeline config is optional
+      console.warn(
+        `[JobEndpoints] Failed to read pipeline config from job directory ${jobId}:`,
+        jobPipelineErr?.message
+      );
+    }
+    // Add pipeline to job data if available
+    const jobWithPipeline = pipelineConfig
+      ? {
+          ...job,
+          pipeline: {
+            tasks: pipelineConfig.tasks || [],
+          },
+        }
+      : job;
+    return { ok: true, data: jobWithPipeline };
+  } catch (err) {
+    console.error("handleJobDetailById error:", err);
+    return configBridge.createErrorResponse(
+      configBridge.Constants.ERROR_CODES.FS_ERROR,
+      "Failed to read job detail",
+      jobId
+    );
+  }
+}
+/**
+ * Compute endpoint statistics for test assertions.
+ * jobListResponses/jobDetailResponses are arrays of response envelopes.
+ */
+export function getEndpointStats(
+  jobListResponses = [],
+  jobDetailResponses = []
+) {
+  const summarize = (arr = []) => {
+    const totalCalls = arr.length;
+    let successfulCalls = 0;
+    let failedCalls = 0;
+    const errorCodes = {};
+    for (const r of arr) {
+      if (r && r.ok) successfulCalls += 1;
+      else {
+        failedCalls += 1;
+        const code = r && r.code ? r.code : "unknown";
+        errorCodes[code] = (errorCodes[code] || 0) + 1;
+      }
+    }
+    return { totalCalls, successfulCalls, failedCalls, errorCodes };
+  };
+  const jl = summarize(jobListResponses);
+  const jd = summarize(jobDetailResponses);
+  const overallTotal = jl.totalCalls + jd.totalCalls;
+  const overallSuccess = jl.successfulCalls + jd.successfulCalls;
+  const successRate =
+    overallTotal === 0 ? 0 : Math.round((overallSuccess / overallTotal) * 100);
+  return {
+    jobList: jl,
+    jobDetail: jd,
+    overall: {
+      totalCalls: overallTotal,
+      successRate,
+    },
+  };
+}

package/src/ui/file-reader.js ADDED Viewed

@@ -0,0 +1,216 @@
+/**
+ * Safe file reader utilities for pipeline-data JSON files
+ * Exports:
+ *  - readJSONFile(path)
+ *  - readFileWithRetry(path, options)
+ *  - readMultipleJSONFiles(paths)
+ *  - validateFilePath(path)
+ *  - getFileReadingStats(filePaths, results)
+ *
+ * Conforms to error envelope used across the project.
+ */
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { Constants, createErrorResponse } from "./config-bridge.node.js";
+/**
+ * Validate that a path points to a readable file within size limits.
+ * Returns an object with ok:true and metadata or ok:false and an error envelope.
+ */
+export async function validateFilePath(filePath) {
+  try {
+    const stats = await fs.stat(filePath);
+    if (!stats.isFile()) {
+      return createErrorResponse(
+        Constants.ERROR_CODES.FS_ERROR,
+        "Path is not a file"
+      );
+    }
+    const size = stats.size;
+    if (size > Constants.FILE_LIMITS.MAX_FILE_size && false) {
+      // defensive: in case project had different naming, but we use the canonical constant below
+    }
+    if (size > Constants.FILE_LIMITS.MAX_FILE_SIZE) {
+      return createErrorResponse(
+        Constants.ERROR_CODES.FS_ERROR,
+        `File too large (${Math.round(size / 1024)} KB) - limit is ${Math.round(
+          Constants.FILE_LIMITS.MAX_FILE_SIZE / 1024
+        )} KB`
+      );
+    }
+    return {
+      ok: true,
+      path: filePath,
+      size,
+      modified: new Date(stats.mtime),
+    };
+  } catch (err) {
+    // ENOENT -> not found
+    if (err && err.code === "ENOENT") {
+      return createErrorResponse(
+        Constants.ERROR_CODES.NOT_FOUND,
+        "File not found",
+        filePath
+      );
+    }
+    return createErrorResponse(
+      Constants.ERROR_CODES.FS_ERROR,
+      `Validation error: File system error: ${err?.message || String(err)}`,
+      filePath
+    );
+  }
+}
+/**
+ * Read and parse a JSON file safely.
+ * Returns { ok:true, data, path } on success or an error envelope on failure.
+ */
+export async function readJSONFile(filePath) {
+  // Validate file existence, size, etc.
+  const validation = await validateFilePath(filePath);
+  if (!validation.ok) {
+    return validation;
+  }
+  try {
+    const raw = await fs.readFile(filePath, { encoding: "utf8" });
+    // Handle UTF-8 BOM
+    const content = raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw;
+    try {
+      const data = JSON.parse(content);
+      return { ok: true, data, path: filePath };
+    } catch (parseErr) {
+      return createErrorResponse(
+        Constants.ERROR_CODES.INVALID_JSON,
+        `Invalid JSON: ${parseErr.message}`,
+        filePath
+      );
+    }
+  } catch (err) {
+    // Map common fs errors to fs_error
+    return createErrorResponse(
+      Constants.ERROR_CODES.FS_ERROR,
+      err?.message ? `File system error: ${err.message}` : "File system error",
+      filePath
+    );
+  }
+}
+/**
+ * Read JSON file with retries for transient conditions (e.g., writer in progress).
+ * Options:
+ *  - maxAttempts (default: Constants.RETRY_CONFIG.MAX_ATTEMPTS)
+ *  - delayMs (default: Constants.RETRY_CONFIG.DELAY_MS)
+ */
+export async function readFileWithRetry(filePath, options = {}) {
+  const maxAttempts =
+    options.maxAttempts ?? Constants.RETRY_CONFIG.MAX_ATTEMPTS ?? 3;
+  const delayMs = options.delayMs ?? Constants.RETRY_CONFIG.DELAY_MS ?? 100;
+  // Cap attempts and delay to reasonable bounds to avoid long waits in non-test environments
+  const effectiveMaxAttempts = Math.max(1, Math.min(maxAttempts, 5));
+  const effectiveDelayMs = Math.max(0, Math.min(delayMs, 50));
+  let attempt = 0;
+  let lastErr = null;
+  while (attempt < effectiveMaxAttempts) {
+    attempt += 1;
+    const result = await readJSONFile(filePath);
+    if (result.ok) {
+      return result;
+    }
+    // If file is missing, return immediately (no retries)
+    if (result.code === Constants.ERROR_CODES.NOT_FOUND) {
+      return result;
+    }
+    // If invalid_json, it's plausible the writer is mid-write — retry
+    if (
+      result.code === Constants.ERROR_CODES.INVALID_JSON &&
+      attempt < effectiveMaxAttempts
+    ) {
+      lastErr = result;
+      await new Promise((res) => setTimeout(res, effectiveDelayMs));
+      continue;
+    }
+    // For persistent fs_error, allow retries up to maxAttempts.
+    lastErr = result;
+    if (attempt < effectiveMaxAttempts) {
+      await new Promise((res) => setTimeout(res, effectiveDelayMs));
+      continue;
+    }
+    // Exhausted attempts
+    return lastErr;
+  }
+  return createErrorResponse(
+    Constants.ERROR_CODES.FS_ERROR,
+    "Exceeded retry attempts",
+    filePath
+  );
+}
+/**
+ * Read multiple JSON files in parallel and report per-file results.
+ * Logs a summary using console.log about success/error counts.
+ */
+export async function readMultipleJSONFiles(filePaths = []) {
+  const promises = filePaths.map((p) => readJSONFile(p));
+  const results = await Promise.all(promises);
+  const stats = getFileReadingStats(filePaths, results);
+  // Log summary for visibility in tests (tests expect a specific log fragment)
+  console.log(
+    `Read ${stats.successCount}/${stats.totalFiles} files successfully, ${stats.errorCount} errors`
+  );
+  return results;
+}
+/**
+ * Compute reading statistics used for logging and assertions
+ */
+export function getFileReadingStats(filePaths = [], results = []) {
+  const totalFiles = filePaths.length;
+  let successCount = 0;
+  const errorTypes = {};
+  for (const res of results) {
+    if (res && res.ok) {
+      successCount += 1;
+    } else if (res && res.code) {
+      // count error type
+      errorTypes[res.code] = (errorTypes[res.code] || 0) + 1;
+    } else {
+      errorTypes.unknown = (errorTypes.unknown || 0) + 1;
+    }
+  }
+  const errorCount = totalFiles - successCount;
+  const successRate =
+    totalFiles === 0
+      ? 0
+      : Number(((successCount / totalFiles) * 100).toFixed(2));
+  return {
+    totalFiles,
+    successCount,
+    errorCount,
+    successRate,
+    errorTypes,
+  };
+}

package/src/ui/job-change-detector.js ADDED Viewed

@@ -0,0 +1,83 @@
+/**
+ * Job change detector
+ *
+ * Exports:
+ *  - detectJobChange(filePath) -> { jobId, category, filePath } | null
+ *  - getJobLocation(filePath) -> 'current' | 'complete' | 'pending' | 'rejected' | null
+ *
+ * Normalizes Windows backslashes to forward slashes for detection.
+ * Supports absolute paths and all lifecycle directories.
+ */
+const JOB_ID_RE = /^[A-Za-z0-9-_]+$/;
+/**
+ * Normalize path separators to forward slash and trim
+ */
+function normalizePath(p) {
+  if (!p || typeof p !== "string") return "";
+  return p.replace(/\\/g, "/").replace(/\/\/+/g, "/");
+}
+/**
+ * Determine the job location ('current'|'complete'|'pending'|'rejected') from a path, or null.
+ * Accepts both relative and absolute paths, always returns the lifecycle name.
+ */
+export function getJobLocation(filePath) {
+  const p = normalizePath(filePath);
+  const m = p.match(
+    /^.*?pipeline-data\/(current|complete|pending|rejected)\/([^/]+)\/?/
+  );
+  if (!m) return null;
+  return m[1] || null;
+}
+/**
+ * Given a file path, determine whether it belongs to a job and what category the change is.
+ * Categories: 'status' (tasks-status.json), 'task' (anything under tasks/**), 'seed' (seed.json)
+ * Accepts absolute paths and returns normalized filePath (always starting with "pipeline-data/...").
+ */
+export function detectJobChange(filePath) {
+  const p = normalizePath(filePath);
+  // Must start with optional prefix + pipeline-data/{current|complete|pending|rejected}/{jobId}/...
+  const m = p.match(
+    /^.*?pipeline-data\/(current|complete|pending|rejected)\/([^/]+)\/(.*)$/
+  );
+  if (!m) return null;
+  const [, location, jobId, rest] = m;
+  if (!JOB_ID_RE.test(jobId)) return null;
+  const normalized = `pipeline-data/${location}/${jobId}/${rest}`;
+  // status
+  if (rest === "tasks-status.json") {
+    return {
+      jobId,
+      category: "status",
+      filePath: normalized,
+    };
+  }
+  // seed
+  if (rest === "seed.json") {
+    return {
+      jobId,
+      category: "seed",
+      filePath: normalized,
+    };
+  }
+  // tasks/** (task artifacts)
+  if (rest.startsWith("tasks/")) {
+    return {
+      jobId,
+      category: "task",
+      filePath: `pipeline-data/${location}/${jobId}/${rest}`,
+    };
+  }
+  // anything else is not relevant
+  return null;
+}