@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.3.0

package/README.md

@@ -38,7 +38,7 @@ The outer pipeline manages runs, state, and isolation. It is responsible for:
 
 - Assigning a pipeline run ID for each new submission.
 - Creating predictable directories for pending seeds, active runs, and completed runs.
-- Spawning isolated processes for each task (so one failure doesn’t crash others).
+- Spawning isolated processes for each task (so one failure doesn't crash others).
 - Tracking progress in a run‑scoped status file.
 - Promoting completed runs into a repository of results with audit metadata.
 
@@ -57,8 +57,8 @@ my-project/
 ```mermaid
 flowchart TD
   A["pipeline-data/pending/*-seed.json"] --> B[Orchestrator]
-  B --> C["create pipeline-data/current/<id>/seed.json"]
-  B --> D["init pipeline-data/current/<id>/tasks-status.json"]
+  B --> C["create pipeline-data/current/{jobId}/seed.json"]
+  B --> D["init pipeline-data/current/{jobId}/tasks-status.json"]
   B --> E[Read pipeline-config/pipeline.json]
   E --> F[Spawn task runner]
   F --> G["write tasks/<task>/letter.json"]
@@ -68,7 +68,7 @@ flowchart TD
   J --> K{More tasks?}
   K -->|yes| F
   K -->|no| L[Promote to complete]
-  L --> M["pipeline-data/complete/<id>/**"]
+  L --> M["pipeline-data/complete/{jobId}/**"]
   L --> N["append pipeline-data/complete/runs.jsonl"]
 ```
 
@@ -163,13 +163,20 @@ flowchart TD
 ```
 my-project/
 ├── pipeline-config/
-│   ├── pipeline.json              # Pipeline definition (ordered list of task IDs)
-│   └── tasks/                     # Task implementations
-│       ├── index.js               # Task registry (maps task IDs → modules)
-│       ├── task-a/
-│       │   └── index.js
-│       └── task-b/
-│           └── index.js
+│   ├── registry.json              # Pipeline registry (maps slugs → configurations)
+│   └── pipelines/                 # Pipeline definitions (slugged layout)
+│       ├── content/
+│       │   ├── pipeline.json      # Pipeline definition (ordered list of task IDs)
+│       │   └── tasks/             # Task implementations
+│       │       ├── index.js       # Task registry (maps task IDs → modules)
+│       │       ├── task-a/
+│       │       │   └── index.js
+│       │       └── task-b/
+│       │           └── index.js
+│       └── analytics/             # Additional pipeline (example)
+│           ├── pipeline.json
+│           └── tasks/
+│               └── index.js
 ├── pipeline-data/                 # Runtime directories (auto‑created/managed)
 │   ├── pending/
 │   ├── current/
@@ -178,7 +185,28 @@ my-project/
 └── .pipelinerc.json              # Optional CLI config
 ```
 
-**`pipeline.json` (example)**
+**`pipeline-config/registry.json` (example)**
+
+```json
+{
+  "pipelines": {
+    "content": {
+      "name": "Content Generation Pipeline",
+      "description": "Generates and processes content using LLM tasks",
+      "pipelinePath": "pipeline-config/content/pipeline.json",
+      "taskRegistryPath": "pipeline-config/content/tasks/index.js"
+    },
+    "analytics": {
+      "name": "Analytics Pipeline",
+      "description": "Processes data for analytics and reporting",
+      "pipelinePath": "pipeline-config/analytics/pipeline.json",
+      "taskRegistryPath": "pipeline-config/analytics/tasks/index.js"
+    }
+  }
+}
+```
+
+**`pipeline-config/pipelines/content/pipeline.json` (example)**
 
 ```json
 {
@@ -186,7 +214,7 @@ my-project/
 }
 ```
 
-**`pipeline-config/tasks/index.js` (example registry)**
+**`pipeline-config/pipelines/content/tasks/index.js` (example registry)**
 
 ```js
 // ESM registry mapping task IDs to loader functions or modules
@@ -196,7 +224,7 @@ export default {
 };
 ```
 
-> The orchestrator resolves task IDs from `pipeline.json` using this registry.
+> The orchestrator resolves pipeline slugs from `registry.json` and loads the corresponding pipeline configuration and task registry.
 
 ### Install & scripts
 
@@ -218,8 +246,8 @@ Add the package and scripts to your consumer project:
 
 ### CLI overview
 
-- **`pipeline-orchestrator init`** – scaffolds `pipeline-config/` and `pipeline-data/` if missing.
-- **`pipeline-orchestrator start`** – starts the orchestrator; watches `pipeline-data/pending/` for new seeds and processes them according to `pipeline-config/pipeline.json`.
+- **`pipeline-orchestrator init`** – scaffolds `pipeline-config/` with registry and default pipeline, plus `pipeline-data/` if missing.
+- **`pipeline-orchestrator start`** – starts the orchestrator; watches `pipeline-data/pending/` for new seeds and processes them using the default pipeline from `pipeline-config/registry.json`.
 - **`pipeline-orchestrator start --ui`** – starts the orchestrator and the optional UI server.
 - **`pipeline-orchestrator submit [path]`** – submits a seed into `pipeline-data/pending/` (path can point to a JSON file).
 
@@ -238,13 +266,234 @@ If present in the project root, this file can provide defaults for the CLI (e.g.
 
 _(Keys and defaults may vary by version; prefer `--help` for authoritative options.)_
 
+### Seed format
+
+All seeds must include a `pipeline` field that references a valid pipeline slug from the registry. The pipeline field is mandatory and no fallbacks are allowed.
+
+**Minimal seed example:**
+
+```json
+{
+  "name": "my-job",
+  "pipeline": "content",
+  "data": {
+    "type": "content-creation",
+    "topic": "AI-Powered Development Tools"
+  }
+}
+```
+
+**Required fields:**
+
+- `name`: Unique identifier for the job (alphanumeric, hyphens, and underscores only)
+- `pipeline`: Valid pipeline slug from `pipeline-config/registry.json`
+- `data`: Object containing the input data for the pipeline
+
 ### Example flow in a consumer project
 
-1. **Initialize**: `npm run pipeline:init` to ensure folders exist.
-2. **Define**: Edit `pipeline-config/pipeline.json` and implement tasks under `pipeline-config/tasks/`.
-3. **Run**: `npm run pipeline` (or `npm run pipeline:ui` for the UI).
-4. **Submit**: Add a seed JSON to `pipeline-data/pending/` or run `npm run pipeline:submit -- ./path/to/seed.json`.
-5. **Inspect**: Watch `pipeline-data/current/<runId>` for in‑progress artifacts and `pipeline-data/complete/<runId>` for results.
+1. **Initialize**: `npm run pipeline:init` to create the registry and default pipeline structure.
+2. **Define**: Edit `pipeline-config/pipelines/{slug}/pipeline.json` and implement tasks under `pipeline-config/pipelines/{slug}/tasks/`.
+3. **Configure**: Update `pipeline-config/registry.json` to add new pipelines or change the default.
+4. **Run**: `npm run pipeline` (or `npm run pipeline:ui` for the UI).
+5. **Submit**: Add a seed JSON to `pipeline-data/pending/` or run `npm run pipeline:submit -- ./path/to/seed.json`.
+6. **Inspect**: Watch `pipeline-data/current/{jobId}` for in‑progress artifacts and `pipeline-data/complete/{jobId}` for results.
+
+---
+
+## Section C — UI and JobId-Only Navigation
+
+This project includes a web UI for monitoring pipeline execution and inspecting results.
+
+### JobId-Only Policy
+
+**Important**: The UI uses JobId-only navigation. All pipeline detail pages use `/pipeline/:jobId` URLs with no slug-based fallbacks.
+
+#### Directory Structure
+
+The UI uses ID-based storage exclusively:
+
+```
+pipeline-data/
+├── pending/
+│   ├── {jobId}/
+│   │   ├── seed.json
+│   │   └── ...
+├── current/
+│   ├── {jobId}/
+│   │   ├── seed.json
+│   │   ├── tasks-status.json
+│   │   └── ...
+├── complete/
+│   ├── {jobId}/
+│   │   ├── seed.json
+│   │   ├── tasks-status.json
+│   │   └── ...
+└── rejected/
+    ├── {jobId}/
+    │   ├── seed.json
+    │   └── ...
+```
+
+#### Accessing Pipeline Details
+
+- **Valid**: `/pipeline/abc123def456` - Loads job with ID `abc123def456`
+- **Invalid**: `/pipeline/content-generation` - Shows "Invalid job ID" error
+
+#### Error Handling
+
+- **Invalid job ID**: Shows "Invalid job ID" for malformed IDs
+- **Job not found**: Shows "Job not found" for valid IDs that don't exist
+- **Network errors**: Shows appropriate network error messages
+
+---
+
+## Section D — File I/O System (New)
+
+### Scoped File Operations
+
+The pipeline now includes a **scoped file I/O system** that provides each task with isolated file operations through a `context.files` API. This replaces the legacy artifacts system with a more organized approach.
+
+#### File Structure
+
+Each task gets its own directory structure:
+
+```
+pipeline-data/current/{jobId}/
+├── tasks/
+│   └── {taskName}/
+│       ├── artifacts/     # Generated outputs (replace mode)
+│       ├── logs/          # Process logs (append mode)
+│       └── tmp/           # Temporary files (replace mode
+└── tasks-status.json      # Updated with files.* arrays
+```
+
+#### File I/O API
+
+Tasks receive a `context.files` object with these methods:
+
+```javascript
+// Write artifacts (default: replace mode)
+await context.files.writeArtifact("output.json", data);
+await context.files.writeArtifact("report.txt", content, { mode: "replace" });
+
+// Write logs (default: append mode)
+await context.files.writeLog("process.log", "Starting process\n");
+await context.files.writeLog("debug.log", error, { mode: "append" });
+
+// Write temporary files (default: replace mode)
+await context.files.writeTmp("temp.json", intermediateData);
+
+// Read files
+const artifact = await context.files.readArtifact("output.json");
+const logs = await context.files.readLog("process.log");
+const temp = await context.files.readTmp("temp.json");
+```
+
+#### Status Schema Updates
+
+The `tasks-status.json` now includes `files.*` arrays:
+
+```json
+{
+  "id": "example-pipeline",
+  "current": "analysis",
+  "files": {
+    "artifacts": ["raw-research.json", "analysis-output.json", "summary.txt"],
+    "logs": ["ingestion.log", "integration.log"],
+    "tmp": ["temp-data.json"]
+  },
+  "tasks": {
+    "analysis": {
+      "state": "complete",
+      "files": {
+        "artifacts": ["raw-research.json", "analysis-output.json"],
+        "logs": ["ingestion.log"],
+        "tmp": []
+      }
+    }
+  }
+}
+```
+
+#### Migration from Legacy Artifacts
+
+The new system **breaks backward compatibility** intentionally:
+
+- **Old**: `task.artifacts` array with file objects
+- **New**: `task.files.artifacts` array with filenames only
+- **Old**: Files stored in task root directory
+- **New**: Files organized in `artifacts/`, `logs/`, `tmp/` subdirectories
+
+To migrate existing demo data:
+
+```bash
+node scripts/migrate-demo-files.js
+```
+
+#### Verification
+
+To verify the file I/O system is working:
+
+1. **Check test suite**: All 882 tests should pass
+2. **Run demo pipeline**: Files should appear in correct subdirectories
+3. **Inspect tasks-status.json**: Should contain `files.*` arrays
+4. **Check UI**: Job details should show files from new schema
+
+#### Example Task Usage
+
+```javascript
+export async function ingestion(context) {
+  const researchContent = context.seed.data.content;
+
+  // Log the start of ingestion
+  await context.files.writeLog(
+    "ingestion.log",
+    `[${new Date().toISOString()}] Starting data ingestion\n`
+  );
+
+  // Store raw research data
+  await context.files.writeArtifact(
+    "raw-research.json",
+    JSON.stringify(
+      {
+        content: researchContent,
+        type: context.seed.data.type,
+        ingestedAt: new Date().toISOString(),
+      },
+      null,
+      2
+    )
+  );
+
+  return { output: { researchContent } };
+}
+
+export async function integration(context) {
+  const { analysisContent } = context.output;
+
+  // Store final analysis output
+  await context.files.writeArtifact(
+    "analysis-output.json",
+    JSON.stringify(
+      {
+        content: analysisContent,
+        timestamp: new Date().toISOString(),
+        taskName: context.taskName,
+      },
+      null,
+      2
+    )
+  );
+
+  // Log completion
+  await context.files.writeLog(
+    "integration.log",
+    `[${new Date().toISOString()}] ✓ Analysis integration completed\n`
+  );
+
+  return { output: { analysis: { content: analysisContent } } };
+}
+```
 
 ---
 
@@ -252,8 +501,72 @@ _(Keys and defaults may vary by version; prefer `--help` for authoritative optio
 
 - **Determinism** – each task persists its inputs/outputs; you can re‑run or debug any stage.
 - **Isolation** – tasks run in separate processes when appropriate.
-- **Artifacts** – tasks write structured artifacts (e.g., `letter.json`, `output.json`) to their run directory.
-- **Status** – a `tasks-status.json` file tracks progress and outcomes across the pipeline.
+- **Scoped File I/O** – tasks use `context.files` API for organized file operations.
+- **Status** – a `tasks-status.json` file tracks progress and file inventories across the pipeline.
+- **JobId-only** – all job identification and navigation uses unique job IDs, not pipeline names.
+
+---
+
+## Section E — Logging and Debugging
+
+### Per-Stage Logging
+
+The pipeline captures console output from each stage execution to dedicated log files. This helps with debugging specific stages and understanding the flow of data through the pipeline.
+
+#### Log File Locations
+
+```
+pipeline-data/current/{jobId}/
+├── files/
+│   └── logs/
+│       ├── stage-validateStructure.log
+│       ├── stage-critique.log
+│       └── stage-refine.log
+└── tasks-status.json
+```
+
+#### Log Contents
+
+Each `stage-{stageName}.log` file contains:
+
+- All console output from that stage (console.log, console.error, console.warn, console.info)
+- Timestamped entries for debugging
+- Error messages and stack traces if the stage fails
+
+#### Status File
+
+The `tasks-status.json` file contains the complete execution state:
+
+- `data` object with all stage outputs (`data.validateStructure`, `data.critique`, etc.)
+- `flags` object with accumulated pipeline flags (`validationFailed`, `critiqueComplete`, etc.)
+- `logs` array with audit trail entries
+- `files` arrays tracking all generated files
+
+#### Debugging Examples
+
+**Check stage output:**
+
+```bash
+# View validation stage logs
+cat pipeline-data/current/{jobId}/files/logs/stage-validateStructure.log
+
+# View critique stage logs
+cat pipeline-data/current/{jobId}/files/logs/stage-critique.log
+```
+
+**Inspect execution state:**
+
+```bash
+# View complete pipeline state
+cat pipeline-data/current/{jobId}/tasks-status.json | jq '.data, .flags'
+```
+
+**Debug failed stages:**
+
+```bash
+# Check error logs for specific stage
+grep -i error pipeline-data/current/{jobId}/files/logs/stage-*.log
+```
 
 ---
 
@@ -261,7 +574,82 @@ _(Keys and defaults may vary by version; prefer `--help` for authoritative optio
 
 - **Nothing happens when I submit a seed** → Ensure the orchestrator is running and watching `pipeline-data/pending/`.
 - **Task not found** → Confirm the task ID exists in `pipeline-config/tasks/index.js` and matches `pipeline.json`.
-- **UI doesn’t load** → Try `pipeline-orchestrator start --ui` and check for port conflicts.
+- **UI doesn't load** → Try `pipeline-orchestrator start --ui` and check for port conflicts.
+- **Invalid job ID error** → Ensure you're using a valid job ID from the job list, not a pipeline name.
+- **Stage failed** → Check `pipeline-data/current/{jobId}/files/logs/stage-{stageName}.log` for detailed error messages.
+- **Missing output** → Inspect `tasks-status.json` to see what data and flags were generated by each stage.
+
+---
+
+## Scaffolding
+
+Use the CLI commands to quickly scaffold a new pipeline structure. These examples show the minimal commands and resulting directory trees.
+
+### Initialize pipeline structure
+
+```bash
+pipeline-orchestrator init --root ./pipelines
+```
+
+**Resulting directory tree:**
+
+```
+pipelines/
+├── pipeline-config/
+│   └── registry.json
+└── pipeline-data/
+    ├── pending/.gitkeep
+    ├── current/.gitkeep
+    ├── complete/.gitkeep
+    └── rejected/.gitkeep
+```
+
+### Add a pipeline
+
+```bash
+pipeline-orchestrator add-pipeline content-generation --root ./pipelines
+```
+
+**Resulting directory tree:**
+
+```
+pipelines/
+├── pipeline-config/
+│   ├── registry.json
+│   └── content-generation/
+│       ├── pipeline.json
+│       └── tasks/
+│           └── index.js
+└── pipeline-data/
+    ├── pending/.gitkeep
+    ├── current/.gitkeep
+    ├── complete/.gitkeep
+    └── rejected/.gitkeep
+```
+
+### Add a pipeline task
+
+```bash
+pipeline-orchestrator add-pipeline-task content-generation research --root ./pipelines
+```
+
+**Resulting directory tree:**
+
+```
+pipelines/
+├── pipeline-config/
+│   ├── registry.json
+│   └── content-generation/
+│       ├── pipeline.json
+│       └── tasks/
+│           ├── index.js
+│           └── research.js
+└── pipeline-data/
+    ├── pending/.gitkeep
+    ├── current/.gitkeep
+    ├── complete/.gitkeep
+    └── rejected/.gitkeep
+```
 
 ---
 
@@ -281,6 +669,9 @@ npm run pipeline:ui
 
 # 4) Submit a seed (JSON file)
 npm run pipeline:submit -- ./seeds/example-seed.json
+
+# 5) Access UI (if running with --ui)
+# Navigate to job details using /pipeline/{jobId} URLs
 ```
 
 ---

package/package.json

@@ -1,9 +1,12 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "0.0.1",
+  "version": "0.3.0",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server.js",
+  "bin": {
+    "pipeline-orchestrator": "src/cli/index.js"
+  },
   "files": [
     "src",
     "README.md",
@@ -17,20 +20,54 @@
     "access": "public"
   },
   "scripts": {
-    "test": "vitest run",
-    "ui": "nodemon src/ui/server.js",
-    "ui:prod": "node src/ui/server.js"
+    "test": "vitest run --config ./vite.config.js --root .",
+    "lint": "eslint . --ext .js,.jsx",
+    "backend": "NODE_ENV=development nodemon src/ui/server.js",
+    "ui": "NODE_ENV=development nodemon src/ui/server.js",
+    "ui:dev": "vite",
+    "ui:build": "vite build",
+    "ui:prod": "node src/ui/server.js",
+    "demo:ui": "NODE_ENV=production PO_ROOT=demo node src/ui/server.js",
+    "demo:orchestrator": "PO_ROOT=demo NODE_ENV=production node -e \"import('./src/core/orchestrator.js').then(m => m.startOrchestrator({ dataDir: process.env.PO_ROOT || 'demo' })).catch(err => { console.error(err); process.exit(1) })\"",
+    "demo:all": "NODE_ENV=development PO_ROOT=demo npm run ui:build && concurrently \"npm:demo:ui\" \"npm:demo:orchestrator\" --kill-others-on-fail",
+    "demo:prod": "npm run ui:build && NODE_ENV=production PO_ROOT=demo node src/ui/server.js"
   },
   "dependencies": {
+    "@radix-ui/react-progress": "^1.1.7",
+    "@radix-ui/react-tabs": "^1.1.13",
+    "@radix-ui/react-toast": "^1.2.15",
+    "@radix-ui/react-tooltip": "^1.2.8",
+    "@radix-ui/themes": "^3.2.1",
     "ajv": "^8.17.1",
     "chokidar": "^3.5.3",
-    "dotenv": "^17.2.2",
-    "openai": "^5.23.1"
+    "clsx": "^2.1.1",
+    "commander": "^14.0.2",
+    "dotenv": "^17.2.3",
+    "lucide-react": "^0.544.0",
+    "openai": "^5.23.1",
+    "react": "^19.2.0",
+    "react-dom": "^19.2.0",
+    "react-router-dom": "^7.9.4",
+    "tslib": "^2.8.1"
   },
   "devDependencies": {
+    "@eslint/js": "^9.37.0",
+    "@tailwindcss/postcss": "^4.1.14",
+    "@testing-library/jest-dom": "^6.9.1",
+    "@testing-library/react": "^16.3.0",
+    "@testing-library/user-event": "^14.6.1",
+    "@vitejs/plugin-react": "^5.0.4",
     "@vitest/coverage-v8": "^3.2.4",
-    "nodemon": "^3.0.2",
-    "prettier": "^3.0.0",
+    "concurrently": "^9.2.1",
+    "eslint": "^9.37.0",
+    "eslint-plugin-react": "^7.37.5",
+    "eslint-plugin-react-hooks": "^6.1.1",
+    "eslint-plugin-react-refresh": "^0.4.23",
+    "jsdom": "^27.0.0",
+    "nodemon": "^3.1.10",
+    "prettier": "^3.6.2",
+    "tailwindcss": "^4.1.14",
+    "vite": "^7.1.9",
     "vitest": "^3.2.4"
   },
   "engines": {

package/src/api/files.js

@@ -0,0 +1,48 @@
+/**
+ * Atomic file operations for seed upload
+ * @module api/files
+ */
+
+import { promises as fs } from "fs";
+import { randomUUID } from "crypto";
+
+/**
+ * Write file atomically using temp file then rename
+ * @param {string} filePath - Target file path
+ * @param {string|Buffer} content - File content
+ * @returns {Promise<void>}
+ */
+async function atomicWrite(filePath, content) {
+  const tempPath = `${filePath}.${randomUUID()}.tmp`;
+
+  try {
+    // Write to temp file first
+    await fs.writeFile(tempPath, content);
+
+    // Atomically rename to target path
+    await fs.rename(tempPath, filePath);
+  } catch (error) {
+    // Clean up temp file on any error
+    try {
+      await fs.unlink(tempPath);
+    } catch (cleanupError) {
+      // Ignore cleanup errors
+    }
+    throw error;
+  }
+}
+
+/**
+ * Clean up partial files on failure
+ * @param {string} filePath - File path that may have partial writes
+ * @returns {Promise<void>}
+ */
+async function cleanupOnFailure(filePath) {
+  try {
+    await fs.unlink(filePath);
+  } catch (error) {
+    // File doesn't exist or can't be deleted - ignore
+  }
+}
+
+export { atomicWrite, cleanupOnFailure };