@ryanfw/prompt-orchestration-pipeline 0.16.1 → 0.16.3

package/docs/pop-task-guide.md

@@ -0,0 +1,268 @@
+# POP Pipeline Task Guide
+
+> Unified reference for creating valid pipeline tasks. Only exported stage functions with exact names below are called by the pipeline runner.
+
+---
+
+## Critical Rules
+
+### Valid Stage Names (Exhaustive List)
+
+The pipeline runner **ONLY** calls these 11 exported functions:
+
+| Stage | Required | Purpose |
+|-------|----------|---------|
+| `ingestion` | Yes | Load input from `data.seed` |
+| `preProcessing` | No | Normalize/enrich data |
+| `promptTemplating` | Yes | Build LLM prompts |
+| `inference` | Yes | Call LLM |
+| `parsing` | No | Parse LLM output |
+| `validateStructure` | No | JSON schema validation |
+| `validateQuality` | No | Domain-specific checks |
+| `critique` | No | Analyze failures |
+| `refine` | No | Produce improved output |
+| `finalValidation` | No | Final validation gate |
+| `integration` | No | Persist results |
+
+### Required Contract
+
+Every stage function must:
+1. Be exported: `export const stageName = ...`
+2. Return: `{ output: any, flags: object }`
+
+### Anti-Patterns (Invalid)
+
+```js
+// ❌ WRONG: Helper functions are NEVER called by pipeline
+function formatPrompt(topic) { return `...${topic}...`; }
+
+// ❌ WRONG: Non-standard export names are NEVER called
+export const myCustomStage = () => ({ output: {}, flags: {} });
+
+// ❌ WRONG: Must return { output, flags } object
+export const ingestion = () => "just a string";
+```
+
+---
+
+## Minimal Working Example
+
+A simple 3-stage task (most tasks only need ingestion → promptTemplating → inference):
+
+```js
+export const ingestion = ({
+  data: { seed: { data: { topic } } },
+  flags,
+}) => ({
+  output: { topic },
+  flags,
+});
+
+export const promptTemplating = ({
+  data: { ingestion: { topic } },
+  flags,
+}) => ({
+  output: {
+    system: "You are a helpful assistant. Respond in JSON.",
+    prompt: `Write about: ${topic}\n\nRespond as: { "content": "..." }`,
+  },
+  flags,
+});
+
+export const inference = async ({
+  io,
+  llm: { deepseek },
+  data: { promptTemplating: { system, prompt } },
+  flags,
+}) => {
+  const response = await deepseek.chat({
+    messages: [
+      { role: "system", content: system },
+      { role: "user", content: prompt },
+    ],
+  });
+  
+  const parsed = typeof response.content === "string"
+    ? JSON.parse(response.content)
+    : response.content;
+  
+  await io.writeArtifact("output.json", JSON.stringify(parsed, null, 2));
+  return { output: {}, flags };
+};
+```
+
+---
+
+## Stage Function Signatures
+
+### ingestion
+```js
+export const ingestion = ({ data: { seed }, flags }) => ({
+  output: { /* extracted fields */ },
+  flags,
+});
+```
+
+### promptTemplating
+```js
+export const promptTemplating = ({ data: { ingestion }, flags }) => ({
+  output: { system: "...", prompt: "..." },
+  flags,
+});
+```
+
+### inference
+**Rule**: Read prompts from `data.promptTemplating`, not from other sources.
+```js
+export const inference = async ({
+  io,
+  llm: { provider },
+  data: { promptTemplating: { system, prompt } },
+  flags,
+}) => {
+  const response = await provider.chat({ messages: [...] });
+  const parsed = response.parsed;
+  await io.writeArtifact("output.json", JSON.stringify(parsed, null, 2));
+  return { output: {}, flags };
+};
+```
+
+### validateStructure
+```js
+export const validateStructure = async ({
+  io,
+  flags,
+  validators: { validateWithSchema },
+}) => {
+  const content = await io.readArtifact("output.json");
+  // Provide your JSON schema here, for example the `outputSchema` from the "JSON Schema Export" section.
+  const mySchema = /* your JSON schema object */ {};
+  const result = validateWithSchema(mySchema, content);
+  if (!result.valid) {
+    return { output: {}, flags: { ...flags, validationFailed: true } };
+  }
+  return { output: {}, flags };
+};
+```
+
+---
+
+## IO API
+
+Available on `io` object passed to stages.
+
+| Function | Parameters | Returns | Description |
+|----------|------------|---------|-------------|
+| `io.writeArtifact` | `name, content, { mode? }` | `Promise<string>` | Persist output files |
+| `io.writeLog` | `name, content, { mode? }` | `Promise<string>` | Debug/progress logs |
+| `io.writeTmp` | `name, content, { mode? }` | `Promise<string>` | Scratch data |
+| `io.readArtifact` | `name` | `Promise<string>` | Load artifact |
+| `io.readLog` | `name` | `Promise<string>` | Read log |
+| `io.readTmp` | `name` | `Promise<string>` | Read temp file |
+| `io.getTaskDir` | — | `string` | Current task directory |
+| `io.getDB` | `options?` | `Database` | SQLite for job (WAL mode) |
+| `io.runBatch` | `{ jobs, processor, ... }` | `Promise<{ completed, failed }>` | Concurrent batch processing |
+
+**When to use artifacts vs stage output**: Use `io.writeArtifact` for large outputs, model-native text, values needed by multiple stages, or for auditability. Use stage `output` for small structured values needed immediately by the next stage.
+
+---
+
+## LLM API
+
+Available on `llm` object. Call with messages array:
+
+```js
+const response = await llm.deepseek.chat({
+  messages: [
+    { role: "system", content: "..." },
+    { role: "user", content: "..." },
+  ],
+  temperature: 0.7,      // optional: 0-2
+  maxTokens: 1000,       // optional
+  responseFormat: "json" // optional
+});
+// Returns: { content: any, usage?: object }
+```
+
+### Available Providers
+- `llm.deepseek.chat()`
+- `llm.anthropic.sonnet45()`
+- `llm.openai.gpt5Mini()`
+- `llm.gemini.flash25()`
+
+---
+
+## Validation API
+
+Available via `validators` object in stages that need schema validation.
+
+```js
+validateWithSchema(schema, data) → { valid: boolean, errors?: AjvError[] }
+```
+
+- Accepts string or object (strings parsed as JSON)
+- Uses Ajv with `{ allErrors: true, strict: false }`
+
+---
+
+## JSON Schema Export
+
+Tasks export schemas to validate their output:
+
+```js
+export const outputSchema = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  required: ["content"],
+  properties: {
+    content: { type: "string", minLength: 1 }
+  }
+};
+```
+
+---
+
+## Seed File Format
+
+Pipeline jobs start from a seed file in `pending/`:
+
+```json
+{
+  "name": "unique-job-id",
+  "pipeline": "pipeline-slug",
+  "data": { /* context for tasks */ }
+}
+```
+
+---
+
+## Context Object Reference
+
+Each stage receives:
+
+```js
+{
+  io,                    // File I/O (may be null)
+  llm,                   // LLM client
+  validators,            // { validateWithSchema }
+  flags,                 // Control flags
+  meta: { taskName, workDir, jobId },
+  data: {
+    seed,                // Initial payload
+    ingestion,           // Output from ingestion
+    preProcessing,       // Output from preProcessing
+    promptTemplating,    // Output from promptTemplating
+    // ... other stage outputs
+  },
+  output,                // Previous non-validation stage output
+}
+```
+
+---
+
+## Summary
+
+1. Export only valid stage names: `ingestion`, `preProcessing`, `promptTemplating`, `inference`, `parsing`, `validateStructure`, `validateQuality`, `critique`, `refine`, `finalValidation`, `integration`
+2. Return `{ output, flags }` from every stage
+3. Custom helper functions are valid JavaScript but will not be called by the pipeline—only use them if called from within a valid stage
+4. Most simple tasks need only: `ingestion` → `promptTemplating` → `inference`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "0.16.1",
+  "version": "0.16.3",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server.js",
@@ -9,6 +9,7 @@
   },
   "files": [
     "src",
+    "docs/pop-task-guide.md",
     "README.md",
     "LICENSE"
   ],

package/src/providers/base.js

@@ -35,7 +35,7 @@ export async function sleep(ms) {
 
 /**
  * Strip markdown code fences from text unconditionally.
- * Handles ```json, ```JSON, and plain ``` with or without newlines.
+ * Handles any language identifier (```json, ```javascript, etc.) or plain ```.
  * @param {string} text - The text to strip fences from
  * @returns {string} The cleaned text, or original if not a string
  */
@@ -43,8 +43,8 @@ export function stripMarkdownFences(text) {
   if (typeof text !== "string") return text;
   const trimmed = text.trim();
   if (trimmed.startsWith("```")) {
-    // Remove opening fence (```json, ```JSON, or just ```)
-    let cleaned = trimmed.replace(/^```(?:json|JSON)?\s*\n?/, "");
+    // Remove opening fence with any language identifier
+    let cleaned = trimmed.replace(/^```[a-zA-Z]*\s*\n?/, "");
     // Remove closing fence
     cleaned = cleaned.replace(/\n?```\s*$/, "");
     return cleaned.trim();

package/src/task-analysis/enrichers/schema-deducer.js

@@ -54,6 +54,10 @@ export async function deduceArtifactSchema(taskCode, artifact) {
   }
 
   // Validate the generated example against the generated schema
+  // Remove any existing schema with the same $id to avoid "schema already exists" error
+  if (schema.$id && ajv.getSchema(schema.$id)) {
+    ajv.removeSchema(schema.$id);
+  }
   const validate = ajv.compile(schema);
   if (!validate(example)) {
     throw new Error(

package/src/ui/endpoints/task-creation-endpoint.js

@@ -1,49 +1,116 @@
 import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
 import { streamSSE } from "../lib/sse.js";
 import { createHighLevelLLM } from "../../llm/index.js";
+import { parseMentions } from "../lib/mention-parser.js";
+import {
+  loadSchemaContext,
+  buildSchemaPromptSection,
+} from "../lib/schema-loader.js";
+import { createLogger } from "../../core/logger.js";
 
-export async function handleTaskPlan(req, res) {
-  console.log("[task-creation-endpoint] Request received");
+const logger = createLogger("TaskCreationEndpoint");
 
-  const { messages, pipelineSlug } = req.body;
+// Resolve path relative to this module for NPM distribution
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const guidelinesPath = path.resolve(__dirname, "../../../docs/pop-task-guide.md");
 
-  console.log("[task-creation-endpoint] Request details:", {
-    hasMessages: !!messages,
-    messageCount: Array.isArray(messages) ? messages.length : 0,
-    pipelineSlug,
-    bodyKeys: Object.keys(req.body),
-  });
+export async function handleTaskPlan(req, res) {
+  const { messages, pipelineSlug } = req.body;
 
   // Validate input
   if (!Array.isArray(messages)) {
-    console.error(
-      "[task-creation-endpoint] Validation failed: messages is not an array"
-    );
     res.status(400).json({ error: "messages must be an array" });
     return;
   }
 
-  console.log(
-    "[task-creation-endpoint] Loading guidelines from docs/pipeline-task-guidelines.md..."
-  );
-
   // Load guidelines - let it throw if missing
-  const guidelinesPath = "docs/pipeline-task-guidelines.md";
   const guidelines = fs.readFileSync(guidelinesPath, "utf-8");
 
-  console.log(
-    "[task-creation-endpoint] Guidelines loaded, length:",
-    guidelines.length
-  );
+  // Parse @mentions and load schema contexts for enrichment
+  const mentionedFiles = parseMentions(messages);
+  const schemaContexts = [];
+  // Load schema contexts sequentially to avoid unbounded concurrent file I/O
+  for (const fileName of mentionedFiles) {
+    // eslint-disable-next-line no-await-in-loop
+    const context = await loadSchemaContext(pipelineSlug, fileName);
+    if (context) {
+      schemaContexts.push(context);
+    }
+  }
+  const schemaEnrichment = buildSchemaPromptSection(schemaContexts);
 
   // Build LLM messages array
-  const systemPrompt = `You are a pipeline task assistant. Help users create task definitions following these guidelines:
+  const systemPrompt = `You are a pipeline task assistant. You help users understand the POP (Prompt Orchestration Pipeline) system and create task definitions.
+
+## How to Answer Questions
+
+When users ask questions, identify which topic area applies and reference the relevant section of knowledge below:
+
+- **LLM/Provider questions** → See "Available LLM Providers" section
+- **Stage/Function questions** → See "Valid Stage Names" and "Stage Function Signatures" sections  
+- **IO/Database questions** → See "IO API" section
+- **Validation questions** → See "Validation API" and "JSON Schema Export" sections
+- **Task creation requests** → Use all sections to build a complete task
+
+Be concise and direct. Use code examples when helpful. Reference specific API signatures.
+
+---
+
+# KNOWLEDGE BASE
 
 ${guidelines}
+${schemaEnrichment ? `\n${schemaEnrichment}\n` : ""}
+
+---
+
+## Quick Reference: Common Questions
+
+**Q: What LLM models/providers are available?**
+Available providers via the \`llm\` object:
+- \`llm.deepseek.chat()\` - DeepSeek model
+- \`llm.anthropic.sonnet45()\` - Anthropic Claude Sonnet 4.5
+- \`llm.openai.gpt5Mini()\` - OpenAI GPT-5 Mini
+- \`llm.gemini.flash25()\` - Google Gemini Flash 2.5
+
+**Q: What functions/stages do I need to define?**
+Minimum required: \`ingestion\`, \`promptTemplating\`, \`inference\`
+Optional: \`preProcessing\`, \`parsing\`, \`validateStructure\`, \`validateQuality\`, \`critique\`, \`refine\`, \`finalValidation\`, \`integration\`
+
+**Q: How do I use the database?**
+Use \`io.getDB()\` to get a SQLite database instance (WAL mode):
+\`\`\`js
+const db = io.getDB();
+db.exec('CREATE TABLE IF NOT EXISTS results (id INTEGER PRIMARY KEY, data TEXT)');
+db.prepare('INSERT INTO results (data) VALUES (?)').run(JSON.stringify(myData));
+\`\`\`
+
+**Q: How do I read/write files?**
+Use the \`io\` object:
+- \`io.writeArtifact(name, content)\` - Persist output files
+- \`io.readArtifact(name)\` - Load artifact
+- \`io.writeTmp(name, content)\` - Scratch data
+- \`io.writeLog(name, content)\` - Debug/progress logs
+
+---
+
+## Task Proposal Guidelines
 
 Provide complete, working code. Use markdown code blocks.
 
-When you have completed a task definition that the user wants to create, wrap it in this format:
+ONLY use the [TASK_PROPOSAL] wrapper when ALL of these conditions are met:
+1. The user has explicitly requested you create/build/write a task for them
+2. You have a complete, production-ready task definition (not an example or illustration)
+3. The user has confirmed their requirements or iterated to a final version
+
+DO NOT use [TASK_PROPOSAL] for:
+- Answering questions about capabilities or how tasks work
+- Showing illustrative examples or code snippets
+- Explaining concepts with sample code
+- Incomplete or draft task definitions still being discussed
+
+When you DO output a [TASK_PROPOSAL], use this format:
 [TASK_PROPOSAL]
 FILENAME: <filename.js>
 TASKNAME: <task-name>
@@ -55,21 +122,13 @@ CODE:
 
   const llmMessages = [{ role: "system", content: systemPrompt }, ...messages];
 
-  console.log("[task-creation-endpoint] LLM messages array created:", {
-    totalMessages: llmMessages.length,
-    systemPromptLength: systemPrompt.length,
-  });
-
   // Create SSE stream
-  console.log("[task-creation-endpoint] Creating SSE stream...");
   const sse = streamSSE(res);
 
   try {
-    console.log("[task-creation-endpoint] Creating LLM instance...");
     // Get LLM instance (uses default provider from config)
     const llm = createHighLevelLLM();
 
-    console.log("[task-creation-endpoint] Calling LLM chat with streaming...");
     // Call LLM with streaming enabled
     const response = await llm.chat({
       messages: llmMessages,
@@ -77,38 +136,20 @@ CODE:
       stream: true,
     });
 
-    console.log("[task-creation-endpoint] LLM response received:", {
-      isStream: typeof response[Symbol.asyncIterator] !== "undefined",
-    });
-
     // Stream is an async generator
-    let chunkCount = 0;
     for await (const chunk of response) {
       if (chunk?.content) {
         sse.send("chunk", { content: chunk.content });
-        chunkCount++;
       }
     }
 
-    console.log("[task-creation-endpoint] Sent", chunkCount, "chunks via SSE");
-
     // Send done event
-    console.log("[task-creation-endpoint] Sending 'done' event...");
     sse.send("done", {});
-    console.log("[task-creation-endpoint] Ending SSE stream...");
     sse.end();
-    console.log("[task-creation-endpoint] Request completed successfully");
   } catch (error) {
-    console.error("[task-creation-endpoint] Error occurred:", {
-      message: error.message,
-      stack: error.stack,
-      name: error.name,
-    });
+    logger.error("LLM streaming failed", error);
     // Send error event
     sse.send("error", { message: error.message });
-    console.log(
-      "[task-creation-endpoint] Error sent via SSE, ending stream..."
-    );
     sse.end();
   }
-}
+}

package/src/ui/endpoints/task-save-endpoint.js

@@ -2,6 +2,7 @@ import path from "node:path";
 import { promises as fs } from "node:fs";
 import { getConfig } from "../../core/config.js";
 import { sendJson } from "../utils/http-utils.js";
+import { reviewAndCorrectTask } from "../lib/task-reviewer.js";
 
 /**
  * Handle task creation requests
@@ -52,7 +53,24 @@ export async function handleTaskSave(req, res) {
     if (!taskFilePath.startsWith(tasksDir)) {
       return sendJson(res, 400, { error: "Invalid filename" });
     }
-    await fs.writeFile(taskFilePath, code, "utf8");
+
+    // Self-correct code before saving
+    let finalCode = code;
+    try {
+      const guidelinesPath = path.join(
+        rootDir,
+        "docs/pipeline-task-guidelines.md"
+      );
+      const guidelines = await fs.readFile(guidelinesPath, "utf8");
+      finalCode = await reviewAndCorrectTask(code, guidelines);
+    } catch (reviewError) {
+      console.warn(
+        "Task review failed, using original code:",
+        reviewError.message
+      );
+    }
+
+    await fs.writeFile(taskFilePath, finalCode, "utf8");
 
     // Update index.js to export new task
     const indexPath = taskRegistryPath;

package/src/ui/lib/mention-parser.js

@@ -0,0 +1,24 @@
+/**
+ * Parse @[display](id) mentions from chat messages.
+ * Used to extract referenced artifact files for schema enrichment.
+ */
+
+const MENTION_REGEX = /@\[([^\]]+)\]\(([^)]+)\)/g;
+
+/**
+ * Extract unique filenames from @mentions in messages.
+ * @param {Array<{ role: string, content: string }>} messages
+ * @returns {string[]} Array of unique filenames
+ */
+export function parseMentions(messages) {
+  const filenames = new Set();
+
+  for (const msg of messages) {
+    if (!msg.content) continue;
+    for (const match of msg.content.matchAll(MENTION_REGEX)) {
+      filenames.add(match[2]);
+    }
+  }
+
+  return [...filenames];
+}

package/src/ui/lib/schema-loader.js

@@ -0,0 +1,69 @@
+/**
+ * Schema loader utility for task creation prompt enrichment.
+ * Loads JSON Schema, sample data, and metadata for referenced artifact files.
+ */
+
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { getPipelineConfig } from "../../core/config.js";
+
+/**
+ * Load schema context for a referenced artifact file.
+ * @param {string} pipelineSlug - Pipeline identifier
+ * @param {string} fileName - Artifact filename (e.g., "analysis-output.json")
+ * @returns {Promise<{ fileName: string, schema: object, sample: object, meta?: object } | null>}
+ */
+export async function loadSchemaContext(pipelineSlug, fileName) {
+  try {
+    const pipelineConfig = getPipelineConfig(pipelineSlug);
+    const pipelineDir = path.dirname(pipelineConfig.pipelineJsonPath);
+    const baseName = path.parse(fileName).name;
+    const schemasDir = path.join(pipelineDir, "schemas");
+
+    const schemaPath = path.join(schemasDir, `${baseName}.schema.json`);
+    const samplePath = path.join(schemasDir, `${baseName}.sample.json`);
+    const metaPath = path.join(schemasDir, `${baseName}.meta.json`);
+
+    // Schema is required - return null if missing
+    const schemaContent = await fs.readFile(schemaPath, "utf8");
+    const schema = JSON.parse(schemaContent);
+
+    // Sample is required - return null if missing
+    const sampleContent = await fs.readFile(samplePath, "utf8");
+    const sample = JSON.parse(sampleContent);
+
+    // Meta is optional
+    let meta;
+    try {
+      const metaContent = await fs.readFile(metaPath, "utf8");
+      meta = JSON.parse(metaContent);
+    } catch {
+      // Meta file missing or invalid - that's fine
+    }
+
+    return { fileName, schema, sample, meta };
+  } catch {
+    // Any error (pipeline not found, file missing, JSON parse error) -> return null
+    return null;
+  }
+}
+
+/**
+ * Build markdown prompt section from schema contexts.
+ * @param {Array<{ fileName: string, schema: object, sample: object, meta?: object }>} contexts
+ * @returns {string} Markdown formatted section for system prompt
+ */
+export function buildSchemaPromptSection(contexts) {
+  if (!contexts || contexts.length === 0) {
+    return "";
+  }
+
+  const sections = contexts.map((ctx) => {
+    let section = `### @${ctx.fileName}\n\n`;
+    section += `**JSON Schema:**\n\n\`\`\`json\n${JSON.stringify(ctx.schema, null, 2)}\n\`\`\`\n\n`;
+    section += `**Sample Data:**\n\n\`\`\`json\n${JSON.stringify(ctx.sample, null, 2)}\n\`\`\``;
+    return section;
+  });
+
+  return `## Referenced Files\n\n${sections.join("\n\n")}`;
+}

package/src/ui/lib/task-reviewer.js

@@ -0,0 +1,51 @@
+import { createHighLevelLLM } from "../../llm/index.js";
+import { stripMarkdownFences } from "../../providers/base.js";
+
+/**
+ * Review and correct task code using LLM
+ * @param {string} code - The task code to review
+ * @param {string} guidelines - Pipeline task guidelines
+ * @returns {Promise<string>} - Returns the original code if the LLM responds with
+ * NO_CHANGES_NEEDED; otherwise returns the LLM's corrected code output (after
+ * markdown fence stripping), which may be empty or invalid if the LLM response
+ * or formatting is unexpected.
+ */
+export async function reviewAndCorrectTask(code, guidelines) {
+  const llm = createHighLevelLLM();
+
+  const prompt = `Review this pipeline task code for:
+1. JavaScript syntax errors
+2. Logic flaws or bugs
+3. Violations of the pipeline task guidelines below
+4. Missing error handling for io/llm operations
+
+If the code is correct, respond with exactly: NO_CHANGES_NEEDED
+
+If corrections are needed, respond with only the corrected code (no explanation).
+
+## Guidelines
+
+${guidelines}
+
+## Code to Review
+
+\`\`\`javascript
+${code}
+\`\`\``;
+
+  const messages = [{ role: "user", content: prompt }];
+
+  const response = await llm.chat({ messages, responseFormat: "text" });
+  const content = response.content || "";
+  const trimmedContent = content.trim();
+
+  // If the LLM returned no usable content, keep the original code
+  if (!trimmedContent) {
+    return code;
+  }
+
+  if (trimmedContent.includes("NO_CHANGES_NEEDED")) {
+    return code;
+  }
+  return stripMarkdownFences(content);
+}