@ryanfw/prompt-orchestration-pipeline 0.12.0 → 0.13.1

package/src/task-analysis/enrichers/schema-deducer.js

@@ -0,0 +1,145 @@
+import { chat } from "../../llm/index.js";
+import Ajv from "ajv";
+
+const ajv = new Ajv();
+
+/**
+ * Deduce JSON schema for an artifact using LLM with structured output.
+ *
+ * @param {string} taskCode - Full source code of the task file
+ * @param {object} artifact - Artifact info { fileName, stage }
+ * @returns {Promise<{ schema, example, reasoning }>}
+ */
+export async function deduceArtifactSchema(taskCode, artifact) {
+  const response = await chat({
+    provider: "deepseek",
+    model: "deepseek-chat",
+    temperature: 0,
+    responseFormat: { type: "json_object" },
+    messages: [
+      { role: "system", content: buildSystemPrompt() },
+      { role: "user", content: buildUserPrompt(taskCode, artifact) },
+    ],
+  });
+
+  if (!response || typeof response !== "object") {
+    throw new Error(
+      `LLM response is missing or not an object when deducing artifact schema for "${artifact.fileName}".`
+    );
+  }
+
+  const result = response.content;
+
+  if (!result || typeof result !== "object") {
+    throw new Error(
+      `LLM response.content is missing or not an object when deducing artifact schema for "${artifact.fileName}".`
+    );
+  }
+
+  const { schema, example, reasoning } = result;
+
+  if (
+    !schema ||
+    typeof schema !== "object" ||
+    !example ||
+    typeof example !== "object" ||
+    typeof reasoning !== "string"
+  ) {
+    throw new Error(
+      `LLM returned invalid structured output when deducing artifact schema for "${artifact.fileName}". ` +
+        `Expected properties: { schema: object, example: object, reasoning: string }.`
+    );
+  }
+
+  // Validate the generated example against the generated schema
+  const validate = ajv.compile(schema);
+  if (!validate(example)) {
+    throw new Error(
+      `Generated example does not validate against schema: ${JSON.stringify(validate.errors)}`
+    );
+  }
+
+  return {
+    schema,
+    example,
+    reasoning,
+  };
+}
+
+function buildSystemPrompt() {
+  return `You are a code analysis expert that deduces JSON schemas from JavaScript source code.
+
+Your task: Given a pipeline task's source code and a target artifact filename, extract the JSON schema that describes that artifact's structure.
+
+ANALYSIS STRATEGY (follow this order):
+1. FIRST: Look for an exported schema constant (e.g. \`export const <name>Schema = {...}\`) that matches the artifact
+2. SECOND: Find the io.writeArtifact() call for this artifact and trace the data being written
+3. THIRD: Look for JSON structure hints in LLM prompts, JSON.parse usage, or data transformations
+4. FOURTH: If the artifact is read and validated elsewhere, check validation code for schema hints
+
+OUTPUT REQUIREMENTS:
+- Schema must be valid JSON Schema Draft-07
+- Schema must include "$schema": "http://json-schema.org/draft-07/schema#"
+- Schema must include "type", "properties", and "required" fields
+- Example must be realistic data that validates against the schema
+- Reasoning must explain your analysis steps
+
+You must respond with a JSON object matching this exact structure:
+{
+  "schema": { <valid JSON Schema Draft-07> },
+  "example": { <realistic example data> },
+  "reasoning": "<step-by-step explanation of how you determined the schema>"
+}`;
+}
+
+function buildUserPrompt(taskCode, artifact) {
+  return `## Task Source Code
+
+\`\`\`javascript
+${taskCode}
+\`\`\`
+
+## Target Artifact
+- Filename: ${artifact.fileName}
+- Written in stage: ${artifact.stage}
+
+## Few-Shot Example
+
+For a task that writes "user-profile.json" with code like:
+\`\`\`javascript
+await io.writeArtifact("user-profile.json", JSON.stringify({
+  name: user.name,
+  email: user.email,
+  preferences: { theme: "dark" }
+}));
+\`\`\`
+
+The correct output would be:
+{
+  "schema": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "required": ["name", "email", "preferences"],
+    "properties": {
+      "name": { "type": "string" },
+      "email": { "type": "string", "format": "email" },
+      "preferences": {
+        "type": "object",
+        "properties": {
+          "theme": { "type": "string", "enum": ["light", "dark"] }
+        }
+      }
+    }
+  },
+  "example": {
+    "name": "Jane Doe",
+    "email": "jane@example.com",
+    "preferences": { "theme": "dark" }
+  },
+  "reasoning": "Found io.writeArtifact call with inline object literal. Traced property types from the object structure. Added format:email based on property name convention."
+}
+
+## Your Task
+
+Analyze the source code and produce the schema, example, and reasoning for the artifact "${artifact.fileName}" written in stage "${artifact.stage}".`;
+}

package/src/task-analysis/enrichers/schema-writer.js

@@ -0,0 +1,74 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Write schema, sample, and meta files for an artifact.
+ *
+ * Key design: Schema files are PURE JSON Schema with no extra keys.
+ * Metadata is stored in a separate .meta.json file.
+ *
+ * @param {string} pipelinePath - Path to pipeline directory
+ * @param {string} artifactName - Artifact filename (e.g., "output.json")
+ * @param {object} deducedData - Object containing { schema, example, reasoning }
+ */
+export async function writeSchemaFiles(
+  pipelinePath,
+  artifactName,
+  deducedData
+) {
+  // Validate that deducedData contains all required properties
+  if (!deducedData || typeof deducedData !== "object") {
+    throw new Error(
+      `Invalid deducedData: expected an object but got ${typeof deducedData}`
+    );
+  }
+
+  if (!deducedData.schema || typeof deducedData.schema !== "object") {
+    throw new Error(
+      `Invalid deducedData.schema: expected an object but got ${typeof deducedData.schema}`
+    );
+  }
+
+  if (deducedData.example === undefined || deducedData.example === null) {
+    throw new Error(
+      `Invalid deducedData.example: expected a value but got ${deducedData.example}`
+    );
+  }
+
+  if (typeof deducedData.reasoning !== "string") {
+    throw new Error(
+      `Invalid deducedData.reasoning: expected a string but got ${typeof deducedData.reasoning}`
+    );
+  }
+
+  const schemasDir = path.join(pipelinePath, "schemas");
+  await fs.mkdir(schemasDir, { recursive: true });
+
+  const baseName = path.parse(artifactName).name;
+
+  // 1. Write pure schema (valid JSON Schema Draft-07)
+  await fs.writeFile(
+    path.join(schemasDir, `${baseName}.schema.json`),
+    JSON.stringify(deducedData.schema, null, 2)
+  );
+
+  // 2. Write sample data (plain JSON)
+  await fs.writeFile(
+    path.join(schemasDir, `${baseName}.sample.json`),
+    JSON.stringify(deducedData.example, null, 2)
+  );
+
+  // 3. Write metadata separately (doesn't pollute schema/example)
+  await fs.writeFile(
+    path.join(schemasDir, `${baseName}.meta.json`),
+    JSON.stringify(
+      {
+        source: "llm-deduction",
+        generatedAt: new Date().toISOString(),
+        reasoning: deducedData.reasoning,
+      },
+      null,
+      2
+    )
+  );
+}

package/src/task-analysis/extractors/artifacts.js

@@ -0,0 +1,137 @@
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+const traverse =
+  require("@babel/traverse").default ?? require("@babel/traverse");
+const generate =
+  require("@babel/generator").default ?? require("@babel/generator");
+import * as t from "@babel/types";
+import { isInsideTryCatch, getStageName } from "../utils/ast.js";
+
+/**
+ * Extract io.readArtifact calls from the AST.
+ *
+ * @param {import("@babel/types").File} ast - The AST to analyze
+ * @returns {Array<{fileName: string, stage: string, required: boolean}>} Array of artifact read references
+ * @throws {Error} If io.readArtifact call is found outside an exported function
+ */
+export function extractArtifactReads(ast) {
+  const reads = [];
+
+  traverse(ast, {
+    CallExpression(path) {
+      const { callee } = path.node;
+
+      // Match: io.readArtifact("file.json") or io.readArtifact`file.json`
+      if (
+        t.isMemberExpression(callee) &&
+        t.isIdentifier(callee.object, { name: "io" }) &&
+        t.isIdentifier(callee.property, { name: "readArtifact" })
+      ) {
+        // Extract fileName from first argument
+        const fileName = extractFileName(path.node.arguments[0]);
+
+        if (!fileName) {
+          throw new Error(
+            `io.readArtifact requires a string literal or template literal argument at ${path.node.loc?.start?.line}:${path.node.loc?.start?.column}`
+          );
+        }
+
+        // Get the stage name (must be in an exported function)
+        const stage = getStageName(path);
+
+        if (!stage) {
+          throw new Error(
+            `io.readArtifact call found outside an exported function at ${path.node.loc?.start?.line}:${path.node.loc?.start?.column}`
+          );
+        }
+
+        // Check if inside try/catch to determine if required
+        const required = !isInsideTryCatch(path);
+
+        reads.push({ fileName, stage, required });
+      }
+    },
+  });
+
+  return reads;
+}
+
+/**
+ * Extract io.writeArtifact calls from the AST.
+ *
+ * @param {import("@babel/types").File} ast - The AST to analyze
+ * @returns {Array<{fileName: string, stage: string}>} Array of artifact write references
+ * @throws {Error} If io.writeArtifact call is found outside an exported function
+ */
+export function extractArtifactWrites(ast) {
+  const writes = [];
+
+  traverse(ast, {
+    CallExpression(path) {
+      const { callee } = path.node;
+
+      // Match: io.writeArtifact("file.json", content)
+      if (
+        t.isMemberExpression(callee) &&
+        t.isIdentifier(callee.object, { name: "io" }) &&
+        t.isIdentifier(callee.property, { name: "writeArtifact" })
+      ) {
+        // Extract fileName from first argument
+        const fileName = extractFileName(path.node.arguments[0]);
+
+        if (!fileName) {
+          throw new Error(
+            `io.writeArtifact requires a string literal or template literal argument at ${path.node.loc?.start?.line}:${path.node.loc?.start?.column}`
+          );
+        }
+
+        // Get the stage name (must be in an exported function)
+        const stage = getStageName(path);
+
+        if (!stage) {
+          throw new Error(
+            `io.writeArtifact call found outside an exported function at ${path.node.loc?.start?.line}:${path.node.loc?.start?.column}`
+          );
+        }
+
+        writes.push({ fileName, stage });
+      }
+    },
+  });
+
+  return writes;
+}
+
+/**
+ * Extract filename from a string literal or template literal.
+ *
+ * @param {import("@babel/types").Node} node - The argument node
+ * @returns {string | null} The extracted filename or null if not a string/template literal
+ */
+function extractFileName(node) {
+  // Handle string literals: "file.json"
+  if (t.isStringLiteral(node)) {
+    return node.value;
+  }
+
+  // Handle template literals: `file.json` or `file-${name}.json`
+  if (t.isTemplateLiteral(node)) {
+    // If there are no expressions, use the simple approach
+    if (!node.expressions || node.expressions.length === 0) {
+      return node.quasis.map((q) => q.value.cooked).join("");
+    }
+
+    // For template literals with expressions, use @babel/generator to preserve them
+    // This ensures dynamic filenames like `file-${name}.json` are preserved as-is
+    const generated = generate(node, { concise: true });
+    // Remove the backticks from the generated code
+    const code = generated.code;
+    if (code.startsWith("`") && code.endsWith("`")) {
+      return code.slice(1, -1);
+    }
+    // Fallback in case the generated code doesn't have backticks
+    return code;
+  }
+
+  return null;
+}

package/src/task-analysis/extractors/llm-calls.js

@@ -0,0 +1,176 @@
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+const traverse =
+  require("@babel/traverse").default ?? require("@babel/traverse");
+import * as t from "@babel/types";
+import { getStageName } from "../utils/ast.js";
+
+/**
+ * Extract LLM method calls from the AST.
+ *
+ * Matches direct calls like:
+ * - llm.deepseek.chat(...)
+ * - llm.openai.gpt5Mini(...)
+ * - llm.anthropic.sonnet45(...)
+ * - llm.gemini.flash25(...)
+ *
+ * And destructured calls like:
+ * - const { deepseek } = llm; deepseek.chat(...)
+ *
+ * @param {import("@babel/types").File} ast - The AST to analyze
+ * @returns {Array<{provider: string, method: string, stage: string}>} Array of LLM call references
+ * @throws {Error} If LLM call is found outside an exported function
+ */
+export function extractLLMCalls(ast) {
+  const calls = [];
+
+  traverse(ast, {
+    CallExpression(path) {
+      const { callee } = path.node;
+
+      // Match: llm.provider.method(...)
+      if (isDirectLLMCall(callee)) {
+        const provider = callee.object.property.name;
+        const method = callee.property.name;
+        const stage = getStageName(path);
+
+        if (!stage) {
+          throw new Error(
+            `LLM call found outside an exported function at ${path.node.loc?.start?.line}:${path.node.loc?.start?.column}`
+          );
+        }
+
+        calls.push({ provider, method, stage });
+      }
+
+      // Match: provider.method(...) where provider was destructured from llm
+      if (isDestructuredLLMCall(callee, path)) {
+        const provider = callee.object.name;
+        const method = callee.property.name;
+        const stage = getStageName(path);
+
+        if (!stage) {
+          throw new Error(
+            `LLM call found outside an exported function at ${path.node.loc?.start?.line}:${path.node.loc?.start?.column}`
+          );
+        }
+
+        calls.push({ provider, method, stage });
+      }
+    },
+  });
+
+  return calls;
+}
+
+/**
+ * Check if a callee node is a direct LLM call pattern.
+ *
+ * Matches nested member expressions: llm.provider.method
+ *
+ * @param {import("@babel/types").Node} callee - The callee node to check
+ * @returns {boolean} True if the callee is a direct LLM call pattern
+ */
+function isDirectLLMCall(callee) {
+  // Must be a member expression (e.g., llm.provider.method)
+  if (!t.isMemberExpression(callee)) {
+    return false;
+  }
+
+  // The object must also be a member expression (e.g., llm.provider)
+  if (!t.isMemberExpression(callee.object)) {
+    return false;
+  }
+
+  // The root object must be identifier "llm"
+  if (!t.isIdentifier(callee.object.object, { name: "llm" })) {
+    return false;
+  }
+
+  // The object property must be an identifier (the provider)
+  if (!t.isIdentifier(callee.object.property)) {
+    return false;
+  }
+
+  // The callee property must be an identifier (the method)
+  if (!t.isIdentifier(callee.property)) {
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Check if a callee node is a destructured LLM call pattern.
+ *
+ * Matches: provider.method(...) where provider was destructured from llm
+ * Uses scope analysis to verify the identifier was actually destructured from llm
+ * in the current scope, avoiding false positives from same-named identifiers in different scopes.
+ *
+ * @param {import("@babel/types").Node} callee - The callee node to check
+ * @param {import("@babel/traverse").NodePath} path - The path of the call expression
+ * @returns {boolean} True if the callee is a destructured LLM call pattern
+ */
+function isDestructuredLLMCall(callee, path) {
+  // Must be a member expression (e.g., provider.method)
+  if (!t.isMemberExpression(callee)) {
+    return false;
+  }
+
+  // The object must be an identifier (the destructured provider)
+  if (!t.isIdentifier(callee.object)) {
+    return false;
+  }
+
+  // The callee property must be an identifier (the method)
+  if (!t.isIdentifier(callee.property)) {
+    return false;
+  }
+
+  // Get the binding for this identifier in the current scope
+  const binding = path.scope.getBinding(callee.object.name);
+  if (!binding) {
+    return false;
+  }
+
+  // Check if it was destructured from llm in a variable declaration
+  if (t.isVariableDeclarator(binding.path.node)) {
+    const { id, init } = binding.path.node;
+    // Match: const { provider } = llm
+    if (t.isObjectPattern(id) && t.isIdentifier(init, { name: "llm" })) {
+      return true;
+    }
+  }
+
+  // Check if it was destructured from llm in function parameters
+  // Match: ({ llm: { provider } }) => {}
+  if (binding.kind === "param" && t.isObjectPattern(binding.path.node)) {
+    // The binding points to the entire parameter ObjectPattern
+    // Look for a property with key "llm" whose value is an ObjectPattern
+    const llmProperty = binding.path.node.properties.find(
+      (prop) =>
+        t.isObjectProperty(prop) &&
+        t.isIdentifier(prop.key, { name: "llm" }) &&
+        t.isObjectPattern(prop.value)
+    );
+
+    if (llmProperty) {
+      // Check if the provider identifier is in the nested ObjectPattern
+      // Handles both shorthand ({ llm: { provider } }) and renamed ({ llm: { provider: alias } }) patterns
+      const providerInPattern = llmProperty.value.properties.some((innerProp) =>
+        t.isObjectProperty(innerProp)
+          ? // Check key for shorthand pattern: { provider }
+            t.isIdentifier(innerProp.key, { name: callee.object.name }) ||
+            // Check value for renamed pattern: { provider: alias } where we're looking for alias
+            t.isIdentifier(innerProp.value, { name: callee.object.name })
+          : false
+      );
+
+      if (providerInPattern) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}

package/src/task-analysis/extractors/stages.js

@@ -0,0 +1,51 @@
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+const traverse =
+  require("@babel/traverse").default ?? require("@babel/traverse");
+
+/**
+ * Extract exported function stages from an AST.
+ *
+ * Visits ExportNamedDeclaration nodes and extracts stage information
+ * including name, line order, and async status.
+ *
+ * @param {import("@babel/types").File} ast - The parsed AST
+ * @returns {Array<{name: string, order: number, isAsync: boolean}>}
+ *          Array of stages sorted by order (line number)
+ */
+export function extractStages(ast) {
+  const stages = [];
+
+  traverse(ast, {
+    ExportNamedDeclaration(path) {
+      const declaration = path.node.declaration;
+
+      // Handle: export function name() {}
+      if (declaration?.type === "FunctionDeclaration") {
+        stages.push({
+          name: declaration.id.name,
+          order: path.node.loc?.start.line ?? 0,
+          isAsync: declaration.async ?? false,
+        });
+        return;
+      }
+
+      // Handle: export const name = () => {} or export const name = async () => {}
+      // or export const name = function() {}
+      if (declaration?.type === "VariableDeclaration") {
+        const declarator = declaration.declarations[0];
+        const init = declarator?.init;
+
+        if (init?.type === "ArrowFunctionExpression" || init?.type === "FunctionExpression") {
+          stages.push({
+            name: declarator.id.name,
+            order: path.node.loc?.start.line ?? 0,
+            isAsync: init.async ?? false,
+          });
+        }
+      }
+    },
+  });
+
+  return stages.sort((a, b) => a.order - b.order);
+}

package/src/task-analysis/index.js

@@ -0,0 +1,103 @@
+import { parseTaskSource } from "./parser.js";
+import { extractStages } from "./extractors/stages.js";
+import {
+  extractArtifactReads,
+  extractArtifactWrites,
+} from "./extractors/artifacts.js";
+import { extractLLMCalls } from "./extractors/llm-calls.js";
+import { writeAnalysisFile } from "./enrichers/analysis-writer.js";
+
+/**
+ * Analyze task source code and extract metadata.
+ *
+ * This is the main entry point for the task analysis library. It parses
+ * the source code and extracts:
+ * - Stages (exported functions with order and async status)
+ * - Artifacts (read/write operations with stage context)
+ * - Models (LLM provider and method calls with stage context)
+ *
+ * @param {string} code - The task source code to analyze
+ * @param {string|null} taskFilePath - Path to the task file (optional)
+ * @returns {TaskAnalysis} Complete analysis of the task
+ * @throws {Error} If parsing or extraction fails
+ *
+ * @example
+ * const code = `
+ *   export function ingestion({ io, llm, data, flags }) {
+ *     const content = io.readArtifact("input.json");
+ *     const result = llm.deepseek.chat(content);
+ *     io.writeArtifact("output.json", result);
+ *   }
+ * `;
+ * const analysis = analyzeTask(code, "/path/to/task.js");
+ * // Returns:
+ * // {
+ * //   taskFilePath: "/path/to/task.js",
+ * //   stages: [{ name: "ingestion", order: 2, isAsync: false }],
+ * //   artifacts: {
+ * //     reads: [{ fileName: "input.json", stage: "ingestion", required: true }],
+ * //     writes: [{ fileName: "output.json", stage: "ingestion" }]
+ * //   },
+ * //   models: [{ provider: "deepseek", method: "chat", stage: "ingestion" }]
+ * // }
+ */
+export { writeAnalysisFile };
+
+export function analyzeTask(code, taskFilePath = null) {
+  // Parse the source code into an AST
+  const ast = parseTaskSource(code);
+
+  // Extract all metadata from the AST
+  const stages = extractStages(ast);
+  const reads = extractArtifactReads(ast);
+  const writes = extractArtifactWrites(ast);
+  const models = extractLLMCalls(ast);
+
+  // Compose into the TaskAnalysis object
+  return {
+    taskFilePath,
+    stages,
+    artifacts: {
+      reads,
+      writes,
+    },
+    models,
+  };
+}
+
+/**
+ * @typedef {Object} TaskAnalysis
+ * @property {string|null} taskFilePath - Path to the task file
+ * @property {Array<Stage>} stages - Array of exported function stages
+ * @property {Object} artifacts - Artifact operations
+ * @property {Array<ArtifactRead>} artifacts.reads - Artifact read operations
+ * @property {Array<ArtifactWrite>} artifacts.writes - Artifact write operations
+ * @property {Array<ModelCall>} models - LLM method calls
+ */
+
+/**
+ * @typedef {Object} Stage
+ * @property {string} name - Stage function name
+ * @property {number} order - Line number for execution order
+ * @property {boolean} isAsync - Whether the stage is async
+ */
+
+/**
+ * @typedef {Object} ArtifactRead
+ * @property {string} fileName - Name of the artifact file
+ * @property {string} stage - Stage name where read occurs
+ * @property {boolean} required - Whether the read is required (not wrapped in try/catch)
+ */
+
+/**
+ * @typedef {Object} ArtifactWrite
+ * @property {string} fileName - Name of the artifact file
+ * @property {string} stage - Stage name where write occurs
+ */
+
+/**
+ * @typedef {Object} ModelCall
+ * @property {string} provider - LLM provider name (e.g., "deepseek", "openai")
+ * @property {string} method - LLM method name (e.g., "chat", "gpt5Mini")
+ * @property {string} stage - Stage name where LLM call occurs
+ */