@ryanfw/prompt-orchestration-pipeline 0.4.0 → 0.6.0

package/src/core/task-runner.js

@@ -4,11 +4,28 @@ import fs from "fs";
 import { createLLM, getLLMEvents } from "../llm/index.js";
 import { loadFreshModule } from "./module-loader.js";
 import { loadEnvironment } from "./environment.js";
-import { getConfig } from "./config.js";
 import { createTaskFileIO } from "./file-io.js";
 import { writeJobStatus } from "./status-writer.js";
 import { computeDeterministicProgress } from "./progress.js";
 
+/**
+ * Derives model key and token counts from LLM metric event.
+ * Returns a tuple: [modelKey, inputTokens, outputTokens].
+ *
+ * @param {Object} metric - The LLM metric event from llm:request:complete
+ * @returns {Array<string, number, number>} [modelKey, inputTokens, outputTokens]
+ */
+export function deriveModelKeyAndTokens(metric) {
+  const provider = metric?.provider || "undefined";
+  const model = metric?.model || "undefined";
+  const modelKey = metric?.metadata?.alias || `${provider}:${model}`;
+  const input = Number.isFinite(metric?.promptTokens) ? metric.promptTokens : 0;
+  const output = Number.isFinite(metric?.completionTokens)
+    ? metric.completionTokens
+    : 0;
+  return [modelKey, input, output];
+}
+
 /**
  * Validates that a value is a plain object (not array, null, or class instance).
  * @param {*} value - The value to check
@@ -365,12 +382,45 @@ export async function runPipeline(modulePath, initialContext = {}) {
   const llmMetrics = [];
   const llmEvents = getLLMEvents();
 
+  // Per-run write queue for serializing tokenUsage appends
+  let tokenWriteQueue = Promise.resolve();
+
+  /**
+   * Appends token usage tuple to tasks-status.json with serialized writes.
+   * @param {string} workDir - Working directory path
+   * @param {string} taskName - Task identifier
+   * @param {Array<string, number, number>} tuple - [modelKey, inputTokens, outputTokens]
+   */
+  function appendTokenUsage(workDir, taskName, tuple) {
+    tokenWriteQueue = tokenWriteQueue
+      .then(() =>
+        writeJobStatus(workDir, (snapshot) => {
+          if (!snapshot.tasks[taskName]) {
+            snapshot.tasks[taskName] = {};
+          }
+          const task = snapshot.tasks[taskName];
+          if (!Array.isArray(task.tokenUsage)) {
+            task.tokenUsage = [];
+          }
+          task.tokenUsage.push(tuple);
+          return snapshot;
+        })
+      )
+      .catch((e) => console.warn("[task-runner] tokenUsage append failed:", e));
+  }
+
   const onLLMComplete = (metric) => {
     llmMetrics.push({
       ...metric,
       task: context.meta.taskName,
       stage: context.currentStage,
     });
+
+    // Append token usage immediately for each successful LLM completion
+    if (context.meta.workDir && context.meta.taskName) {
+      const tuple = deriveModelKeyAndTokens(metric);
+      appendTokenUsage(context.meta.workDir, context.meta.taskName, tuple);
+    }
   };
 
   llmEvents.on("llm:request:complete", onLLMComplete);
@@ -531,6 +581,8 @@ export async function runPipeline(modulePath, initialContext = {}) {
               error: errInfo,
               refinementCycle: refinementCount,
             });
+            await tokenWriteQueue.catch(() => {});
+            llmEvents.off("llm:request:complete", onLLMComplete);
             return {
               ok: false,
               failedStage: s,
@@ -824,6 +876,9 @@ export async function runPipeline(modulePath, initialContext = {}) {
           }
         }
 
+        await tokenWriteQueue.catch(() => {});
+        llmEvents.off("llm:request:complete", onLLMComplete);
+
         // For non-validation stages or when refinements are exhausted, fail immediately
         return {
           ok: false,
@@ -859,6 +914,8 @@ export async function runPipeline(modulePath, initialContext = {}) {
     typeof tasks.validateQuality === "function";
 
   if (context.flags.validationFailed && hasValidation) {
+    await tokenWriteQueue.catch(() => {});
+    llmEvents.off("llm:request:complete", onLLMComplete);
     return {
       ok: false,
       failedStage: "final-validation",
@@ -869,6 +926,9 @@ export async function runPipeline(modulePath, initialContext = {}) {
     };
   }
 
+  // Flush any trailing token usage appends before cleanup
+  await tokenWriteQueue.catch(() => {}); // absorb last error to not mask pipeline result
+
   llmEvents.off("llm:request:complete", onLLMComplete);
 
   // Write final status with currentStage: null to indicate completion

package/src/llm/index.js

@@ -18,8 +18,12 @@ export function registerMockProvider(provider) {
 
 // Auto-register mock provider in test mode when default provider is "mock"
 function autoRegisterMockProvider() {
-  const config = getConfig();
-  if (config.llm.defaultProvider === "mock" && !mockProviderInstance) {
+  // Skip config check in tests to avoid PO_ROOT requirement
+  const isTest =
+    process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+  const defaultProvider = isTest ? "mock" : getConfig().llm.defaultProvider;
+
+  if (defaultProvider === "mock" && !mockProviderInstance) {
     // Auto-register a basic mock provider for testing
     mockProviderInstance = {
       chat: async () => ({
@@ -92,6 +96,11 @@ export async function chat(options) {
     temperature,
     maxTokens,
     metadata = {},
+    topP,
+    frequencyPenalty,
+    presencePenalty,
+    stop,
+    responseFormat,
     ...rest
   } = options;
 
@@ -157,52 +166,90 @@ export async function chat(options) {
         totalTokens: result.usage.total_tokens,
       };
     } else if (provider === "openai") {
-      const result = await openaiChat({
+      const openaiArgs = {
         messages,
         model: model || "gpt-5-chat-latest",
-        maxTokens,
         temperature,
+        maxTokens,
         ...rest,
-      });
+      };
+      if (responseFormat !== undefined)
+        openaiArgs.responseFormat = responseFormat;
+      if (topP !== undefined) openaiArgs.topP = topP;
+      if (frequencyPenalty !== undefined)
+        openaiArgs.frequencyPenalty = frequencyPenalty;
+      if (presencePenalty !== undefined)
+        openaiArgs.presencePenalty = presencePenalty;
+      if (stop !== undefined) openaiArgs.stop = stop;
+
+      const result = await openaiChat(openaiArgs);
 
       response = {
-        content: typeof result === "string" ? result : JSON.stringify(result),
-        raw: result,
+        content:
+          result?.content ??
+          (typeof result === "string" ? result : String(result)),
+        raw: result?.raw ?? result,
       };
 
-      // Estimate tokens since GPT-5 responses API might not return usage
-      const promptTokens = estimateTokens(systemMsg + userMsg);
-      const completionTokens = estimateTokens(response.content);
-      usage = {
-        promptTokens,
-        completionTokens,
-        totalTokens: promptTokens + completionTokens,
-      };
+      // Use provider usage if available; otherwise estimate tokens
+      if (result?.usage) {
+        const { prompt_tokens, completion_tokens, total_tokens } = result.usage;
+        usage = {
+          promptTokens: prompt_tokens,
+          completionTokens: completion_tokens,
+          totalTokens: total_tokens,
+        };
+      } else {
+        const promptTokens = estimateTokens(systemMsg + userMsg);
+        const completionTokens = estimateTokens(response.content);
+        usage = {
+          promptTokens,
+          completionTokens,
+          totalTokens: promptTokens + completionTokens,
+        };
+      }
     } else if (provider === "deepseek") {
-      const result = await deepseekChat(
-        {
-          messages,
-          model: "deepseek-chat",
-        }
-
-        // systemMsg,
-        // userMsg,
-        // model || "deepseek-reasoner"
-      );
+      const deepseekArgs = {
+        messages,
+        model: model || "deepseek-reasoner",
+        temperature,
+        maxTokens,
+        ...rest,
+      };
+      if (topP !== undefined) deepseekArgs.topP = topP;
+      if (frequencyPenalty !== undefined)
+        deepseekArgs.frequencyPenalty = frequencyPenalty;
+      if (presencePenalty !== undefined)
+        deepseekArgs.presencePenalty = presencePenalty;
+      if (stop !== undefined) deepseekArgs.stop = stop;
+      if (responseFormat !== undefined)
+        deepseekArgs.responseFormat = responseFormat;
+
+      const result = await deepseekChat(deepseekArgs);
 
       response = {
         content: result.content,
       };
 
-      const promptTokens = estimateTokens(systemMsg + userMsg);
-      const completionTokens = estimateTokens(
-        typeof result === "string" ? result : JSON.stringify(result)
-      );
-      usage = {
-        promptTokens,
-        completionTokens,
-        totalTokens: promptTokens + completionTokens,
-      };
+      // Use actual usage from deepseek API if available; otherwise estimate
+      if (result?.usage) {
+        const { prompt_tokens, completion_tokens, total_tokens } = result.usage;
+        usage = {
+          promptTokens: prompt_tokens,
+          completionTokens: completion_tokens,
+          totalTokens: total_tokens,
+        };
+      } else {
+        const promptTokens = estimateTokens(systemMsg + userMsg);
+        const completionTokens = estimateTokens(
+          typeof result === "string" ? result : JSON.stringify(result)
+        );
+        usage = {
+          promptTokens,
+          completionTokens,
+          totalTokens: promptTokens + completionTokens,
+        };
+      }
     } else {
       throw new Error(`Provider ${provider} not yet implemented`);
     }
@@ -222,8 +269,11 @@ export async function chat(options) {
       timestamp: new Date().toISOString(),
     });
 
-    // Return clean response - no metrics attached!
-    return response;
+    // Return clean response with usage - no metrics attached!
+    return {
+      ...response,
+      usage,
+    };
   } catch (error) {
     const duration = Date.now() - startTime;
 
@@ -299,8 +349,11 @@ function buildProviderFunctions(models) {
 
 // Helper function for single prompt completion
 export async function complete(prompt, options = {}) {
-  const config = getConfig();
-  const defaultProvider = options.provider || config.llm.defaultProvider;
+  // Skip config check in tests to avoid PO_ROOT requirement
+  const isTest =
+    process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+  const defaultProvider =
+    options.provider || (isTest ? "openai" : getConfig().llm.defaultProvider);
 
   return chat({
     provider: defaultProvider,
@@ -417,8 +470,12 @@ export function createLLM() {
 
 // Separate function for high-level LLM interface (used by llm.test.js)
 export function createHighLevelLLM(options = {}) {
-  const config = getConfig();
-  const defaultProvider = options.defaultProvider || config.llm.defaultProvider;
+  // Skip config check in tests to avoid PO_ROOT requirement
+  const isTest =
+    process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+  const config = isTest ? { llm: { models: {} } } : getConfig();
+  const defaultProvider =
+    options.defaultProvider || (isTest ? "openai" : config.llm.defaultProvider);
 
   // Build functions from registry
   const providerFunctions = buildProviderFunctions(config.llm.models);

package/src/pages/Code.jsx

@@ -0,0 +1,297 @@
+import React, { useState, useEffect } from "react";
+import { Box, Heading, Table, Code, Text } from "@radix-ui/themes";
+import Layout from "../components/Layout.jsx";
+import PageSubheader from "../components/PageSubheader.jsx";
+import { Button } from "../components/ui/button.jsx";
+
+const ioFunctions = [
+  {
+    name: "writeArtifact",
+    description: "Write an artifact file",
+    params:
+      'name: string, content: string, options?: { mode?: "replace"|"append"=replace }',
+    returns: "Promise<string>",
+    notes: "Writes to {workDir}/files/artifacts; updates tasks-status.json",
+  },
+  {
+    name: "writeLog",
+    description: "Write a log file",
+    params:
+      'name: string, content: string, options?: { mode?: "append"|"replace"=append }',
+    returns: "Promise<string>",
+    notes:
+      "Writes to {workDir}/files/logs; default append; updates tasks-status.json",
+  },
+  {
+    name: "writeTmp",
+    description: "Write a temporary file",
+    params:
+      'name: string, content: string, options?: { mode?: "replace"|"append"=replace }',
+    returns: "Promise<string>",
+    notes: "Writes to {workDir}/files/tmp; updates tasks-status.json",
+  },
+  {
+    name: "readArtifact",
+    description: "Read an artifact file",
+    params: "name: string",
+    returns: "Promise<string>",
+    notes: "Reads from {workDir}/files/artifacts",
+  },
+  {
+    name: "readLog",
+    description: "Read a log file",
+    params: "name: string",
+    returns: "Promise<string>",
+    notes: "Reads from {workDir}/files/logs",
+  },
+  {
+    name: "readTmp",
+    description: "Read a temporary file",
+    params: "name: string",
+    returns: "Promise<string>",
+    notes: "Reads from {workDir}/files/tmp",
+  },
+  {
+    name: "getTaskDir",
+    description: "Get the task directory path",
+    params: "",
+    returns: "string",
+    notes: "Returns {workDir}/tasks/{taskName}",
+  },
+  {
+    name: "getCurrentStage",
+    description: "Get the current stage name",
+    params: "",
+    returns: "string",
+    notes: "Calls injected getStage()",
+  },
+];
+
+const sampleSeed = {
+  name: "some-name",
+  pipeline: "content-generation",
+  data: {
+    type: "some-type",
+    contentType: "blog-post",
+    targetAudience: "software-developers",
+    tone: "professional-yet-accessible",
+    length: "1500-2000 words",
+    outputFormat: "blog-post",
+  },
+};
+
+export default function CodePage() {
+  const [llmFunctions, setLlmFunctions] = useState(null);
+
+  useEffect(() => {
+    fetch("/api/llm/functions")
+      .then((res) => res.json())
+      .then(setLlmFunctions)
+      .catch(console.error);
+  }, []);
+
+  const breadcrumbs = [{ label: "Home", href: "/" }, { label: "Code" }];
+
+  const handleCopySeed = () => {
+    navigator.clipboard.writeText(JSON.stringify(sampleSeed, null, 2));
+  };
+
+  return (
+    <Layout>
+      <PageSubheader breadcrumbs={breadcrumbs} />
+      <Box>
+        {/* Seed File Example Section */}
+        <Box mb="8">
+          <Heading size="6" mb="4">
+            Seed File Example
+          </Heading>
+          <Text as="p" mb="3" size="2">
+            A seed file is a JSON object used to start a new pipeline job. It
+            defines the job name, the pipeline to run, and any contextual data
+            the pipeline requires to begin.
+          </Text>
+          <Text as="p" mb="3" size="2" weight="bold">
+            Required fields:
+          </Text>
+          <ul className="list-disc list-inside mb-4 space-y-1">
+            <li className="text-sm text-gray-700">
+              <Text as="span" weight="bold">
+                name
+              </Text>{" "}
+              (string): Human-friendly title; non-empty, printable only, ≤120
+              chars; must be unique.
+            </li>
+            <li className="text-sm text-gray-700">
+              <Text as="span" weight="bold">
+                pipeline
+              </Text>{" "}
+              (string): Pipeline slug defined in your registry (e.g.,
+              content-generation).
+            </li>
+            <li className="text-sm text-gray-700">
+              <Text as="span" weight="bold">
+                data
+              </Text>{" "}
+              (object): Required but flexible; include any arbitrary keys your
+              pipeline tasks expect.
+            </li>
+          </ul>
+          <Box mb="3">
+            <Button
+              size="1"
+              onClick={handleCopySeed}
+              data-testid="copy-seed-example"
+            >
+              Copy
+            </Button>
+          </Box>
+          <pre className="text-xs bg-gray-50 p-3 rounded overflow-auto max-h-60 border border-gray-200">
+            {JSON.stringify(sampleSeed, null, 2)}
+          </pre>
+        </Box>
+
+        <Heading size="6" mb="4">
+          Pipeline Task IO API
+        </Heading>
+        <Box overflowX="auto">
+          <Table.Root>
+            <Table.Header>
+              <Table.Row>
+                <Table.ColumnHeaderCell>Function</Table.ColumnHeaderCell>
+                <Table.ColumnHeaderCell>Parameters</Table.ColumnHeaderCell>
+                <Table.ColumnHeaderCell>Returns</Table.ColumnHeaderCell>
+                <Table.ColumnHeaderCell>Notes</Table.ColumnHeaderCell>
+              </Table.Row>
+            </Table.Header>
+            <Table.Body>
+              {ioFunctions.map((fn) => (
+                <Table.Row key={fn.name}>
+                  <Table.RowHeaderCell>
+                    <Code size="3">io.{fn.name}</Code>
+                  </Table.RowHeaderCell>
+                  <Table.Cell>
+                    <Code size="3">{fn.params || "—"}</Code>
+                  </Table.Cell>
+                  <Table.Cell>
+                    <Code size="3">{fn.returns}</Code>
+                  </Table.Cell>
+                  <Table.Cell>
+                    {fn.description}
+                    <br />
+                    {fn.notes}
+                  </Table.Cell>
+                </Table.Row>
+              ))}
+            </Table.Body>
+          </Table.Root>
+        </Box>
+
+        <Heading size="6" mt="8" mb="4">
+          Pipeline Task LLM API
+        </Heading>
+        <Box mb="4">
+          <Heading size="4" mb="2">
+            Arguments
+          </Heading>
+          <Code size="3" mb="4">
+            {`{
+  messages: Array<{role: "system"|"user"|"assistant", content: string }>,
+  temperature?: number,
+  maxTokens?: number,
+  responseFormat?: "json" | { type: "json_object" | { type: "json_schema", name: string, json_schema: object } },
+  stop?: string | string[],
+  topP?: number,
+  frequencyPenalty?: number,
+  presencePenalty?: number,
+  tools?: Array<{type: "function", function: object}>,
+  toolChoice?: "auto" | "required" | { type: "function", function: { name: string } },
+  seed?: number,
+  provider?: string,
+  model?: string,
+  metadata?: object,
+  maxRetries?: number
+}`}
+          </Code>
+          <Heading size="4" mb="2">
+            Returns
+          </Heading>
+          <Code size="3">{`Promise<{ content: any, usage?: object, raw?: any }>`}</Code>
+        </Box>
+
+        {llmFunctions && (
+          <Box overflowX="auto">
+            <Table.Root>
+              <Table.Header>
+                <Table.Row>
+                  <Table.ColumnHeaderCell>Function</Table.ColumnHeaderCell>
+                  <Table.ColumnHeaderCell>Model</Table.ColumnHeaderCell>
+                </Table.Row>
+              </Table.Header>
+              <Table.Body>
+                {Object.entries(llmFunctions).map(([provider, functions]) =>
+                  functions.map((fn) => (
+                    <Table.Row key={fn.fullPath}>
+                      <Table.RowHeaderCell>
+                        <Code size="3">{fn.fullPath}</Code>
+                      </Table.RowHeaderCell>
+                      <Table.Cell>
+                        <Code size="3">{fn.model}</Code>
+                      </Table.Cell>
+                    </Table.Row>
+                  ))
+                )}
+              </Table.Body>
+            </Table.Root>
+          </Box>
+        )}
+
+        <Heading size="6" mt="8" mb="4">
+          Environment Configuration
+        </Heading>
+        <Box mb="4">
+          <Heading size="4" mb="2">
+            Example .env Configuration
+          </Heading>
+          <Box overflowX="auto">
+            <Table.Root>
+              <Table.Header>
+                <Table.Row>
+                  <Table.ColumnHeaderCell>
+                    Environment Variable
+                  </Table.ColumnHeaderCell>
+                </Table.Row>
+              </Table.Header>
+              <Table.Body>
+                <Table.Row>
+                  <Table.RowHeaderCell>
+                    <Code size="3">OPENAI_API_KEY=</Code>
+                  </Table.RowHeaderCell>
+                </Table.Row>
+                <Table.Row>
+                  <Table.RowHeaderCell>
+                    <Code size="3">DEEPSEEK_API_KEY=</Code>
+                  </Table.RowHeaderCell>
+                </Table.Row>
+                <Table.Row>
+                  <Table.RowHeaderCell>
+                    <Code size="3">GEMINI_API_KEY=</Code>
+                  </Table.RowHeaderCell>
+                </Table.Row>
+                <Table.Row>
+                  <Table.RowHeaderCell>
+                    <Code size="3">ANTHROPIC_API_KEY=</Code>
+                  </Table.RowHeaderCell>
+                </Table.Row>
+                <Table.Row>
+                  <Table.RowHeaderCell>
+                    <Code size="3">Z_API_KEY=</Code>
+                  </Table.RowHeaderCell>
+                </Table.Row>
+              </Table.Body>
+            </Table.Root>
+          </Box>
+        </Box>
+      </Box>
+    </Layout>
+  );
+}