@outputai/cli 0.11.1-next.90d8cc0.0 → 0.11.1-next.a07f3e4.0

package/dist/assets/docker/docker-compose-dev.yml

@@ -80,7 +80,7 @@ services:
         condition: service_healthy
       worker:
         condition: service_healthy
-    image: outputai/api:${OUTPUT_API_VERSION:-0.11.1-next.90d8cc0.0}
+    image: outputai/api:${OUTPUT_API_VERSION:-0.11.1-next.a07f3e4.0}
     init: true
     networks:
       - main

package/dist/generated/framework_version.json

@@ -1,3 +1,3 @@
 {
-    "framework": "0.11.1-next.90d8cc0.0"
+    "framework": "0.11.1-next.a07f3e4.0"
 }

package/dist/services/cost_calculator.js

@@ -15,6 +15,8 @@ function lineRate(type, pricing) {
     const rates = {
         input: pricing.input ?? 0,
         input_cached: pricing.cached_input ?? 0,
+        input_cache_read: pricing.cached_input ?? pricing.input ?? 0,
+        input_cache_write: pricing.input ?? 0,
         output: pricing.output ?? 0,
         reasoning: pricing.reasoning ?? pricing.output ?? 0
     };
@@ -42,6 +44,81 @@ function eventTokenUsage(lines) {
         reasoningTokens: sumOf('reasoning')
     };
 }
+function normalizedItemType(item) {
+    if (item.group === 'input') {
+        if (item.label === null || item.label === 'no_cache') {
+            return 'input';
+        }
+        if (item.label === 'cache_read') {
+            return 'input_cache_read';
+        }
+        if (item.label === 'cache_write') {
+            return 'input_cache_write';
+        }
+    }
+    if (item.group === 'output') {
+        if (item.label === null || item.label === 'text') {
+            return 'output';
+        }
+        if (item.label === 'reasoning') {
+            return 'reasoning';
+        }
+    }
+    return item.label ? `${item.group}_${item.label}` : item.group;
+}
+function normalizedItemTokenUsage(items) {
+    const sumOf = (group, labels) => items
+        .filter(item => item.group === group && (!labels || labels.includes(item.label)))
+        .reduce((sum, item) => sum + item.amount, 0);
+    return {
+        inputTokens: sumOf('input'),
+        cachedInputTokens: sumOf('input', ['cache_read']),
+        outputTokens: sumOf('output', [null, 'output', 'text']),
+        reasoningTokens: sumOf('output', ['reasoning'])
+    };
+}
+function parseLegacyLLMUsageEvent(node, stepName, event) {
+    return {
+        stepName: stepName || node.name || 'unknown',
+        llmName: node.name || 'llm',
+        model: event.modelId || 'unknown',
+        usage: eventTokenUsage(event.usage ?? []),
+        originalCost: event.total,
+        lines: event.usage ?? []
+    };
+}
+function parseNormalizedLLMUsage(node, stepName, event) {
+    const lines = event.items.map(item => ({
+        type: normalizedItemType(item),
+        ppm: 0,
+        amount: item.amount,
+        total: 0
+    }));
+    return {
+        stepName: stepName || node.name || 'unknown',
+        llmName: node.name || 'llm',
+        model: event.modelId || 'unknown',
+        usage: normalizedItemTokenUsage(event.items),
+        originalCost: 0,
+        lines
+    };
+}
+function parseLLMCostEvent(node, stepName, event) {
+    const lines = event.items.map(item => ({
+        type: normalizedItemType(item),
+        ppm: item.ppm ?? 0,
+        amount: item.amount,
+        total: item.total ?? 0
+    }));
+    return {
+        stepName: stepName || node.name || 'unknown',
+        llmName: node.name || 'llm',
+        model: event.modelId || 'unknown',
+        usage: normalizedItemTokenUsage(event.items),
+        originalCost: event.total ?? 0,
+        lines
+    };
+}
 export function extractValue(obj, path) {
     if (!path || !obj) {
         return obj;
@@ -106,20 +183,20 @@ function findCalls(node, match, extract, parentStepName = null, seenIds = new Se
     }
     return calls;
 }
-// Only nodes carrying an llm:usage event are priced — the event holds the
-// as-charged cost and the per-token-type amounts. Traces from SDKs that
-// predate cost attributes report no LLM costs.
+// Prefer generation cost, then normalized generation usage, then legacy usage.
 export function findLLMCalls(node, parentStepName = null, seenIds = new Set()) {
-    return findCalls(node, n => n.kind === 'llm' && !!n.attributes?.['llm:usage'], (n, stepName) => {
-        const event = n.attributes['llm:usage'];
-        return {
-            stepName: stepName || n.name || 'unknown',
-            llmName: n.name || 'llm',
-            model: event.modelId || 'unknown',
-            usage: eventTokenUsage(event.usage ?? []),
-            originalCost: event.total,
-            lines: event.usage ?? []
-        };
+    return findCalls(node, n => n.kind === 'llm' && !!(n.attributes?.['llm:generation:cost'] ||
+        n.attributes?.['llm:generation:usage'] ||
+        n.attributes?.['llm:usage']), (n, stepName) => {
+        const cost = n.attributes?.['llm:generation:cost'];
+        if (cost) {
+            return parseLLMCostEvent(n, stepName, cost);
+        }
+        const usage = n.attributes?.['llm:generation:usage'];
+        if (usage) {
+            return parseNormalizedLLMUsage(n, stepName, usage);
+        }
+        return parseLegacyLLMUsageEvent(n, stepName, n.attributes['llm:usage']);
     }, parentStepName, seenIds);
 }
 export function findHTTPCalls(node, parentStepName = null, seenIds = new Set()) {
@@ -360,7 +437,9 @@ function findModelPricing(model, models) {
     if (models[model]) {
         return models[model];
     }
-    return Object.entries(models).find(([key]) => model.startsWith(key))?.[1];
+    return Object.entries(models)
+        .filter(([key]) => model.startsWith(key))
+        .sort(([left], [right]) => right.length - left.length)[0]?.[1];
 }
 function aggregateLLMCosts(llmCalls, config) {
     const results = [];

package/dist/services/cost_calculator.spec.js

@@ -37,6 +37,50 @@ function llmEventNode(id, model, lines) {
         }
     };
 }
+function llmCostNode(id, model, items) {
+    const totalOf = (group) => items.filter(item => item.group === group).reduce((sum, item) => sum + (item.total ?? 0), 0);
+    const input = totalOf('input');
+    const output = totalOf('output');
+    return {
+        id,
+        kind: 'llm',
+        name: 'gen',
+        attributes: {
+            'llm:generation:cost': {
+                type: 'llm:generation:cost',
+                providerId: 'test-provider',
+                modelId: model,
+                input,
+                output,
+                total: input + output,
+                status: 'precise',
+                items
+            }
+        }
+    };
+}
+function llmUsageNode(id, model, items) {
+    const totalOf = (group) => items.filter(item => item.group === group).reduce((sum, item) => sum + item.amount, 0);
+    const input = totalOf('input');
+    const output = totalOf('output');
+    return {
+        id,
+        kind: 'llm',
+        name: 'gen',
+        attributes: {
+            'llm:generation:usage': {
+                type: 'llm:generation:usage',
+                providerId: 'test-provider',
+                modelId: model,
+                input,
+                output,
+                total: input + output,
+                status: 'complete',
+                items
+            }
+        }
+    };
+}
 function httpEventNode(id, url, method, total, output = { status: 200, body: {} }, inputBody) {
     return {
         id,
@@ -182,7 +226,109 @@ describe('findLLMCalls', () => {
         expect(calls[0].usage.outputTokens).toBe(7275);
         expect(calls[0].originalCost).toBeCloseTo(0.35525, 5);
     });
-    it('ignores llm nodes without a llm:usage event', () => {
+    it('does not interpret transitional component names in legacy llm:usage events', () => {
+        const node = llmEventNode('transitional-components', 'legacy-model', [
+            { type: 'input', ppm: 1, amount: 10, total: 0.00001 },
+            { type: 'input_cached', ppm: 0.1, amount: 20, total: 0.000002 },
+            { type: 'output', ppm: 5, amount: 30, total: 0.00015 },
+            { type: 'reasoning', ppm: 5, amount: 40, total: 0.0002 },
+            { type: 'input_cache_read', ppm: 0.1, amount: 100, total: 0.00001 },
+            { type: 'input_cache_write', ppm: 1.25, amount: 200, total: 0.00025 },
+            { type: 'output_text', ppm: 5, amount: 300, total: 0.0015 },
+            { type: 'output_reasoning', ppm: 5, amount: 400, total: 0.002 }
+        ]);
+        const calls = findLLMCalls({ kind: 'workflow', children: [node] });
+        expect(calls[0].usage).toEqual({
+            inputTokens: 30,
+            cachedInputTokens: 20,
+            outputTokens: 30,
+            reasoningTokens: 40
+        });
+    });
+    it('reads every supported item from an llm:generation:cost attribute', () => {
+        const items = [
+            { group: 'input', label: null, amount: 100, ppm: 1, total: 0.0001, status: 'ok' },
+            { group: 'input', label: 'no_cache', amount: 200, ppm: 1, total: 0.0002, status: 'ok' },
+            { group: 'input', label: 'cache_write', amount: 300, ppm: 1.25, total: 0.000375, status: 'ok' },
+            { group: 'input', label: 'cache_read', amount: 400, ppm: 0.1, total: 0.00004, status: 'ok' },
+            { group: 'output', label: null, amount: 50, ppm: 5, total: 0.00025, status: 'ok' },
+            { group: 'output', label: 'text', amount: 60, ppm: 5, total: 0.0003, status: 'ok' },
+            { group: 'output', label: 'reasoning', amount: 70, ppm: 10, total: 0.0007, status: 'ok' }
+        ];
+        const calls = findLLMCalls({
+            kind: 'workflow',
+            children: [llmCostNode('llm-cost', 'new-model', items)]
+        });
+        expect(calls).toHaveLength(1);
+        expect(calls[0]).toMatchObject({
+            model: 'new-model',
+            usage: {
+                inputTokens: 1000,
+                cachedInputTokens: 400,
+                outputTokens: 110,
+                reasoningTokens: 70
+            }
+        });
+        expect(calls[0].lines.map(line => line.type)).toEqual([
+            'input',
+            'input',
+            'input_cache_write',
+            'input_cache_read',
+            'output',
+            'output',
+            'reasoning'
+        ]);
+        expect(calls[0].originalCost).toBeCloseTo(0.001965, 8);
+    });
+    it('reads normalized usage without interpreting token totals as cost', () => {
+        const items = [
+            { group: 'input', label: null, amount: 100 },
+            { group: 'input', label: 'no_cache', amount: 200 },
+            { group: 'input', label: 'cache_write', amount: 300 },
+            { group: 'input', label: 'cache_read', amount: 400 },
+            { group: 'output', label: null, amount: 50 },
+            { group: 'output', label: 'text', amount: 60 },
+            { group: 'output', label: 'reasoning', amount: 70 }
+        ];
+        const calls = findLLMCalls({
+            kind: 'workflow',
+            children: [llmUsageNode('llm-usage', 'new-model', items)]
+        });
+        expect(calls).toHaveLength(1);
+        expect(calls[0]).toMatchObject({
+            model: 'new-model',
+            usage: {
+                inputTokens: 1000,
+                cachedInputTokens: 400,
+                outputTokens: 110,
+                reasoningTokens: 70
+            },
+            originalCost: 0
+        });
+        expect(calls[0].lines).toEqual([
+            { type: 'input', ppm: 0, amount: 100, total: 0 },
+            { type: 'input', ppm: 0, amount: 200, total: 0 },
+            { type: 'input_cache_write', ppm: 0, amount: 300, total: 0 },
+            { type: 'input_cache_read', ppm: 0, amount: 400, total: 0 },
+            { type: 'output', ppm: 0, amount: 50, total: 0 },
+            { type: 'output', ppm: 0, amount: 60, total: 0 },
+            { type: 'reasoning', ppm: 0, amount: 70, total: 0 }
+        ]);
+    });
+    it('prefers generation cost when both generation attributes are present', () => {
+        const node = llmCostNode('both', 'new-model', [
+            { group: 'input', label: null, amount: 100, ppm: 1, total: 0.0001, status: 'ok' }
+        ]);
+        node.attributes['llm:generation:usage'] = llmUsageNode('usage', 'usage-model', [
+            { group: 'input', label: null, amount: 10 }
+        ])
+            .attributes['llm:generation:usage'];
+        const calls = findLLMCalls({ kind: 'workflow', children: [node] });
+        expect(calls).toHaveLength(1);
+        expect(calls[0].model).toBe('new-model');
+        expect(calls[0].originalCost).toBe(0.0001);
+    });
+    it('ignores llm nodes without a cost event', () => {
         const trace = {
             kind: 'workflow',
             name: 'test',
@@ -314,20 +460,67 @@ describe('calculateCost', () => {
         const report = calculateCost(duplicateTrace, testConfig, 'test.json');
         expect(report.llmCalls).toHaveLength(1);
     });
-    it('matches versioned model names by prefix', () => {
+    it('uses the longest matching prefix for versioned model names', () => {
         const trace = {
             kind: 'workflow',
             name: 'test',
             children: [
-                llmEventNode('llm-1', 'claude-sonnet-4-5-20250514', llmLines(1000, 3, 500, 15))
+                llmEventNode('llm-1', 'gpt-4.1-mini-20250514', llmLines(1_000_000, 9, 1_000_000, 9))
             ]
         };
-        const report = calculateCost(trace, testConfig, 'test.json');
-        // priced at the claude-sonnet-4-5 prefix rates: 1000@$3/M + 500@$15/M
-        expect(report.llmCalls[0].adjustedCost).toBeCloseTo(0.0105, 5);
+        const config = {
+            ...testConfig,
+            models: {
+                'gpt-4.1': { provider: 'openai', input: 2, output: 8 },
+                'gpt-4.1-mini': { provider: 'openai', input: 0.4, output: 1.6 }
+            }
+        };
+        const report = calculateCost(trace, config, 'test.json');
+        expect(report.llmCalls[0].adjustedCost).toBeCloseTo(2, 8);
     });
 });
 describe('event-driven LLM costs (original vs adjusted)', () => {
+    it('re-prices normalized usage when generation cost is absent', () => {
+        const trace = {
+            kind: 'workflow',
+            name: 'test',
+            children: [llmUsageNode('usage-only', 'claude-sonnet-4-5', [
+                    { group: 'input', label: 'no_cache', amount: 1_000_000 },
+                    { group: 'input', label: 'cache_read', amount: 1_000_000 },
+                    { group: 'input', label: 'cache_write', amount: 1_000_000 },
+                    { group: 'output', label: 'text', amount: 1_000_000 },
+                    { group: 'output', label: 'reasoning', amount: 1_000_000 }
+                ])]
+        };
+        const report = calculateCost(trace, testConfig, 'test.json');
+        expect(report.llmCalls[0]).toMatchObject({
+            input: 3_000_000,
+            cached: 1_000_000,
+            output: 1_000_000,
+            reasoning: 1_000_000,
+            originalCost: 0,
+            adjustedCost: 36.3
+        });
+        expect(report.llmOriginalCost).toBe(0);
+        expect(report.llmAdjustedCost).toBeCloseTo(36.3, 8);
+    });
+    it('falls back to input pricing for normalized cache reads', () => {
+        const trace = {
+            kind: 'workflow',
+            name: 'test',
+            children: [llmUsageNode('cache-read', 'cache-model', [
+                    { group: 'input', label: 'cache_read', amount: 1_000_000 }
+                ])]
+        };
+        const config = {
+            ...testConfig,
+            models: {
+                'cache-model': { provider: 'test', input: 2, output: 10 }
+            }
+        };
+        const report = calculateCost(trace, config, 'test.json');
+        expect(report.llmCalls[0].adjustedCost).toBeCloseTo(2, 8);
+    });
     it('matches original when costs.yml rate equals the event rate', () => {
         // haiku event priced at the same rate as testConfig (input 1 / output 5)
         const trace = {
@@ -362,19 +555,18 @@ describe('event-driven LLM costs (original vs adjusted)', () => {
         expect(report.llmCalls[0].originalCost).toBeCloseTo(0.35525, 5);
         expect(report.llmCalls[0].adjustedCost).toBeCloseTo(0.35525, 5);
     });
-    it('prices an unknown line type at its as-charged total, not $0', () => {
-        const lines = [
-            ...llmLines(1_000_000, 1, 1_000_000, 5),
-            { type: 'input_cache_write', ppm: 1.25, amount: 200_000, total: 0.25 }
-        ];
+    it('re-prices normalized cache writes at the configured input rate', () => {
         const trace = {
             kind: 'workflow',
             name: 'test',
-            children: [llmEventNode('cw', 'claude-haiku-4-5', lines)]
+            children: [llmCostNode('cw', 'claude-haiku-4-5', [
+                    { group: 'input', label: 'no_cache', amount: 1_000_000, ppm: 1, total: 1, status: 'ok' },
+                    { group: 'input', label: 'cache_write', amount: 200_000, ppm: 1.25, total: 0.25, status: 'ok' },
+                    { group: 'output', label: 'text', amount: 1_000_000, ppm: 5, total: 5, status: 'ok' }
+                ])]
         };
         const report = calculateCost(trace, testConfig, 'test.json');
-        // known lines reprice to $6 at config rates; unknown line passes through at $0.25
-        expect(report.llmCalls[0].adjustedCost).toBeCloseTo(6.25, 5);
+        expect(report.llmCalls[0].adjustedCost).toBeCloseTo(6.2, 5);
         expect(report.llmCalls[0].originalCost).toBeCloseTo(6.25, 5);
     });
     it('reports inputTokens including cached tokens for event traces', () => {

package/dist/templates/project/src/workflows/blog_evaluator/evaluators.ts.template

@@ -1,6 +1,6 @@
 import { evaluator, EvaluationNumberResult } from '@outputai/core';
 import type { EvaluationResultArgs } from '@outputai/core';
-import { generateText, Output } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/llm';
 import { blogContentSchema } from './types.js';
 import type { BlogContent } from './types.js';
 
@@ -15,7 +15,7 @@ export const evaluateSignalToNoise = evaluator( {
         title: input.title,
         content: input.content
       },
-      output: Output.object( { schema: EvaluationNumberResult.schema } )
+      output: aiSdk.Output.object( { schema: EvaluationNumberResult.schema } )
     } );
 
     return new EvaluationNumberResult( output as EvaluationResultArgs<number> );

package/dist/templates/project/src/workflows/blog_evaluator/prompts/signal_noise@v1.prompt.template

@@ -3,7 +3,7 @@ provider: anthropic
 # current as of 2026-05-04 — run output-dev-model-selection for the latest
 model: claude-sonnet-4-6
 temperature: 0.3
-maxTokens: 4096
+maxOutputTokens: 4096
 ---
 
 <system>

package/dist/templates/workflow/README.md.template

@@ -123,7 +123,7 @@ Define steps in `steps.ts` with schemas:
 
 ```typescript
 import { step, z } from '@outputai/core';
-import { generateText, Output } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/llm';
 
 export const myStep = step( {
   name: 'my_step',
@@ -138,7 +138,7 @@ export const myStep = step( {
     const { output } = await generateText( {
       prompt: 'my_prompt@v1',
       variables: { text },
-      output: Output.object( {
+      output: aiSdk.Output.object( {
         schema: z.object( { result: z.string() } )
       } )
     } );
@@ -153,7 +153,7 @@ Define evaluators in `evaluators.ts`:
 
 ```typescript
 import { evaluator, z, EvaluationVerdictResult } from '@outputai/core';
-import { generateText, Output } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/llm';
 
 export const evaluateQuality = evaluator( {
   name: 'evaluate_quality',
@@ -166,7 +166,7 @@ export const evaluateQuality = evaluator( {
     const { output: evaluation } = await generateText( {
       prompt: 'evaluate@v1',
       variables: { input, output },
-      output: Output.object( { schema: EvaluationVerdictResult.schema } )
+      output: aiSdk.Output.object( { schema: EvaluationVerdictResult.schema } )
     } );
 
     return new EvaluationVerdictResult( evaluation );
@@ -178,8 +178,13 @@ export const evaluateQuality = evaluator( {
 
 Create prompt files in `prompts/` following the pattern:
 - File naming: `promptName@v1.prompt`
-- Include YAML frontmatter with provider, model, temperature, and maxTokens
+- Include YAML frontmatter with provider, model, temperature, and maxOutputTokens
 - Use LiquidJS syntax for variables: `{{ variableName }}`
+- Start the rendered body with a role tag for message mode, or with plain text for instruction mode
+
+In message mode, use only `<system>`, `<user>`, and `<assistant>` at the top level. Keep all prose inside those blocks, close every block, and escape literal same-role examples such as `&lt;user&gt;...&lt;/user&gt;`.
+
+Structured tool messages are runtime history supplied through Agent `messages` or `messageStore`; they are not authored in prompt files.
 
 Example prompt file:
 ```
@@ -188,7 +193,7 @@ provider: anthropic
 # current as of 2026-05-04 — run output-dev-model-selection for the latest
 model: claude-sonnet-4-6
 temperature: 0.3
-maxTokens: 1024
+maxOutputTokens: 1024
 ---
 
 <system>

package/dist/templates/workflow/evaluators.ts.template

@@ -1,5 +1,5 @@
 import { evaluator, z, EvaluationVerdictResult } from '@outputai/core';
-import { generateText, Output } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/llm';
 
 // TODO: Update the evaluator to assess your workflow's output quality
 export const evaluate{{WorkflowName}} = evaluator( {
@@ -13,7 +13,7 @@ export const evaluate{{WorkflowName}} = evaluator( {
     const { output: evaluation } = await generateText( {
       prompt: 'example@v1',
       variables: { text: `Input: ${input}\nOutput: ${output}\n\nDoes the output adequately address the input?` },
-      output: Output.object( { schema: EvaluationVerdictResult.schema } )
+      output: aiSdk.Output.object( { schema: EvaluationVerdictResult.schema } )
     } );
 
     return new EvaluationVerdictResult( evaluation );

package/dist/templates/workflow/prompts/example@v1.prompt.template

@@ -3,7 +3,7 @@ provider: anthropic
 # current as of 2026-05-04 — run output-dev-model-selection for the latest
 model: claude-sonnet-4-6
 temperature: 0.3
-maxTokens: 4096
+maxOutputTokens: 4096
 ---
 
 <system>

package/dist/types/cost.d.ts

@@ -13,8 +13,7 @@ export interface TokenUsage {
     cachedInputTokens?: number;
     reasoningTokens?: number;
 }
-export type { LLMUsageEvent } from '@outputai/llm';
-import type { LLMUsageEvent } from '@outputai/llm';
+import type { LLMGenerationCost, LLMGenerationUsage, LLMUsageEvent } from '@outputai/llm';
 export type LLMUsageLine = LLMUsageEvent['usage'][number];
 export interface HTTPCostEvent {
     type: 'http:request:cost';
@@ -29,6 +28,8 @@ export interface HTTPCountEvent {
 }
 export interface NodeAttributes {
     'llm:usage'?: LLMUsageEvent;
+    'llm:generation:usage'?: LLMGenerationUsage;
+    'llm:generation:cost'?: LLMGenerationCost;
     'http:request:cost'?: HTTPCostEvent;
     'http:request:count'?: HTTPCountEvent;
 }

package/oclif.manifest.json

@@ -1752,5 +1752,5 @@
       ]
     }
   },
-  "version": "0.11.1-next.90d8cc0.0"
+  "version": "0.11.1-next.a07f3e4.0"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/cli",
-  "version": "0.11.1-next.90d8cc0.0",
+  "version": "0.11.1-next.a07f3e4.0",
   "description": "CLI for Output.ai workflow generation",
   "type": "module",
   "main": "dist/index.js",
@@ -36,9 +36,9 @@
     "react": "19.2.5",
     "semver": "7.7.4",
     "undici": "8.9.0",
-    "@outputai/credentials": "0.11.1-next.90d8cc0.0",
-    "@outputai/evals": "0.11.1-next.90d8cc0.0",
-    "@outputai/llm": "0.11.1-next.90d8cc0.0"
+    "@outputai/evals": "0.11.1-next.a07f3e4.0",
+    "@outputai/credentials": "0.11.1-next.a07f3e4.0",
+    "@outputai/llm": "0.11.1-next.a07f3e4.0"
   },
   "devDependencies": {
     "@types/cli-progress": "3.11.6",