@outputai/cli 0.15.1-next.4a437bf.0 → 0.15.1-next.5db73d3.0

package/dist/assets/docker/docker-compose-dev.yml

@@ -80,7 +80,7 @@ services:
         condition: service_healthy
       worker:
         condition: service_healthy
-    image: outputai/api:${OUTPUT_API_VERSION:-0.15.1-next.4a437bf.0}
+    image: outputai/api:${OUTPUT_API_VERSION:-0.15.1-next.5db73d3.0}
     init: true
     networks:
       - main

package/dist/commands/workflow/test.d.ts

@@ -1,5 +1,5 @@
 import { Command } from '@oclif/core';
-import type { EvalOutput } from '@outputai/evals';
+import type { EvalOutput } from '@outputai/core/evals';
 export default class WorkflowTest extends Command {
     static description: string;
     static enableJsonFlag: boolean;

package/dist/commands/workflow/test.js

@@ -5,7 +5,9 @@ import { readAllDatasets, writeDataset } from '#services/datasets.js';
 import { fetchWorkflowCatalog } from '#api/workflow_catalog.js';
 import { diagnoseMissingEvalWorkflow } from '#utils/eval_diagnostics.js';
 import { handleApiError } from '#utils/error_handler.js';
-import { getEvalWorkflowName, renderEvalOutput, computeExitCode, EvalOutputSchema } from '@outputai/evals';
+import { getEvalWorkflowName } from '#utils/eval_naming.js';
+import { renderEvalOutput, computeExitCode } from '#utils/eval_render.js';
+import { EvalOutputSchema } from '@outputai/core/evals';
 export default class WorkflowTest extends Command {
     static description = 'Run evaluations against a workflow using its datasets';
     static enableJsonFlag = true;

package/dist/commands/workflow/test.spec.js

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { getEvalWorkflowName, renderEvalOutput } from '@outputai/evals';
+import { getEvalWorkflowName } from '#utils/eval_naming.js';
+import { renderEvalOutput } from '#utils/eval_render.js';
 vi.mock('#api/generated/api.js', () => ({
     postWorkflowRun: vi.fn()
 }));

package/dist/generated/framework_version.json

@@ -1,3 +1,3 @@
 {
-    "framework": "0.15.1-next.4a437bf.0"
+    "framework": "0.15.1-next.5db73d3.0"
 }

package/dist/services/cost_calculator.js

@@ -32,7 +32,7 @@ function priceLines(lines, pricing) {
     }, 0);
 }
 // Token counts for display. The producer's 'input' line excludes cached tokens
-// (sdk/llm emits input − cached), while AI SDK reports inputTokens as the
+// (core/llm emits input − cached), while AI SDK reports inputTokens as the
 // total — add cached back so the columns match what providers report.
 function eventTokenUsage(lines) {
     const sumOf = (type) => lines.filter(l => l.type === type).reduce((s, l) => s + l.amount, 0);

package/dist/services/datasets.d.ts

@@ -1,4 +1,4 @@
-import type { Dataset } from '@outputai/evals';
+import type { Dataset } from '@outputai/core/evals';
 export interface DatasetInfo {
     name: string;
     path: string;

package/dist/services/datasets.js

@@ -2,7 +2,7 @@ import { readFile, writeFile, readdir, mkdir } from 'node:fs/promises';
 import { existsSync } from 'node:fs';
 import { resolve, join } from 'node:path';
 import { load as parseYaml, dump as stringifyYaml } from 'js-yaml';
-import { DatasetSchema } from '@outputai/evals';
+import { DatasetSchema } from '@outputai/core/evals';
 import { getTrace } from '#services/trace_reader.js';
 import { sanitizeSecrets } from '#utils/secret_sanitizer.js';
 import { resolveWorkflowDir, WORKFLOWS_PATHS } from '#utils/workflow_dir.js';

package/dist/services/workflow_planner.js

@@ -1,5 +1,5 @@
 import { initializeAgentConfig } from './coding_agents.js';
-import { generateText } from '@outputai/llm';
+import { generateText } from '@outputai/core/llm';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { config } from '#config.js';

package/dist/services/workflow_planner.spec.js

@@ -1,10 +1,10 @@
 import { describe, it, expect, beforeEach, vi } from 'vitest';
 import { generatePlanName, writePlanFile, updateAgentTemplates } from './workflow_planner.js';
 import { initializeAgentConfig } from './coding_agents.js';
-import { generateText } from '@outputai/llm';
+import { generateText } from '@outputai/core/llm';
 import fs from 'node:fs/promises';
 vi.mock('./coding_agents.js');
-vi.mock('@outputai/llm');
+vi.mock('@outputai/core/llm');
 vi.mock('node:fs/promises');
 const mockGenerateTextResult = (text) => ({
     text,

package/dist/templates/agent_instructions/CLAUDE.md.template

@@ -35,7 +35,7 @@ src/
       types.ts       # Zod schemas and TypeScript types
       prompts/       # .prompt files (Liquid.js templates with YAML frontmatter)
       scenarios/     # Test scenario JSON files
-  clients/           # Shared HTTP clients (use @outputai/http, not fetch/axios)
+  clients/           # Shared HTTP clients (use @outputai/core/http, not fetch/axios)
   shared/            # Shared utilities across workflows
 config/
   costs.yml              # Token/API pricing overrides
@@ -46,8 +46,8 @@ config/
 ## Key Conventions
 
 - **Workflows are deterministic**: No I/O, no `Date.now()`, no `Math.random()` in `workflow.ts`. All side effects go in steps or evaluators.
-- **HTTP clients**: Use `outputFetch` or `createKyClient` from `@outputai/http` instead of raw `fetch` or `axios`. Requests are automatically traced; for paid APIs, attach spend via `addRequestCost` in an `afterResponse` hook (`output-dev-http-client-create`) and forward cost events with `output-dev-cost-hooks`.
-- **LLM calls**: Use `generateText` from `@outputai/llm` with `.prompt` files. Never call LLM APIs directly.
+- **HTTP clients**: Use `outputFetch` or `createKyClient` from `@outputai/core/http` instead of raw `fetch` or `axios`. Requests are automatically traced; for paid APIs, attach spend via `addRequestCost` in an `afterResponse` hook (`output-dev-http-client-create`) and forward cost events with `output-dev-cost-hooks`.
+- **LLM calls**: Use `generateText` from `@outputai/core/llm` with `.prompt` files. Never call LLM APIs directly.
 
 ---
 

package/dist/templates/project/README.md.template

@@ -27,7 +27,7 @@ src/
 
 ### Clients Directory
 
-The `src/clients/` directory contains API clients using `@outputai/http` for external services.
+The `src/clients/` directory contains API clients using `@outputai/core/http` for external services.
 
 ### Shared Directory
 

package/dist/templates/project/package.json.template

@@ -14,6 +14,8 @@
     "output:dev": "output dev"
   },
   "dependencies": {
+    "@ai-sdk/anthropic": "4.0.61",
+    "@ai-sdk/openai": "4.0.73",
     "@outputai/output": "{{frameworkVersion}}"
   },
   "devDependencies": {

package/dist/templates/project/src/clients/jina.ts.template

@@ -1,4 +1,4 @@
-import { createKyClient } from '@outputai/http';
+import { createKyClient } from '@outputai/core/http';
 
 export interface JinaReaderResponse {
   code: number;

package/dist/templates/project/src/workflows/blog_evaluator/evaluators.ts.template

@@ -1,6 +1,6 @@
 import { evaluator, EvaluationNumberResult } from '@outputai/core';
 import type { EvaluationResultArgs } from '@outputai/core';
-import { generateText, aiSdk } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/core/llm';
 import { blogContentSchema } from './types.js';
 import type { BlogContent } from './types.js';
 

package/dist/templates/workflow/README.md.template

@@ -123,7 +123,7 @@ Define steps in `steps.ts` with schemas:
 
 ```typescript
 import { step, z } from '@outputai/core';
-import { generateText, aiSdk } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/core/llm';
 
 export const myStep = step( {
   name: 'my_step',
@@ -153,7 +153,7 @@ Define evaluators in `evaluators.ts`:
 
 ```typescript
 import { evaluator, z, EvaluationVerdictResult } from '@outputai/core';
-import { generateText, aiSdk } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/core/llm';
 
 export const evaluateQuality = evaluator( {
   name: 'evaluate_quality',

package/dist/templates/workflow/evaluators.ts.template

@@ -1,5 +1,5 @@
 import { evaluator, z, EvaluationVerdictResult } from '@outputai/core';
-import { generateText, aiSdk } from '@outputai/llm';
+import { generateText, aiSdk } from '@outputai/core/llm';
 
 // TODO: Update the evaluator to assess your workflow's output quality
 export const evaluate{{WorkflowName}} = evaluator( {

package/dist/templates/workflow/steps.ts.template

@@ -1,5 +1,5 @@
 import { step, z } from '@outputai/core';
-import { generateText } from '@outputai/llm';
+import { generateText } from '@outputai/core/llm';
 
 export const processText = step( {
   name: 'process_text',

package/dist/test_helpers/mocks.d.ts

@@ -6,7 +6,7 @@ export declare const mockClaudeAgentSDK: {
     Agent: Mock;
 };
 /**
- * Mock for @outputai/llm
+ * Mock for @outputai/core/llm
  */
 export declare const mockLLM: {
     generateText: Mock;

package/dist/test_helpers/mocks.js

@@ -13,7 +13,7 @@ export const mockClaudeAgentSDK = {
     }))
 };
 /**
- * Mock for @outputai/llm
+ * Mock for @outputai/core/llm
  */
 export const mockLLM = {
     generateText: vi.fn().mockResolvedValue({

package/dist/types/cost.d.ts

@@ -13,7 +13,7 @@ export interface TokenUsage {
     cachedInputTokens?: number;
     reasoningTokens?: number;
 }
-import type { LLMGenerationCost, LLMGenerationUsage, LLMUsageEvent } from '@outputai/llm';
+import type { LLMGenerationCost, LLMGenerationUsage, LLMUsageEvent } from '@outputai/core/llm';
 export type LLMUsageLine = LLMUsageEvent['usage'][number];
 export interface HTTPCostEvent {
     type: 'http:request:cost';

package/dist/utils/eval_diagnostics.js

@@ -1,7 +1,7 @@
 import { existsSync } from 'node:fs';
 import { resolve } from 'node:path';
-import { getEvalWorkflowName } from '@outputai/evals';
 import { resolveWorkflowDir } from '#utils/workflow_dir.js';
+import { getEvalWorkflowName } from '#utils/eval_naming.js';
 const EVAL_WORKFLOW_FILES = ['tests/evals/workflow.ts', 'tests/evals/workflow.js'];
 /**
  * Explain why a `<wf>_eval` workflow isn't registered. When the eval source

package/dist/utils/eval_naming.d.ts

@@ -0,0 +1 @@
+export declare const getEvalWorkflowName: (workflowName: string) => string;

package/dist/utils/eval_naming.js

@@ -0,0 +1,2 @@
+const EVAL_SUFFIX = '_eval';
+export const getEvalWorkflowName = (workflowName) => `${workflowName}${EVAL_SUFFIX}`;

package/dist/utils/eval_naming.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/eval_naming.spec.js

@@ -0,0 +1,10 @@
+import { describe, it, expect } from 'vitest';
+import { getEvalWorkflowName } from './eval_naming.js';
+describe('naming', () => {
+    describe('getEvalWorkflowName', () => {
+        it('appends _eval suffix', () => {
+            expect(getEvalWorkflowName('simple')).toBe('simple_eval');
+            expect(getEvalWorkflowName('my_workflow')).toBe('my_workflow_eval');
+        });
+    });
+});

package/dist/utils/eval_render.d.ts

@@ -0,0 +1,3 @@
+import type { EvalOutput } from '@outputai/core/evals';
+export declare function renderEvalOutput(evalOutput: EvalOutput, evalName?: string): string;
+export declare function computeExitCode(evalOutput: EvalOutput): number;

package/dist/utils/eval_render.js

@@ -0,0 +1,75 @@
+import { VERDICT, CRITICALITY } from '@outputai/core/evals';
+const COLORS = {
+    reset: '\x1b[0m',
+    green: '\x1b[32m',
+    yellow: '\x1b[33m',
+    red: '\x1b[31m',
+    dim: '\x1b[2m',
+    bold: '\x1b[1m',
+    cyan: '\x1b[36m'
+};
+const VERDICT_STYLE = {
+    [VERDICT.PASS]: { label: 'PASS', color: COLORS.green },
+    [VERDICT.PARTIAL]: { label: 'PARTIAL', color: COLORS.yellow },
+    [VERDICT.FAIL]: { label: 'FAIL', color: COLORS.red }
+};
+function colorize(text, color) {
+    return `${color}${text}${COLORS.reset}`;
+}
+function padWithDots(label, width = 30) {
+    const dotCount = Math.max(2, width - label.length);
+    return ` ${'.'.repeat(dotCount)} `;
+}
+function renderEvaluator(evaluator) {
+    const lines = [];
+    const style = VERDICT_STYLE[evaluator.verdict];
+    const prefix = evaluator.criticality === CRITICALITY.INFORMATIONAL ?
+        `${COLORS.dim}(info)${COLORS.reset} ` :
+        '';
+    lines.push(`    ${prefix}${evaluator.name}${padWithDots(evaluator.name, 24)}${colorize(evaluator.verdict, style.color)}`);
+    if (evaluator.reasoning && evaluator.verdict !== VERDICT.PASS) {
+        lines.push(`      ${COLORS.dim}-> ${evaluator.reasoning}${COLORS.reset}`);
+    }
+    if (evaluator.feedback) {
+        for (const fb of evaluator.feedback) {
+            const item = fb;
+            if (item.issue) {
+                lines.push(`      ${COLORS.dim}-> ${item.issue}${COLORS.reset}`);
+            }
+        }
+    }
+    return lines;
+}
+function renderCase(evalCase) {
+    const caseStyle = VERDICT_STYLE[evalCase.verdict];
+    const lines = [
+        `  ${evalCase.datasetName}${padWithDots(evalCase.datasetName)}${colorize(caseStyle.label, caseStyle.color)}`
+    ];
+    evalCase.evaluators.forEach(evaluator => lines.push(...renderEvaluator(evaluator)));
+    lines.push('');
+    return lines;
+}
+export function renderEvalOutput(evalOutput, evalName) {
+    const lines = [];
+    if (evalName) {
+        lines.push(colorize(evalName, COLORS.bold));
+        lines.push('');
+    }
+    evalOutput.cases.forEach(evalCase => lines.push(...renderCase(evalCase)));
+    const { summary } = evalOutput;
+    const parts = [];
+    if (summary.passed > 0) {
+        parts.push(colorize(`${summary.passed} passed`, COLORS.green));
+    }
+    if (summary.partial > 0) {
+        parts.push(colorize(`${summary.partial} partial`, COLORS.yellow));
+    }
+    if (summary.failed > 0) {
+        parts.push(colorize(`${summary.failed} failed`, COLORS.red));
+    }
+    lines.push(`${parts.join(', ')} (${Math.round(summary.acceptableRate * 100)}% acceptable)`);
+    return lines.join('\n');
+}
+export function computeExitCode(evalOutput) {
+    return evalOutput.cases.some(c => c.verdict === VERDICT.FAIL) ? 1 : 0;
+}

package/dist/utils/eval_render.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/eval_render.spec.js

@@ -0,0 +1,171 @@
+import { describe, it, expect } from 'vitest';
+import { renderEvalOutput, computeExitCode } from './eval_render.js';
+const makeOutput = (cases, overrides) => {
+    const passed = cases.filter(c => c.verdict === 'pass').length;
+    const partial = cases.filter(c => c.verdict === 'partial').length;
+    const failed = cases.filter(c => c.verdict === 'fail').length;
+    const total = cases.length;
+    return {
+        cases,
+        summary: {
+            total,
+            passed,
+            partial,
+            failed,
+            acceptableRate: total > 0 ? (passed + partial) / total : 0,
+            ...overrides
+        }
+    };
+};
+describe('computeExitCode', () => {
+    it('returns 0 when all cases pass', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] }
+        ]);
+        expect(computeExitCode(output)).toBe(0);
+    });
+    it('returns 0 when cases are partial but none fail', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'partial', evaluators: [] }
+        ]);
+        expect(computeExitCode(output)).toBe(0);
+    });
+    it('returns 1 when any case fails', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] },
+            { datasetName: 'b', verdict: 'fail', evaluators: [] }
+        ]);
+        expect(computeExitCode(output)).toBe(1);
+    });
+    it('returns 0 for empty cases', () => {
+        const output = makeOutput([]);
+        expect(computeExitCode(output)).toBe(0);
+    });
+});
+describe('renderEvalOutput', () => {
+    it('includes eval name when provided', () => {
+        const output = makeOutput([
+            { datasetName: 'test', verdict: 'pass', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output, 'my_eval');
+        expect(rendered).toContain('my_eval');
+    });
+    it('includes dataset names', () => {
+        const output = makeOutput([
+            { datasetName: 'basic_input', verdict: 'pass', evaluators: [] },
+            { datasetName: 'edge_case', verdict: 'fail', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('basic_input');
+        expect(rendered).toContain('edge_case');
+    });
+    it('includes verdict labels', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] },
+            { datasetName: 'b', verdict: 'fail', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('PASS');
+        expect(rendered).toContain('FAIL');
+    });
+    it('shows reasoning for failed evaluators', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'fail',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'fail',
+                        criticality: 'required',
+                        reasoning: 'Expected 15, got null'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('Expected 15, got null');
+    });
+    it('hides reasoning for passing evaluators', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'pass',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'pass',
+                        criticality: 'required',
+                        reasoning: 'All good'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).not.toContain('All good');
+    });
+    it('shows summary with acceptable rate', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] },
+            { datasetName: 'b', verdict: 'fail', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('1 passed');
+        expect(rendered).toContain('1 failed');
+        expect(rendered).toContain('50% acceptable');
+    });
+    it('includes PARTIAL verdict label', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'partial', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('PARTIAL');
+    });
+    it('shows reasoning for partial evaluators', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'partial',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'partial',
+                        criticality: 'required',
+                        reasoning: 'Close but not quite'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('Close but not quite');
+    });
+    it('renders feedback issues', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'fail',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'fail',
+                        criticality: 'required',
+                        feedback: [
+                            { issue: 'Missing required field' },
+                            { issue: 'Invalid format' }
+                        ]
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('Missing required field');
+        expect(rendered).toContain('Invalid format');
+    });
+    it('marks informational evaluators with (info) prefix', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'pass',
+                evaluators: [{
+                        name: 'info_check',
+                        verdict: 'fail',
+                        criticality: 'informational'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('(info)');
+    });
+});

package/oclif.manifest.json

@@ -1772,5 +1772,5 @@
       ]
     }
   },
-  "version": "0.15.1-next.4a437bf.0"
+  "version": "0.15.1-next.5db73d3.0"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/cli",
-  "version": "0.15.1-next.4a437bf.0",
+  "version": "0.15.1-next.5db73d3.0",
   "description": "CLI for Output.ai workflow generation",
   "type": "module",
   "main": "dist/index.js",
@@ -17,6 +17,7 @@
     "output": "./bin/run.js"
   },
   "dependencies": {
+    "@ai-sdk/anthropic": "4.0.61",
     "@anthropic-ai/claude-agent-sdk": "0.3.281",
     "@hackylabs/deep-redact": "3.0.5",
     "@inquirer/prompts": "8.7.2",
@@ -35,10 +36,8 @@
     "ky": "2.0.2",
     "react": "19.3.0",
     "semver": "7.8.5",
-    "undici": "8.11.0",
-    "@outputai/evals": "0.15.1-next.4a437bf.0",
-    "@outputai/core": "0.15.1-next.4a437bf.0",
-    "@outputai/llm": "0.15.1-next.4a437bf.0"
+    "undici": "8.10.2",
+    "@outputai/core": "0.15.1-next.5db73d3.0"
   },
   "devDependencies": {
     "@types/cli-progress": "3.11.6",