@output.ai/cli 0.7.6 → 0.7.8-dev.pr263-a59dd0e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/http_client.d.ts +13 -0
- package/dist/api/http_client.js +20 -0
- package/dist/assets/docker/docker-compose-dev.yml +1 -1
- package/dist/commands/workflow/run.js +2 -1
- package/dist/services/messages.js +1 -1
- package/dist/templates/agent_instructions/dotoutputai/AGENTS.md.template +231 -24
- package/dist/templates/project/README.md.template +38 -1
- package/dist/templates/project/src/shared/clients/pokeapi.ts.template +29 -0
- package/dist/templates/project/src/shared/utils/string.ts.template +3 -0
- package/dist/templates/project/src/workflows/poke_battle/evaluators.ts.template +44 -0
- package/dist/templates/project/src/workflows/poke_battle/prompts/evaluate_realism@v1.prompt.template +24 -0
- package/dist/templates/project/src/workflows/poke_battle/prompts/generate_screenplay@v1.prompt.template +18 -0
- package/dist/templates/project/src/workflows/poke_battle/scenarios/pikachu_vs_charmander.json.template +8 -0
- package/dist/templates/project/src/workflows/poke_battle/steps.ts.template +37 -0
- package/dist/templates/project/src/workflows/poke_battle/utils.ts.template +7 -0
- package/dist/templates/project/src/workflows/poke_battle/workflow.ts.template +47 -0
- package/dist/templates/workflow/README.md.template +96 -75
- package/dist/templates/workflow/evaluators.ts.template +23 -0
- package/dist/utils/error_handler.js +22 -0
- package/dist/utils/paths.d.ts +1 -1
- package/dist/utils/paths.js +1 -1
- package/package.json +1 -1
- package/dist/templates/project/src/workflows/example_question/prompts/answer_question@v1.prompt.template +0 -13
- package/dist/templates/project/src/workflows/example_question/scenarios/question_ada_lovelace.json.template +0 -3
- package/dist/templates/project/src/workflows/example_question/steps.ts.template +0 -16
- package/dist/templates/project/src/workflows/example_question/workflow.ts.template +0 -22
|
@@ -2,6 +2,19 @@
|
|
|
2
2
|
* Custom ky-based HTTP client for Orval-generated API
|
|
3
3
|
*/
|
|
4
4
|
import type { Options as KyOptions } from 'ky';
|
|
5
|
+
/**
|
|
6
|
+
* Custom error class for HTTP errors with response details
|
|
7
|
+
*/
|
|
8
|
+
export declare class HttpError extends Error {
|
|
9
|
+
response: {
|
|
10
|
+
status: number;
|
|
11
|
+
data?: unknown;
|
|
12
|
+
};
|
|
13
|
+
constructor(message: string, response: {
|
|
14
|
+
status: number;
|
|
15
|
+
data?: unknown;
|
|
16
|
+
});
|
|
17
|
+
}
|
|
5
18
|
/**
|
|
6
19
|
* Custom API request options that extend RequestInit with additional config
|
|
7
20
|
*/
|
package/dist/api/http_client.js
CHANGED
|
@@ -3,6 +3,17 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import ky from 'ky';
|
|
5
5
|
import { config } from '#config.js';
|
|
6
|
+
/**
|
|
7
|
+
* Custom error class for HTTP errors with response details
|
|
8
|
+
*/
|
|
9
|
+
export class HttpError extends Error {
|
|
10
|
+
response;
|
|
11
|
+
constructor(message, response) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.response = response;
|
|
14
|
+
this.name = 'HttpError';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
6
17
|
const api = ky.create({
|
|
7
18
|
prefixUrl: config.apiUrl,
|
|
8
19
|
timeout: config.requestTimeout,
|
|
@@ -46,5 +57,14 @@ const wrapResponse = (response, data) => ({
|
|
|
46
57
|
export const customFetchInstance = async (url, options) => {
|
|
47
58
|
const response = await api(stripLeadingSlash(url), buildKyOptions(options));
|
|
48
59
|
const data = await response.json().catch(() => undefined);
|
|
60
|
+
// Throw for non-2xx responses so catch handlers can process errors
|
|
61
|
+
if (!response.ok) {
|
|
62
|
+
const errorData = data;
|
|
63
|
+
const message = errorData?.message || `HTTP ${response.status} error`;
|
|
64
|
+
throw new HttpError(message, {
|
|
65
|
+
status: response.status,
|
|
66
|
+
data: errorData
|
|
67
|
+
});
|
|
68
|
+
}
|
|
49
69
|
return wrapResponse(response, data);
|
|
50
70
|
};
|
|
@@ -65,7 +65,8 @@ export default class WorkflowRun extends Command {
|
|
|
65
65
|
}
|
|
66
66
|
async catch(error) {
|
|
67
67
|
return handleApiError(error, (...args) => this.error(...args), {
|
|
68
|
-
404: 'Workflow not found. Check the workflow name.'
|
|
68
|
+
404: 'Workflow not found. Check the workflow name.',
|
|
69
|
+
500: 'Workflow execution failed.'
|
|
69
70
|
});
|
|
70
71
|
}
|
|
71
72
|
}
|
|
@@ -177,7 +177,7 @@ export const getProjectSuccessMessage = (folderName, installSuccess, envConfigur
|
|
|
177
177
|
note: 'Launches Temporal, Redis, PostgreSQL, API, Worker, and UI'
|
|
178
178
|
}, {
|
|
179
179
|
step: 'Run example workflow',
|
|
180
|
-
command: 'npx output workflow run
|
|
180
|
+
command: 'npx output workflow run poke_battle --input src/workflows/poke_battle/scenarios/pikachu_vs_charmander.json',
|
|
181
181
|
note: 'Execute in a new terminal after services are running'
|
|
182
182
|
}, {
|
|
183
183
|
step: 'Monitor workflows',
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## Overview
|
|
4
4
|
|
|
5
|
-
This project uses Output Framework to build durable, LLM-powered workflows orchestrated by Temporal. Output Framework provides abstractions for creating reliable AI workflows with automatic retry, tracing, and error handling. Developers use it to build workflows like fact checkers, content generators, data extractors, research assistants, and multi-step AI agents.
|
|
5
|
+
This project uses Output.ai Framework to build durable, LLM-powered workflows orchestrated by Temporal. Output Framework provides abstractions for creating reliable AI workflows with automatic retry, tracing, and error handling. Developers use it to build workflows like fact checkers, content generators, data extractors, research assistants, and multi-step AI agents mixing API clients.
|
|
6
6
|
|
|
7
7
|
### Project Overview
|
|
8
8
|
|
|
@@ -26,17 +26,127 @@ Each workflow is self-contained in a single folder with a predictable structure:
|
|
|
26
26
|
- **Steps**: All external operations (APIs, DBs, LLMs) must be wrapped in steps
|
|
27
27
|
- **Schemas**: Use Zod (`z`) from `@output.ai/core` to define input/output schemas
|
|
28
28
|
|
|
29
|
+
## Code Reuse Rules
|
|
30
|
+
|
|
31
|
+
**IMPORTANT: Workflows are isolated by design.** Do not reuse code directly between sibling workflows.
|
|
32
|
+
|
|
33
|
+
### Allowed Code Sharing Locations:
|
|
34
|
+
- `src/clients/` - HTTP API clients shared across ALL workflows
|
|
35
|
+
- `src/workflows/{name}/shared/` - Shared utilities within a workflow group
|
|
36
|
+
- `src/workflows/{name}/lib/` - Internal libraries for a workflow group
|
|
37
|
+
|
|
38
|
+
### Forbidden:
|
|
39
|
+
- Importing from sibling workflow folders (e.g., `../other_workflow/steps.js`)
|
|
40
|
+
- Copying step implementations between workflows
|
|
41
|
+
- Referencing types from other workflow folders
|
|
42
|
+
|
|
43
|
+
Each workflow must remain independently deployable and testable. Cross-workflow imports create hidden dependencies that break this isolation.
|
|
44
|
+
|
|
29
45
|
## Project Structure
|
|
30
46
|
|
|
31
47
|
```
|
|
32
|
-
src/
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
48
|
+
src/
|
|
49
|
+
clients/ # Shared HTTP API clients (one file per external service)
|
|
50
|
+
jina.ts # Example: Jina Reader API client
|
|
51
|
+
stripe.ts # Example: Stripe API client
|
|
52
|
+
workflows/{name}/
|
|
53
|
+
types.ts # Zod schemas and TypeScript types (should hold ALL types for the workflow)
|
|
54
|
+
workflow.ts # Orchestration logic (deterministic)
|
|
55
|
+
steps.ts # I/O operations (APIs, LLM, DB)
|
|
56
|
+
evaluators.ts # LLM-as-a-judge analysis steps returning EvaluationResult
|
|
57
|
+
prompts/*.prompt # LLM prompts (name@v1.prompt)
|
|
58
|
+
scenarios/*.json # Test scenarios
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## API Clients Pattern
|
|
62
|
+
|
|
63
|
+
External API integrations should be placed in `src/clients/` as reusable modules. Each client wraps `@output.ai/http` for automatic tracing and retries.
|
|
64
|
+
|
|
65
|
+
```typescript
|
|
66
|
+
// src/clients/jina.ts
|
|
67
|
+
import { httpClient } from '@output.ai/http';
|
|
68
|
+
|
|
69
|
+
const client = httpClient({
|
|
70
|
+
prefixUrl: 'https://r.jina.ai',
|
|
71
|
+
timeout: 30000
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
export const jinaClient = {
|
|
75
|
+
read: async (url: string): Promise<string> => {
|
|
76
|
+
const response = await client.get(url);
|
|
77
|
+
return response.text();
|
|
78
|
+
}
|
|
79
|
+
};
|
|
38
80
|
```
|
|
39
81
|
|
|
82
|
+
**Usage in steps:**
|
|
83
|
+
```typescript
|
|
84
|
+
// src/workflows/my_workflow/steps.ts
|
|
85
|
+
import { jinaClient } from '../../clients/jina.js';
|
|
86
|
+
|
|
87
|
+
export const scrapeUrl = step({
|
|
88
|
+
name: 'scrapeUrl',
|
|
89
|
+
// ...
|
|
90
|
+
fn: async (url) => jinaClient.read(url)
|
|
91
|
+
});
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**Key requirements:**
|
|
95
|
+
- One file per external service in `src/clients/`
|
|
96
|
+
- Always use `@output.ai/http` (never axios or fetch directly)
|
|
97
|
+
- Export a named client object with typed methods
|
|
98
|
+
- Import clients in steps using relative paths (`../../clients/`)
|
|
99
|
+
|
|
100
|
+
## Types Pattern
|
|
101
|
+
|
|
102
|
+
Every workflow MUST have a `types.ts` file containing all Zod schemas and TypeScript types. This ensures type safety and schema reusability across workflow components.
|
|
103
|
+
|
|
104
|
+
```typescript
|
|
105
|
+
import { z } from '@output.ai/core';
|
|
106
|
+
|
|
107
|
+
// Workflow input/output schemas
|
|
108
|
+
export const WorkflowInputSchema = z.object({
|
|
109
|
+
query: z.string().describe('The search query'),
|
|
110
|
+
maxResults: z.number().optional().default(10)
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
export const WorkflowOutputSchema = z.object({
|
|
114
|
+
results: z.array(z.object({
|
|
115
|
+
title: z.string(),
|
|
116
|
+
content: z.string()
|
|
117
|
+
})),
|
|
118
|
+
totalCount: z.number()
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
// Step schemas
|
|
122
|
+
export const FetchDataInputSchema = z.object({
|
|
123
|
+
url: z.string().url()
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
export const FetchDataOutputSchema = z.object({
|
|
127
|
+
data: z.unknown(),
|
|
128
|
+
status: z.number()
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
// Inferred TypeScript types
|
|
132
|
+
export type WorkflowInput = z.infer<typeof WorkflowInputSchema>;
|
|
133
|
+
export type WorkflowOutput = z.infer<typeof WorkflowOutputSchema>;
|
|
134
|
+
export type FetchDataInput = z.infer<typeof FetchDataInputSchema>;
|
|
135
|
+
export type FetchDataOutput = z.infer<typeof FetchDataOutputSchema>;
|
|
136
|
+
|
|
137
|
+
// Shared interfaces (non-Zod types used within the workflow)
|
|
138
|
+
export interface ApiResponse {
|
|
139
|
+
code: number;
|
|
140
|
+
data: unknown;
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
**Key requirements:**
|
|
145
|
+
- All Zod schemas used in `workflow.ts` and `steps.ts` MUST be defined in `types.ts`
|
|
146
|
+
- Export both schemas (for runtime validation) and inferred types (for TypeScript)
|
|
147
|
+
- Use `.describe()` on schema fields for documentation
|
|
148
|
+
- Keep API response interfaces separate from Zod schemas
|
|
149
|
+
|
|
40
150
|
## Commands
|
|
41
151
|
|
|
42
152
|
```bash
|
|
@@ -53,6 +163,8 @@ npx output workflow result <workflowId> # Get result when complete
|
|
|
53
163
|
npx output workflow stop <workflowId> # Cancel running workflow
|
|
54
164
|
```
|
|
55
165
|
|
|
166
|
+
**When running workflows for users**: After execution completes, try to format the result nicely for readability. Use markdown formatting, tables for structured data, and highlight key values. Don't just dump raw JSON.
|
|
167
|
+
|
|
56
168
|
## Workflow Pattern
|
|
57
169
|
|
|
58
170
|
Workflows orchestrate steps. They must be deterministic (no direct I/O).
|
|
@@ -74,7 +186,7 @@ export default workflow({
|
|
|
74
186
|
});
|
|
75
187
|
```
|
|
76
188
|
|
|
77
|
-
**Allowed imports**: steps.ts, evaluators.ts,
|
|
189
|
+
**Allowed imports**: steps.ts, evaluators.ts, ../../shared/steps/*.ts, types.ts, consts.ts, utils.ts
|
|
78
190
|
|
|
79
191
|
**Forbidden in workflows**: Direct API calls, Math.random(), Date.now(), dynamic imports
|
|
80
192
|
|
|
@@ -116,7 +228,7 @@ temperature: 0.7
|
|
|
116
228
|
maxTokens: 2000
|
|
117
229
|
---
|
|
118
230
|
<system>You are a helpful assistant.</system>
|
|
119
|
-
<user>Summarize:
|
|
231
|
+
<user>Summarize: {{ content }}</user>
|
|
120
232
|
```
|
|
121
233
|
|
|
122
234
|
**Step using prompt**:
|
|
@@ -129,10 +241,11 @@ export const summarize = step({
|
|
|
129
241
|
inputSchema: z.object({ content: z.string() }),
|
|
130
242
|
outputSchema: z.string(),
|
|
131
243
|
fn: async ({ content }) => {
|
|
132
|
-
|
|
244
|
+
const { result } = await generateText({
|
|
133
245
|
prompt: 'summarize@v1',
|
|
134
246
|
variables: { content }
|
|
135
247
|
});
|
|
248
|
+
return result;
|
|
136
249
|
}
|
|
137
250
|
});
|
|
138
251
|
|
|
@@ -142,11 +255,12 @@ export const extractInfo = step({
|
|
|
142
255
|
inputSchema: z.object({ text: z.string() }),
|
|
143
256
|
outputSchema: z.object({ title: z.string(), summary: z.string() }),
|
|
144
257
|
fn: async ({ text }) => {
|
|
145
|
-
|
|
258
|
+
const { result } = await generateObject({
|
|
146
259
|
prompt: 'extract@v1',
|
|
147
260
|
variables: { text },
|
|
148
261
|
schema: z.object({ title: z.string(), summary: z.string() })
|
|
149
262
|
});
|
|
263
|
+
return result;
|
|
150
264
|
}
|
|
151
265
|
});
|
|
152
266
|
```
|
|
@@ -175,7 +289,7 @@ const result = await client.post('endpoint', { json: payload }).json();
|
|
|
175
289
|
|
|
176
290
|
## Evaluator Pattern
|
|
177
291
|
|
|
178
|
-
Evaluators
|
|
292
|
+
Evaluators are LLM-as-a-judge for analyzing data and return confidence-scored results. They are highly recommended for anything that is high-value involving LLMs and can benefit from self-improvement loops or scoring for logging the results on tracing.
|
|
179
293
|
|
|
180
294
|
```typescript
|
|
181
295
|
import { evaluator, EvaluationStringResult } from '@output.ai/core';
|
|
@@ -205,23 +319,116 @@ throw new FatalError('Critical failure - do not retry');
|
|
|
205
319
|
throw new ValidationError('Invalid input format');
|
|
206
320
|
```
|
|
207
321
|
|
|
208
|
-
##
|
|
322
|
+
## Creating New Workflows
|
|
323
|
+
|
|
324
|
+
**IMPORTANT**: When creating a new workflow, you MUST use the following agents and commands in order. Do not skip steps.
|
|
325
|
+
|
|
326
|
+
### Mandatory Workflow Creation Process
|
|
327
|
+
|
|
328
|
+
1. **Plan** → `/outputai:plan_workflow` or `workflow-planner` agent
|
|
329
|
+
- Defines workflow architecture, steps, and data flow
|
|
330
|
+
- Identifies required external APIs and LLM operations
|
|
331
|
+
- MUST be run first before any implementation
|
|
332
|
+
|
|
333
|
+
2. **Build** → `/outputai:build_workflow`
|
|
334
|
+
- Creates the workflow folder structure
|
|
335
|
+
- Generates `types.ts`, `workflow.ts`, `steps.ts`
|
|
336
|
+
- Sets up test scenarios
|
|
337
|
+
|
|
338
|
+
3. **Prompts** → `workflow-prompt-writer` agent
|
|
339
|
+
- Creates `.prompt` files for all LLM operations
|
|
340
|
+
- Reviews and optimizes prompt templates
|
|
341
|
+
- Ensures proper Liquid.js templating
|
|
342
|
+
|
|
343
|
+
4. **Quality** → `workflow-quality` agent
|
|
344
|
+
- Validates implementation against best practices
|
|
345
|
+
- Checks for proper error handling and retries
|
|
346
|
+
- Ensures schema consistency across components
|
|
347
|
+
|
|
348
|
+
### Available Sub-Agents
|
|
349
|
+
|
|
350
|
+
| Agent | Purpose |
|
|
351
|
+
|-------|---------|
|
|
352
|
+
| `workflow-planner` | Architecture design, step breakdown, data flow planning |
|
|
353
|
+
| `workflow-prompt-writer` | Create and review `.prompt` files with proper templates |
|
|
354
|
+
| `workflow-quality` | Validate best practices, error handling, schema consistency |
|
|
355
|
+
| `workflow-context-fetcher` | Retrieve project context (used by other agents) |
|
|
356
|
+
| `workflow-debugger` | Debug workflow issues, analyze execution traces |
|
|
357
|
+
|
|
358
|
+
### Available Commands
|
|
359
|
+
|
|
360
|
+
| Command | When to Use |
|
|
361
|
+
|---------|-------------|
|
|
362
|
+
| `/outputai:plan_workflow` | **ALWAYS FIRST** - Plan new workflow architecture |
|
|
363
|
+
| `/outputai:build_workflow` | Implement the planned workflow |
|
|
364
|
+
| `/outputai:debug_workflow` | Debug failing or misbehaving workflows |
|
|
365
|
+
|
|
366
|
+
## Working with This Codebase
|
|
367
|
+
|
|
368
|
+
**CRITICAL: Trust the documentation.** When creating or modifying workflows:
|
|
369
|
+
|
|
370
|
+
1. **Do NOT scan the entire codebase** - The patterns and examples in this document are authoritative
|
|
371
|
+
2. **Follow the documented patterns** - Use the step, workflow, and evaluator patterns exactly as shown
|
|
372
|
+
3. **Use the sub-agents** - They have the context needed to create correct implementations
|
|
373
|
+
|
|
374
|
+
### What to Read:
|
|
375
|
+
- This current file (you're reading it)
|
|
376
|
+
- The specific workflow folder you're modifying
|
|
377
|
+
- `src/clients/` for available HTTP clients
|
|
378
|
+
|
|
379
|
+
### What NOT to Do:
|
|
380
|
+
- Grep through all workflows looking for "examples"
|
|
381
|
+
- Read multiple workflow implementations to "understand patterns"
|
|
382
|
+
- Second-guess the documented patterns based on existing code variations
|
|
383
|
+
|
|
384
|
+
## Common Issues
|
|
385
|
+
|
|
386
|
+
### Restarting Worker After Adding Workflows
|
|
387
|
+
|
|
388
|
+
After creating a new workflow, you likely need restart the worker container for changes to take effect.
|
|
389
|
+
|
|
390
|
+
```bash
|
|
391
|
+
# Check running containers
|
|
392
|
+
docker ps --filter "name=output" --format "{{.Names}}: {{.Status}}"
|
|
393
|
+
|
|
394
|
+
# Restart the worker (adjust container name based on your project)
|
|
395
|
+
docker restart <project-name>-worker-1
|
|
396
|
+
|
|
397
|
+
# Wait for worker to restart, then run the workflow
|
|
398
|
+
sleep 5 && npx output workflow run <workflow_name> --input '<json>'
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
### Payload Size Limits
|
|
402
|
+
|
|
403
|
+
Plan for size limits when designing workflows.**
|
|
404
|
+
|
|
405
|
+
- **Temporal limit**: ~2MB per workflow input/output payload
|
|
406
|
+
- **gRPC limit**: ~4MB per message
|
|
407
|
+
|
|
408
|
+
When planning workflows that process large data (documents, images, API responses):
|
|
409
|
+
- Chunk large arrays and process in batches
|
|
410
|
+
- Summarize or extract only needed fields from large API responses
|
|
411
|
+
- When dealing with files prompt the user about cloud storage (S3, etc)
|
|
209
412
|
|
|
210
|
-
|
|
413
|
+
### Docker-Based Development
|
|
211
414
|
|
|
212
|
-
|
|
213
|
-
- workflow-quality: Workflow quality and best practices specialist
|
|
214
|
-
- workflow-prompt-writer: Prompt file creation and review specialist
|
|
215
|
-
- workflow-context-fetcher: Efficient context retrieval (used by other agents)
|
|
216
|
-
- workflow-debugger: Workflow debugging specialist
|
|
415
|
+
`npx output dev` runs services in Docker containers. For debugging:
|
|
217
416
|
|
|
218
|
-
|
|
417
|
+
```bash
|
|
418
|
+
# View worker logs
|
|
419
|
+
docker logs -f output-worker-1
|
|
420
|
+
|
|
421
|
+
# View API logs
|
|
422
|
+
docker logs -f output-api-1
|
|
219
423
|
|
|
220
|
-
|
|
424
|
+
# View Temporal logs
|
|
425
|
+
docker logs -f output-temporal-1
|
|
426
|
+
|
|
427
|
+
# Shell into a container
|
|
428
|
+
docker exec -it output-worker-1 sh
|
|
429
|
+
```
|
|
221
430
|
|
|
222
|
-
-
|
|
223
|
-
- /outputai:build_workflow: Workflow Implementation command
|
|
224
|
-
- /outputai:debug_workflow: Workflow Debugging command
|
|
431
|
+
Logs do not appear in the terminal directly - check container logs when debugging workflow issues.
|
|
225
432
|
|
|
226
433
|
## Configuration
|
|
227
434
|
|
|
@@ -7,6 +7,43 @@
|
|
|
7
7
|
- Node.js >= 24.3
|
|
8
8
|
- Docker and Docker Compose (for local development)
|
|
9
9
|
|
|
10
|
+
## Project Structure
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
src/
|
|
14
|
+
├── shared/ # Shared code across workflows
|
|
15
|
+
│ ├── clients/ # API clients (e.g., pokeapi.ts)
|
|
16
|
+
│ └── utils/ # Utility functions (e.g., string.ts)
|
|
17
|
+
└── workflows/ # Workflow definitions
|
|
18
|
+
└── poke_battle/ # Example workflow
|
|
19
|
+
├── workflow.ts # Main workflow
|
|
20
|
+
├── steps.ts # Workflow steps
|
|
21
|
+
├── evaluators.ts # Quality evaluators
|
|
22
|
+
├── utils.ts # Local utilities
|
|
23
|
+
├── prompts/ # LLM prompts
|
|
24
|
+
└── scenarios/ # Test scenarios
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
### Shared Directory
|
|
28
|
+
|
|
29
|
+
The `src/shared/` directory contains code shared across multiple workflows:
|
|
30
|
+
|
|
31
|
+
- **`shared/clients/`** - API clients using `@output.ai/http` for external services
|
|
32
|
+
- **`shared/utils/`** - Helper functions and utilities
|
|
33
|
+
|
|
34
|
+
### Import Rules
|
|
35
|
+
|
|
36
|
+
**Workflows** can import from:
|
|
37
|
+
- Local steps, evaluators, and utilities
|
|
38
|
+
- Shared steps, evaluators, clients, and utilities
|
|
39
|
+
|
|
40
|
+
**Steps and Evaluators** can import from:
|
|
41
|
+
- Local utilities and clients
|
|
42
|
+
- Shared utilities and clients
|
|
43
|
+
|
|
44
|
+
**Steps and Evaluators cannot** import from:
|
|
45
|
+
- Other steps or evaluators (Temporal activity isolation)
|
|
46
|
+
|
|
10
47
|
## Getting Started
|
|
11
48
|
|
|
12
49
|
### 1. Install Dependencies
|
|
@@ -44,7 +81,7 @@ This starts:
|
|
|
44
81
|
In a new terminal:
|
|
45
82
|
|
|
46
83
|
```bash
|
|
47
|
-
npx output workflow run
|
|
84
|
+
npx output workflow run poke_battle --input src/workflows/poke_battle/scenarios/pikachu_vs_charmander.json
|
|
48
85
|
```
|
|
49
86
|
|
|
50
87
|
### 5. Stop Services
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { httpClient, HttpClientOptions } from '@output.ai/http';
|
|
2
|
+
|
|
3
|
+
export interface Pokemon {
|
|
4
|
+
name: string;
|
|
5
|
+
types: string[];
|
|
6
|
+
abilities: string[];
|
|
7
|
+
stats: { name: string; value: number }[];
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface PokeApiPokemon {
|
|
11
|
+
name: string;
|
|
12
|
+
types: object[];
|
|
13
|
+
abilities: object[];
|
|
14
|
+
stats: object[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const basePokeApiClient = httpClient( {
|
|
18
|
+
prefixUrl: 'https://pokeapi.co/api/v2'
|
|
19
|
+
} );
|
|
20
|
+
|
|
21
|
+
const pokemonClient = basePokeApiClient.extend( options => ( {
|
|
22
|
+
prefixUrl: `${options.prefixUrl}/pokemon`,
|
|
23
|
+
timeout: 3000
|
|
24
|
+
} ) as HttpClientOptions );
|
|
25
|
+
|
|
26
|
+
export async function getPokemon( name: string ): Promise<Pokemon> {
|
|
27
|
+
const response = await pokemonClient.get<PokeApiPokemon>( name );
|
|
28
|
+
return response.json() as Promise<Pokemon>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { evaluator, z, EvaluationNumberResult } from '@output.ai/core';
|
|
2
|
+
import { generateObject } from '@output.ai/llm';
|
|
3
|
+
import type { Pokemon } from '../../shared/clients/pokeapi.js';
|
|
4
|
+
|
|
5
|
+
export const evaluateBattleRealism = evaluator( {
|
|
6
|
+
name: 'evaluate_battle_realism',
|
|
7
|
+
description: 'Evaluate how realistic the Pokemon battle screenplay is',
|
|
8
|
+
inputSchema: z.object( {
|
|
9
|
+
screenplay: z.string(),
|
|
10
|
+
pokemon1: z.object( {
|
|
11
|
+
name: z.string(),
|
|
12
|
+
types: z.array( z.string() ),
|
|
13
|
+
abilities: z.array( z.string() ),
|
|
14
|
+
stats: z.array( z.object( { name: z.string(), value: z.number() } ) )
|
|
15
|
+
} ),
|
|
16
|
+
pokemon2: z.object( {
|
|
17
|
+
name: z.string(),
|
|
18
|
+
types: z.array( z.string() ),
|
|
19
|
+
abilities: z.array( z.string() ),
|
|
20
|
+
stats: z.array( z.object( { name: z.string(), value: z.number() } ) )
|
|
21
|
+
} )
|
|
22
|
+
} ),
|
|
23
|
+
fn: async ( input: { screenplay: string; pokemon1: Pokemon; pokemon2: Pokemon } ) => {
|
|
24
|
+
const { result } = await generateObject( {
|
|
25
|
+
prompt: 'evaluate_realism@v1',
|
|
26
|
+
variables: {
|
|
27
|
+
pokemon1Name: input.pokemon1.name,
|
|
28
|
+
pokemon1Types: input.pokemon1.types.join( '/' ),
|
|
29
|
+
pokemon1Stats: input.pokemon1.stats.map( s => `${s.name}: ${s.value}` ).join( ', ' ),
|
|
30
|
+
pokemon2Name: input.pokemon2.name,
|
|
31
|
+
pokemon2Types: input.pokemon2.types.join( '/' ),
|
|
32
|
+
pokemon2Stats: input.pokemon2.stats.map( s => `${s.name}: ${s.value}` ).join( ', ' ),
|
|
33
|
+
screenplay: input.screenplay
|
|
34
|
+
},
|
|
35
|
+
schema: z.object( {
|
|
36
|
+
score: z.number().min( 0 ).max( 100 ).describe( 'Realism score 0-100' )
|
|
37
|
+
} )
|
|
38
|
+
} );
|
|
39
|
+
return new EvaluationNumberResult( {
|
|
40
|
+
value: result.score,
|
|
41
|
+
confidence: 0.9
|
|
42
|
+
} );
|
|
43
|
+
}
|
|
44
|
+
} );
|
package/dist/templates/project/src/workflows/poke_battle/prompts/evaluate_realism@v1.prompt.template
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
---
|
|
2
|
+
provider: anthropic
|
|
3
|
+
model: claude-haiku-4-5
|
|
4
|
+
temperature: 0.7
|
|
5
|
+
maxTokens: 8192
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
Evaluate this Pokemon battle screenplay for realism.
|
|
9
|
+
|
|
10
|
+
Consider:
|
|
11
|
+
- Are the moves/abilities used consistent with each Pokemon's actual abilities?
|
|
12
|
+
- Is the battle outcome realistic given their types and stats?
|
|
13
|
+
- Does the pacing feel like a real battle?
|
|
14
|
+
|
|
15
|
+
\{{ pokemon1Name }}'s Stat Sheet:
|
|
16
|
+
|
|
17
|
+
\{{ pokemon1Description }}
|
|
18
|
+
|
|
19
|
+
\{{ pokemon2Name }}'s Stat Sheet:
|
|
20
|
+
|
|
21
|
+
\{{ pokemon2Description }}
|
|
22
|
+
|
|
23
|
+
Screenplay:
|
|
24
|
+
\{{ screenplay }}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
---
|
|
2
|
+
provider: anthropic
|
|
3
|
+
model: claude-haiku-4-5
|
|
4
|
+
temperature: 0.7
|
|
5
|
+
maxTokens: 8192
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
Write a dramatic 5-minute screenplay of a Pokemon battle between \{{ pokemon1Name }} and \{{ pokemon2Name }}.
|
|
9
|
+
|
|
10
|
+
\{{ pokemon1Name }}'s Stat Sheet:
|
|
11
|
+
|
|
12
|
+
\{{ pokemon1Description }}
|
|
13
|
+
|
|
14
|
+
\{{ pokemon2Name }}'s Stat Sheet:
|
|
15
|
+
|
|
16
|
+
\{{ pokemon2Description }}
|
|
17
|
+
|
|
18
|
+
Include dramatic moments, skill usage that matches their actual abilities, and a decisive outcome based on their type matchups. Format it as a proper screenplay with scene descriptions and dialogue.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { step, z } from '@output.ai/core';
|
|
2
|
+
import { generateText } from '@output.ai/llm';
|
|
3
|
+
import type { Pokemon } from '../../shared/clients/pokeapi.js';
|
|
4
|
+
|
|
5
|
+
export const generateScreenplay = step( {
|
|
6
|
+
name: 'generate_screenplay',
|
|
7
|
+
description: 'Generate a dramatic 5-minute Pokemon battle screenplay',
|
|
8
|
+
inputSchema: z.object( {
|
|
9
|
+
pokemon1: z.object( {
|
|
10
|
+
name: z.string(),
|
|
11
|
+
types: z.array( z.object() ),
|
|
12
|
+
abilities: z.array( z.object() ),
|
|
13
|
+
stats: z.array( z.object() )
|
|
14
|
+
} ),
|
|
15
|
+
pokemon2: z.object( {
|
|
16
|
+
name: z.string(),
|
|
17
|
+
types: z.array( z.object() ),
|
|
18
|
+
abilities: z.array( z.object() ),
|
|
19
|
+
stats: z.array( z.object() )
|
|
20
|
+
} )
|
|
21
|
+
} ),
|
|
22
|
+
outputSchema: z.object( {
|
|
23
|
+
screenplay: z.string()
|
|
24
|
+
} ),
|
|
25
|
+
fn: async ( input: { pokemon1: Pokemon; pokemon2: Pokemon } ) => {
|
|
26
|
+
const { result } = await generateText( {
|
|
27
|
+
prompt: 'generate_screenplay@v1',
|
|
28
|
+
variables: {
|
|
29
|
+
pokemon1Name: input.pokemon1.name,
|
|
30
|
+
pokemon2Name: input.pokemon2.name,
|
|
31
|
+
pokemon1Description: JSON.stringify( input.pokemon1 ),
|
|
32
|
+
pokemon2Description: JSON.stringify( input.pokemon2 )
|
|
33
|
+
}
|
|
34
|
+
} );
|
|
35
|
+
return { screenplay: result };
|
|
36
|
+
}
|
|
37
|
+
} );
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { workflow, z } from '@output.ai/core';
|
|
2
|
+
import { lowercase } from '../../shared/utils/string.js';
|
|
3
|
+
import { getPokemon } from '../../shared/clients/pokeapi.js';
|
|
4
|
+
import { generateScreenplay } from './steps.js';
|
|
5
|
+
import { evaluateBattleRealism } from './evaluators.js';
|
|
6
|
+
import { createWorkflowOutput } from './utils.js';
|
|
7
|
+
|
|
8
|
+
export default workflow( {
|
|
9
|
+
name: 'poke_battle',
|
|
10
|
+
description: '{{description}}',
|
|
11
|
+
inputSchema: z.object( {
|
|
12
|
+
pokemon1Name: z.string().describe( 'Name of the first Pokemon' ),
|
|
13
|
+
pokemon2Name: z.string().describe( 'Name of the second Pokemon' )
|
|
14
|
+
} ),
|
|
15
|
+
outputSchema: z.object( {
|
|
16
|
+
screenplay: z.string().describe( 'A 5-minute battle screenplay' ),
|
|
17
|
+
confidenceScore: z.number().describe( 'Realism confidence score 0-100' ),
|
|
18
|
+
summary: z.string().describe( 'Battle summary with confidence' )
|
|
19
|
+
} ),
|
|
20
|
+
fn: async input => {
|
|
21
|
+
// Use shared util to normalize names
|
|
22
|
+
const name1 = lowercase( input.pokemon1Name );
|
|
23
|
+
const name2 = lowercase( input.pokemon2Name );
|
|
24
|
+
|
|
25
|
+
// Use shared client to fetch Pokemon data
|
|
26
|
+
const pokemon1 = await getPokemon( name1 );
|
|
27
|
+
const pokemon2 = await getPokemon( name2 );
|
|
28
|
+
|
|
29
|
+
// Generate the battle screenplay
|
|
30
|
+
const { screenplay } = await generateScreenplay( { pokemon1, pokemon2 } );
|
|
31
|
+
|
|
32
|
+
// Evaluate realism
|
|
33
|
+
const evaluation = await evaluateBattleRealism( {
|
|
34
|
+
screenplay,
|
|
35
|
+
pokemon1,
|
|
36
|
+
pokemon2
|
|
37
|
+
} );
|
|
38
|
+
|
|
39
|
+
// Use local util to format output
|
|
40
|
+
return createWorkflowOutput( screenplay, evaluation.value );
|
|
41
|
+
},
|
|
42
|
+
options: {
|
|
43
|
+
retry: {
|
|
44
|
+
maximumAttempts: 3
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
} );
|
|
@@ -10,8 +10,50 @@ This workflow was generated using the Output SDK CLI. It provides a starting poi
|
|
|
10
10
|
|
|
11
11
|
- `workflow.ts` - Main workflow definition with input/output schemas
|
|
12
12
|
- `steps.ts` - Activity/step definitions with input/output schemas
|
|
13
|
-
- `
|
|
14
|
-
-
|
|
13
|
+
- `evaluators.ts` - Quality evaluators for workflow outputs
|
|
14
|
+
- `prompts/` - LLM prompt templates
|
|
15
|
+
|
|
16
|
+
## File Organization
|
|
17
|
+
|
|
18
|
+
You can organize your workflow files in two ways:
|
|
19
|
+
|
|
20
|
+
**Flat files:**
|
|
21
|
+
```
|
|
22
|
+
workflow/
|
|
23
|
+
├── workflow.ts
|
|
24
|
+
├── steps.ts
|
|
25
|
+
├── evaluators.ts
|
|
26
|
+
└── utils.ts
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
**Folder-based:**
|
|
30
|
+
```
|
|
31
|
+
workflow/
|
|
32
|
+
├── workflow.ts
|
|
33
|
+
├── steps/
|
|
34
|
+
│ ├── fetch_data.ts
|
|
35
|
+
│ └── process_data.ts
|
|
36
|
+
├── evaluators/
|
|
37
|
+
│ └── quality.ts
|
|
38
|
+
└── utils/
|
|
39
|
+
└── helpers.ts
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Import Rules
|
|
43
|
+
|
|
44
|
+
**Important:** Steps and evaluators are Temporal activities. Activities cannot call other activities.
|
|
45
|
+
|
|
46
|
+
**Steps can import from:**
|
|
47
|
+
- Local utilities (`./utils.ts`, `./utils/*.ts`)
|
|
48
|
+
- Shared utilities (`../../shared/utils/*.ts`)
|
|
49
|
+
- Shared clients (`../../shared/clients/*.ts`)
|
|
50
|
+
|
|
51
|
+
**Steps cannot import from:**
|
|
52
|
+
- Other steps or evaluators (activity isolation)
|
|
53
|
+
- Workflow files
|
|
54
|
+
|
|
55
|
+
**Workflows can import from:**
|
|
56
|
+
- Steps, evaluators, and utilities (local and shared)
|
|
15
57
|
|
|
16
58
|
## Setup
|
|
17
59
|
|
|
@@ -63,121 +105,109 @@ Example:
|
|
|
63
105
|
|
|
64
106
|
### Workflow Structure
|
|
65
107
|
|
|
66
|
-
The workflow follows the
|
|
108
|
+
The workflow follows the Output SDK conventions:
|
|
67
109
|
|
|
68
110
|
```typescript
|
|
69
|
-
import { workflow } from '@output.ai/core';
|
|
70
|
-
import { myStep
|
|
71
|
-
|
|
72
|
-
const inputSchema = {
|
|
73
|
-
type: 'object',
|
|
74
|
-
properties: {
|
|
75
|
-
// Define your input properties
|
|
76
|
-
}
|
|
77
|
-
};
|
|
78
|
-
|
|
79
|
-
const outputSchema = {
|
|
80
|
-
type: 'object',
|
|
81
|
-
properties: {
|
|
82
|
-
// Define your output properties
|
|
83
|
-
}
|
|
84
|
-
};
|
|
111
|
+
import { workflow, z } from '@output.ai/core';
|
|
112
|
+
import { myStep } from './steps.js';
|
|
113
|
+
import { evaluateQuality } from './evaluators.js';
|
|
85
114
|
|
|
86
115
|
export default workflow( {
|
|
87
116
|
name: 'workflowName',
|
|
88
117
|
description: 'Workflow description',
|
|
89
|
-
inputSchema,
|
|
90
|
-
outputSchema,
|
|
118
|
+
inputSchema: z.object( { /* ... */ } ),
|
|
119
|
+
outputSchema: z.object( { /* ... */ } ),
|
|
91
120
|
fn: async ( input ) => {
|
|
92
|
-
// Call steps directly
|
|
93
121
|
const result = await myStep( input );
|
|
94
|
-
|
|
122
|
+
const { score } = await evaluateQuality( { input, output: result } );
|
|
123
|
+
return { result, qualityScore: score };
|
|
95
124
|
}
|
|
96
125
|
} );
|
|
97
126
|
```
|
|
98
127
|
|
|
99
128
|
### Adding New Steps
|
|
100
129
|
|
|
101
|
-
|
|
130
|
+
Define steps in `steps.ts` with schemas:
|
|
102
131
|
|
|
103
132
|
```typescript
|
|
104
|
-
import { step } from '@output.ai/core';
|
|
105
|
-
|
|
106
|
-
const inputSchema = {
|
|
107
|
-
type: 'object',
|
|
108
|
-
properties: {
|
|
109
|
-
value: { type: 'number' }
|
|
110
|
-
},
|
|
111
|
-
required: ['value']
|
|
112
|
-
};
|
|
113
|
-
|
|
114
|
-
const outputSchema = {
|
|
115
|
-
type: 'object',
|
|
116
|
-
properties: {
|
|
117
|
-
result: { type: 'string' }
|
|
118
|
-
}
|
|
119
|
-
};
|
|
133
|
+
import { step, z } from '@output.ai/core';
|
|
120
134
|
|
|
121
135
|
export const myStep = step( {
|
|
122
136
|
name: 'myStep',
|
|
123
137
|
description: 'Description of what this step does',
|
|
124
|
-
inputSchema
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
138
|
+
inputSchema: z.object( {
|
|
139
|
+
value: z.number()
|
|
140
|
+
} ),
|
|
141
|
+
outputSchema: z.object( {
|
|
142
|
+
result: z.string()
|
|
143
|
+
} ),
|
|
144
|
+
fn: async ( input ) => {
|
|
128
145
|
return { result: `Processed ${input.value}` };
|
|
129
146
|
}
|
|
130
147
|
} );
|
|
131
148
|
```
|
|
132
149
|
|
|
133
|
-
|
|
150
|
+
### Adding Evaluators
|
|
151
|
+
|
|
152
|
+
Define evaluators in `evaluators.ts`:
|
|
134
153
|
|
|
135
154
|
```typescript
|
|
136
|
-
import {
|
|
155
|
+
import { evaluator, z } from '@output.ai/core';
|
|
156
|
+
import { generateText } from '@output.ai/llm';
|
|
137
157
|
|
|
138
|
-
|
|
139
|
-
|
|
158
|
+
export const evaluateQuality = evaluator( {
|
|
159
|
+
name: 'evaluate_quality',
|
|
160
|
+
description: 'Evaluate output quality',
|
|
161
|
+
inputSchema: z.object( {
|
|
162
|
+
input: z.any(),
|
|
163
|
+
output: z.any()
|
|
164
|
+
} ),
|
|
165
|
+
outputSchema: z.object( {
|
|
166
|
+
score: z.number().min( 0 ).max( 100 )
|
|
167
|
+
} ),
|
|
168
|
+
fn: async ( data ) => {
|
|
169
|
+
const { result } = await generateText( {
|
|
170
|
+
prompt: 'evaluate@v1',
|
|
171
|
+
variables: { input: data.input, output: data.output }
|
|
172
|
+
} );
|
|
173
|
+
return { score: parseInt( result, 10 ) };
|
|
174
|
+
}
|
|
175
|
+
} );
|
|
140
176
|
```
|
|
141
177
|
|
|
142
178
|
### Using LLM in Steps
|
|
143
179
|
|
|
144
|
-
The template includes an example of using LLM with prompts:
|
|
145
|
-
|
|
146
180
|
```typescript
|
|
147
181
|
import { generateText } from '@output.ai/llm';
|
|
148
182
|
|
|
149
183
|
export const llmStep = step( {
|
|
150
184
|
name: 'llmStep',
|
|
151
185
|
description: 'Generate text using LLM',
|
|
152
|
-
inputSchema: {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
outputSchema: { type: 'string' },
|
|
159
|
-
fn: async ( input: { userInput: string } ) => {
|
|
160
|
-
const response = await generateText( {
|
|
186
|
+
inputSchema: z.object( {
|
|
187
|
+
userInput: z.string()
|
|
188
|
+
} ),
|
|
189
|
+
outputSchema: z.string(),
|
|
190
|
+
fn: async ( input ) => {
|
|
191
|
+
const { result } = await generateText( {
|
|
161
192
|
prompt: 'prompt@v1',
|
|
162
193
|
variables: { userInput: input.userInput }
|
|
163
194
|
} );
|
|
164
|
-
return
|
|
195
|
+
return result;
|
|
165
196
|
}
|
|
166
197
|
} );
|
|
167
198
|
```
|
|
168
199
|
|
|
169
200
|
### Creating Prompt Templates
|
|
170
201
|
|
|
171
|
-
Create
|
|
202
|
+
Create prompt files in `prompts/` following the pattern:
|
|
172
203
|
- File naming: `promptName@v1.prompt`
|
|
173
|
-
- Include YAML frontmatter with
|
|
204
|
+
- Include YAML frontmatter with model
|
|
174
205
|
- Use LiquidJS syntax for variables: `{{ variableName }}`
|
|
175
206
|
|
|
176
207
|
Example prompt file:
|
|
177
208
|
```
|
|
178
209
|
---
|
|
179
|
-
|
|
180
|
-
model: claude-3-5-sonnet-latest
|
|
210
|
+
model: anthropic/claude-sonnet-4-20250514
|
|
181
211
|
---
|
|
182
212
|
|
|
183
213
|
{{ userInput }}
|
|
@@ -195,19 +225,10 @@ To test your workflow:
|
|
|
195
225
|
|
|
196
226
|
Example execution:
|
|
197
227
|
```bash
|
|
198
|
-
|
|
199
|
-
-H "Content-Type: application/json" \
|
|
200
|
-
-d '{
|
|
201
|
-
"workflowName": "{{workflowName}}",
|
|
202
|
-
"input": {
|
|
203
|
-
"prompt": "Tell me about workflows",
|
|
204
|
-
"data": { "value": 42, "type": "example" }
|
|
205
|
-
}
|
|
206
|
-
}'
|
|
228
|
+
npx output workflow run {{workflowName}} --input '{"prompt": "Hello"}'
|
|
207
229
|
```
|
|
208
230
|
|
|
209
231
|
## Resources
|
|
210
232
|
|
|
211
|
-
- [Output SDK Documentation](https://
|
|
233
|
+
- [Output SDK Documentation](https://docs.output.ai)
|
|
212
234
|
- [Temporal Documentation](https://docs.temporal.io)
|
|
213
|
-
- [AI SDK Documentation](https://sdk.vercel.ai/docs)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { evaluator, z } from '@output.ai/core';
|
|
2
|
+
|
|
3
|
+
// Example evaluator - customize for your workflow
|
|
4
|
+
export const evaluate{{WorkflowName}} = evaluator( {
|
|
5
|
+
name: 'evaluate_{{workflowName}}',
|
|
6
|
+
description: 'Evaluate the quality of {{workflowName}} output',
|
|
7
|
+
inputSchema: z.object( {
|
|
8
|
+
input: z.any(),
|
|
9
|
+
output: z.any()
|
|
10
|
+
} ),
|
|
11
|
+
outputSchema: z.object( {
|
|
12
|
+
score: z.number().min( 0 ).max( 100 ).describe( 'Quality score 0-100' ),
|
|
13
|
+
feedback: z.string().describe( 'Feedback on the output quality' )
|
|
14
|
+
} ),
|
|
15
|
+
fn: async () => {
|
|
16
|
+
// TODO: Implement evaluation logic
|
|
17
|
+
// Use LLM or custom logic to score the output quality
|
|
18
|
+
return {
|
|
19
|
+
score: 100,
|
|
20
|
+
feedback: 'Evaluation not yet implemented'
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
} );
|
|
@@ -3,8 +3,24 @@ const DEFAULT_MESSAGES = {
|
|
|
3
3
|
ECONNREFUSED: `Connection refused to ${config.apiUrl}. Is the API server running?`,
|
|
4
4
|
401: 'Authentication failed. Check your API_AUTH_TOKEN.',
|
|
5
5
|
404: 'Resource not found.',
|
|
6
|
+
500: 'Server error.',
|
|
6
7
|
UNKNOWN: 'An unknown error occurred.'
|
|
7
8
|
};
|
|
9
|
+
/**
|
|
10
|
+
* Extract error type and message from API response data
|
|
11
|
+
*/
|
|
12
|
+
function extractApiErrorDetails(data) {
|
|
13
|
+
const errorData = data;
|
|
14
|
+
if (!errorData?.error && !errorData?.message) {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
const errorType = errorData.error || 'Error';
|
|
18
|
+
const baseMsg = errorData.message || 'Unknown error';
|
|
19
|
+
const rootCauseLine = errorData.rootCause ?
|
|
20
|
+
`\n${errorData.rootCause.error}: ${errorData.rootCause.message}` :
|
|
21
|
+
'';
|
|
22
|
+
return { errorType, errorMsg: `${baseMsg}.${rootCauseLine}` };
|
|
23
|
+
}
|
|
8
24
|
/**
|
|
9
25
|
* Extract detailed error information from fetch errors and their causes
|
|
10
26
|
*/
|
|
@@ -39,6 +55,12 @@ export function handleApiError(error, errorFn, overrides = {}) {
|
|
|
39
55
|
}
|
|
40
56
|
if (apiError.response?.status) {
|
|
41
57
|
const status = apiError.response.status;
|
|
58
|
+
// Extract error details from response body
|
|
59
|
+
const apiErrorDetails = extractApiErrorDetails(apiError.response.data);
|
|
60
|
+
if (apiErrorDetails) {
|
|
61
|
+
const { errorType, errorMsg } = apiErrorDetails;
|
|
62
|
+
errorFn(`${errorType}: ${errorMsg}`, { exit: 1 });
|
|
63
|
+
}
|
|
42
64
|
const message = errorMessages[status];
|
|
43
65
|
if (message) {
|
|
44
66
|
errorFn(message, { exit: 1 });
|
package/dist/utils/paths.d.ts
CHANGED
package/dist/utils/paths.js
CHANGED
package/package.json
CHANGED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
provider: anthropic
|
|
3
|
-
model: claude-opus-4-1-20250805
|
|
4
|
-
temperature: 0.7
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
<system>
|
|
8
|
-
You are a helpful assistant. Answer the user's question concisely and clearly.
|
|
9
|
-
</system>
|
|
10
|
-
|
|
11
|
-
<user>
|
|
12
|
-
Answer the following question: \{{ question }}
|
|
13
|
-
</user>
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { step, z } from '@output.ai/core';
|
|
2
|
-
import { generateText } from '@output.ai/llm';
|
|
3
|
-
|
|
4
|
-
export const answerQuestion = step( {
|
|
5
|
-
name: 'answerQuestion',
|
|
6
|
-
description: 'Answer a question using an LLM',
|
|
7
|
-
inputSchema: z.string(),
|
|
8
|
-
outputSchema: z.string(),
|
|
9
|
-
fn: async question => {
|
|
10
|
-
const { result } = await generateText( {
|
|
11
|
-
prompt: 'answer_question@v1',
|
|
12
|
-
variables: { question }
|
|
13
|
-
} );
|
|
14
|
-
return result;
|
|
15
|
-
}
|
|
16
|
-
} );
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { workflow, z } from '@output.ai/core';
|
|
2
|
-
import { answerQuestion } from './steps.js';
|
|
3
|
-
|
|
4
|
-
export default workflow( {
|
|
5
|
-
name: 'example_question',
|
|
6
|
-
description: '{{description}}',
|
|
7
|
-
inputSchema: z.object( {
|
|
8
|
-
question: z.string().describe( 'A question to answer' )
|
|
9
|
-
} ),
|
|
10
|
-
outputSchema: z.object( {
|
|
11
|
-
answer: z.string().describe( 'The answer to the question' )
|
|
12
|
-
} ),
|
|
13
|
-
fn: async input => {
|
|
14
|
-
const answer = await answerQuestion( input.question );
|
|
15
|
-
return { answer };
|
|
16
|
-
},
|
|
17
|
-
options: {
|
|
18
|
-
retry: {
|
|
19
|
-
maximumAttempts: 3
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
} );
|