llmz 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.workerd.js +1 -1
- package/package.json +7 -3
- package/CLAUDE.md +0 -362
- package/docs/TODO.md +0 -919
- package/eslint.config.mjs +0 -17
package/dist/index.cjs
CHANGED
package/dist/index.js
CHANGED
package/dist/index.workerd.js
CHANGED
|
@@ -15759,7 +15759,7 @@ var init_execute = __esm({
|
|
|
15759
15759
|
});
|
|
15760
15760
|
|
|
15761
15761
|
// package.json
|
|
15762
|
-
var version = "0.3.
|
|
15762
|
+
var version = "0.3.1";
|
|
15763
15763
|
|
|
15764
15764
|
// src/index.ts
|
|
15765
15765
|
init_tool();
|
package/package.json
CHANGED
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
"name": "llmz",
|
|
3
3
|
"type": "module",
|
|
4
4
|
"description": "LLMz - An LLM-native Typescript VM built on top of Zui",
|
|
5
|
-
"version": "0.3.
|
|
5
|
+
"version": "0.3.1",
|
|
6
|
+
"files": [
|
|
7
|
+
"dist",
|
|
8
|
+
"DOCS.md"
|
|
9
|
+
],
|
|
6
10
|
"types": "./dist/index.d.ts",
|
|
7
11
|
"main": "./dist/index.cjs",
|
|
8
12
|
"module": "./dist/index.js",
|
|
@@ -64,8 +68,8 @@
|
|
|
64
68
|
"tsx": "^4.19.2"
|
|
65
69
|
},
|
|
66
70
|
"peerDependencies": {
|
|
67
|
-
"@botpress/client": "
|
|
68
|
-
"@botpress/cognitive": "1.0.
|
|
71
|
+
"@botpress/client": "2.0.0",
|
|
72
|
+
"@botpress/cognitive": "1.0.1",
|
|
69
73
|
"@bpinternal/thicktoken": "^3.1.0",
|
|
70
74
|
"@bpinternal/zui": "^2.4.0"
|
|
71
75
|
},
|
package/CLAUDE.md
DELETED
|
@@ -1,362 +0,0 @@
|
|
|
1
|
-
# CLAUDE.md
|
|
2
|
-
|
|
3
|
-
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
-
|
|
5
|
-
## What is LLMz?
|
|
6
|
-
|
|
7
|
-
LLMz is a revolutionary TypeScript AI agent framework that fundamentally changes how AI agents work. Like other agent frameworks, LLMz calls LLM models in a loop to achieve desired outcomes with access to tools and memory. However, LLMz is **code-first** – meaning it generates and runs TypeScript code in a sandbox rather than using traditional JSON tool calling.
|
|
8
|
-
|
|
9
|
-
**Core Philosophy**: Stop chaining tools. Start generating real code.
|
|
10
|
-
|
|
11
|
-
### Why Code Generation Works Better
|
|
12
|
-
|
|
13
|
-
Traditional agent frameworks rely on JSON tool calling, which has significant limitations:
|
|
14
|
-
|
|
15
|
-
- Hard-to-parse JSON schemas for LLMs
|
|
16
|
-
- Incapable of complex logic, loops, and conditionals
|
|
17
|
-
- Multiple expensive roundtrips for each tool call
|
|
18
|
-
- Unreliable beyond simple scenarios
|
|
19
|
-
|
|
20
|
-
LLMz leverages the fact that models have been trained extensively on millions of TypeScript codebases, making them incredibly reliable at generating working code. This enables:
|
|
21
|
-
|
|
22
|
-
- Complex logic and multi-tool orchestration in **one call**
|
|
23
|
-
- Native LLM thinking via comments and code structure
|
|
24
|
-
- Complete type safety and predictable schemas
|
|
25
|
-
- Seamless scaling in production environments
|
|
26
|
-
|
|
27
|
-
## Core Concepts
|
|
28
|
-
|
|
29
|
-
### Execution Loop
|
|
30
|
-
|
|
31
|
-
LLMz exposes a single method (`execute`) that runs in a loop until one of these conditions:
|
|
32
|
-
|
|
33
|
-
1. **An Exit is returned** - Agent completes with structured result
|
|
34
|
-
2. **Agent waits for user input** (Chat Mode) - Returns control to user
|
|
35
|
-
3. **Maximum iterations reached** - Safety limit to prevent infinite loops
|
|
36
|
-
|
|
37
|
-
The loop automatically handles:
|
|
38
|
-
|
|
39
|
-
- Tool calling and result processing
|
|
40
|
-
- Thinking about outputs and context
|
|
41
|
-
- Error recovery and retry logic
|
|
42
|
-
- Variable state persistence across iterations
|
|
43
|
-
|
|
44
|
-
### Generated Code Structure
|
|
45
|
-
|
|
46
|
-
Every LLMz code block follows a predictable structure:
|
|
47
|
-
|
|
48
|
-
**Return Statement (Required)**:
|
|
49
|
-
|
|
50
|
-
```tsx
|
|
51
|
-
// Chat mode - give turn back to user
|
|
52
|
-
return { action: 'listen' }
|
|
53
|
-
|
|
54
|
-
// Worker mode - complete with result
|
|
55
|
-
return { action: 'done', result: calculatedValue }
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
**Tool Calls with Logic**:
|
|
59
|
-
|
|
60
|
-
```tsx
|
|
61
|
-
// Complex logic impossible with JSON tool calling
|
|
62
|
-
const price = await getTicketPrice({ from: 'quebec', to: 'new york' })
|
|
63
|
-
|
|
64
|
-
if (price > 500) {
|
|
65
|
-
throw new Error('Price too high')
|
|
66
|
-
} else {
|
|
67
|
-
const ticketId = await buyTicket({ from: 'quebec', to: 'new york' })
|
|
68
|
-
return { action: 'done', result: ticketId }
|
|
69
|
-
}
|
|
70
|
-
```
|
|
71
|
-
|
|
72
|
-
**Comments for Planning**:
|
|
73
|
-
|
|
74
|
-
```tsx
|
|
75
|
-
// Comments help LLM think step-by-step and plan ahead
|
|
76
|
-
// Check user's budget first before proceeding with purchase
|
|
77
|
-
const budget = await getUserBudget()
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
### Execution Modes
|
|
81
|
-
|
|
82
|
-
**Chat Mode**: Interactive conversational agents
|
|
83
|
-
|
|
84
|
-
- Enabled when `chat` is provided to `execute()`
|
|
85
|
-
- Agents can `yield` React components to user
|
|
86
|
-
- Special `ListenExit` automatically available for user interaction
|
|
87
|
-
- Transcript management for conversation history
|
|
88
|
-
|
|
89
|
-
**Worker Mode**: Automated execution environments
|
|
90
|
-
|
|
91
|
-
- Enabled when `chat` is omitted from `execute()`
|
|
92
|
-
- Focus on computational tasks and data processing
|
|
93
|
-
- Uses `DefaultExit` if no custom exits provided
|
|
94
|
-
- Sandboxed execution with security isolation
|
|
95
|
-
|
|
96
|
-
### Chat Components (Chat Mode Only)
|
|
97
|
-
|
|
98
|
-
Agents can yield React components for rich user interaction:
|
|
99
|
-
|
|
100
|
-
```tsx
|
|
101
|
-
// Multi-line text support
|
|
102
|
-
yield <Text>
|
|
103
|
-
Hello, world!
|
|
104
|
-
This is a second line.
|
|
105
|
-
</Text>
|
|
106
|
-
|
|
107
|
-
// Composed/nested components
|
|
108
|
-
yield <Message>
|
|
109
|
-
<Text>What do you prefer?</Text>
|
|
110
|
-
<Button>Cats</Button>
|
|
111
|
-
<Button>Dogs</Button>
|
|
112
|
-
</Message>
|
|
113
|
-
|
|
114
|
-
return { action: 'listen' }
|
|
115
|
-
```
|
|
116
|
-
|
|
117
|
-
Components map to communication channels (Webchat, SMS, etc.) and can be completely custom.
|
|
118
|
-
|
|
119
|
-
### Tools and Objects
|
|
120
|
-
|
|
121
|
-
**Tools**: Type-safe functions with Zod/Zui schemas
|
|
122
|
-
|
|
123
|
-
- Input/output validation and TypeScript inference
|
|
124
|
-
- Synchronous and asynchronous support
|
|
125
|
-
- Retry logic and error handling
|
|
126
|
-
- Tool aliases for multiple names
|
|
127
|
-
|
|
128
|
-
**Objects**: Namespaced containers for related tools and variables
|
|
129
|
-
|
|
130
|
-
- Group related functionality together
|
|
131
|
-
- Support both readonly and writable variables
|
|
132
|
-
- Variable type validation with schemas
|
|
133
|
-
- Mutation tracking across iterations
|
|
134
|
-
|
|
135
|
-
### Advanced Features
|
|
136
|
-
|
|
137
|
-
**Snapshots**: Pausable/resumable execution
|
|
138
|
-
|
|
139
|
-
- Throw `SnapshotSignal` to halt and serialize execution state
|
|
140
|
-
- Resume later from exact same point
|
|
141
|
-
- Useful for long-running workflows and async operations
|
|
142
|
-
|
|
143
|
-
**Thinking**: Agent reflection and variable inspection
|
|
144
|
-
|
|
145
|
-
- Agent-initiated: `return { action: 'think' }`
|
|
146
|
-
- Tool-initiated: Throw `ThinkSignal` to force reflection
|
|
147
|
-
- Helps agents avoid rushed decisions and process complex information
|
|
148
|
-
|
|
149
|
-
**Hooks**: Custom logic at execution points
|
|
150
|
-
|
|
151
|
-
- `onTrace`: Non-blocking monitoring and logging
|
|
152
|
-
- `onExit`: Validate exits and implement guardrails
|
|
153
|
-
- `onBeforeExecution`: Code mutation and security checks
|
|
154
|
-
- `onIterationEnd`: State augmentation between iterations
|
|
155
|
-
|
|
156
|
-
## Project Overview
|
|
157
|
-
|
|
158
|
-
LLMz operates as an LLM-native TypeScript VM built on top of Zui (Botpress's internal schema library), battle-tested in production powering millions of AI agents worldwide.
|
|
159
|
-
|
|
160
|
-
## Architecture
|
|
161
|
-
|
|
162
|
-
LLMz operates as an LLM-native TypeScript VM built on top of Zui (Botpress's internal schema library). The framework includes two primary execution modes:
|
|
163
|
-
|
|
164
|
-
LLMz operates in two primary modes:
|
|
165
|
-
|
|
166
|
-
- **Chat Mode**: Interactive conversational agents with tools and user interaction. Chat mode is enabled when `chat` is provided to `execute()`
|
|
167
|
-
- **Worker Mode**: Automated execution environments with sandboxing and security. Worker mode is enabled when `chat` is omitted from `execute()`.
|
|
168
|
-
|
|
169
|
-
### Core Components
|
|
170
|
-
|
|
171
|
-
**Main Framework** (`src/`):
|
|
172
|
-
|
|
173
|
-
- `llmz.ts`: Core execution engine and context management
|
|
174
|
-
- `vm.ts`: Virtual machine with isolated-vm and Node.js execution environments
|
|
175
|
-
- `compiler/`: TypeScript compilation pipeline with AST transformations
|
|
176
|
-
- `chat.ts`: Chat interface and message handling
|
|
177
|
-
- `tool.ts`: Tool definition and execution system
|
|
178
|
-
- `component.ts`: UI component system for interactive elements
|
|
179
|
-
|
|
180
|
-
**Execution Pipeline**:
|
|
181
|
-
|
|
182
|
-
- `prompts/`: Dual-mode prompt system (chat-mode/, worker-mode/)
|
|
183
|
-
- `transcript.ts`: Conversation history and context management
|
|
184
|
-
- `context.ts`: Execution context and iteration tracking
|
|
185
|
-
- `result.ts`: Execution result types and error handling
|
|
186
|
-
|
|
187
|
-
**Safety & Security**:
|
|
188
|
-
|
|
189
|
-
- `errors.ts`: Comprehensive error handling and signals
|
|
190
|
-
- `stack-traces.ts`: Stack trace sanitization
|
|
191
|
-
- `snapshots.ts`: Execution state snapshots
|
|
192
|
-
- `truncator.ts`: Content truncation for token limits
|
|
193
|
-
|
|
194
|
-
## Development Commands
|
|
195
|
-
|
|
196
|
-
### Building and Testing
|
|
197
|
-
|
|
198
|
-
```bash
|
|
199
|
-
# Build the package (TypeScript compilation + bundling)
|
|
200
|
-
pnpm build
|
|
201
|
-
|
|
202
|
-
# Type checking without emitting files
|
|
203
|
-
pnpm check:type
|
|
204
|
-
|
|
205
|
-
# Development mode with hot reloading
|
|
206
|
-
pnpm watch
|
|
207
|
-
|
|
208
|
-
# Run test suite
|
|
209
|
-
pnpm test
|
|
210
|
-
|
|
211
|
-
# Run tests in watch mode
|
|
212
|
-
pnpm test:watch
|
|
213
|
-
|
|
214
|
-
# Update test snapshots
|
|
215
|
-
pnpm test:update
|
|
216
|
-
|
|
217
|
-
# Generate prompt files (.md.ts) files after changing them
|
|
218
|
-
pnpm generate
|
|
219
|
-
```
|
|
220
|
-
|
|
221
|
-
### Testing Configuration
|
|
222
|
-
|
|
223
|
-
- Tests use Vitest with custom configuration for LLM testing
|
|
224
|
-
- Retry mechanism (2 retries) due to LLM non-determinism
|
|
225
|
-
- Extended timeout (60s) for LLM response times
|
|
226
|
-
- Custom snapshot serializers for stack traces
|
|
227
|
-
- Markdown and text file loaders for prompt testing
|
|
228
|
-
|
|
229
|
-
## Key Dependencies
|
|
230
|
-
|
|
231
|
-
**Core Runtime**:
|
|
232
|
-
|
|
233
|
-
- `isolated-vm`: Secure JavaScript execution environment
|
|
234
|
-
- `@babel/*`: TypeScript/JSX compilation pipeline
|
|
235
|
-
- `@botpress/client`: Botpress API integration
|
|
236
|
-
- `@bpinternal/zui`: Schema validation and TypeScript generation
|
|
237
|
-
|
|
238
|
-
**Development**:
|
|
239
|
-
|
|
240
|
-
- `tsup`: Fast TypeScript bundler (ESM + CJS output)
|
|
241
|
-
- `vitest`: Modern test framework with TypeScript support
|
|
242
|
-
- `prettier`: Code formatting
|
|
243
|
-
|
|
244
|
-
## Code Architecture Patterns
|
|
245
|
-
|
|
246
|
-
### Execution Flow
|
|
247
|
-
|
|
248
|
-
1. **Prompt Generation**: Dual-mode prompts based on execution type
|
|
249
|
-
2. **Code Generation**: LLM generates TypeScript code with embedded tools
|
|
250
|
-
3. **Compilation**: Babel-based AST transformation with custom plugins
|
|
251
|
-
4. **Execution**: Isolated VM or Node.js execution with monitoring
|
|
252
|
-
5. **Result Processing**: Type-safe result handling and error recovery
|
|
253
|
-
|
|
254
|
-
### Key Plugins (`src/compiler/plugins/`):
|
|
255
|
-
|
|
256
|
-
- `track-tool-calls.ts`: Instrument tool calls for monitoring
|
|
257
|
-
- `variable-extraction.ts`: Extract and track variable usage
|
|
258
|
-
- `line-tracking.ts`: Source map generation and error tracking
|
|
259
|
-
- `jsx-preserve-newlines.ts`: Maintain formatting in JSX components
|
|
260
|
-
|
|
261
|
-
### VM Execution Strategy
|
|
262
|
-
|
|
263
|
-
- **Production**: Uses `isolated-vm` for security isolation
|
|
264
|
-
- **CI/Development**: Falls back to Node.js VM for compatibility
|
|
265
|
-
- **Browser**: Uses standard JavaScript execution
|
|
266
|
-
|
|
267
|
-
## Tool System
|
|
268
|
-
|
|
269
|
-
Tools are defined using Zui schemas and can be:
|
|
270
|
-
|
|
271
|
-
- **Synchronous/Asynchronous**: Both execution types supported
|
|
272
|
-
- **Type-Safe**: Full TypeScript inference and validation
|
|
273
|
-
- **Retryable**: Built-in retry logic with exponential backoff
|
|
274
|
-
- **Traceable**: Comprehensive execution monitoring
|
|
275
|
-
|
|
276
|
-
Example tool definition:
|
|
277
|
-
|
|
278
|
-
```typescript
|
|
279
|
-
const tool = new Tool({
|
|
280
|
-
name: 'exampleTool',
|
|
281
|
-
description: 'Tool description',
|
|
282
|
-
input: z.object({ param: z.string() }),
|
|
283
|
-
output: z.object({ result: z.string() }),
|
|
284
|
-
handler: async ({ param }) => ({ result: `Processed: ${param}` }),
|
|
285
|
-
})
|
|
286
|
-
```
|
|
287
|
-
|
|
288
|
-
## Component System
|
|
289
|
-
|
|
290
|
-
LLMz includes a React-like component system for building interactive UIs:
|
|
291
|
-
|
|
292
|
-
- **JSX Support**: Full JSX compilation and rendering
|
|
293
|
-
- **Type Safety**: Components are fully typed with Zui schemas
|
|
294
|
-
- **Rendering**: Server-side rendering to various output formats
|
|
295
|
-
|
|
296
|
-
## Security Considerations
|
|
297
|
-
|
|
298
|
-
- **Isolated Execution**: VM isolation prevents access to host system
|
|
299
|
-
- **Stack Trace Cleaning**: Removes internal framework details from errors
|
|
300
|
-
- **Tool Access Control**: Granular permissions for tool execution
|
|
301
|
-
- **Content Truncation**: Automatic handling of token limits
|
|
302
|
-
- **Error Recovery**: Graceful handling of execution failures
|
|
303
|
-
|
|
304
|
-
## File Structure Conventions
|
|
305
|
-
|
|
306
|
-
- Tests co-located with source files (`.test.ts` suffix)
|
|
307
|
-
- Markdown prompts compiled to TypeScript modules
|
|
308
|
-
- Source maps preserved for debugging
|
|
309
|
-
- TypeScript declaration files generated for exports
|
|
310
|
-
|
|
311
|
-
## Environment Variables
|
|
312
|
-
|
|
313
|
-
- `VM_DRIVER`: Choose VM execution environment ('isolated-vm' | 'node')
|
|
314
|
-
- `CI`: Automatically detected, affects VM driver selection
|
|
315
|
-
|
|
316
|
-
## Performance Considerations
|
|
317
|
-
|
|
318
|
-
- Code splitting with dynamic imports for faster startup
|
|
319
|
-
- LRU caching for compiled code and results
|
|
320
|
-
- Lazy loading of heavy dependencies
|
|
321
|
-
- Bundle size optimization with tsup configuration
|
|
322
|
-
|
|
323
|
-
## Exit System
|
|
324
|
-
|
|
325
|
-
LLMz uses a sophisticated exit system for controlling agent termination:
|
|
326
|
-
|
|
327
|
-
- **Built-in Exits**: `ThinkExit`, `ListenExit`, `DefaultExit` for common patterns
|
|
328
|
-
- **Custom Exits**: Domain-specific exits with typed schemas using Zod/Zui
|
|
329
|
-
- **Type Safety**: `result.is(exit)` for compile-time type checking
|
|
330
|
-
- **Agent Usage**: Agents call `return { action: 'exit_name', ...data }` to exit
|
|
331
|
-
|
|
332
|
-
## Prompt System Architecture
|
|
333
|
-
|
|
334
|
-
The dual-mode prompt system (`src/prompts/`) generates different prompts based on execution mode:
|
|
335
|
-
|
|
336
|
-
- **Chat Mode**: Uses prompts from `chat-mode/` for interactive conversations
|
|
337
|
-
- **Worker Mode**: Uses prompts from `worker-mode/` for automated execution
|
|
338
|
-
- **Markdown Templates**: Prompts are written in Markdown and compiled to TypeScript
|
|
339
|
-
- **Dynamic Content**: Tool definitions, schemas, and context injected at runtime
|
|
340
|
-
|
|
341
|
-
## Compiler Pipeline
|
|
342
|
-
|
|
343
|
-
The Babel-based compilation system transforms generated code:
|
|
344
|
-
|
|
345
|
-
1. **AST Parsing**: TypeScript/JSX code parsed into Abstract Syntax Tree
|
|
346
|
-
2. **Plugin Transformation**: Custom plugins modify the AST for execution
|
|
347
|
-
3. **Code Generation**: Modified AST compiled back to executable JavaScript
|
|
348
|
-
4. **Source Maps**: Generated for debugging and error tracking
|
|
349
|
-
|
|
350
|
-
Key transformations:
|
|
351
|
-
|
|
352
|
-
- Tool call instrumentation for monitoring
|
|
353
|
-
- Variable extraction and tracking
|
|
354
|
-
- JSX component handling
|
|
355
|
-
- Line number preservation for stack traces
|
|
356
|
-
|
|
357
|
-
# important-instruction-reminders
|
|
358
|
-
|
|
359
|
-
Do what has been asked; nothing more, nothing less.
|
|
360
|
-
NEVER create files unless they're absolutely necessary for achieving your goal.
|
|
361
|
-
ALWAYS prefer editing an existing file to creating a new one.
|
|
362
|
-
NEVER proactively create documentation files (\*.md) or README files. Only create documentation files if explicitly requested by the User.
|
package/docs/TODO.md
DELETED
|
@@ -1,919 +0,0 @@
|
|
|
1
|
-
# How LLMz works
|
|
2
|
-
|
|
3
|
-
Like many other agent frameworks, LLMz is an agentic framework that calls LLM models in a loop to achieve a desired outcome, with optional access to tools and memory.
|
|
4
|
-
|
|
5
|
-
Unlike other agent frameworks, LLMz is code-first – meaning it generates and runs Typescript code in a sandbox to communicate and execute tools rather than using rudimentary JSON tool calling and text responses. This is what makes agents built on LLMz more reliable and capable than other agents.
|
|
6
|
-
|
|
7
|
-
## Execution Loop
|
|
8
|
-
|
|
9
|
-
At its simplest, LLMz exposes a single method (`execute`) that will run in a loop until one of the following conditions are met:
|
|
10
|
-
|
|
11
|
-
- An `Exit` is returned
|
|
12
|
-
- The agent waits for the user input (in Chat Mode)
|
|
13
|
-
- or the maximum number of iterations has been reached
|
|
14
|
-
|
|
15
|
-
The loop will iterate by calling tools, thinking about tool outputs and recovering from errors automatically.
|
|
16
|
-
|
|
17
|
-
## Code Generation
|
|
18
|
-
|
|
19
|
-
Unlike traditional tool-calling agents, LLMz defines tools using Typescript and runs real code in a VM.
|
|
20
|
-
Concretely, that means LLMz does not require and rely on tool-calling, JSON output or any other feature of LLMs outside of text generation.
|
|
21
|
-
|
|
22
|
-
Because models have been trained extensively on code bases, models are usually much better at generating working code than calling tools using JSON.
|
|
23
|
-
|
|
24
|
-
### Structure of an LLMz Code Bloc
|
|
25
|
-
|
|
26
|
-
#### Return statement
|
|
27
|
-
|
|
28
|
-
At the minimum, an LLMz response _must_ contain a return statement with an Exit.
|
|
29
|
-
|
|
30
|
-
```tsx
|
|
31
|
-
// in chat mode, this gives back the turn to the user
|
|
32
|
-
return { action: 'listen' }
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
```tsx
|
|
36
|
-
// assuming an Exit named 'done' with output schema <number> has been declared
|
|
37
|
-
return { action: 'done', result: 666 }
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
#### Tool calls
|
|
41
|
-
|
|
42
|
-
Because LLMz generates standard Typescript code and because the VM has access to tools passed to `execute()`. This allows the combination of multiple tool calls, conditional logic and error handling. You'll also notice that tools are type-safe and provide an output schema, which the generated code can easily use to combine tools.
|
|
43
|
-
|
|
44
|
-
```tsx
|
|
45
|
-
// The user wants to fly from Quebec to New York with a max budget of $500
|
|
46
|
-
const price = await getTicketPrice({ from: 'quebec', to: 'new york' })
|
|
47
|
-
|
|
48
|
-
if (price > 500) {
|
|
49
|
-
throw new Error('Price too high')
|
|
50
|
-
} else {
|
|
51
|
-
const ticketId = await buyTicket({ from: 'quebec', to: 'new york' })
|
|
52
|
-
return { action: 'done', result: ticketId }
|
|
53
|
-
}
|
|
54
|
-
```
|
|
55
|
-
|
|
56
|
-
#### Comments
|
|
57
|
-
|
|
58
|
-
The use of comments in the code helps the LLM to think "step-by-step" and use the tools correctly. It helps the LLM plan ahead of writing the code.
|
|
59
|
-
|
|
60
|
-
#### React Components (Chat Mode)
|
|
61
|
-
|
|
62
|
-
In Chat Mode, the code can `yield` React components to respond to the user. Unlike tool calls, components have many benefits.
|
|
63
|
-
|
|
64
|
-
They support multi-line text:
|
|
65
|
-
|
|
66
|
-
```tsx
|
|
67
|
-
yield <Text>
|
|
68
|
-
Hello, world!
|
|
69
|
-
This is a second line.
|
|
70
|
-
</Text>
|
|
71
|
-
|
|
72
|
-
return { action: 'listen' }
|
|
73
|
-
```
|
|
74
|
-
|
|
75
|
-
And they can be composed / nested:
|
|
76
|
-
|
|
77
|
-
```tsx
|
|
78
|
-
yield <Message>
|
|
79
|
-
<Text>What do you prefer ?</Text>
|
|
80
|
-
<Button>Cats</Button>
|
|
81
|
-
<Button>Dogs</Button>
|
|
82
|
-
</Message>
|
|
83
|
-
|
|
84
|
-
return { action: 'listen' }
|
|
85
|
-
```
|
|
86
|
-
|
|
87
|
-
# Modes: Chat vs Worker
|
|
88
|
-
|
|
89
|
-
## Chat Mode
|
|
90
|
-
|
|
91
|
-
- When using Chat Mode, you need to implement the base Chat class exported by the `llmz` package
|
|
92
|
-
- In the examples folder, we implemented a basic `CLIChat` class that provides a basic CLI-base chat application
|
|
93
|
-
- The chat class must provide 3 things:
|
|
94
|
-
- providing/fetching the chat transcript (an array of messages in the conversation)
|
|
95
|
-
- providing a list of components the agent can respond with (for example "text" or "button")
|
|
96
|
-
- a `handler` to send the agent messages to the user
|
|
97
|
-
- Note that the `transcript` and `components` are called every iteration, therefore can be either static or dynamic. LLMz accepts a static property but also an async getter.
|
|
98
|
-
|
|
99
|
-
### Chat Components (Chat Mode)
|
|
100
|
-
|
|
101
|
-
A chat component is a type of message your agent can reply with. Typically, components should map to the messages supported by the channel the communication occurs in. For example, on Botpress Webchat, a Carousel, Card, Buttons, Text, Image, Video etc. On SMS, usually only Text and Image are supported.
|
|
102
|
-
|
|
103
|
-
Because Chat Components are just React components (TSX), you can define custom components for anything. For example, you could implement a PlaneTicket component (see example 10) and render the message however you want in the channel in the `Chat` handler method.
|
|
104
|
-
|
|
105
|
-
### ListenExit
|
|
106
|
-
|
|
107
|
-
In chat mode, the special ListenExit is automatically added. this is what allows your agent to stop looping and wait for the user to respond. Read more about Exit below.
|
|
108
|
-
|
|
109
|
-
### Transcript
|
|
110
|
-
|
|
111
|
-
A transcript is an array of Transcript.Message
|
|
112
|
-
|
|
113
|
-
```
|
|
114
|
-
// See transcript.d.ts
|
|
115
|
-
import { Transcript } from 'llmz'
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
Message types:
|
|
119
|
-
|
|
120
|
-
- User: a message in the conversation coming from the user
|
|
121
|
-
- Assistant: a message the agent sent in the conversation
|
|
122
|
-
- Event: an event represents something that happened in the context of the conversation but isn't necessarily something the user or assistant said. It could be a user interaction event, such as button click; or a notification like an extended silence or the result of an async operation.
|
|
123
|
-
- Summary: when a conversation gets too long, you can generate a summary of the conversation thus far and replace the messages by a summary message (transcript compression).
|
|
124
|
-
|
|
125
|
-
## Worker Mode
|
|
126
|
-
|
|
127
|
-
In worker mode, the only possible outcome is the return of one of the provided Exit (or max iteration error).
|
|
128
|
-
If no Exit is passed to `execute()`, then llmz's DefaultExit will be used.
|
|
129
|
-
|
|
130
|
-
# Execute Props / Input
|
|
131
|
-
|
|
132
|
-
## Cancellation / Abort Signal
|
|
133
|
-
|
|
134
|
-
You can provide an AbortSignal to `execute({ signal })` to abort the execution of llmz. This will immediately abort LLM requests as well as the VM sandbox code execution.
|
|
135
|
-
|
|
136
|
-
## Options
|
|
137
|
-
|
|
138
|
-
### Loop
|
|
139
|
-
|
|
140
|
-
This is how many iterations maximum `execute` will loop before erroring when failing to `exit` gracefully.
|
|
141
|
-
|
|
142
|
-
### Model
|
|
143
|
-
|
|
144
|
-
This is which LLM model the cognitive client will use for the LLM generations.
|
|
145
|
-
|
|
146
|
-
### Timeout
|
|
147
|
-
|
|
148
|
-
This is how how long the code execution can run for in milliseconds.
|
|
149
|
-
|
|
150
|
-
### Temperature
|
|
151
|
-
|
|
152
|
-
This is the LLM model temperature (between 0 and 1).
|
|
153
|
-
|
|
154
|
-
## Dynamic Inputs
|
|
155
|
-
|
|
156
|
-
Most parameters to `execute` can be either static or dynamic.
|
|
157
|
-
For example, `instructions` and `tools` can be async functions that return the parameter.
|
|
158
|
-
|
|
159
|
-
```tsx
|
|
160
|
-
await execute({
|
|
161
|
-
client,
|
|
162
|
-
|
|
163
|
-
// Use function-based instructions that evaluate at runtime
|
|
164
|
-
instructions: () => { return '<instructions>' }
|
|
165
|
-
|
|
166
|
-
tools: async () => {
|
|
167
|
-
// execute async operations
|
|
168
|
-
return [toolA, toolB]
|
|
169
|
-
}
|
|
170
|
-
})
|
|
171
|
-
```
|
|
172
|
-
|
|
173
|
-
# Hooks
|
|
174
|
-
|
|
175
|
-
LLMz provides a comprehensive hook system that allows you to inject custom logic at various points during execution. Hooks are categorized as either blocking (execution waits) or non-blocking, and either mutation (can modify data) or non-mutation.
|
|
176
|
-
|
|
177
|
-
## onTrace (non-blocking, non-mutation)
|
|
178
|
-
|
|
179
|
-
Called for each trace generated during iteration. Useful for logging, debugging, or monitoring execution progress.
|
|
180
|
-
|
|
181
|
-
**Characteristics:**
|
|
182
|
-
|
|
183
|
-
- **Non-blocking**: Execution continues without waiting for this hook
|
|
184
|
-
- **Non-mutation**: Cannot modify traces
|
|
185
|
-
- **Called**: For every trace event during iteration
|
|
186
|
-
|
|
187
|
-
**Usage:**
|
|
188
|
-
|
|
189
|
-
```tsx
|
|
190
|
-
await execute({
|
|
191
|
-
onTrace: ({ trace, iteration }) => {
|
|
192
|
-
console.log(`Iteration ${iteration}: ${trace.type}`, trace)
|
|
193
|
-
},
|
|
194
|
-
// ... other props
|
|
195
|
-
})
|
|
196
|
-
```
|
|
197
|
-
|
|
198
|
-
**Available Trace Types:**
|
|
199
|
-
|
|
200
|
-
- `abort_signal`: Abort signal received
|
|
201
|
-
- `comment`: Comment found in generated code
|
|
202
|
-
- `llm_call_success`: LLM generation completed successfully
|
|
203
|
-
- `property`: Object property accessed or modified
|
|
204
|
-
- `think_signal`: ThinkSignal thrown
|
|
205
|
-
- `tool_call`: Tool executed
|
|
206
|
-
- `yield`: Component yielded in chat mode
|
|
207
|
-
- `log`: General logging event
|
|
208
|
-
|
|
209
|
-
## onIterationEnd (blocking, non-mutation)
|
|
210
|
-
|
|
211
|
-
Called after each iteration ends, regardless of status. Useful for logging, cleanup, or controlling iteration timing.
|
|
212
|
-
|
|
213
|
-
**Characteristics:**
|
|
214
|
-
|
|
215
|
-
- **Blocking**: Execution waits until this hook resolves
|
|
216
|
-
- **Non-mutation**: Cannot modify iteration result or status
|
|
217
|
-
- **Called**: After every iteration completion
|
|
218
|
-
|
|
219
|
-
**Usage:**
|
|
220
|
-
|
|
221
|
-
```tsx
|
|
222
|
-
await execute({
|
|
223
|
-
onIterationEnd: async (iteration, controller) => {
|
|
224
|
-
console.log(`Iteration ${iteration.id} ended with status: ${iteration.status.type}`)
|
|
225
|
-
|
|
226
|
-
// Add delays, cleanup, or conditional logic
|
|
227
|
-
if (iteration.status.type === 'execution_error') {
|
|
228
|
-
await logError(iteration.error)
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
// Can use controller to abort execution if needed
|
|
232
|
-
// controller.abort('Custom abort reason')
|
|
233
|
-
},
|
|
234
|
-
// ... other props
|
|
235
|
-
})
|
|
236
|
-
```
|
|
237
|
-
|
|
238
|
-
## onExit (blocking, non-mutation)
|
|
239
|
-
|
|
240
|
-
Called when an exit is reached. Useful for logging, notifications, or implementing guardrails by throwing errors to prevent exit.
|
|
241
|
-
|
|
242
|
-
**Characteristics:**
|
|
243
|
-
|
|
244
|
-
- **Blocking**: Execution waits until this hook resolves
|
|
245
|
-
- **Non-mutation**: Cannot modify exit result value
|
|
246
|
-
- **Called**: When any exit is reached
|
|
247
|
-
- **Guardrails**: Can throw error to prevent exit and continue iteration
|
|
248
|
-
|
|
249
|
-
**Usage:**
|
|
250
|
-
|
|
251
|
-
```tsx
|
|
252
|
-
await execute({
|
|
253
|
-
onExit: async (result) => {
|
|
254
|
-
console.log(`Exiting with: ${result.exit.name}`, result.result)
|
|
255
|
-
|
|
256
|
-
// Implement guardrails
|
|
257
|
-
if (result.exit.name === 'approve_loan' && result.result.amount > 10000) {
|
|
258
|
-
throw new Error('Manager approval required for loans over $10,000')
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
// Send notifications
|
|
262
|
-
await notifyStakeholders(result)
|
|
263
|
-
},
|
|
264
|
-
// ... other props
|
|
265
|
-
})
|
|
266
|
-
```
|
|
267
|
-
|
|
268
|
-
## onBeforeExecution (blocking, mutation)
|
|
269
|
-
|
|
270
|
-
Called after LLM generates code but before execution. Allows code modification and guardrails implementation.
|
|
271
|
-
|
|
272
|
-
**Characteristics:**
|
|
273
|
-
|
|
274
|
-
- **Blocking**: Execution waits until this hook resolves
|
|
275
|
-
- **Mutation**: Can modify the code to be executed
|
|
276
|
-
- **Called**: After code generation, before VM execution
|
|
277
|
-
- **Guardrails**: Can throw error to trigger new iteration
|
|
278
|
-
|
|
279
|
-
**Usage:**
|
|
280
|
-
|
|
281
|
-
```tsx
|
|
282
|
-
await execute({
|
|
283
|
-
onBeforeExecution: async (iteration, controller) => {
|
|
284
|
-
console.log('Generated code:', iteration.code)
|
|
285
|
-
|
|
286
|
-
// Code modification
|
|
287
|
-
if (iteration.code?.includes('dangerousOperation')) {
|
|
288
|
-
return {
|
|
289
|
-
code: iteration.code.replace('dangerousOperation', 'safeOperation'),
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
// Guardrails - throw to prevent execution
|
|
294
|
-
if (iteration.code?.includes('forbidden')) {
|
|
295
|
-
throw new Error('Forbidden operation detected')
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
// Add security checks, logging, etc.
|
|
299
|
-
await auditCodeGeneration(iteration.code)
|
|
300
|
-
},
|
|
301
|
-
// ... other props
|
|
302
|
-
})
|
|
303
|
-
```
|
|
304
|
-
|
|
305
|
-
## onBeforeTool (blocking, mutation)
|
|
306
|
-
|
|
307
|
-
Called before any tool execution. Allows input modification and tool execution control.
|
|
308
|
-
|
|
309
|
-
**Characteristics:**
|
|
310
|
-
|
|
311
|
-
- **Blocking**: Execution waits until this hook resolves
|
|
312
|
-
- **Mutation**: Can modify tool input
|
|
313
|
-
- **Called**: Before every tool execution
|
|
314
|
-
- **Control**: Can prevent tool execution by throwing error
|
|
315
|
-
|
|
316
|
-
**Usage:**
|
|
317
|
-
|
|
318
|
-
```tsx
|
|
319
|
-
await execute({
|
|
320
|
-
onBeforeTool: async ({ iteration, tool, input, controller }) => {
|
|
321
|
-
console.log(`Executing tool: ${tool.name}`, input)
|
|
322
|
-
|
|
323
|
-
// Input modification
|
|
324
|
-
if (tool.name === 'sendEmail') {
|
|
325
|
-
return {
|
|
326
|
-
input: {
|
|
327
|
-
...input,
|
|
328
|
-
subject: `[Automated] ${input.subject}`, // Add prefix
|
|
329
|
-
},
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
// Access control
|
|
334
|
-
if (tool.name === 'deleteFile' && !hasPermission(input.path)) {
|
|
335
|
-
throw new Error('Insufficient permissions to delete file')
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
// Rate limiting, validation, etc.
|
|
339
|
-
await validateToolUsage(tool, input)
|
|
340
|
-
},
|
|
341
|
-
// ... other props
|
|
342
|
-
})
|
|
343
|
-
```
|
|
344
|
-
|
|
345
|
-
## onAfterTool (blocking, mutation)
|
|
346
|
-
|
|
347
|
-
Called after tool execution. Allows output modification and post-processing.
|
|
348
|
-
|
|
349
|
-
**Characteristics:**
|
|
350
|
-
|
|
351
|
-
- **Blocking**: Execution waits until this hook resolves
|
|
352
|
-
- **Mutation**: Can modify tool output
|
|
353
|
-
- **Called**: After every tool execution
|
|
354
|
-
- **Processing**: Can transform results before they reach the LLM
|
|
355
|
-
|
|
356
|
-
**Usage:**
|
|
357
|
-
|
|
358
|
-
```tsx
|
|
359
|
-
await execute({
|
|
360
|
-
onAfterTool: async ({ iteration, tool, input, output, controller }) => {
|
|
361
|
-
console.log(`Tool ${tool.name} completed`, { input, output })
|
|
362
|
-
|
|
363
|
-
// Output modification
|
|
364
|
-
if (tool.name === 'fetchUserData') {
|
|
365
|
-
return {
|
|
366
|
-
output: {
|
|
367
|
-
...output,
|
|
368
|
-
// Remove sensitive data before LLM sees it
|
|
369
|
-
ssn: undefined,
|
|
370
|
-
creditCard: undefined,
|
|
371
|
-
},
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
// Result enhancement
|
|
376
|
-
if (tool.name === 'calculatePrice') {
|
|
377
|
-
return {
|
|
378
|
-
output: {
|
|
379
|
-
...output,
|
|
380
|
-
currency: 'USD',
|
|
381
|
-
timestamp: Date.now(),
|
|
382
|
-
},
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
// Logging, caching, notifications
|
|
387
|
-
await cacheResult(tool.name, input, output)
|
|
388
|
-
},
|
|
389
|
-
// ... other props
|
|
390
|
-
})
|
|
391
|
-
```
|
|
392
|
-
|
|
393
|
-
## Hook Execution Order
|
|
394
|
-
|
|
395
|
-
1. **onTrace**: Throughout execution (non-blocking)
|
|
396
|
-
2. **onBeforeExecution**: After code generation, before execution
|
|
397
|
-
3. **onBeforeTool**: Before each tool call
|
|
398
|
-
4. **onAfterTool**: After each tool call
|
|
399
|
-
5. **onExit**: When exit is reached
|
|
400
|
-
6. **onIterationEnd**: After iteration completes
|
|
401
|
-
|
|
402
|
-
## Best Practices
|
|
403
|
-
|
|
404
|
-
- **Error Handling**: Always wrap hook logic in try-catch for production
|
|
405
|
-
- **Performance**: Keep hooks lightweight, especially onTrace
|
|
406
|
-
- **Security**: Use onBeforeExecution and onBeforeTool for security validation
|
|
407
|
-
- **Debugging**: Leverage onTrace for comprehensive execution monitoring
|
|
408
|
-
- **Guardrails**: Implement business logic validation in onExit
|
|
409
|
-
- **Data Transformation**: Use onBeforeTool/onAfterTool for input/output processing
|
|
410
|
-
|
|
411
|
-
# Execution Result
|
|
412
|
-
|
|
413
|
-
Every call to `execute()` returns an ExecutionResult that provides type-safe access to the execution outcome. LLMz execution can result in three different types of results: Success, Error, or Interrupted.
|
|
414
|
-
|
|
415
|
-
## Result Types
|
|
416
|
-
|
|
417
|
-
### SuccessExecutionResult
|
|
418
|
-
|
|
419
|
-
Agent completed successfully with an Exit. This is the most common positive outcome containing the structured data produced by the agent.
|
|
420
|
-
|
|
421
|
-
### ErrorExecutionResult
|
|
422
|
-
|
|
423
|
-
Execution failed with an unrecoverable error such as:
|
|
424
|
-
|
|
425
|
-
- User aborted via AbortSignal
|
|
426
|
-
- Maximum iterations exceeded without reaching an exit
|
|
427
|
-
- Critical system failures
|
|
428
|
-
|
|
429
|
-
### PartialExecutionResult
|
|
430
|
-
|
|
431
|
-
Execution was interrupted by a SnapshotSignal for pauseable operations. Contains a snapshot that can be used to resume execution later.
|
|
432
|
-
|
|
433
|
-
## Basic Status Checking
|
|
434
|
-
|
|
435
|
-
Use type guard methods to safely access result data:
|
|
436
|
-
|
|
437
|
-
```tsx
|
|
438
|
-
const result = await execute({
|
|
439
|
-
instructions: 'Calculate the sum of numbers 1 to 100',
|
|
440
|
-
client,
|
|
441
|
-
})
|
|
442
|
-
|
|
443
|
-
// Check execution status
|
|
444
|
-
if (result.isSuccess()) {
|
|
445
|
-
console.log('Success:', result.output)
|
|
446
|
-
console.log('Generated code:', result.iteration.code)
|
|
447
|
-
} else if (result.isError()) {
|
|
448
|
-
console.error('Error:', result.error)
|
|
449
|
-
console.error('Failed iteration:', result.iteration?.error)
|
|
450
|
-
} else if (result.isInterrupted()) {
|
|
451
|
-
console.log('Interrupted:', result.signal.message)
|
|
452
|
-
console.log('Snapshot available:', !!result.snapshot)
|
|
453
|
-
}
|
|
454
|
-
```
|
|
455
|
-
|
|
456
|
-
## Type-Safe Exit Checking
|
|
457
|
-
|
|
458
|
-
Use `result.is(exit)` for type-safe access to specific exit data:
|
|
459
|
-
|
|
460
|
-
```tsx
|
|
461
|
-
const dataExit = new Exit({
|
|
462
|
-
name: 'dataProcessed',
|
|
463
|
-
schema: z.object({
|
|
464
|
-
recordCount: z.number(),
|
|
465
|
-
processingTime: z.number(),
|
|
466
|
-
}),
|
|
467
|
-
})
|
|
468
|
-
|
|
469
|
-
const errorExit = new Exit({
|
|
470
|
-
name: 'processingError',
|
|
471
|
-
schema: z.object({
|
|
472
|
-
errorCode: z.string(),
|
|
473
|
-
details: z.string(),
|
|
474
|
-
}),
|
|
475
|
-
})
|
|
476
|
-
|
|
477
|
-
const result = await execute({
|
|
478
|
-
instructions: 'Process the data',
|
|
479
|
-
exits: [dataExit, errorExit],
|
|
480
|
-
client,
|
|
481
|
-
})
|
|
482
|
-
|
|
483
|
-
// Type-safe exit handling with automatic output typing
|
|
484
|
-
if (result.is(dataExit)) {
|
|
485
|
-
// TypeScript knows result.output has { recordCount: number, processingTime: number }
|
|
486
|
-
console.log(`Processed ${result.output.recordCount} records`)
|
|
487
|
-
console.log(`Processing took ${result.output.processingTime}ms`)
|
|
488
|
-
} else if (result.is(errorExit)) {
|
|
489
|
-
// TypeScript knows result.output has { errorCode: string, details: string }
|
|
490
|
-
console.error(`Error ${result.output.errorCode}: ${result.output.details}`)
|
|
491
|
-
}
|
|
492
|
-
```
|
|
493
|
-
|
|
494
|
-
## Accessing Execution Details
|
|
495
|
-
|
|
496
|
-
### Iterations and Execution Flow
|
|
497
|
-
|
|
498
|
-
```tsx
|
|
499
|
-
const result = await execute({ ... })
|
|
500
|
-
|
|
501
|
-
// Access the final iteration
|
|
502
|
-
const lastIteration = result.iteration
|
|
503
|
-
if (lastIteration) {
|
|
504
|
-
console.log('Generated code:', lastIteration.code)
|
|
505
|
-
console.log('Status:', lastIteration.status.type)
|
|
506
|
-
console.log('Duration:', lastIteration.duration)
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
// Access all iterations to see full execution flow
|
|
510
|
-
result.iterations.forEach((iteration, index) => {
|
|
511
|
-
console.log(`Iteration ${index + 1}:`)
|
|
512
|
-
console.log(' Status:', iteration.status.type)
|
|
513
|
-
console.log(' Code length:', iteration.code?.length || 0)
|
|
514
|
-
console.log(' Variables:', Object.keys(iteration.variables).length)
|
|
515
|
-
})
|
|
516
|
-
|
|
517
|
-
// Find specific iteration types
|
|
518
|
-
const errorIterations = result.iterations.filter(
|
|
519
|
-
iter => iter.status.type === 'execution_error'
|
|
520
|
-
)
|
|
521
|
-
|
|
522
|
-
const thinkingIterations = result.iterations.filter(
|
|
523
|
-
iter => iter.status.type === 'thinking_requested'
|
|
524
|
-
)
|
|
525
|
-
```
|
|
526
|
-
|
|
527
|
-
### Variables and Declarations
|
|
528
|
-
|
|
529
|
-
Agent-generated variables are accessible in the iteration object:
|
|
530
|
-
|
|
531
|
-
```tsx
|
|
532
|
-
// If agent generates: const hello = '1234'
|
|
533
|
-
const lastIteration = result.iteration
|
|
534
|
-
if (lastIteration) {
|
|
535
|
-
console.log(lastIteration.variables.hello) // '1234'
|
|
536
|
-
|
|
537
|
-
// Access all variables from the final iteration
|
|
538
|
-
Object.entries(lastIteration.variables).forEach(([name, value]) => {
|
|
539
|
-
console.log(`Variable ${name}:`, value)
|
|
540
|
-
})
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
// Variables persist across thinking iterations
|
|
544
|
-
result.iterations.forEach((iteration) => {
|
|
545
|
-
if (iteration.status.type === 'thinking_requested') {
|
|
546
|
-
console.log('Variables during thinking:', iteration.variables)
|
|
547
|
-
}
|
|
548
|
-
})
|
|
549
|
-
```
|
|
550
|
-
|
|
551
|
-
### Tool Calls and Traces
|
|
552
|
-
|
|
553
|
-
```tsx
|
|
554
|
-
const result = await execute({ ... })
|
|
555
|
-
|
|
556
|
-
// Access tool calls from all iterations
|
|
557
|
-
const allToolCalls = result.iterations.flatMap(iter =>
|
|
558
|
-
iter.traces.filter(trace => trace.type === 'tool_call')
|
|
559
|
-
)
|
|
560
|
-
|
|
561
|
-
console.log('Total tool calls:', allToolCalls.length)
|
|
562
|
-
|
|
563
|
-
// Access other trace types
|
|
564
|
-
const lastIteration = result.iteration
|
|
565
|
-
if (lastIteration) {
|
|
566
|
-
const yields = lastIteration.traces.filter(trace => trace.type === 'yield')
|
|
567
|
-
const comments = lastIteration.traces.filter(trace => trace.type === 'comment')
|
|
568
|
-
const propertyAccess = lastIteration.traces.filter(trace => trace.type === 'property')
|
|
569
|
-
}
|
|
570
|
-
```
|
|
571
|
-
|
|
572
|
-
### Context and Metadata
|
|
573
|
-
|
|
574
|
-
```tsx
|
|
575
|
-
if (result.isSuccess()) {
|
|
576
|
-
// Access original execution parameters
|
|
577
|
-
console.log('Instructions:', result.context.instructions)
|
|
578
|
-
console.log('Loop limit:', result.context.loop)
|
|
579
|
-
console.log('Temperature:', result.context.temperature)
|
|
580
|
-
console.log('Model:', result.context.model)
|
|
581
|
-
|
|
582
|
-
// Access tools and exits that were available
|
|
583
|
-
console.log(
|
|
584
|
-
'Available tools:',
|
|
585
|
-
result.context.tools?.map((t) => t.name)
|
|
586
|
-
)
|
|
587
|
-
console.log(
|
|
588
|
-
'Available exits:',
|
|
589
|
-
result.context.exits?.map((e) => e.name)
|
|
590
|
-
)
|
|
591
|
-
}
|
|
592
|
-
```
|
|
593
|
-
|
|
594
|
-
## Snapshot Handling (Advanced)
|
|
595
|
-
|
|
596
|
-
Handle interrupted executions with snapshot resumption:
|
|
597
|
-
|
|
598
|
-
```tsx
|
|
599
|
-
const result = await execute({
|
|
600
|
-
instructions: 'Process large dataset with pauseable operation',
|
|
601
|
-
tools: [snapshotCapableTool],
|
|
602
|
-
client,
|
|
603
|
-
})
|
|
604
|
-
|
|
605
|
-
if (result.isInterrupted()) {
|
|
606
|
-
console.log('Execution paused:', result.signal.message)
|
|
607
|
-
console.log('Reason:', result.signal.longMessage)
|
|
608
|
-
|
|
609
|
-
// Serialize snapshot for persistence
|
|
610
|
-
const serialized = result.snapshot.toJSON()
|
|
611
|
-
await database.saveSnapshot('execution-123', serialized)
|
|
612
|
-
|
|
613
|
-
// Later, resume from snapshot
|
|
614
|
-
const snapshot = Snapshot.fromJSON(serialized)
|
|
615
|
-
snapshot.resolve({ resumeData: 'Operation completed' })
|
|
616
|
-
|
|
617
|
-
const continuation = await execute({
|
|
618
|
-
snapshot,
|
|
619
|
-
instructions: result.context.instructions,
|
|
620
|
-
tools: result.context.tools,
|
|
621
|
-
exits: result.context.exits,
|
|
622
|
-
client,
|
|
623
|
-
})
|
|
624
|
-
|
|
625
|
-
// Continuation will resume from exactly where it left off
|
|
626
|
-
if (continuation.isSuccess()) {
|
|
627
|
-
console.log('Resumed execution completed:', continuation.output)
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
```
|
|
631
|
-
|
|
632
|
-
## Built-in Exits
|
|
633
|
-
|
|
634
|
-
```tsx
|
|
635
|
-
import { ListenExit, DefaultExit, ThinkExit } from 'llmz'
|
|
636
|
-
|
|
637
|
-
const result = await execute({ ... })
|
|
638
|
-
|
|
639
|
-
// Check for built-in exits
|
|
640
|
-
if (result.is(ListenExit)) {
|
|
641
|
-
console.log('Agent is waiting for user input')
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
if (result.is(DefaultExit)) {
|
|
645
|
-
// DefaultExit has success/failure discriminated union
|
|
646
|
-
if (result.output.success) {
|
|
647
|
-
console.log('Completed successfully:', result.output.result)
|
|
648
|
-
} else {
|
|
649
|
-
console.error('Completed with error:', result.output.error)
|
|
650
|
-
}
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
if (result.is(ThinkExit)) {
|
|
654
|
-
console.log('Agent requested thinking time')
|
|
655
|
-
console.log('Current variables:', result.output.variables)
|
|
656
|
-
}
|
|
657
|
-
```
|
|
658
|
-
|
|
659
|
-
## Error Analysis
|
|
660
|
-
|
|
661
|
-
````tsx
|
|
662
|
-
if (result.isError()) {
|
|
663
|
-
console.error('Execution failed:', result.error)
|
|
664
|
-
|
|
665
|
-
// Analyze the failure progression
|
|
666
|
-
const failedIteration = result.iteration
|
|
667
|
-
if (failedIteration) {
|
|
668
|
-
switch (failedIteration.status.type) {
|
|
669
|
-
case 'execution_error':
|
|
670
|
-
console.error('Code execution failed:', failedIteration.status.execution_error.message)
|
|
671
|
-
console.error('Stack trace:', failedIteration.status.execution_error.stack)
|
|
672
|
-
console.error('Failed code:', failedIteration.code)
|
|
673
|
-
break
|
|
674
|
-
|
|
675
|
-
case 'generation_error':
|
|
676
|
-
console.error('LLM generation failed:', failedIteration.status.generation_error.message)
|
|
677
|
-
break
|
|
678
|
-
|
|
679
|
-
case 'invalid_code_error':
|
|
680
|
-
console.error('Invalid code generated:', failedIteration.status.invalid_code_error.message)
|
|
681
|
-
console.error('Invalid code:', failedIteration.code)
|
|
682
|
-
break
|
|
683
|
-
|
|
684
|
-
case 'aborted':
|
|
685
|
-
console.error('Execution aborted:', failedIteration.status.aborted.reason)
|
|
686
|
-
break
|
|
687
|
-
}
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
// Review all iterations to understand failure progression
|
|
691
|
-
console.log('Iterations before failure:', result.iterations.length)
|
|
692
|
-
result.iterations.forEach((iter, i) => {
|
|
693
|
-
console.log(`Iteration ${i + 1}: ${iter.status.type}`)
|
|
694
|
-
})
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
# Tools
|
|
698
|
-
|
|
699
|
-
- todo: Tool class props
|
|
700
|
-
- todo: input and outpout schema
|
|
701
|
-
- todo: tips: schemas and descriptions are really important for LLMz to generate good code, thus, good tool usage.
|
|
702
|
-
- todo: use `tool.getTypings()` to see the code generated for the LLM
|
|
703
|
-
- todo: cloning a tool and mutate the input, output, name, description or handler (see example 19, tool wrap)
|
|
704
|
-
- todo: static inputs to "freeze" and force inputs on a tool
|
|
705
|
-
- todo: aliases, so the same tool can be called with multiple names
|
|
706
|
-
|
|
707
|
-
# Objects
|
|
708
|
-
|
|
709
|
-
- todo: explain what an object is. it's like a namespace. it groups related tools together. but objects can also do more: they can contain variables.
|
|
710
|
-
|
|
711
|
-
## Variables
|
|
712
|
-
|
|
713
|
-
- todo: readonly vs writable variables
|
|
714
|
-
- todo: variable types (schema)
|
|
715
|
-
- todo: variable usage within the VM code
|
|
716
|
-
- todo: type and schema validation – show an example of how the VM will throw an error when trying to assign a value to a variable that doesn't pass the schema validation. see example 09.
|
|
717
|
-
- todo: tracking mutations
|
|
718
|
-
- todo: persisting variables across executions
|
|
719
|
-
|
|
720
|
-
## Tools
|
|
721
|
-
|
|
722
|
-
- todo: identical to global tools, except they are available under the object namespace. for example `myObject.myTool()`.
|
|
723
|
-
|
|
724
|
-
# Snapshots (advanced)
|
|
725
|
-
|
|
726
|
-
## SnapshotSignal
|
|
727
|
-
|
|
728
|
-
- todo: inside a tool, throw a SnapshotSignal to halt the execution of llmz and take a serializable snapshot of the execution.
|
|
729
|
-
- todo: snapshots are built to persist the state of an executuon with the goal of resuming it in the future
|
|
730
|
-
- todo: thisi s useful for long-running tools for examples, like workflows etc
|
|
731
|
-
|
|
732
|
-
## Snapshot object
|
|
733
|
-
|
|
734
|
-
- todo: getting the snapshot from result (result.isInterrupted() and result.snapshot)
|
|
735
|
-
- todo: serialize snapshot to JSON
|
|
736
|
-
- todo: restoring a snapshot from JSON
|
|
737
|
-
- todo: resolve a snapshot (success)
|
|
738
|
-
- todo: reject a snapshot (reject)
|
|
739
|
-
|
|
740
|
-
# Thinking
|
|
741
|
-
|
|
742
|
-
## ThinkSignal
|
|
743
|
-
|
|
744
|
-
- todo: throw new ThinkSignal to force the iteraiton and looking at variables. this is useful for tools to force LLMs to have a look at the results before responding for example.
|
|
745
|
-
|
|
746
|
-
## return { action: 'think' }
|
|
747
|
-
|
|
748
|
-
- todo: special return type to think and inspect variables and iterate
|
|
749
|
-
|
|
750
|
-
## Citations
|
|
751
|
-
|
|
752
|
-
CitationsManager is a helper provided by LLMz to standardize the registration of sources/snippets of text and referencing them in agent responses. It's particularly useful for RAG (Retrieval-Augmented Generation) systems where you need to track the source of information and provide proper attribution.
|
|
753
|
-
|
|
754
|
-
### Core Concepts
|
|
755
|
-
|
|
756
|
-
Citations use rare Unicode symbols (`【】`) as markers that are unlikely to appear in natural text, making them safe to use in LLM prompts and responses. The system supports:
|
|
757
|
-
|
|
758
|
-
- **Source Registration**: Register any object as a citation source
|
|
759
|
-
- **Tag Generation**: Automatic creation of unique citation tags like `【0】`, `【1】`
|
|
760
|
-
- **Content Processing**: Extract and clean citation tags from text
|
|
761
|
-
- **Multiple Citations**: Support for multi-source citations like `【0,1,3】`
|
|
762
|
-
|
|
763
|
-
### Basic Usage
|
|
764
|
-
|
|
765
|
-
```tsx
|
|
766
|
-
import { CitationsManager } from 'llmz'
|
|
767
|
-
|
|
768
|
-
const citations = new CitationsManager()
|
|
769
|
-
|
|
770
|
-
// Register sources and get citation tags
|
|
771
|
-
const source1 = citations.registerSource({
|
|
772
|
-
file: 'document.pdf',
|
|
773
|
-
page: 5,
|
|
774
|
-
title: 'Company Policy'
|
|
775
|
-
})
|
|
776
|
-
const source2 = citations.registerSource({
|
|
777
|
-
url: 'https://example.com/article',
|
|
778
|
-
title: 'Best Practices'
|
|
779
|
-
})
|
|
780
|
-
|
|
781
|
-
console.log(source1.tag) // "【0】"
|
|
782
|
-
console.log(source2.tag) // "【1】"
|
|
783
|
-
|
|
784
|
-
// Use tags in content
|
|
785
|
-
const content = `The policy states that employees must arrive on time${source1.tag}. However, best practices suggest flexibility${source2.tag}.`
|
|
786
|
-
|
|
787
|
-
// Extract and process citations
|
|
788
|
-
const { cleaned, citations: found } = citations.extractCitations(content, (citation) => {
|
|
789
|
-
return `[${citation.id + 1}]` // Convert to numbered format
|
|
790
|
-
})
|
|
791
|
-
|
|
792
|
-
console.log(cleaned) // "The policy states that employees must arrive on time[1]. However, best practices suggest flexibility[2]."
|
|
793
|
-
console.log(found) // Array of citation objects with source data
|
|
794
|
-
````
|
|
795
|
-
|
|
796
|
-
### RAG Implementation Example
|
|
797
|
-
|
|
798
|
-
Here's how citations are used in a real RAG system (from example 20):
|
|
799
|
-
|
|
800
|
-
```tsx
|
|
801
|
-
// Tool for searching knowledge base with citations
|
|
802
|
-
const ragTool = new Tool({
|
|
803
|
-
name: 'search',
|
|
804
|
-
description: 'Searches in the knowledge base for relevant information.',
|
|
805
|
-
input: z.string().describe('The query to search in the knowledge base.'),
|
|
806
|
-
async handler(query) {
|
|
807
|
-
// Perform semantic search
|
|
808
|
-
const { passages } = await client.searchFiles({
|
|
809
|
-
query,
|
|
810
|
-
tags: { purpose: RAG_TAG },
|
|
811
|
-
limit: 20,
|
|
812
|
-
contextDepth: 3,
|
|
813
|
-
consolidate: true,
|
|
814
|
-
})
|
|
815
|
-
|
|
816
|
-
// Handle no results
|
|
817
|
-
if (!passages.length) {
|
|
818
|
-
throw new ThinkSignal(
|
|
819
|
-
'No results were found',
|
|
820
|
-
'No results were found in the knowledge bases. You can try rephrasing your question or asking something else. Do NOT answer the question as no results were found.'
|
|
821
|
-
)
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
// Build response with citations
|
|
825
|
-
let message: string[] = ['Here are the search results from the knowledge base:']
|
|
826
|
-
let { tag: example } = chat.citations.registerSource({}) // Example citation
|
|
827
|
-
|
|
828
|
-
// Register each retrieved passage as a source
|
|
829
|
-
for (const passage of passages) {
|
|
830
|
-
const { tag } = chat.citations.registerSource({
|
|
831
|
-
file: passage.file.key,
|
|
832
|
-
title: passage.file.tags.title,
|
|
833
|
-
})
|
|
834
|
-
|
|
835
|
-
message.push(`<${tag} file="${passage.file.key}">`)
|
|
836
|
-
message.push(`**${passage.file.tags.title}**`)
|
|
837
|
-
message.push(passage.content)
|
|
838
|
-
message.push(`</${tag}>`)
|
|
839
|
-
}
|
|
840
|
-
|
|
841
|
-
// Provide context with citation instructions
|
|
842
|
-
throw new ThinkSignal(
|
|
843
|
-
`We got the search results. When answering the question, you MUST add inline citations (eg: "Yes, the price is $10${example} ...")`,
|
|
844
|
-
message.join('\n').trim()
|
|
845
|
-
)
|
|
846
|
-
},
|
|
847
|
-
})
|
|
848
|
-
```
|
|
849
|
-
|
|
850
|
-
### Chat Integration
|
|
851
|
-
|
|
852
|
-
The CLIChat utility demonstrates how to integrate citations into chat interfaces:
|
|
853
|
-
|
|
854
|
-
```tsx
|
|
855
|
-
class CLIChat extends Chat {
|
|
856
|
-
public citations: CitationsManager = new CitationsManager()
|
|
857
|
-
|
|
858
|
-
private async sendMessage(input: RenderedComponent) {
|
|
859
|
-
// ... component handling ...
|
|
860
|
-
|
|
861
|
-
if (text.length > 0) {
|
|
862
|
-
let sources: string[] = []
|
|
863
|
-
|
|
864
|
-
// Extract citations and format them for display
|
|
865
|
-
const { cleaned } = this.citations.extractCitations(text, (citation) => {
|
|
866
|
-
let idx = chalk.bgGreenBright.black.bold(` ${sources.length + 1} `)
|
|
867
|
-
sources.push(`${idx}: ${JSON.stringify(citation.source)}`)
|
|
868
|
-
return `${idx}` // Replace 【0】 with [1]
|
|
869
|
-
})
|
|
870
|
-
|
|
871
|
-
// Display cleaned text and sources
|
|
872
|
-
console.log(`🤖 Agent: ${cleaned}`)
|
|
873
|
-
|
|
874
|
-
if (sources.length) {
|
|
875
|
-
console.log(chalk.dim('Citations'))
|
|
876
|
-
console.log(chalk.dim('========='))
|
|
877
|
-
console.log(chalk.dim(sources.join('\n')))
|
|
878
|
-
}
|
|
879
|
-
}
|
|
880
|
-
}
|
|
881
|
-
}
|
|
882
|
-
```
|
|
883
|
-
|
|
884
|
-
### Advanced Features
|
|
885
|
-
|
|
886
|
-
#### Multiple Citation Support
|
|
887
|
-
|
|
888
|
-
```tsx
|
|
889
|
-
// Agent can reference multiple sources in one citation
|
|
890
|
-
const content = 'This fact is supported by multiple studies【0,1,3】'
|
|
891
|
-
|
|
892
|
-
const { cleaned, citations } = manager.extractCitations(content)
|
|
893
|
-
// citations array will contain entries for sources 0, 1, and 3
|
|
894
|
-
```
|
|
895
|
-
|
|
896
|
-
#### Object Citation Processing
|
|
897
|
-
|
|
898
|
-
```tsx
|
|
899
|
-
// Remove citations from complex objects
|
|
900
|
-
const dataWithCitations = {
|
|
901
|
-
summary: 'The report shows positive trends【0】',
|
|
902
|
-
details: {
|
|
903
|
-
revenue: 'Increased by 15%【1】',
|
|
904
|
-
costs: 'Reduced by 8%【2】',
|
|
905
|
-
},
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
const [cleanData, extractedCitations] = manager.removeCitationsFromObject(dataWithCitations)
|
|
909
|
-
// cleanData has citations removed, extractedCitations contains path + citation info
|
|
910
|
-
```
|
|
911
|
-
|
|
912
|
-
#### Citation Stripping
|
|
913
|
-
|
|
914
|
-
```tsx
|
|
915
|
-
// Remove all citation tags from content
|
|
916
|
-
const textWithCitations = 'This statement【0】 has multiple【1,2】 citations.'
|
|
917
|
-
const cleaned = CitationsManager.stripCitationTags(textWithCitations)
|
|
918
|
-
// Result: "This statement has multiple citations."
|
|
919
|
-
```
|
package/eslint.config.mjs
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import rootConfig from '../../eslint.config.mjs'
|
|
2
|
-
|
|
3
|
-
export default [
|
|
4
|
-
...rootConfig,
|
|
5
|
-
{
|
|
6
|
-
ignores: ['e2e/__tests__/**/*', 'tsup.config.ts', 'vitest.*.ts'],
|
|
7
|
-
languageOptions: {
|
|
8
|
-
parserOptions: {
|
|
9
|
-
project: ['./tsconfig.json'],
|
|
10
|
-
tsconfigRootDir: import.meta.dirname,
|
|
11
|
-
},
|
|
12
|
-
},
|
|
13
|
-
},
|
|
14
|
-
{
|
|
15
|
-
ignores: ['examples/**/*'],
|
|
16
|
-
},
|
|
17
|
-
]
|