@eigenpal/automations 0.20.69 → 0.20.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # @eigenpal/automations
2
2
 
3
+ ## 0.20.71
4
+
5
+ ### Minor Changes
6
+
7
+ - 28d79f3: Define agents with inline or file instructions, immutable .withFilesystem() variants with bounded seed files and exact sandbox packages, custom typed tools, working directories, and composable execution guards.
8
+
9
+ Define knowledge, rules, templates and directory skills as imported TypeScript resources. Portable resource text works in both agent modes; skills require filesystem access. Capture resources at the agent release commit instead of maintaining unchecked version strings.
10
+
3
11
  ## 0.20.49
4
12
 
5
13
  ### Patch Changes
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Open-source TypeScript authoring SDK for EigenPal agents. This package defines
4
4
  configuration; it does not itself change agent discovery, releases, or execution.
5
- Worker authoring will extend the same package in a later release.
5
+ Filesystem access is an explicit agent capability.
6
6
 
7
7
  ```sh
8
8
  npm install @eigenpal/automations zod
@@ -29,14 +29,20 @@ export default invoice;
29
29
  `AGENT.md`, skills, rules, knowledge, and templates remain ordinary package files.
30
30
  The existing folder path determines agent identity; `name` is a display name.
31
31
  Configuration includes existing API/email triggers, exact-version workspace
32
- dependencies, and evaluator composition. There are no workflow or worker APIs,
33
- tool callbacks, model execution, secret values, or administrative operations.
32
+ dependencies, and evaluator composition. There are no workflow authoring APIs, model execution in this public package,
33
+ secret values, or administrative operations.
34
+
35
+ ## Explore complete examples
36
+
37
+ Open [`examples/agents`](examples/agents/README.md) for a standalone IDE project with
38
+ functional and filesystem agents, typed tools, evaluations, and all four directory-backed
39
+ shared resource kinds. Its README includes local setup, type checks and Git installation steps.
34
40
 
35
41
  ## Compilation boundary
36
42
 
37
43
  ```ts
38
44
  import { compileAgent } from '@eigenpal/automations/compiler';
39
- import invoice from './agent.js';
45
+ import invoice from './agent.ts';
40
46
 
41
47
  const compiled = compileAgent(invoice);
42
48
  // { schemaVersion: 1, kind: 'agent', manifest, inputSchema?, outputSchema? }
@@ -69,8 +75,8 @@ the existing platform contract. The returned definition envelope is frozen;
69
75
  compilation revalidates its configuration.
70
76
 
71
77
  The versioned `CompiledDefinition<K>` envelope separates source syntax from
72
- runtime data. A later worker definition can share that envelope and schema
73
- types without changing agent configuration or adding a worker engine here.
78
+ runtime data. Execution capabilities extend the agent envelope without introducing another
79
+ automation kind.
74
80
 
75
81
  Agents are defined in `agent.ts`. Source conversion is internal migration
76
82
  tooling, not part of this authoring API.
@@ -93,3 +99,478 @@ The public repository is a release source mirror, maintained under Apache-2.0.
93
99
  Release tags use the platform release version. The platform release workflow
94
100
  publishes to npm with GitHub Actions OIDC, then mirrors the published source here.
95
101
  Please report SDK issues in the public repository.
102
+
103
+ ## Define an agent
104
+
105
+ Put `agent.ts` in `agents/invoice-reviewer/` in the organization repository:
106
+
107
+ ```ts
108
+ import { createAgent, createTool, maxTurns, maxTimeMs, maxCredits } from '@eigenpal/automations';
109
+ import { z } from 'zod';
110
+
111
+ const invoiceReviewer = createAgent({
112
+ name: 'Invoice reviewer',
113
+ instructions: 'Call calculateTotal with the supplied subtotal. Return its result.',
114
+ inputSchema: z.object({ subtotal: z.number().nonnegative() }),
115
+ outputSchema: z.strictObject({ total: z.number() }),
116
+ tools: {
117
+ calculateTotal: createTool({
118
+ description: 'Calculate the total with 20% tax.',
119
+ inputSchema: z.object({ subtotal: z.number().nonnegative() }),
120
+ outputSchema: z.strictObject({ total: z.number() }),
121
+ execute: ({ subtotal }) => ({ total: Math.round(subtotal * 120) / 100 }),
122
+ }),
123
+ },
124
+ guards: [maxTurns(10), maxTimeMs(60_000), maxCredits(100)],
125
+ });
126
+
127
+ export default invoiceReviewer;
128
+ ```
129
+
130
+ This runs without a filesystem sandbox. Tools and schemas are inferred from Zod;
131
+ `{ subtotal: 100 }` can produce `{ total: 120 }`. The repository folder determines
132
+ agent identity; the display name can change without changing that identity.
133
+
134
+ `createAgent` properties:
135
+
136
+ - `name`: required display name.
137
+ - `description`: optional description for discovery.
138
+ - `model`: optional configured organization model identifier; omission uses the existing platform selection.
139
+ - `instructions`: optional nonblank text, up to 128 KiB. If omitted, use the exact package-local `AGENT.md` from the pinned Git commit. Inline text overrides that file; the two are not concatenated. Execution requires nonblank resolved instructions in either mode.
140
+ - `inputSchema`: optional object schema for run input. Prefer Zod 4. Raw JSON Schema remains accepted for existing contracts.
141
+ - `outputSchema`: optional object schema for the validated final answer. Prefer `z.strictObject` to reject unexpected fields.
142
+ - `tools`: optional map of names to `createTool(...)` definitions. Names become model-facing tool names. At most 16; built-in names such as `read`, `write`, `bash` and `finish` are reserved.
143
+ - `guards`: optional array of positive limits. `maxTurns(n)` bounds model requests, including retries; `maxTimeMs(n)` bounds active execution time, including preparation but excluding time parked for human review; `maxCredits(n)` bounds model credits only. One credit is one cent. `maxTimeMs(n)` bounds active sandbox execution time; sandbox compute is billed separately. Repeated limits use the strictest value and continue across human-review resumes. Platform ceilings remain 128 turns, 20 active minutes and 10,000 model credits, with any tighter deployment settings still applying. Conservative request admission can stop before the exact limit when the remaining balance cannot cover the next bounded request.
144
+ - `triggers`: optional `{ api, email: { enabled, aliases } }`. API defaults to enabled; absent email is disabled. Aliases may be addresses or objects with `address`, `allowlist`, `replyConfig`, `requireSenderAuth` and `enabled`. An omitted allowlist preserves platform ownership; `[]` denies all senders. Existing email routing rules continue to apply.
145
+ - `resources`: optional map of imported knowledge, rules and templates. Keys are local aliases; TypeScript checks each imported definition. See shared resources below. Evaluators are registered through the chain, never a `createAgent` property.
146
+
147
+ `createTool` properties:
148
+
149
+ - `description`: required nonblank model-facing explanation of when to call the tool.
150
+ - `inputSchema`: required object schema; validates arguments before invocation and infers the callback input.
151
+ - `outputSchema`: required object schema; validates the returned value and checks its TypeScript shape.
152
+ - `execute`: required synchronous TypeScript callback. Local helpers and captured serializable constants are bundled. The existing QuickJS script sandbox executes the callback with bounded time/memory/output and without host credentials, filesystem, network or arbitrary Node packages. Async callbacks are rejected. This boundary stays the same in both agent modes.
153
+
154
+ Zod schemas must be representable as JSON Schema. Transforms, coercion, defaults,
155
+ custom refinements and parsing effects such as `.trim()` are rejected rather than
156
+ silently ignored. Use portable constraints such as `.min()` and `.nonnegative()`.
157
+
158
+ ## Add filesystem access
159
+
160
+ Use the same agent with an added capability:
161
+
162
+ ```ts
163
+ export default invoiceReviewer.withFilesystem({
164
+ // Optional, relative to the repository root. Default: this agent's folder.
165
+ cwd: 'agents/invoice-reviewer',
166
+
167
+ // Optional UTF-8 seed files, relative to cwd.
168
+ files: { 'scratch/tax.json': '{"rate":0.2}' },
169
+
170
+ // Optional extra packages installed inside the sandbox before execution.
171
+ packages: {
172
+ python: { openpyxl: '3.1.5', numpy: '2.3.3' },
173
+ npm: { 'decimal.js': '10.6.0' },
174
+ },
175
+ });
176
+ ```
177
+
178
+ `.withFilesystem()` without arguments is enough to opt in. It returns a new
179
+ immutable definition, preserving the input/output types, tools, instructions,
180
+ triggers and guards. The original agent stays filesystem-free. Repeating the
181
+ method replaces its filesystem options; it does not silently merge them.
182
+ There is one public spelling for opt-in, with no separate agent type or
183
+ `filesystem: true` authoring option.
184
+
185
+ For an omitted or string `cwd`, the runtime clones the repository at the run's
186
+ exact Git commit and starts the existing filesystem agent with read/write/edit/shell capabilities. The capability
187
+ adds filesystem tools to the same agent contract. It does not move custom
188
+ TypeScript callbacks into an unrestricted Node process.
189
+
190
+ To start empty instead of checking out the project:
191
+
192
+ ```ts
193
+ export default invoiceReviewer.withFilesystem({
194
+ cwd: null,
195
+ files: { 'invoice.json': '{"subtotal":100}' },
196
+ });
197
+ ```
198
+
199
+ The definition and instructions still come from the pinned source snapshot, but
200
+ repository files, manifests and repository environment declarations are not loaded.
201
+ Use `files` for initial content and `packages` for additional installed libraries.
202
+ Explicitly attached resources and skills remain available with `cwd: null`.
203
+ Platform tools and run input/output paths remain outside the working directory.
204
+
205
+ Filesystem options:
206
+
207
+ - `cwd`: omit it to start in the folder exporting the agent; use a repository-relative existing directory to override it, or `'.'` for the repository root. Set `null` to skip the execution checkout and start with a working directory populated by `files` and explicitly attached resources. An empty string is rejected. Paths use ASCII letters, digits, `_`, `.`, `-` and `/` only; spaces and Unicode names are not supported in workspace paths. String paths must remain inside the checkout after symlink resolution. The option selects initial content/location, not sandbox filesystem permissions.
208
+ - `skills`: optional map of imported `createSkill` definitions. Directory skills and their helper files also work with `cwd: null`.
209
+ - `files`: optional map of relative paths to text. At most 100 files and 256 KiB total UTF-8 paths/content. Creates missing directories. Identical existing content is accepted; differing content, symlinks, traversal, absolute paths and managed directories (`.git`, `.eigenpal`, `node_modules`, `eigenpal_modules`) fail before model execution. Binary assets and larger files belong in Git or normal run file inputs. Seed files initialize the workspace; they do not update the Git repository.
210
+ - `packages.python`: optional package-to-exact-version map, such as `{ numpy: '2.3.3' }`. Combined with `requirements.txt` in the selected directory in one `uv` install, so conflicting requirements fail.
211
+ - `packages.npm`: optional package-to-exact-version map, such as `{ 'decimal.js': '10.6.0' }`. Combined with the selected directory's `package.json`; conflicting declared versions fail. Existing `bun.lock` is reused by the current installer. This pins declared direct versions; it does not promise a fully locked transitive environment.
212
+
213
+ There are at most 32 additional packages across both maps. Python accepts numeric
214
+ release versions with two to four components; npm accepts exact three-component
215
+ versions, including prereleases. Ranges, URLs, install flags and alternate package
216
+ indexes are not accepted in these maps. Existing repository manifests still work
217
+ without declaring any extra packages. Install failures stop setup before the model
218
+ runs. Packages install only inside the sandbox, never on the application host.
219
+
220
+ There is no `language` flag: shell scripts can use both Python and JavaScript in
221
+ one run. A Python package makes Python scripts available to the filesystem agent;
222
+ it does not turn `createTool.execute` into a Python callback. Python callbacks and
223
+ Monty are outside this PR.
224
+
225
+ On deployments without a sandbox provider, ordinary agents still run. Starting a
226
+ filesystem agent returns `agent_capability_unavailable` before model execution or
227
+ sandbox provisioning; it never silently downgrades the agent.
228
+
229
+ ## Shared resources
230
+
231
+ From your agent directory, use `eigenpal agents validate` to capture and validate
232
+ the definition against your configured deployment. In the provided example project, `bun run typecheck` checks
233
+ authoring types locally. The deployment must include directory-resource support.
234
+ The low-level `compileAgent` API does not discover files; it expects already captured
235
+ resources. The public example's `bun run check` verifies its bundled synthetic fixtures
236
+ locally, without credentials, deployment access or model calls.
237
+
238
+ For a new project, the CLI creates the SDK/Zod/TypeScript dependencies, a strict
239
+ no-emit tsconfig supporting `.ts` imports, and a `typecheck` script. Existing
240
+ project configuration is preserved:
241
+
242
+ ```sh
243
+ eigenpal agents init invoice --template agent
244
+ bun install
245
+ bun run typecheck
246
+ cd agents/invoice
247
+ eigenpal agents validate
248
+ ```
249
+
250
+ A resource is a directory with a default-exported `resource.ts`. Import the definition;
251
+ EigenPal captures its accompanying files from the same Git commit as the agent.
252
+ No asset imports, `content`, `files` maps, YAML manifests or dependency-version strings.
253
+
254
+ ```ts
255
+ // resources/knowledge/tax-policy/resource.ts
256
+ import { createKnowledge } from '@eigenpal/automations';
257
+ export default createKnowledge({ name: 'Tax policy' });
258
+ // Add README.md, rates.csv, guidance.pdf, or nested document folders beside it.
259
+
260
+ // resources/rules/invoice-style/resource.ts
261
+ import { createRule } from '@eigenpal/automations';
262
+ export default createRule({ name: 'Invoice style' });
263
+ // Put instructions in RULE.md beside it.
264
+
265
+ // resources/templates/invoice/resource.ts
266
+ import { createTemplate } from '@eigenpal/automations';
267
+ export default createTemplate({ name: 'Invoice' });
268
+ // Put template.docx, template.xlsx or template.md beside it, exactly one.
269
+
270
+ // resources/skills/invoice-checks/resource.ts
271
+ import { createSkill } from '@eigenpal/automations';
272
+ export default createSkill({ name: 'Invoice checks' });
273
+ // Keep SKILL.md and scripts/, references/, or other helper files beside it.
274
+ ```
275
+
276
+ All factories accept `name` and optional `description`. Templates additionally accept
277
+ `file`, such as `createTemplate({ name: 'Invoice', file: 'invoice.docx' })`, when the
278
+ entry uses another filename. It must remain inside the resource directory. Ambiguous
279
+ or missing entries fail validation. Markdown templates use `{{field}}` and nested
280
+ `{{customer.name}}` variables; Liquid control tags are unsupported. DOCX/XLSX use the
281
+ platform Office template syntax:
282
+
283
+ - DOCX scalar: `Supplier: {supplier}`.
284
+ - DOCX loop: `{#items}{name}: {amount}{/items}` with `items: [{ name, amount }]`.
285
+ - DOCX conditional: `{#approved}Approved{/approved}` with a boolean value.
286
+ - XLSX scalar: `{supplier}` in a cell.
287
+ - XLSX table: `{table:items.name}` and `{table:items.amount}` in adjacent cells.
288
+
289
+ Office placeholders use single braces. Raw XML insertion and XLSX image placeholders are unsupported by resource
290
+ tools. Template expansion, extracted text and artifact bytes are bounded before output
291
+ registration; generated files never overwrite source templates.
292
+
293
+ ```ts
294
+ // agents/invoice/agent.ts
295
+ import { createAgent } from '@eigenpal/automations';
296
+ import policy from '../../resources/knowledge/tax-policy/resource.ts';
297
+ import style from '../../resources/rules/invoice-style/resource.ts';
298
+ import invoice from '../../resources/templates/invoice/resource.ts';
299
+
300
+ export default createAgent({
301
+ name: 'Invoice reviewer',
302
+ instructions: 'Check the invoice using the tax policy and fill its summary.',
303
+ resources: { policy, style, invoice },
304
+ });
305
+ ```
306
+
307
+ Rules enter the instructions directly. Knowledge and templates add a small catalog
308
+ and built-in tools, rather than flooding model context with document text:
309
+
310
+ - `resource_list({ resource })` lists captured file paths for any attached resource.
311
+ - `resource_search({ resource, query })` performs case-insensitive literal search,
312
+ returning up to ten excerpts with paths and character offsets. No vector service is required.
313
+ - `resource_read({ resource, path, offset?, limit? })` reads extracted text (or registers a downloadable artifact for a binary companion file), up to
314
+ 12,000 characters per call, and returns `nextOffset` for paging.
315
+ - `template_inspect({ resource })` discovers template placeholders.
316
+ - `template_fill({ resource, data, filename? })` fills the captured template and
317
+ registers a downloadable run artifact, returning its `fileId` and `name`.
318
+
319
+ `resource` is the imported alias (`policy` or `invoice` above). Tool schemas enumerate
320
+ only attached aliases. Browse tools also expose rule/template companion files. Built-in names are reserved so custom tools cannot shadow them.
321
+ Knowledge supports Markdown, UTF-8 text, CSV, JSON, PDF, DOCX and XLSX. PDF access
322
+ uses the deployment's existing bounded text extractor; scanned PDFs need a text layer.
323
+ Document contents are untrusted reference data, not additional agent instructions.
324
+ Missing template values, unsupported formats and oversized outputs fail explicitly.
325
+
326
+ Filesystem agents gain the same tools and receive the original resource directories:
327
+
328
+ ```ts
329
+ import checks from '../../resources/skills/invoice-checks/resource.ts';
330
+ // Extend the agent configuration before registering evaluators:
331
+ // createAgent({ ... }).withFilesystem({ skills: { checks } });
332
+ ```
333
+
334
+ Files are installed under `/workspace/agent/.eigenpal-resources/<alias>/`. Skills
335
+ include `SKILL.md` and their helpers; helper code is data until explicitly executed
336
+ inside the filesystem sandbox. Ordinary agents cannot attach directory skills. Filesystem agents receive a skill catalog and the path to SKILL.md, then read instructions/helpers as needed. Scripts do not run automatically.
337
+ `cwd: null` skips the repository checkout but still installs explicitly attached
338
+ resources and skills, alongside seed files. Changing `cwd` does not relocate resources.
339
+ Conflicting files and symlinks fail instead of being overwritten.
340
+
341
+ Capture recursively excludes hidden files/directories, dependency folders,
342
+ `resource.ts`, package manifests/locks and private-key file extensions. The filename `resource.ts` and its default export are required; directory names and imported aliases are your choice. It preserves
343
+ asset bytes and executable flags. Unsupported knowledge files fail validation.
344
+ The existing compiler limits apply: 128 captured files, 2 MiB source and bounded
345
+ compiled resource data. Large knowledge collections belong outside this Git snapshot.
346
+
347
+ Capture is performed by trusted source readers. Factories never read the host disk.
348
+ The compiled snapshot includes all captured data; historical runs/resumes do not
349
+ consult a newer resource directory. Resource edits require a new agent release.
350
+ SDK callbacks still run in bounded QuickJS without host filesystem/network access;
351
+ the browse/fill tools are fixed platform operations, not authored host callbacks.
352
+
353
+ ## Run, edit and resume
354
+
355
+ Use the existing run API, client SDK and CLI with the same agent identity:
356
+
357
+ ```http
358
+ POST /api/v1/runs
359
+ Authorization: Bearer <api-key>
360
+ Content-Type: application/json
361
+
362
+ {"target":"agents.invoice-reviewer","input":{"subtotal":100}}
363
+ ```
364
+
365
+ The public authoring package defines agents; existing execution clients start and
366
+ inspect runs. A run captures the selected source definition and isolated tool
367
+ bundle. Human-review resumes retain that definition, conversation and remaining
368
+ guards. This PR adds neither `.asTool()` nor new conversational follow-up APIs.
369
+
370
+ In the UI, TypeScript source uses syntax highlighting. API/email enablement edits
371
+ change the `agent.ts` draft and take effect when that version is released. Source
372
+ edits preserve callback code and filesystem options; stale writes are rejected.
373
+ Literal options and top-level const definitions are supported; dynamic configuration
374
+ is refused safely and can be edited directly in source.
375
+ Email alias management retains its existing routing-management behavior.
376
+
377
+ Under the hood, the isolated compiler turns the TypeScript authoring definition
378
+ into validated runtime configuration and JSON Schemas, with tool code kept in a
379
+ separate isolated executable bundle. TypeScript remains the source of truth;
380
+ compiled data pins reproducible runs/resumes and lets non-TypeScript services
381
+ validate the same contracts. There is no cross-run compilation cache.
382
+
383
+ ## Define evaluations
384
+
385
+ Keep datasets and documents in the existing dataset storage. An agent definition
386
+ contains reusable evaluation logic, not the dataset itself. Configure the agent
387
+ first, register named evaluators next, then finish with one pass decision.
388
+
389
+ This complete `agent.ts` starts with an empty workspace seeded with an invoice,
390
+ adds a synchronous calculation tool, and exports a weighted evaluation decision:
391
+
392
+ ```ts
393
+ import {
394
+ createAgent,
395
+ createTool,
396
+ exactDiff,
397
+ llmJudge,
398
+ weightedScore,
399
+ maxTurns,
400
+ maxTimeMs,
401
+ maxCredits,
402
+ } from '@eigenpal/automations';
403
+ import { z } from 'zod';
404
+
405
+ const invoice = createAgent({
406
+ name: 'Invoice reviewer',
407
+ instructions:
408
+ 'Review the supplied invoice, call calculateTotal, and return the supplier, amounts, and a factual summary.',
409
+ inputSchema: z.object({ invoice: z.string() }),
410
+ outputSchema: z.strictObject({
411
+ supplier: z.strictObject({ name: z.string(), taxId: z.string() }),
412
+ amounts: z.strictObject({ subtotal: z.number(), tax: z.number(), total: z.number() }),
413
+ summary: z.string(),
414
+ }),
415
+ tools: {
416
+ calculateTotal: createTool({
417
+ description: 'Calculate invoice tax and total from subtotal and tax rate.',
418
+ inputSchema: z.object({ subtotal: z.number().nonnegative(), rate: z.number().nonnegative() }),
419
+ outputSchema: z.strictObject({ tax: z.number(), total: z.number() }),
420
+ execute: ({ subtotal, rate }) => {
421
+ const tax = Math.round(subtotal * rate * 100) / 100;
422
+ return { tax, total: subtotal + tax };
423
+ },
424
+ }),
425
+ },
426
+ guards: [maxTurns(10), maxTimeMs(60_000), maxCredits(100)],
427
+ }).withFilesystem({
428
+ cwd: null,
429
+ files: { 'invoice.txt': 'Supplier: Example Ltd; tax ID: EX123; subtotal: 100; tax rate: 0.2' },
430
+ });
431
+
432
+ // Finish configuration above before the first defineEvaluator call.
433
+ const evaluated = invoice
434
+ .defineEvaluator(
435
+ 'fields',
436
+ exactDiff({
437
+ select: (output) => ({
438
+ supplierTaxId: output.supplier?.taxId,
439
+ total: output.amounts?.total,
440
+ }),
441
+ options: { rules: { total: { numericTolerance: 0.01 } } },
442
+ })
443
+ )
444
+ .defineEvaluator(
445
+ 'meaning',
446
+ llmJudge({
447
+ prompt: 'Score whether the actual summary preserves the expected invoice facts.',
448
+ select: ({ actual, expected }) => ({
449
+ actual: actual.summary,
450
+ expected: expected?.summary,
451
+ }),
452
+ })
453
+ )
454
+ .defineEvaluator('balanced', ({ actual }) =>
455
+ Math.abs(actual.amounts.subtotal + actual.amounts.tax - actual.amounts.total) <= 0.01 ? 1 : 0
456
+ );
457
+
458
+ export default evaluated.defineEvalPass(
459
+ weightedScore({
460
+ weights: { fields: 2, meaning: 1, balanced: 3 },
461
+ threshold: 0.9,
462
+ })
463
+ );
464
+
465
+ // Alternatively, replace the default export above with a boolean decision:
466
+ // export default evaluated.defineEvalPass(
467
+ // ({ fields, meaning, balanced }) => balanced === 1 && fields === 1 && meaning >= 0.8
468
+ // );
469
+ ```
470
+
471
+ No output types or evaluator score maps need manual annotations. In custom and
472
+ judge callbacks, `actual.amounts.total` is a `number`; `expected` can be absent,
473
+ and each nested expected field can also be absent. Use `expected?.amounts?.total`
474
+ when selecting a nested expected value. The exact selector runs on both actual
475
+ and partial expected outputs, so it uses `output.amounts?.total` even though the
476
+ actual output is complete. Optional chaining makes the selector type-safe; it
477
+ does not make missing expected values compare successfully. Supply the selected
478
+ expected fields in the dataset to avoid an evaluation error.
479
+
480
+ Pass callbacks only see registered score names. Misspelled output properties and
481
+ score names fail TypeScript checking. Use a Zod output schema for full inference;
482
+ raw JSON Schema and absent output schemas intentionally infer `unknown` and
483
+ require narrowing. Change tools, schemas, guards, or filesystem options on
484
+ `invoice` before registering evaluators; the `evaluated` chain only accepts more
485
+ evaluators and its final pass decision. Either pass branch returns a complete
486
+ agent definition that can be compiled.
487
+
488
+ ### Authoring phases
489
+
490
+ All methods return new immutable definitions. The original base agent is reusable.
491
+
492
+ 1. `createAgent` and `.withFilesystem` configure behavior.
493
+ 2. The first `.defineEvaluator(name, evaluate)` closes configuration. Register up
494
+ to 32 uniquely named evaluators. Names must be stable string literals, begin
495
+ with a letter and contain only letters, digits, `_` or `-`, up to 64 characters.
496
+ Prototype-related names are reserved. Runtime strings and unions are rejected
497
+ by TypeScript because they cannot establish a reliable score registry.
498
+ 3. One terminal `.defineEvalPass(predicateOrWeightedScore)` closes registration.
499
+ Its result has no configuration, evaluator-registration or pass methods.
500
+ An unfinished evaluator chain cannot be compiled or released.
501
+
502
+ Agents with no evaluation configuration require no pass rule. Evaluation-enabled
503
+ agents require an explicit decision; there is no implicit threshold. Evaluation
504
+ configuration works with filesystem and filesystem-free agents. Ordinary runs
505
+ never invoke evaluators automatically. Agent execution guards cover inference
506
+ and tool execution. Evaluation scripts have separate fixed execution limits;
507
+ model judging uses the existing evaluation model connection and billing.
508
+
509
+ ### Evaluator callbacks
510
+
511
+ `defineEvaluator` takes a name and either a synchronous callback or helper:
512
+
513
+ - **Custom callback:** `({ actual, expected }) => number`. Return a finite score
514
+ between 0 and 1. Inputs are deeply readonly. `actual` is the validated agent
515
+ output; `expected` is optional and deeply partial because stored examples can
516
+ omit expected fields. Conditional JSON Schema constraints (`if`, `not`, and
517
+ dependent schemas) are refused for partial expectations rather than weakened. Use optional chaining or explicitly guard missing values.
518
+ - **`exactDiff({ select, options? })`:** applies the same synchronous projection
519
+ to actual and expected outputs. `select` receives a deeply readonly partial
520
+ output, so nested optional chaining is required where expected data can be
521
+ absent. Return a JSON object, array or primitive. Selecting an object gives
522
+ readable diagnostic labels. Missing expected data or undefined selected values
523
+ cause an explicit evaluation error, rather than silently matching. Empty
524
+ selections are rejected. Comparison reuses the platform comparator.
525
+ - **`llmJudge({ prompt, select, model? })`:** `select` receives typed
526
+ `{ actual, expected }` and returns only the JSON data to expose to the judge.
527
+ `prompt` is required, 10 to 131072 characters. `model` optionally selects a
528
+ configured organization model; omission uses existing model selection. Dataset
529
+ expected answers never become agent input. The worker calls the judge through
530
+ model connections; selectors do not receive credentials.
531
+
532
+ Selectors must synchronously return JSON-compatible data. Promises, functions
533
+ and host objects are not supported. Callback code runs in the bounded script
534
+ runtime without ambient network, filesystem or host credentials. Errors,
535
+ timeouts, invalid scores and missing selected expected values are recorded as
536
+ evaluation errors and prevent an overall pass. Exceptions are never silently
537
+ skipped or counted as perfect scores.
538
+
539
+ Advanced `exactDiff.options` preserve existing comparison semantics:
540
+
541
+ - `rules`: a map of selected projection paths (or `$` for the entire projection)
542
+ to comparison rules. Paths are inferred from the selected shape up to four
543
+ nesting levels. Each rule supports `ignore: true`, `order: 'ordered' |
544
+ 'unordered'`, `items: 'at-least' | 'at-most' | 'exactly'`, `matchBy` (relative
545
+ item identity path or array of paths), `allowExtraFields`, and nonnegative
546
+ `numericTolerance`. `matchBy` requires `order: 'unordered'`; ignore rules
547
+ cannot contain other settings. Match identity candidates are inferred from selected
548
+ array item shapes; the comparison engine validates their use on the actual array.
549
+ - For compatibility with existing comparison behavior, `paths`, `unorderedPaths`,
550
+ `numericTolerance` and `allowExtraFields` remain available as global options.
551
+ Do not combine them with `rules`. Path lists are inferred from the projection.
552
+
553
+ These options apply to the projection, not to the unselected original output.
554
+ Explicit `null` and empty strings in selected expected values are compared.
555
+ Omitted expected properties remain partial; selecting an undefined value is an
556
+ error. Numeric tolerance is an absolute difference. Omission preserves existing
557
+ comparison defaults, including historical relative numeric epsilon, allowed extra
558
+ object fields, and positional-prefix array matching. Use `items: 'exactly'` and
559
+ `allowExtraFields: false` when extra values must fail.
560
+
561
+ ### A UI-friendly weighted decision
562
+
563
+ Use `weightedScore` when a weighted average is enough. In the complete example
564
+ above, the average is `(fields * 2 + meaning + balanced * 3) / 6`, with an inclusive
565
+ threshold of `0.9`. Only listed evaluators contribute.
566
+ Weights must reference registered names, be finite and nonnegative, with a
567
+ positive finite total. The threshold is inclusive and between 0 and 1. Unknown
568
+ names fail TypeScript checking and runtime validation. This descriptor can render
569
+ as ordinary weight/threshold controls in the UI. Predicate decisions render as
570
+ source with the overall verdict; individual scores and diagnostics remain
571
+ available in either form. Weighting is never silently applied to predicates.
572
+
573
+ A predicate receives the complete readonly score map and must synchronously
574
+ return `boolean`. Use ordinary `&&`, `||`, comparisons and arithmetic for custom
575
+ logic. A second pass definition, additional evaluator or filesystem change after
576
+ the pass is a type error and unavailable at runtime.
@@ -0,0 +1,40 @@
1
+ import { z } from 'zod';
2
+ import type { DefinitionSchema, InferSchema } from './definition.js';
3
+ export declare const executionGuardSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
4
+ kind: z.ZodLiteral<"turns">;
5
+ value: z.ZodNumber;
6
+ }, z.core.$strict>, z.ZodObject<{
7
+ kind: z.ZodLiteral<"timeMs">;
8
+ value: z.ZodNumber;
9
+ }, z.core.$strict>, z.ZodObject<{
10
+ kind: z.ZodLiteral<"credits">;
11
+ value: z.ZodNumber;
12
+ }, z.core.$strict>], "kind">;
13
+ export type ExecutionGuard = z.infer<typeof executionGuardSchema>;
14
+ export declare const maxTurns: (value: number) => ExecutionGuard;
15
+ export declare const maxTimeMs: (value: number) => ExecutionGuard;
16
+ export declare const maxCredits: (value: number) => ExecutionGuard;
17
+ export interface ToolDefinition<I = unknown, O = unknown> {
18
+ readonly description: string;
19
+ readonly inputSchema: DefinitionSchema;
20
+ readonly outputSchema: DefinitionSchema;
21
+ readonly execute: (input: I) => O;
22
+ }
23
+ /** Validate callbacks without invoking author code. Runtime also rejects returned promises. */
24
+ export declare function assertSynchronousTool(execute: unknown, field?: string): void;
25
+ /** Define a typed calculation tool. Callbacks run in bounded QuickJS, without Node, network or filesystem access. */
26
+ export declare function createTool<I extends DefinitionSchema, O extends DefinitionSchema>(options: {
27
+ /** Model-facing explanation of when this tool is useful. */
28
+ description: string;
29
+ /** Zod 4 or raw JSON object schema. Zod infers arguments; raw schemas infer unknown. */
30
+ inputSchema: I;
31
+ /** Zod 4 or raw JSON object schema. Zod checks return types; both validate the returned JSON at runtime. */
32
+ outputSchema: O;
33
+ /** Synchronous, JSON-compatible computation. No host APIs or asynchronous I/O. */
34
+ execute: (input: InferSchema<I>) => InferSchema<O>;
35
+ }): ToolDefinition<InferSchema<I>, InferSchema<O>>;
36
+ export interface CompiledTool {
37
+ description: string;
38
+ inputSchema: Record<string, unknown>;
39
+ outputSchema: Record<string, unknown>;
40
+ }
@@ -0,0 +1,27 @@
1
+ import { z } from 'zod';
2
+ const positiveInteger = z.number().int().positive().max(Number.MAX_SAFE_INTEGER);
3
+ export const executionGuardSchema = z.discriminatedUnion('kind', [
4
+ z.strictObject({ kind: z.literal('turns'), value: positiveInteger }),
5
+ z.strictObject({ kind: z.literal('timeMs'), value: positiveInteger }),
6
+ z.strictObject({ kind: z.literal('credits'), value: z.number().positive().finite() }),
7
+ ]);
8
+ export const maxTurns = (value) => executionGuardSchema.parse({ kind: 'turns', value });
9
+ export const maxTimeMs = (value) => executionGuardSchema.parse({ kind: 'timeMs', value });
10
+ export const maxCredits = (value) => executionGuardSchema.parse({ kind: 'credits', value });
11
+ /** Validate callbacks without invoking author code. Runtime also rejects returned promises. */
12
+ export function assertSynchronousTool(execute, field = 'execute') {
13
+ if (typeof execute !== 'function' ||
14
+ [
15
+ '[object AsyncFunction]',
16
+ '[object AsyncGeneratorFunction]',
17
+ '[object GeneratorFunction]',
18
+ ].includes(Object.prototype.toString.call(execute)))
19
+ throw new Error(`${field} must be a synchronous callback returning JSON data`);
20
+ }
21
+ /** Define a typed calculation tool. Callbacks run in bounded QuickJS, without Node, network or filesystem access. */
22
+ export function createTool(options) {
23
+ if (!options.description.trim() || typeof options.execute !== 'function')
24
+ throw new Error('Tools require a description and execute callback');
25
+ assertSynchronousTool(options.execute);
26
+ return Object.freeze({ ...options });
27
+ }
@@ -1,3 +1,11 @@
1
- import type { AgentDefinition, CompiledAgent } from './definition.js';
1
+ import type { AgentCore, CompiledAgent, DefinitionSchema, JsonSchema } from './definition.js';
2
+ import { type AgentEvaluation } from './evaluation.js';
3
+ export declare function compileSchema(schema: DefinitionSchema, field: string): JsonSchema;
4
+ type CompilableAgent = Omit<AgentCore, 'evaluation'> & {
5
+ readonly evaluation?: AgentEvaluation<never, never>;
6
+ readonly defineEvalPass?: never;
7
+ };
2
8
  /** Accepts an already loaded definition. Loading/executing tenant TS is a separate boundary. */
3
- export declare function compileAgent(definition: AgentDefinition): CompiledAgent;
9
+ export declare function compileAgent(definition: CompilableAgent): CompiledAgent;
10
+ export { resolveDirectoryResource } from './directory-resources.js';
11
+ export { resourceTools } from './resource-tools.js';