@eigenpal/automations 0.20.70 → 0.20.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +487 -6
- package/dist/capabilities.d.ts +40 -0
- package/dist/capabilities.js +27 -0
- package/dist/compiler.d.ts +10 -2
- package/dist/compiler.js +103 -2
- package/dist/config.d.ts +28 -9
- package/dist/config.js +57 -30
- package/dist/definition.d.ts +56 -12
- package/dist/directory-resources.d.ts +3 -0
- package/dist/directory-resources.js +51 -0
- package/dist/evaluation.d.ts +118 -0
- package/dist/evaluation.js +118 -0
- package/dist/index.d.ts +8 -2
- package/dist/index.js +83 -9
- package/dist/resource-tools.d.ts +5 -0
- package/dist/resource-tools.js +50 -0
- package/dist/resources.d.ts +140 -0
- package/dist/resources.js +74 -0
- package/package.json +1 -1
package/dist/compiler.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { assertSynchronousTool } from './capabilities.js';
|
|
2
3
|
import { agentConfigSchema } from './config.js';
|
|
4
|
+
import { normalizeEvaluator, validateEvaluatorName, validateWeightedPass, } from './evaluation.js';
|
|
3
5
|
import { copyJsonSchema } from './json-schema.js';
|
|
6
|
+
import { RESOURCE_TOOL_NAMES, resourceTools } from './resource-tools.js';
|
|
7
|
+
import { resourceMapSchema } from './resources.js';
|
|
4
8
|
const portableChecks = new Set([
|
|
5
9
|
'less_than',
|
|
6
10
|
'greater_than',
|
|
@@ -11,7 +15,7 @@ const portableChecks = new Set([
|
|
|
11
15
|
'length_equals',
|
|
12
16
|
'string_format',
|
|
13
17
|
]);
|
|
14
|
-
function compileSchema(schema, field) {
|
|
18
|
+
export function compileSchema(schema, field) {
|
|
15
19
|
try {
|
|
16
20
|
const json = schema instanceof z.ZodType
|
|
17
21
|
? z.toJSONSchema(schema, {
|
|
@@ -42,10 +46,105 @@ function compileSchema(schema, field) {
|
|
|
42
46
|
export function compileAgent(definition) {
|
|
43
47
|
if (definition.kind !== 'agent')
|
|
44
48
|
throw new Error('compileAgent expects an agent definition');
|
|
49
|
+
if (definition.config.workspace !== undefined && definition.config.filesystem !== true)
|
|
50
|
+
throw new Error('workspace requires filesystem capability');
|
|
51
|
+
if (Object.values(definition.resources ?? {}).some((resource) => !('content' in resource)))
|
|
52
|
+
throw new Error('Directory resources require source capture. Run eigenpal agents validate from the agent folder against a deployment with directory resource support. compileAgent does not read local files.');
|
|
53
|
+
const resources = resourceMapSchema.parse(definition.resources ?? {});
|
|
54
|
+
if (Object.values(resources).some((resource) => resource.kind === 'skill') &&
|
|
55
|
+
definition.config.filesystem !== true)
|
|
56
|
+
throw new Error('Skills require .withFilesystem()');
|
|
57
|
+
const tools = Object.fromEntries(Object.entries(definition.tools ?? {}).map(([name, tool]) => {
|
|
58
|
+
if (!/^[a-zA-Z][a-zA-Z0-9_]{0,63}$/.test(name) ||
|
|
59
|
+
[
|
|
60
|
+
'__proto__',
|
|
61
|
+
'constructor',
|
|
62
|
+
'prototype',
|
|
63
|
+
'finish',
|
|
64
|
+
...RESOURCE_TOOL_NAMES,
|
|
65
|
+
'read',
|
|
66
|
+
'write',
|
|
67
|
+
'edit',
|
|
68
|
+
'bash',
|
|
69
|
+
'grep',
|
|
70
|
+
'find',
|
|
71
|
+
'ls',
|
|
72
|
+
'request_human_review',
|
|
73
|
+
'report_step_progress',
|
|
74
|
+
'web_search',
|
|
75
|
+
'fetch_url',
|
|
76
|
+
'write_workflow_output_json',
|
|
77
|
+
'write_workflow_output_file',
|
|
78
|
+
].includes(name))
|
|
79
|
+
throw new Error(`Invalid tool name tools.${name}: use a letter followed by up to 63 letters, digits or underscores; built-in names are reserved`);
|
|
80
|
+
if (!tool.description?.trim() || typeof tool.execute !== 'function')
|
|
81
|
+
throw new Error(`Invalid tool definition tools.${name}: provide a description and synchronous execute callback`);
|
|
82
|
+
assertSynchronousTool(tool.execute, `tools.${name}.execute`);
|
|
83
|
+
return [
|
|
84
|
+
name,
|
|
85
|
+
{
|
|
86
|
+
description: tool.description,
|
|
87
|
+
inputSchema: compileSchema(tool.inputSchema, `tools.${name}.inputSchema`),
|
|
88
|
+
outputSchema: compileSchema(tool.outputSchema, `tools.${name}.outputSchema`),
|
|
89
|
+
},
|
|
90
|
+
];
|
|
91
|
+
}));
|
|
92
|
+
if (Object.keys(tools).length > 16)
|
|
93
|
+
throw new Error('At most 16 tools are supported');
|
|
94
|
+
Object.assign(tools, resourceTools(resources));
|
|
95
|
+
const evaluation = definition.evaluation;
|
|
96
|
+
if (evaluation && !evaluation.pass)
|
|
97
|
+
throw new Error('Evaluation registration must end with defineEvalPass');
|
|
98
|
+
if (evaluation) {
|
|
99
|
+
const names = Object.keys(evaluation.evaluators);
|
|
100
|
+
if (!names.length || names.length > 32)
|
|
101
|
+
throw new Error('Evaluations require 1 to 32 evaluators');
|
|
102
|
+
for (const name of names) {
|
|
103
|
+
validateEvaluatorName(name);
|
|
104
|
+
normalizeEvaluator(evaluation.evaluators[name]);
|
|
105
|
+
}
|
|
106
|
+
if (evaluation.pass.kind === 'weighted')
|
|
107
|
+
validateWeightedPass(evaluation.pass, names);
|
|
108
|
+
else if (evaluation.pass.kind !== 'callback' || typeof evaluation.pass.evaluate !== 'function')
|
|
109
|
+
throw new Error('Invalid evaluation pass');
|
|
110
|
+
}
|
|
111
|
+
const compiledEvaluation = evaluation
|
|
112
|
+
? {
|
|
113
|
+
evaluators: Object.fromEntries(Object.entries(evaluation.evaluators).map(([name, evaluator]) => [
|
|
114
|
+
name,
|
|
115
|
+
{
|
|
116
|
+
kind: evaluator.kind,
|
|
117
|
+
...(evaluator.kind === 'exact-diff' && evaluator.options
|
|
118
|
+
? { options: evaluator.options }
|
|
119
|
+
: {}),
|
|
120
|
+
...(evaluator.kind === 'llm-judge'
|
|
121
|
+
? {
|
|
122
|
+
prompt: evaluator.prompt,
|
|
123
|
+
...(evaluator.model ? { model: evaluator.model } : {}),
|
|
124
|
+
}
|
|
125
|
+
: {}),
|
|
126
|
+
},
|
|
127
|
+
])),
|
|
128
|
+
pass: evaluation.pass.kind === 'weighted'
|
|
129
|
+
? {
|
|
130
|
+
kind: 'weighted',
|
|
131
|
+
weights: { ...evaluation.pass.weights },
|
|
132
|
+
threshold: evaluation.pass.threshold,
|
|
133
|
+
}
|
|
134
|
+
: { kind: 'callback' },
|
|
135
|
+
}
|
|
136
|
+
: undefined;
|
|
45
137
|
return {
|
|
138
|
+
...(compiledEvaluation ? { evaluation: compiledEvaluation } : {}),
|
|
139
|
+
...(Object.keys(tools).length ? { tools } : {}),
|
|
140
|
+
...(Object.keys(resources).length ? { resources } : {}),
|
|
46
141
|
schemaVersion: 1,
|
|
47
142
|
kind: 'agent',
|
|
48
|
-
manifest: {
|
|
143
|
+
manifest: {
|
|
144
|
+
schemaVersion: 1,
|
|
145
|
+
...agentConfigSchema.parse(definition.config),
|
|
146
|
+
filesystem: definition.config.filesystem ?? false,
|
|
147
|
+
},
|
|
49
148
|
...(definition.inputSchema === undefined
|
|
50
149
|
? {}
|
|
51
150
|
: { inputSchema: compileSchema(definition.inputSchema, 'inputSchema') }),
|
|
@@ -54,3 +153,5 @@ export function compileAgent(definition) {
|
|
|
54
153
|
: { outputSchema: compileSchema(definition.outputSchema, 'outputSchema') }),
|
|
55
154
|
};
|
|
56
155
|
}
|
|
156
|
+
export { resolveDirectoryResource } from './directory-resources.js';
|
|
157
|
+
export { resourceTools } from './resource-tools.js';
|
package/dist/config.d.ts
CHANGED
|
@@ -1,8 +1,36 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
export declare const filesystemOptionsSchema: z.ZodObject<{
|
|
3
|
+
cwd: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
4
|
+
files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
5
|
+
packages: z.ZodOptional<z.ZodObject<{
|
|
6
|
+
python: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
7
|
+
npm: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
8
|
+
}, z.core.$strict>>;
|
|
9
|
+
}, z.core.$strict>;
|
|
2
10
|
export declare const agentConfigSchema: z.ZodObject<{
|
|
3
11
|
name: z.ZodString;
|
|
4
12
|
description: z.ZodOptional<z.ZodString>;
|
|
5
13
|
model: z.ZodOptional<z.ZodString>;
|
|
14
|
+
instructions: z.ZodOptional<z.ZodString>;
|
|
15
|
+
filesystem: z.ZodOptional<z.ZodBoolean>;
|
|
16
|
+
workspace: z.ZodOptional<z.ZodObject<{
|
|
17
|
+
cwd: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
18
|
+
files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
19
|
+
packages: z.ZodOptional<z.ZodObject<{
|
|
20
|
+
python: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
21
|
+
npm: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
22
|
+
}, z.core.$strict>>;
|
|
23
|
+
}, z.core.$strict>>;
|
|
24
|
+
guards: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
25
|
+
kind: z.ZodLiteral<"turns">;
|
|
26
|
+
value: z.ZodNumber;
|
|
27
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
28
|
+
kind: z.ZodLiteral<"timeMs">;
|
|
29
|
+
value: z.ZodNumber;
|
|
30
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
31
|
+
kind: z.ZodLiteral<"credits">;
|
|
32
|
+
value: z.ZodNumber;
|
|
33
|
+
}, z.core.$strict>], "kind">>>;
|
|
6
34
|
triggers: z.ZodOptional<z.ZodObject<{
|
|
7
35
|
api: z.ZodOptional<z.ZodBoolean>;
|
|
8
36
|
email: z.ZodOptional<z.ZodObject<{
|
|
@@ -16,13 +44,4 @@ export declare const agentConfigSchema: z.ZodObject<{
|
|
|
16
44
|
}, z.core.$strict>]>>>;
|
|
17
45
|
}, z.core.$strict>>;
|
|
18
46
|
}, z.core.$strict>>;
|
|
19
|
-
dependencies: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
20
|
-
evaluators: z.ZodOptional<z.ZodObject<{
|
|
21
|
-
passThreshold: z.ZodOptional<z.ZodNumber>;
|
|
22
|
-
items: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
23
|
-
use: z.ZodString;
|
|
24
|
-
weight: z.ZodOptional<z.ZodNumber>;
|
|
25
|
-
version: z.ZodOptional<z.ZodString>;
|
|
26
|
-
}, z.core.$strict>>>;
|
|
27
|
-
}, z.core.$strict>>;
|
|
28
47
|
}, z.core.$strict>;
|
package/dist/config.js
CHANGED
|
@@ -1,41 +1,75 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
|
|
3
|
-
const
|
|
4
|
-
const dependencyName = z
|
|
5
|
-
.string()
|
|
6
|
-
.regex(new RegExp(`^workspace:(agents|resources)\\.${segment}(\\.${segment})*$`));
|
|
7
|
-
const evaluatorName = z
|
|
8
|
-
.string()
|
|
9
|
-
.refine((value) => (value.startsWith('./evaluators/') &&
|
|
10
|
-
value.endsWith('.yaml') &&
|
|
11
|
-
!value.includes('..') &&
|
|
12
|
-
!value.includes('\\')) ||
|
|
13
|
-
new RegExp(`^workspace:evaluators\\.${segment}(\\.${segment})*$`).test(value), 'Use ./evaluators/<file>.yaml or workspace:evaluators.<name>');
|
|
14
|
-
const evaluatorItem = z
|
|
2
|
+
import { executionGuardSchema } from './capabilities.js';
|
|
3
|
+
export const filesystemOptionsSchema = z
|
|
15
4
|
.strictObject({
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
5
|
+
cwd: z
|
|
6
|
+
.string()
|
|
7
|
+
.max(512)
|
|
8
|
+
.refine((value) => value === '.' || safePath(value), 'Use a repository-relative directory containing only ASCII letters, digits, _, ., - and /, without traversal')
|
|
9
|
+
.nullable()
|
|
10
|
+
.optional(),
|
|
11
|
+
files: z
|
|
12
|
+
.record(z
|
|
13
|
+
.string()
|
|
14
|
+
.refine(safePath, 'Use a relative file path containing only ASCII letters, digits, _, ., - and /, without traversal'), z.string())
|
|
15
|
+
.optional(),
|
|
16
|
+
packages: z
|
|
17
|
+
.strictObject({
|
|
18
|
+
python: z
|
|
19
|
+
.record(z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]*$/), z.string().regex(/^[0-9]+(?:\.[0-9]+){1,3}$/))
|
|
20
|
+
.optional(),
|
|
21
|
+
npm: z
|
|
22
|
+
.record(z.string().regex(/^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/), z.string().regex(/^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$/))
|
|
23
|
+
.optional(),
|
|
24
|
+
})
|
|
25
|
+
.optional(),
|
|
19
26
|
})
|
|
20
|
-
.superRefine((
|
|
21
|
-
|
|
27
|
+
.superRefine((value, context) => {
|
|
28
|
+
const files = Object.entries(value.files ?? {});
|
|
29
|
+
if (files.length > 100 ||
|
|
30
|
+
files.reduce((total, [path, text]) => total + utf8Length(path + text), 0) > 256 * 1024)
|
|
22
31
|
context.addIssue({
|
|
23
32
|
code: 'custom',
|
|
24
|
-
|
|
25
|
-
message: 'Shared evaluators require an exact version',
|
|
33
|
+
message: 'Workspace files allow at most 100 files and 256 KiB total',
|
|
26
34
|
});
|
|
27
|
-
if (
|
|
35
|
+
if (Object.keys(value.packages?.python ?? {}).length +
|
|
36
|
+
Object.keys(value.packages?.npm ?? {}).length >
|
|
37
|
+
32)
|
|
28
38
|
context.addIssue({
|
|
29
39
|
code: 'custom',
|
|
30
|
-
|
|
31
|
-
message: 'Local evaluators do not take a version',
|
|
40
|
+
message: 'Workspace allows at most 32 additional packages',
|
|
32
41
|
});
|
|
33
42
|
});
|
|
43
|
+
// No host globals: the public SDK also runs inside the isolated source compiler.
|
|
44
|
+
function utf8Length(value) {
|
|
45
|
+
let bytes = 0;
|
|
46
|
+
for (const character of value) {
|
|
47
|
+
const code = character.codePointAt(0);
|
|
48
|
+
bytes += code <= 0x7f ? 1 : code <= 0x7ff ? 2 : code <= 0xffff ? 3 : 4;
|
|
49
|
+
}
|
|
50
|
+
return bytes;
|
|
51
|
+
}
|
|
52
|
+
function safePath(value) {
|
|
53
|
+
return (value.length <= 512 &&
|
|
54
|
+
/^[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)*$/.test(value) &&
|
|
55
|
+
!value
|
|
56
|
+
.split('/')
|
|
57
|
+
.some((part) => ['.', '..', '.git', '.eigenpal', 'node_modules', 'eigenpal_modules'].includes(part)));
|
|
58
|
+
}
|
|
34
59
|
// Deliberately no defaults: preserve omitted fields for the existing platform parser.
|
|
35
60
|
export const agentConfigSchema = z.strictObject({
|
|
36
61
|
name: z.string().min(1),
|
|
37
62
|
description: z.string().optional(),
|
|
38
63
|
model: z.string().optional(),
|
|
64
|
+
instructions: z
|
|
65
|
+
.string()
|
|
66
|
+
.min(1)
|
|
67
|
+
.max(128 * 1024)
|
|
68
|
+
.refine((v) => v.trim().length > 0)
|
|
69
|
+
.optional(),
|
|
70
|
+
filesystem: z.boolean().optional(),
|
|
71
|
+
workspace: filesystemOptionsSchema.optional(),
|
|
72
|
+
guards: z.array(executionGuardSchema).max(32).optional(),
|
|
39
73
|
triggers: z
|
|
40
74
|
.strictObject({
|
|
41
75
|
api: z.boolean().optional(),
|
|
@@ -58,11 +92,4 @@ export const agentConfigSchema = z.strictObject({
|
|
|
58
92
|
.optional(),
|
|
59
93
|
})
|
|
60
94
|
.optional(),
|
|
61
|
-
dependencies: z.record(dependencyName, exactVersion).optional(),
|
|
62
|
-
evaluators: z
|
|
63
|
-
.strictObject({
|
|
64
|
-
passThreshold: z.number().min(0).max(1).optional(),
|
|
65
|
-
items: z.array(evaluatorItem).optional(),
|
|
66
|
-
})
|
|
67
|
-
.optional(),
|
|
68
95
|
});
|
package/dist/definition.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import type { z } from 'zod';
|
|
2
|
+
import type { CompiledTool, ExecutionGuard, ToolDefinition } from './capabilities.js';
|
|
3
|
+
import type { AgentEvaluation, CompiledEvaluation, DeepReadonly, EvaluatorInput, WeightedPass } from './evaluation.js';
|
|
4
|
+
import type { PortableResource, ResourceDeclaration, ResourceDefinition, Skill } from './resources.js';
|
|
2
5
|
/** Raw JSON Schema preserves complete contracts and platform-specific annotations. */
|
|
3
6
|
export type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
4
7
|
[key: string]: JsonValue;
|
|
@@ -24,10 +27,29 @@ export interface AgentEmailAlias {
|
|
|
24
27
|
requireSenderAuth?: boolean;
|
|
25
28
|
enabled?: boolean;
|
|
26
29
|
}
|
|
30
|
+
/** Additive sandbox capability. Paths are relative; package versions must be exact. */
|
|
31
|
+
export interface FilesystemOptions {
|
|
32
|
+
/** Omit for the agent folder; use a repo-relative path to override, or null for no checkout. Paths use ASCII letters, digits, _, ., - and / only; no traversal. */
|
|
33
|
+
cwd?: string | null;
|
|
34
|
+
/** Directory skills require the filesystem capability. */
|
|
35
|
+
skills?: Record<string, Skill>;
|
|
36
|
+
/** UTF-8 seed files relative to cwd. Existing differing files are never overwritten. */
|
|
37
|
+
files?: Record<string, string>;
|
|
38
|
+
/** Extra sandbox dependencies, combined with repository package manifests. */
|
|
39
|
+
packages?: {
|
|
40
|
+
python?: Record<string, string>;
|
|
41
|
+
npm?: Record<string, string>;
|
|
42
|
+
};
|
|
43
|
+
}
|
|
27
44
|
export interface AgentConfig {
|
|
28
45
|
name: string;
|
|
29
46
|
description?: string;
|
|
30
47
|
model?: string;
|
|
48
|
+
instructions?: string;
|
|
49
|
+
filesystem?: boolean;
|
|
50
|
+
/** Normalized sandbox capability; author with .withFilesystem(). */
|
|
51
|
+
workspace?: FilesystemOptions;
|
|
52
|
+
guards?: ExecutionGuard[];
|
|
31
53
|
triggers?: {
|
|
32
54
|
api?: boolean;
|
|
33
55
|
email?: {
|
|
@@ -35,22 +57,39 @@ export interface AgentConfig {
|
|
|
35
57
|
aliases?: (string | AgentEmailAlias)[];
|
|
36
58
|
};
|
|
37
59
|
};
|
|
38
|
-
dependencies?: Record<string, string>;
|
|
39
|
-
evaluators?: {
|
|
40
|
-
passThreshold?: number;
|
|
41
|
-
items?: {
|
|
42
|
-
use: string;
|
|
43
|
-
weight?: number;
|
|
44
|
-
version?: string;
|
|
45
|
-
}[];
|
|
46
|
-
};
|
|
47
60
|
}
|
|
48
|
-
export interface AgentOptions<I = DefinitionSchema, O = DefinitionSchema> extends AgentConfig {
|
|
61
|
+
export interface AgentOptions<I = DefinitionSchema, O = DefinitionSchema> extends Omit<AgentConfig, 'filesystem' | 'workspace'> {
|
|
62
|
+
/** Zod 4 or raw JSON object schema for run input. Zod infers types; raw schemas infer unknown. No transforms or defaults. */
|
|
49
63
|
inputSchema?: I;
|
|
64
|
+
/** Zod 4 or raw JSON object schema for the final answer. Zod infers evaluator/selector arguments; raw schemas infer unknown. */
|
|
50
65
|
outputSchema?: O;
|
|
66
|
+
/** Import default-exported resource.ts declarations. Keys are local aliases in resource tools and sandbox paths. Files are captured at release, not read by this factory. */
|
|
67
|
+
resources?: Record<string, PortableResource>;
|
|
68
|
+
/** Model-facing names mapped to createTool definitions. Up to 16; built-in names such as read, bash and resource_read are reserved. */
|
|
69
|
+
tools?: Record<string, ToolDefinition<never, unknown>>;
|
|
70
|
+
}
|
|
71
|
+
export interface AgentCore<I = DefinitionSchema | undefined, O = DefinitionSchema | undefined> extends AutomationDefinition<'agent', I, O> {
|
|
72
|
+
readonly config: DeepReadonly<AgentConfig>;
|
|
73
|
+
readonly resources?: Readonly<Record<string, ResourceDeclaration>>;
|
|
74
|
+
readonly tools?: Readonly<Record<string, ToolDefinition<never, unknown>>>;
|
|
75
|
+
readonly evaluation?: AgentEvaluation<InferSchema<O>, never>;
|
|
76
|
+
}
|
|
77
|
+
type IsUnion<T, Whole = T> = T extends Whole ? ([Whole] extends [T] ? false : true) : never;
|
|
78
|
+
type EvaluatorName<N extends string> = string extends N ? never : true extends IsUnion<N> ? never : N;
|
|
79
|
+
export interface AgentDefinition<I = DefinitionSchema | undefined, O = DefinitionSchema | undefined> extends AgentCore<I, O> {
|
|
80
|
+
/** Opt into a sandbox. Replaces previous filesystem settings; configure before defining evaluators. */
|
|
81
|
+
withFilesystem(options?: FilesystemOptions): AgentDefinition<I, O>;
|
|
82
|
+
/** Register a named score from 0 to 1. Closes agent configuration; output types flow into the callback/selector. */
|
|
83
|
+
defineEvaluator<const N extends string>(name: N & EvaluatorName<N>, evaluate: EvaluatorInput<InferSchema<O>>): AgentEvaluators<I, O, Record<N, number>>;
|
|
51
84
|
}
|
|
52
|
-
|
|
53
|
-
|
|
85
|
+
/** Evaluation registration closes configuration; pass closes evaluation registration. */
|
|
86
|
+
export interface AgentEvaluators<I, O, S extends Record<string, number>> extends AgentCore<I, O> {
|
|
87
|
+
/** Register a named score from 0 to 1. Closes agent configuration; output types flow into the callback/selector. */
|
|
88
|
+
defineEvaluator<const N extends string>(name: N & EvaluatorName<N> & (N extends keyof S ? never : N), evaluate: EvaluatorInput<InferSchema<O>>): AgentEvaluators<I, O, S & Record<N, number>>;
|
|
89
|
+
/** Finish evaluation registration with one typed predicate or weightedScore decision. No further configuration follows. */
|
|
90
|
+
defineEvalPass<const W extends Record<string, number>>(pass: ((scores: Readonly<S>) => boolean) | (WeightedPass<W> & {
|
|
91
|
+
readonly weights: Partial<S> & Record<Exclude<keyof W, keyof S>, never>;
|
|
92
|
+
})): AgentCore<I, O>;
|
|
54
93
|
}
|
|
55
94
|
export type InferInput<T extends AutomationDefinition<string, unknown, unknown>> = InferSchema<NonNullable<T['inputSchema']>>;
|
|
56
95
|
export type InferOutput<T extends AutomationDefinition<string, unknown, unknown>> = InferSchema<NonNullable<T['outputSchema']>>;
|
|
@@ -64,5 +103,10 @@ export interface CompiledDefinition<K extends string> {
|
|
|
64
103
|
export interface CompiledAgent extends CompiledDefinition<'agent'> {
|
|
65
104
|
manifest: AgentConfig & {
|
|
66
105
|
schemaVersion: 1;
|
|
106
|
+
filesystem: boolean;
|
|
67
107
|
};
|
|
108
|
+
resources?: Record<string, ResourceDefinition>;
|
|
109
|
+
tools?: Record<string, CompiledTool>;
|
|
110
|
+
evaluation?: CompiledEvaluation;
|
|
68
111
|
}
|
|
112
|
+
export {};
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { type DirectoryResource, type ResourceDefinition, type ResourceFile, type ResourceKind } from './resources.js';
|
|
2
|
+
/** Resolve only compiler-supplied data. Never reads the host filesystem. */
|
|
3
|
+
export declare function resolveDirectoryResource<K extends ResourceKind>(declaration: DirectoryResource<K>, files: Record<string, ResourceFile>): ResourceDefinition<K>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { directoryResourceSchema, resourceSchema, } from './resources.js';
|
|
2
|
+
/** Resolve only compiler-supplied data. Never reads the host filesystem. */
|
|
3
|
+
export function resolveDirectoryResource(declaration, files) {
|
|
4
|
+
const resource = directoryResourceSchema.parse(declaration);
|
|
5
|
+
const names = Object.keys(files).sort();
|
|
6
|
+
const text = (name) => {
|
|
7
|
+
const value = files[name];
|
|
8
|
+
if (typeof value === 'string')
|
|
9
|
+
return value;
|
|
10
|
+
if (value?.encoding === 'utf8')
|
|
11
|
+
return value.content;
|
|
12
|
+
throw new Error(`Resource "${resource.name}": expected UTF-8 file ${name}`);
|
|
13
|
+
};
|
|
14
|
+
let entry;
|
|
15
|
+
let content;
|
|
16
|
+
if (resource.kind === 'rule' || resource.kind === 'skill') {
|
|
17
|
+
entry = resource.kind === 'rule' ? 'RULE.md' : 'SKILL.md';
|
|
18
|
+
content = text(entry);
|
|
19
|
+
}
|
|
20
|
+
else if (resource.kind === 'template') {
|
|
21
|
+
const candidates = resource.file
|
|
22
|
+
? [resource.file]
|
|
23
|
+
: names.filter((name) => /^template\.(docx|xlsx|md)$/.test(name));
|
|
24
|
+
if (candidates.length !== 1 || !Object.hasOwn(files, candidates[0]))
|
|
25
|
+
throw new Error(`Resource "${resource.name}": provide one template.docx, template.xlsx or template.md, or select file explicitly. Found: ${candidates.slice(0, 5).join(', ') || 'none'}`);
|
|
26
|
+
entry = candidates[0];
|
|
27
|
+
if (!/\.(docx|xlsx|md)$/.test(entry))
|
|
28
|
+
throw new Error('Unsupported template format');
|
|
29
|
+
content = resource.description?.trim()
|
|
30
|
+
? resource.description
|
|
31
|
+
: `Inspect and fill ${entry} with the template tools.`;
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
if (!names.length)
|
|
35
|
+
throw new Error('Knowledge directory has no documents');
|
|
36
|
+
if (names.some((name) => !/\.(md|txt|csv|json|pdf|docx|xlsx)$/i.test(name)))
|
|
37
|
+
throw new Error('Knowledge supports Markdown, text, CSV, JSON, PDF, DOCX and XLSX documents');
|
|
38
|
+
content = resource.description?.trim()
|
|
39
|
+
? resource.description
|
|
40
|
+
: 'Browse, search and read the attached knowledge documents.';
|
|
41
|
+
}
|
|
42
|
+
const assets = Object.fromEntries(Object.entries(files).filter(([name]) => resource.kind !== 'skill' || name !== 'SKILL.md'));
|
|
43
|
+
return Object.freeze(resourceSchema.parse({
|
|
44
|
+
kind: resource.kind,
|
|
45
|
+
name: resource.name,
|
|
46
|
+
...(resource.description ? { description: resource.description } : {}),
|
|
47
|
+
content,
|
|
48
|
+
...(entry ? { entry } : {}),
|
|
49
|
+
files: assets,
|
|
50
|
+
}));
|
|
51
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
export type DeepReadonly<T> = T extends object ? {
|
|
2
|
+
readonly [K in keyof T]: DeepReadonly<T[K]>;
|
|
3
|
+
} : T;
|
|
4
|
+
export type JsonProjection = string | number | boolean | null | undefined | readonly JsonProjection[] | {
|
|
5
|
+
readonly [key: string]: JsonProjection;
|
|
6
|
+
};
|
|
7
|
+
type Join<K extends string, P> = P extends string ? `${K}.${P}` : never;
|
|
8
|
+
type Previous = [never, 0, 1, 2, 3, 4];
|
|
9
|
+
type ProjectionPath<T, Depth extends number = 4> = [Depth] extends [0] ? never : T extends readonly (infer U)[] ? '[]' | Join<'[]', ProjectionPath<U, Previous[Depth]>> : T extends object ? {
|
|
10
|
+
[K in keyof T & string]: K | (NonNullable<T[K]> extends readonly (infer U)[] ? `${K}[]` | Join<`${K}[]`, ProjectionPath<U, Previous[Depth]>> : Join<K, ProjectionPath<NonNullable<T[K]>, Previous[Depth]>>);
|
|
11
|
+
}[keyof T & string] : never;
|
|
12
|
+
type ArrayIdentityPath<P, D extends number = 4> = [D] extends [0] ? never : P extends readonly (infer I)[] ? ProjectionPath<I> : P extends object ? {
|
|
13
|
+
[K in keyof P]: ArrayIdentityPath<NonNullable<P[K]>, Previous[D]>;
|
|
14
|
+
}[keyof P] : never;
|
|
15
|
+
export type SelectedDiffOptions<P> = Omit<ExactDiffOptions, 'rules' | 'paths' | 'unorderedPaths'> & {
|
|
16
|
+
rules?: Partial<Record<'$' | ProjectionPath<P>, Omit<ExactDiffRule, 'matchBy'> & {
|
|
17
|
+
matchBy?: ArrayIdentityPath<P> | ArrayIdentityPath<P>[];
|
|
18
|
+
}>>;
|
|
19
|
+
paths?: ProjectionPath<P>[];
|
|
20
|
+
unorderedPaths?: ProjectionPath<P>[];
|
|
21
|
+
};
|
|
22
|
+
/** Stored expected answers may be partial; they are never exposed to the agent. */
|
|
23
|
+
export type DeepPartial<T> = T extends readonly unknown[] ? {
|
|
24
|
+
[K in keyof T]: DeepPartial<T[K]>;
|
|
25
|
+
} : T extends object ? {
|
|
26
|
+
[K in keyof T]?: DeepPartial<T[K]>;
|
|
27
|
+
} : T;
|
|
28
|
+
export interface EvaluationContext<T> {
|
|
29
|
+
readonly actual: DeepReadonly<T>;
|
|
30
|
+
readonly expected: DeepReadonly<DeepPartial<T>> | undefined;
|
|
31
|
+
}
|
|
32
|
+
export type EvaluationCallback<T> = (context: EvaluationContext<T>) => number;
|
|
33
|
+
export interface ExactDiffRule {
|
|
34
|
+
ignore?: true;
|
|
35
|
+
order?: 'ordered' | 'unordered';
|
|
36
|
+
items?: 'at-least' | 'at-most' | 'exactly';
|
|
37
|
+
matchBy?: string | string[];
|
|
38
|
+
allowExtraFields?: boolean;
|
|
39
|
+
numericTolerance?: number;
|
|
40
|
+
}
|
|
41
|
+
export interface ExactDiffOptions {
|
|
42
|
+
rules?: Record<string, ExactDiffRule>;
|
|
43
|
+
paths?: string[];
|
|
44
|
+
unorderedPaths?: string[];
|
|
45
|
+
numericTolerance?: number;
|
|
46
|
+
allowExtraFields?: boolean;
|
|
47
|
+
}
|
|
48
|
+
export interface ExactDiffEvaluator<T> {
|
|
49
|
+
readonly kind: 'exact-diff';
|
|
50
|
+
readonly options?: ExactDiffOptions;
|
|
51
|
+
/** Applied to actual and expected separately. */
|
|
52
|
+
readonly select: (output: DeepReadonly<DeepPartial<T>>) => JsonProjection;
|
|
53
|
+
}
|
|
54
|
+
export interface LlmJudgeEvaluator<T> {
|
|
55
|
+
readonly kind: 'llm-judge';
|
|
56
|
+
readonly prompt: string;
|
|
57
|
+
readonly select: (context: EvaluationContext<T>) => JsonProjection;
|
|
58
|
+
readonly model?: string;
|
|
59
|
+
}
|
|
60
|
+
export interface CustomEvaluator<T> {
|
|
61
|
+
readonly kind: 'custom';
|
|
62
|
+
readonly evaluate: EvaluationCallback<T>;
|
|
63
|
+
}
|
|
64
|
+
export type Evaluator<T> = ExactDiffEvaluator<T> | LlmJudgeEvaluator<T> | CustomEvaluator<T>;
|
|
65
|
+
export type EvaluatorInput<T> = EvaluationCallback<T> | Evaluator<T>;
|
|
66
|
+
export interface WeightedPass<S extends Record<string, number>> {
|
|
67
|
+
readonly kind: 'weighted';
|
|
68
|
+
readonly weights: S;
|
|
69
|
+
readonly threshold: number;
|
|
70
|
+
}
|
|
71
|
+
export interface CallbackPass<S extends Record<string, number>> {
|
|
72
|
+
readonly kind: 'callback';
|
|
73
|
+
readonly evaluate: (scores: Readonly<S>) => boolean;
|
|
74
|
+
}
|
|
75
|
+
export type EvaluationPass<S extends Record<string, number>> = WeightedPass<S> | CallbackPass<S>;
|
|
76
|
+
export interface AgentEvaluation<T = unknown, S extends Record<string, number> = Record<string, number>> {
|
|
77
|
+
readonly evaluators: Readonly<Record<string, Evaluator<T>>>;
|
|
78
|
+
readonly pass: EvaluationPass<S>;
|
|
79
|
+
}
|
|
80
|
+
export interface CompiledEvaluation {
|
|
81
|
+
evaluators: Record<string, {
|
|
82
|
+
kind: 'custom' | 'exact-diff' | 'llm-judge';
|
|
83
|
+
prompt?: string;
|
|
84
|
+
model?: string;
|
|
85
|
+
options?: ExactDiffOptions;
|
|
86
|
+
}>;
|
|
87
|
+
pass: {
|
|
88
|
+
kind: 'callback';
|
|
89
|
+
} | {
|
|
90
|
+
kind: 'weighted';
|
|
91
|
+
weights: Record<string, number>;
|
|
92
|
+
threshold: number;
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/** Compare a typed projection, keeping full dataset expected answers unchanged. */
|
|
96
|
+
export declare function exactDiff<T, const P extends JsonProjection>(options: {
|
|
97
|
+
select: (output: DeepReadonly<DeepPartial<T>>) => P;
|
|
98
|
+
options?: SelectedDiffOptions<P>;
|
|
99
|
+
}): ExactDiffEvaluator<T>;
|
|
100
|
+
/** Judge only the explicitly selected data through the worker model connection. */
|
|
101
|
+
export declare function llmJudge<T>(options: {
|
|
102
|
+
prompt: string;
|
|
103
|
+
select: (context: EvaluationContext<T>) => JsonProjection;
|
|
104
|
+
model?: string;
|
|
105
|
+
}): LlmJudgeEvaluator<T>;
|
|
106
|
+
/** A declarative weighted average, rendered as controls in the evaluation UI. */
|
|
107
|
+
export declare function weightedScore<const S extends Record<string, number>>(options: {
|
|
108
|
+
weights: S;
|
|
109
|
+
threshold: number;
|
|
110
|
+
}): WeightedPass<S>;
|
|
111
|
+
export declare function validateWeightedPass(options: {
|
|
112
|
+
weights: object;
|
|
113
|
+
threshold: number;
|
|
114
|
+
}, names?: readonly string[]): void;
|
|
115
|
+
export declare function freezeEvaluationData<T>(value: T): T;
|
|
116
|
+
export declare function validateEvaluatorName(name: string): void;
|
|
117
|
+
export declare function normalizeEvaluator<T>(input: EvaluatorInput<T>): Evaluator<T>;
|
|
118
|
+
export {};
|