@axiastudio/aioc 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/agent.d.ts +8 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +3 -1
- package/dist/harness-descriptor.d.ts +8 -2
- package/dist/harness-descriptor.d.ts.map +1 -1
- package/dist/harness-descriptor.js +66 -14
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/run-regression-summary.d.ts +34 -0
- package/dist/run-regression-summary.d.ts.map +1 -0
- package/dist/run-regression-summary.js +71 -0
- package/dist/run-regression.d.ts +71 -0
- package/dist/run-regression.d.ts.map +1 -0
- package/dist/run-regression.js +91 -0
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +8 -3
- package/package.json +3 -1
package/README.md
CHANGED
|
@@ -39,6 +39,7 @@ AIOC `0.2.2` completes the descriptor instruction-composition surface with reusa
|
|
|
39
39
|
AIOC `0.2.3` adds RFC-0010 policy composition helpers for exact-name tool and handoff policy dispatch without changing runtime enforcement semantics.
|
|
40
40
|
AIOC `0.2.4` adds experimental governance-event packages and an OpenTelemetry Logs exporter for reduced, operational observability events derived from `RunRecord` values.
|
|
41
41
|
AIOC `0.2.5` makes replay history-faithful by recording the initial input scope in `RunRecord` and replaying from it by default.
|
|
42
|
+
AIOC `0.2.6` adds descriptor-level conditional agent handoffs with boolean `where` gates that filter handoff tools before provider requests.
|
|
42
43
|
|
|
43
44
|
### Experimental Packages
|
|
44
45
|
|
|
@@ -146,7 +147,7 @@ console.log(result.finalOutput);
|
|
|
146
147
|
- `loadAgentHarnessDescriptorFromFile(...)`
|
|
147
148
|
- `AgentHarnessDescriptor` and related descriptor types
|
|
148
149
|
|
|
149
|
-
The Agent Harness Descriptor is included in the supported `0.2.x` API surface as the `aioc.agent_graph.v0` descriptor contract. Use it for controlled configuration, examples, and evaluation harnesses; keep executable tools, policies, providers, persistence, approvals, and deployment configuration in application code.
|
|
150
|
+
The Agent Harness Descriptor is included in the supported `0.2.x` API surface as the `aioc.agent_graph.v0` descriptor contract. Use it for controlled configuration, examples, and evaluation harnesses; keep executable tools, policies, providers, persistence, approvals, and deployment configuration in application code. Descriptor handoffs may use boolean `where` gates to hide unavailable handoff tools before the provider call; `HandoffPolicy` still owns allow/deny/approval decisions for exposed handoffs.
|
|
150
151
|
|
|
151
152
|
## Policy Gate (Minimal)
|
|
152
153
|
|
|
@@ -263,6 +264,11 @@ This repository also contains `aioc-inspect`, a private reference example UI for
|
|
|
263
264
|
|
|
264
265
|
## Examples
|
|
265
266
|
|
|
267
|
+
Suggested reading path: start with `example:hello`, then policy, approval,
|
|
268
|
+
tool-policy, and policy-composition. Use run-record and harness examples after
|
|
269
|
+
the basics; `example:non-regression` and `example:run-regression` are advanced
|
|
270
|
+
workflows.
|
|
271
|
+
|
|
266
272
|
| Command | Purpose | Needs API key |
|
|
267
273
|
|---|---|---|
|
|
268
274
|
| `npm run example:hello` | Minimal single-agent run | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
|
|
@@ -271,16 +277,20 @@ This repository also contains `aioc-inspect`, a private reference example UI for
|
|
|
271
277
|
| `npm run example:approval-evidence` | Approval evidence passed through context and reevaluated by policy | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
|
|
272
278
|
| `npm run example:tool-policy` | Straight tool + policy flow with allowed execution | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
|
|
273
279
|
| `npm run example:policy-composition` | Exact-name tool policy dispatch with fallback deny | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
|
|
280
|
+
| `npm run example:harness-rerun` | Replay against a modified harness with a mocked new tool output | Yes (`OPENAI_API_KEY`) |
|
|
274
281
|
| `npm run example:run-record` | Run-record persistence with redaction + audit | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
|
|
275
282
|
| `npm run example:rru:01-extract` | Minimal `extractToolCalls(...)` | No |
|
|
276
283
|
| `npm run example:rru:02-compare` | Minimal `compareRunRecords(...)` | No |
|
|
277
284
|
| `npm run example:rru:03-replay-strict` | Minimal strict replay | No |
|
|
278
285
|
| `npm run example:rru:04-replay-hybrid` | Minimal hybrid replay | No |
|
|
279
286
|
| `npm run example:non-regression` | Advanced v1/v2 run-record diff | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
|
|
287
|
+
| `npm run example:run-regression` | Local `runRegressionSuite(...)` over a modified harness | Yes (`OPENAI_API_KEY`) |
|
|
280
288
|
|
|
281
289
|
Notes:
|
|
282
290
|
|
|
283
291
|
- for live-provider examples, set `AIOC_EXAMPLE_PROVIDER` to `openai` or `mistral`; the matching API key must also be available
|
|
292
|
+
- `example:harness-rerun` and `example:run-regression` configure OpenAI directly from `OPENAI_API_KEY`; the model is declared in the YAML descriptor
|
|
293
|
+
- run-record utility examples are deterministic and do not need a provider
|
|
284
294
|
- `example:non-regression` is educational and can be non-deterministic because it uses a live provider.
|
|
285
295
|
- canonical examples guide: `docs/CANONICAL-EXAMPLES.md`.
|
|
286
296
|
|
package/dist/agent.d.ts
CHANGED
|
@@ -3,6 +3,12 @@ import type { OutputGuardrail } from "./guardrails";
|
|
|
3
3
|
import type { Tool } from "./tool";
|
|
4
4
|
import type { ModelSettings } from "./types";
|
|
5
5
|
export type AgentInstructions<TContext = unknown> = string | ((runContext: RunContext<TContext>) => string | Promise<string>);
|
|
6
|
+
export type AgentHandoffCondition<TContext = unknown> = (runContext: RunContext<TContext>) => boolean;
|
|
7
|
+
export interface AgentHandoff<TContext = unknown> {
|
|
8
|
+
agent: Agent<TContext>;
|
|
9
|
+
enabled?: AgentHandoffCondition<TContext>;
|
|
10
|
+
}
|
|
11
|
+
export type AgentHandoffEntry<TContext = unknown> = Agent<TContext> | AgentHandoff<TContext>;
|
|
6
12
|
export interface AgentConfiguration<TContext = unknown> {
|
|
7
13
|
name: string;
|
|
8
14
|
handoffDescription?: string;
|
|
@@ -11,7 +17,7 @@ export interface AgentConfiguration<TContext = unknown> {
|
|
|
11
17
|
model?: string;
|
|
12
18
|
modelSettings?: ModelSettings;
|
|
13
19
|
tools?: Tool<TContext>[];
|
|
14
|
-
handoffs?:
|
|
20
|
+
handoffs?: AgentHandoffEntry<TContext>[];
|
|
15
21
|
outputGuardrails?: OutputGuardrail<TContext>[];
|
|
16
22
|
}
|
|
17
23
|
export declare class Agent<TContext = unknown> {
|
|
@@ -23,6 +29,7 @@ export declare class Agent<TContext = unknown> {
|
|
|
23
29
|
modelSettings?: ModelSettings;
|
|
24
30
|
tools: Tool<TContext>[];
|
|
25
31
|
handoffs: Agent<TContext>[];
|
|
32
|
+
handoffRules: AgentHandoff<TContext>[];
|
|
26
33
|
outputGuardrails: OutputGuardrail<TContext>[];
|
|
27
34
|
constructor(config: AgentConfiguration<TContext>);
|
|
28
35
|
resolveInstructions(runContext: RunContext<TContext>): Promise<string | undefined>;
|
package/dist/agent.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,MAAM,MAAM,iBAAiB,CAAC,QAAQ,GAAG,OAAO,IAC5C,MAAM,GACN,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AAErE,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,OAAO;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,MAAM,MAAM,iBAAiB,CAAC,QAAQ,GAAG,OAAO,IAC5C,MAAM,GACN,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AAErE,MAAM,MAAM,qBAAqB,CAAC,QAAQ,GAAG,OAAO,IAAI,CACtD,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,KAC7B,OAAO,CAAC;AAEb,MAAM,WAAW,YAAY,CAAC,QAAQ,GAAG,OAAO;IAC9C,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,iBAAiB,CAAC,QAAQ,GAAG,OAAO,IAC5C,KAAK,CAAC,QAAQ,CAAC,GACf,YAAY,CAAC,QAAQ,CAAC,CAAC;AAE3B,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,OAAO;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzC,gBAAgB,CAAC,EAAE,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;CAChD;AAED,qBAAa,KAAK,CAAC,QAAQ,GAAG,OAAO;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvC,gBAAgB,EAAE,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAElC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC;IAe1C,mBAAmB,CACvB,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,GAC/B,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;CAS/B"}
|
package/dist/agent.js
CHANGED
|
@@ -10,6 +10,7 @@ class Agent {
|
|
|
10
10
|
modelSettings;
|
|
11
11
|
tools;
|
|
12
12
|
handoffs;
|
|
13
|
+
handoffRules;
|
|
13
14
|
outputGuardrails;
|
|
14
15
|
constructor(config) {
|
|
15
16
|
this.name = config.name;
|
|
@@ -19,7 +20,8 @@ class Agent {
|
|
|
19
20
|
this.model = config.model;
|
|
20
21
|
this.modelSettings = config.modelSettings;
|
|
21
22
|
this.tools = config.tools ?? [];
|
|
22
|
-
this.
|
|
23
|
+
this.handoffRules = (config.handoffs ?? []).map((handoff) => handoff instanceof Agent ? { agent: handoff } : handoff);
|
|
24
|
+
this.handoffs = this.handoffRules.map((rule) => rule.agent);
|
|
23
25
|
this.outputGuardrails = config.outputGuardrails ?? [];
|
|
24
26
|
}
|
|
25
27
|
async resolveInstructions(runContext) {
|
|
@@ -20,11 +20,17 @@ export interface HarnessAgentDescriptor {
|
|
|
20
20
|
model?: string;
|
|
21
21
|
modelSettings?: Record<string, unknown>;
|
|
22
22
|
tools?: string[];
|
|
23
|
-
handoffs?:
|
|
23
|
+
handoffs?: HarnessHandoffEntryDescriptor[];
|
|
24
24
|
}
|
|
25
|
-
export interface
|
|
25
|
+
export interface HarnessWhereDescriptor {
|
|
26
26
|
context: string;
|
|
27
27
|
}
|
|
28
|
+
export type HarnessInstructionWhereDescriptor = HarnessWhereDescriptor;
|
|
29
|
+
export interface HarnessHandoffDescriptor {
|
|
30
|
+
agent: string;
|
|
31
|
+
where?: HarnessWhereDescriptor;
|
|
32
|
+
}
|
|
33
|
+
export type HarnessHandoffEntryDescriptor = string | HarnessHandoffDescriptor;
|
|
28
34
|
export interface HarnessInstructionPartDescriptor {
|
|
29
35
|
text: string;
|
|
30
36
|
where?: HarnessInstructionWhereDescriptor;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harness-descriptor.d.ts","sourceRoot":"","sources":["../src/harness-descriptor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"harness-descriptor.d.ts","sourceRoot":"","sources":["../src/harness-descriptor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAA6C,MAAM,SAAS,CAAC;AAG3E,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEnD,MAAM,WAAW,yBAAyB;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,6BAA6B,CAAC;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,6BAA6B,EAAE,CAAC;CAC5C;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,iCAAiC,GAAG,sBAAsB,CAAC;AAEvE,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,sBAAsB,CAAC;CAChC;AAED,MAAM,MAAM,6BAA6B,GAAG,MAAM,GAAG,wBAAwB,CAAC;AAE9E,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,iCAAiC,CAAC;CAC3C;AAED,MAAM,MAAM,6BAA6B,GACrC,MAAM,GACN,gCAAgC,EAAE,CAAC;AAEvC,MAAM,WAAW,iCAAiC;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,4BAA4B,GACpC,OAAO,GACP,iCAAiC,CAAC;AAEtC,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;IACvD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,4BAA4B,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,sBAAsB;IACrC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,EAAE,yBAAyB,CAAC;IACrC,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAC9C,cAAc,CAAC,EAAE,IAAI,CACnB,sBAAsB,EACtB,OAAO,GAAG,eAAe,GAAG,cAAc,CAC3C,CAAC;IACF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,OAAO;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY,CAAC,QAAQ,GAAG,OAAO;IAC9C,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC1D,aAAa,CAAC,KAAK,CAAC,EAAE,yBAAyB,GAAG,QAAQ,CAAC;CAC5D;AAqoBD,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,sBAAsB,GACjC,MAAM,CAER;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,GAAG,OAAO,EAClD,UAAU,EAAE,sBAAsB,EAClC,QAAQ,GAAE,oBAAoB,CAAC,QAAQ,CAAM,GAC5C,YAAY,CAAC,QAAQ,CAAC,CAiFxB"}
|
|
@@ -30,6 +30,28 @@ function assertNonEmptyString(value, label) {
|
|
|
30
30
|
throw new Error(`${label} must be a non-empty string.`);
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
+
function assertHandoffEntry(value, label) {
|
|
34
|
+
if (typeof value === "string") {
|
|
35
|
+
assertNonEmptyString(value, label);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
if (!isPlainObject(value)) {
|
|
39
|
+
throw new Error(`${label} must be a non-empty string or an object.`);
|
|
40
|
+
}
|
|
41
|
+
assertNonEmptyString(value.agent, `${label}.agent`);
|
|
42
|
+
if (typeof value.where !== "undefined") {
|
|
43
|
+
assertInstructionWhere(value.where, label);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function assertHandoffEntries(value, label) {
|
|
47
|
+
if (typeof value === "undefined") {
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
if (!Array.isArray(value)) {
|
|
51
|
+
throw new Error(`${label} must be an array.`);
|
|
52
|
+
}
|
|
53
|
+
value.forEach((entry, index) => assertHandoffEntry(entry, `${label}[${index}]`));
|
|
54
|
+
}
|
|
33
55
|
function validateDescriptorShape(descriptor) {
|
|
34
56
|
assertPlainObject(descriptor, "Harness descriptor");
|
|
35
57
|
assertPlainObject(descriptor.runtime, "Harness descriptor runtime");
|
|
@@ -49,6 +71,7 @@ function validateDescriptorShape(descriptor) {
|
|
|
49
71
|
for (const [agentId, agentDescriptor] of Object.entries(descriptor.agents)) {
|
|
50
72
|
assertPlainObject(agentDescriptor, `Harness descriptor agent "${agentId}"`);
|
|
51
73
|
assertNoUnresolvedInstructionFile(agentDescriptor, `Harness descriptor agent "${agentId}"`);
|
|
74
|
+
assertHandoffEntries(agentDescriptor.handoffs, `Harness descriptor agent "${agentId}".handoffs`);
|
|
52
75
|
}
|
|
53
76
|
for (const [toolId, toolDescriptor] of Object.entries(descriptor.tools ?? {})) {
|
|
54
77
|
assertPlainObject(toolDescriptor, `Harness descriptor tool "${toolId}"`);
|
|
@@ -202,31 +225,37 @@ function assertDeclaredPromptPaths(agentId, paths, promptAccessRules) {
|
|
|
202
225
|
}
|
|
203
226
|
}
|
|
204
227
|
}
|
|
228
|
+
function assertDeclaredBooleanContextPath(label, path, promptAccessRules) {
|
|
229
|
+
const rule = promptAccessRules.get(path);
|
|
230
|
+
if (!rule) {
|
|
231
|
+
throw new Error(`${label} references undeclared context path "${path}".`);
|
|
232
|
+
}
|
|
233
|
+
if (rule.type !== "boolean") {
|
|
234
|
+
throw new Error(`${label} context path "${path}" must be declared with type "boolean".`);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
205
237
|
function assertDeclaredBooleanWherePaths(agentId, paths, promptAccessRules) {
|
|
206
238
|
for (const path of paths) {
|
|
207
|
-
|
|
208
|
-
if (!rule) {
|
|
209
|
-
throw new Error(`Agent "${agentId}" instruction where references undeclared context path "${path}".`);
|
|
210
|
-
}
|
|
211
|
-
if (rule.type !== "boolean") {
|
|
212
|
-
throw new Error(`Agent "${agentId}" instruction where context path "${path}" must be declared with type "boolean".`);
|
|
213
|
-
}
|
|
239
|
+
assertDeclaredBooleanContextPath(`Agent "${agentId}" instruction where`, path, promptAccessRules);
|
|
214
240
|
}
|
|
215
241
|
}
|
|
216
|
-
function
|
|
217
|
-
if (!
|
|
242
|
+
function shouldIncludeWhere(where, context, label) {
|
|
243
|
+
if (!where) {
|
|
218
244
|
return true;
|
|
219
245
|
}
|
|
220
|
-
const path = normalizePromptPath(
|
|
246
|
+
const path = normalizePromptPath(where.context, "Instruction where.context path");
|
|
221
247
|
const resolved = readWhereContextValue(context, path);
|
|
222
248
|
if (!resolved.exists || typeof resolved.value === "undefined") {
|
|
223
|
-
throw new Error(
|
|
249
|
+
throw new Error(`${label} could not resolve context path "${path}".`);
|
|
224
250
|
}
|
|
225
251
|
if (typeof resolved.value !== "boolean") {
|
|
226
|
-
throw new Error(
|
|
252
|
+
throw new Error(`${label} context path "${path}" must resolve to a boolean.`);
|
|
227
253
|
}
|
|
228
254
|
return resolved.value === true;
|
|
229
255
|
}
|
|
256
|
+
function shouldIncludeInstructionPart(part, context) {
|
|
257
|
+
return shouldIncludeWhere(part.where, context, "Harness descriptor instruction where");
|
|
258
|
+
}
|
|
230
259
|
function renderInstructionParts(instructions, context, promptAccessRules) {
|
|
231
260
|
const parts = [];
|
|
232
261
|
for (const part of instructions) {
|
|
@@ -323,6 +352,22 @@ function resolveTool(toolId, descriptor, registry) {
|
|
|
323
352
|
}
|
|
324
353
|
return toolDefinition;
|
|
325
354
|
}
|
|
355
|
+
function readHandoffEntry(entry) {
|
|
356
|
+
if (typeof entry === "string") {
|
|
357
|
+
return {
|
|
358
|
+
agent: entry,
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
return entry;
|
|
362
|
+
}
|
|
363
|
+
function compileHandoffCondition(fromAgentId, toAgentId, where, promptAccessRules) {
|
|
364
|
+
if (!where) {
|
|
365
|
+
return undefined;
|
|
366
|
+
}
|
|
367
|
+
const path = normalizePromptPath(where.context, `Agent "${fromAgentId}" handoff "${toAgentId}" where.context path`);
|
|
368
|
+
assertDeclaredBooleanContextPath(`Agent "${fromAgentId}" handoff "${toAgentId}" where`, path, promptAccessRules);
|
|
369
|
+
return (runContext) => shouldIncludeWhere(where, runContext.context, `Harness descriptor handoff "${fromAgentId}" to "${toAgentId}" where`);
|
|
370
|
+
}
|
|
326
371
|
function hashAgentHarnessDescriptor(descriptor) {
|
|
327
372
|
return `sha256:${(0, canonical_json_1.hashCanonicalJsonValue)(descriptor)}`;
|
|
328
373
|
}
|
|
@@ -350,13 +395,20 @@ function buildAgentHarness(descriptor, registry = {}) {
|
|
|
350
395
|
if (!agent) {
|
|
351
396
|
continue;
|
|
352
397
|
}
|
|
353
|
-
|
|
398
|
+
const handoffRules = (agentDescriptor.handoffs ?? []).map((entry) => {
|
|
399
|
+
const handoff = readHandoffEntry(entry);
|
|
400
|
+
const handoffId = handoff.agent;
|
|
354
401
|
const handoffAgent = agents.get(handoffId);
|
|
355
402
|
if (!handoffAgent) {
|
|
356
403
|
throw new Error(`Harness descriptor references unknown handoff agent "${handoffId}" from "${agentId}".`);
|
|
357
404
|
}
|
|
358
|
-
return
|
|
405
|
+
return {
|
|
406
|
+
agent: handoffAgent,
|
|
407
|
+
enabled: compileHandoffCondition(agentId, handoffId, handoff.where, promptAccessRules),
|
|
408
|
+
};
|
|
359
409
|
});
|
|
410
|
+
agent.handoffRules = handoffRules;
|
|
411
|
+
agent.handoffs = handoffRules.map((handoff) => handoff.agent);
|
|
360
412
|
}
|
|
361
413
|
const entryAgent = agents.get(descriptor.runtime.entry_agent);
|
|
362
414
|
if (!entryAgent) {
|
package/dist/index.d.ts
CHANGED
|
@@ -18,6 +18,8 @@ export * from "./providers/mistral";
|
|
|
18
18
|
export * from "./providers/openai";
|
|
19
19
|
export * from "./run";
|
|
20
20
|
export * from "./run-output-events";
|
|
21
|
+
export * from "./run-regression";
|
|
22
|
+
export * from "./run-regression-summary";
|
|
21
23
|
export * from "./run-record";
|
|
22
24
|
export * from "./run-record-utils";
|
|
23
25
|
export * from "./run-context";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,OAAO,CAAC;AACtB,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,QAAQ,CAAC;AACvB,cAAc,SAAS,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,OAAO,CAAC;AACtB,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,QAAQ,CAAC;AACvB,cAAc,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -34,6 +34,8 @@ __exportStar(require("./providers/mistral"), exports);
|
|
|
34
34
|
__exportStar(require("./providers/openai"), exports);
|
|
35
35
|
__exportStar(require("./run"), exports);
|
|
36
36
|
__exportStar(require("./run-output-events"), exports);
|
|
37
|
+
__exportStar(require("./run-regression"), exports);
|
|
38
|
+
__exportStar(require("./run-regression-summary"), exports);
|
|
37
39
|
__exportStar(require("./run-record"), exports);
|
|
38
40
|
__exportStar(require("./run-record-utils"), exports);
|
|
39
41
|
__exportStar(require("./run-context"), exports);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { RunRegressionResult, RunRegressionStatus } from "./run-regression";
|
|
2
|
+
export interface RunRegressionCaseSummary {
|
|
3
|
+
name: string;
|
|
4
|
+
status: RunRegressionStatus;
|
|
5
|
+
baselineRunId: string;
|
|
6
|
+
candidateRunId: string;
|
|
7
|
+
signals: {
|
|
8
|
+
statusChanged: boolean;
|
|
9
|
+
toolsChanged: boolean;
|
|
10
|
+
policyChanged: boolean;
|
|
11
|
+
finalOutputChanged: boolean;
|
|
12
|
+
};
|
|
13
|
+
judge?: {
|
|
14
|
+
verdict: RunRegressionStatus;
|
|
15
|
+
summary: string;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
export interface RunRegressionSummary {
|
|
19
|
+
suite?: string;
|
|
20
|
+
status: RunRegressionStatus;
|
|
21
|
+
totals: {
|
|
22
|
+
cases: number;
|
|
23
|
+
passed: number;
|
|
24
|
+
warned: number;
|
|
25
|
+
failed: number;
|
|
26
|
+
};
|
|
27
|
+
cases: RunRegressionCaseSummary[];
|
|
28
|
+
}
|
|
29
|
+
export interface SummarizeRunRegressionResultsOptions<TContext = unknown> {
|
|
30
|
+
suite?: string;
|
|
31
|
+
classifyCase?: (result: RunRegressionResult<TContext>) => RunRegressionStatus;
|
|
32
|
+
}
|
|
33
|
+
export declare function summarizeRunRegressionResults<TContext = unknown>(results: Array<RunRegressionResult<TContext>>, options?: SummarizeRunRegressionResultsOptions<TContext>): RunRegressionSummary;
|
|
34
|
+
//# sourceMappingURL=run-regression-summary.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-regression-summary.d.ts","sourceRoot":"","sources":["../src/run-regression-summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EACnB,mBAAmB,EACpB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,mBAAmB,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE;QACP,aAAa,EAAE,OAAO,CAAC;QACvB,YAAY,EAAE,OAAO,CAAC;QACtB,aAAa,EAAE,OAAO,CAAC;QACvB,kBAAkB,EAAE,OAAO,CAAC;KAC7B,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,mBAAmB,CAAC;QAC7B,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,mBAAmB,CAAC;IAC5B,MAAM,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,KAAK,EAAE,wBAAwB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,oCAAoC,CAAC,QAAQ,GAAG,OAAO;IACtE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,mBAAmB,CAAC,QAAQ,CAAC,KAAK,mBAAmB,CAAC;CAC/E;AA2DD,wBAAgB,6BAA6B,CAAC,QAAQ,GAAG,OAAO,EAC9D,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,EAC7C,OAAO,GAAE,oCAAoC,CAAC,QAAQ,CAAM,GAC3D,oBAAoB,CA2BtB"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.summarizeRunRegressionResults = summarizeRunRegressionResults;
|
|
4
|
+
function defaultClassifyRunRegressionCase(result) {
|
|
5
|
+
if (result.candidate.status === "failed" ||
|
|
6
|
+
result.judge?.verdict === "fail") {
|
|
7
|
+
return "fail";
|
|
8
|
+
}
|
|
9
|
+
if (!result.comparison.equal || result.judge?.verdict === "warn") {
|
|
10
|
+
return "warn";
|
|
11
|
+
}
|
|
12
|
+
return "pass";
|
|
13
|
+
}
|
|
14
|
+
function summarizeCase(result, status) {
|
|
15
|
+
const judge = result.judge
|
|
16
|
+
? {
|
|
17
|
+
verdict: result.judge.verdict,
|
|
18
|
+
summary: result.judge.summary,
|
|
19
|
+
}
|
|
20
|
+
: undefined;
|
|
21
|
+
return {
|
|
22
|
+
name: result.name,
|
|
23
|
+
status,
|
|
24
|
+
baselineRunId: result.baseline.runId,
|
|
25
|
+
candidateRunId: result.candidate.runId,
|
|
26
|
+
signals: {
|
|
27
|
+
statusChanged: result.baseline.status !== result.candidate.status,
|
|
28
|
+
toolsChanged: !result.comparison.summary.sameToolCallShape,
|
|
29
|
+
policyChanged: !result.comparison.summary.samePolicyDecisions,
|
|
30
|
+
finalOutputChanged: !result.comparison.summary.sameFinalResponse,
|
|
31
|
+
},
|
|
32
|
+
...(judge ? { judge } : {}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function toSuiteStatus(totals) {
|
|
36
|
+
if (totals.failed > 0) {
|
|
37
|
+
return "fail";
|
|
38
|
+
}
|
|
39
|
+
if (totals.warned > 0) {
|
|
40
|
+
return "warn";
|
|
41
|
+
}
|
|
42
|
+
return "pass";
|
|
43
|
+
}
|
|
44
|
+
function summarizeRunRegressionResults(results, options = {}) {
|
|
45
|
+
const classifyCase = options.classifyCase ?? (defaultClassifyRunRegressionCase);
|
|
46
|
+
const totals = {
|
|
47
|
+
cases: results.length,
|
|
48
|
+
passed: 0,
|
|
49
|
+
warned: 0,
|
|
50
|
+
failed: 0,
|
|
51
|
+
};
|
|
52
|
+
const cases = results.map((result) => {
|
|
53
|
+
const status = classifyCase(result);
|
|
54
|
+
if (status === "pass") {
|
|
55
|
+
totals.passed += 1;
|
|
56
|
+
}
|
|
57
|
+
else if (status === "warn") {
|
|
58
|
+
totals.warned += 1;
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
totals.failed += 1;
|
|
62
|
+
}
|
|
63
|
+
return summarizeCase(result, status);
|
|
64
|
+
});
|
|
65
|
+
return {
|
|
66
|
+
...(typeof options.suite === "undefined" ? {} : { suite: options.suite }),
|
|
67
|
+
status: toSuiteStatus(totals),
|
|
68
|
+
totals,
|
|
69
|
+
cases,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { RunRecord } from "./run-record";
|
|
2
|
+
import { type RunRegressionSummary, type SummarizeRunRegressionResultsOptions } from "./run-regression-summary";
|
|
3
|
+
import { type CompareRunRecordsOptions, type ReplayFromRunRecordInput, type RunRecordComparison } from "./run-record-utils";
|
|
4
|
+
export type RunRegressionStatus = "pass" | "warn" | "fail";
|
|
5
|
+
export interface RunRegressionExpectation {
|
|
6
|
+
intent?: string;
|
|
7
|
+
shouldUseTools?: string[];
|
|
8
|
+
shouldAvoidTools?: string[];
|
|
9
|
+
shouldPreserve?: string[];
|
|
10
|
+
shouldImprove?: string[];
|
|
11
|
+
notes?: string;
|
|
12
|
+
}
|
|
13
|
+
export interface RunJudgeFinding {
|
|
14
|
+
severity: "info" | "warn" | "error";
|
|
15
|
+
reason: string;
|
|
16
|
+
evidence?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface RunJudgeResult {
|
|
19
|
+
verdict: RunRegressionStatus;
|
|
20
|
+
summary: string;
|
|
21
|
+
findings: RunJudgeFinding[];
|
|
22
|
+
score?: number;
|
|
23
|
+
judgeModel?: string;
|
|
24
|
+
judgePromptVersion?: string;
|
|
25
|
+
}
|
|
26
|
+
export interface RunJudgeInput<TContext = unknown, TDescriptor = unknown> {
|
|
27
|
+
baseline: RunRecord<TContext>;
|
|
28
|
+
candidate: RunRecord<TContext>;
|
|
29
|
+
comparison: RunRecordComparison;
|
|
30
|
+
expectation?: RunRegressionExpectation;
|
|
31
|
+
baselineDescriptor?: TDescriptor;
|
|
32
|
+
candidateDescriptor?: TDescriptor;
|
|
33
|
+
}
|
|
34
|
+
export type RunJudge<TContext = unknown, TDescriptor = unknown> = (input: RunJudgeInput<TContext, TDescriptor>) => Promise<RunJudgeResult> | RunJudgeResult;
|
|
35
|
+
export interface RunRegressionResult<TContext = unknown> {
|
|
36
|
+
name: string;
|
|
37
|
+
baseline: RunRecord<TContext>;
|
|
38
|
+
candidate: RunRecord<TContext>;
|
|
39
|
+
comparison: RunRecordComparison;
|
|
40
|
+
judge?: RunJudgeResult;
|
|
41
|
+
}
|
|
42
|
+
export interface RunRegressionSuiteCase<TContext = unknown> {
|
|
43
|
+
name?: string;
|
|
44
|
+
baseline: RunRecord<TContext>;
|
|
45
|
+
}
|
|
46
|
+
export interface RunRegressionSuite<TContext = unknown> {
|
|
47
|
+
name?: string;
|
|
48
|
+
expectation?: RunRegressionExpectation;
|
|
49
|
+
cases: Array<RunRegressionSuiteCase<TContext>>;
|
|
50
|
+
}
|
|
51
|
+
export interface RunRegressionCaseInput<TContext = unknown> extends Omit<ReplayFromRunRecordInput<TContext>, "sourceRunRecord"> {
|
|
52
|
+
name?: string;
|
|
53
|
+
baseline: RunRecord<TContext>;
|
|
54
|
+
comparisonOptions?: CompareRunRecordsOptions;
|
|
55
|
+
}
|
|
56
|
+
export interface RunRegressionSuiteInput<TContext = unknown, TDescriptor = unknown> extends Omit<RunRegressionCaseInput<TContext>, "baseline" | "name"> {
|
|
57
|
+
suite: RunRegressionSuite<TContext>;
|
|
58
|
+
judge?: RunJudge<TContext, TDescriptor>;
|
|
59
|
+
baselineDescriptor?: TDescriptor;
|
|
60
|
+
candidateDescriptor?: TDescriptor;
|
|
61
|
+
summaryOptions?: Omit<SummarizeRunRegressionResultsOptions<TContext>, "suite">;
|
|
62
|
+
}
|
|
63
|
+
export interface RunRegressionSuiteResult<TContext = unknown> {
|
|
64
|
+
name?: string;
|
|
65
|
+
expectation?: RunRegressionExpectation;
|
|
66
|
+
results: Array<RunRegressionResult<TContext>>;
|
|
67
|
+
summary: RunRegressionSummary;
|
|
68
|
+
}
|
|
69
|
+
export declare function runRegressionCase<TContext = unknown>(input: RunRegressionCaseInput<TContext>): Promise<RunRegressionResult<TContext>>;
|
|
70
|
+
export declare function runRegressionSuite<TContext = unknown, TDescriptor = unknown>(input: RunRegressionSuiteInput<TContext, TDescriptor>): Promise<RunRegressionSuiteResult<TContext>>;
|
|
71
|
+
//# sourceMappingURL=run-regression.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-regression.d.ts","sourceRoot":"","sources":["../src/run-regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,oCAAoC,EAC1C,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAGL,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAC7B,KAAK,mBAAmB,EACzB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,mBAAmB,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAE3D,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,mBAAmB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa,CAAC,QAAQ,GAAG,OAAO,EAAE,WAAW,GAAG,OAAO;IACtE,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC/B,UAAU,EAAE,mBAAmB,CAAC;IAChC,WAAW,CAAC,EAAE,wBAAwB,CAAC;IACvC,kBAAkB,CAAC,EAAE,WAAW,CAAC;IACjC,mBAAmB,CAAC,EAAE,WAAW,CAAC;CACnC;AAED,MAAM,MAAM,QAAQ,CAAC,QAAQ,GAAG,OAAO,EAAE,WAAW,GAAG,OAAO,IAAI,CAChE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,WAAW,CAAC,KACxC,OAAO,CAAC,cAAc,CAAC,GAAG,cAAc,CAAC;AAE9C,MAAM,WAAW,mBAAmB,CAAC,QAAQ,GAAG,OAAO;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC/B,UAAU,EAAE,mBAAmB,CAAC;IAChC,KAAK,CAAC,EAAE,cAAc,CAAC;CACxB;AAED,MAAM,WAAW,sBAAsB,CAAC,QAAQ,GAAG,OAAO;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,OAAO;IACpD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,wBAAwB,CAAC;IACvC,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,sBAAsB,CAAC,QAAQ,GAAG,OAAO,CAAE,SAAQ,IAAI,CACtE,wBAAwB,CAAC,QAAQ,CAAC,EAClC,iBAAiB,CAClB;IACC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,iBAAiB,CAAC,EAAE,wBAAwB,CAAC;CAC9C;AAED,MAAM,WAAW,uBAAuB,CACtC,QAAQ,GAAG,OAAO,EAClB,WAAW,GAAG,OAAO,CACrB,SAAQ,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;IACnE,KAAK,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACpC,KAAK,CAAC,EAAE,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACxC,kBAAkB,CAAC,EAAE,WAAW,CAAC;IACjC,mBAAmB,CAAC,EAAE,WAAW,CAAC;IAClC,cAAc,CAAC,EAAE,IAAI,CACnB,oCAAoC,CAAC,QAAQ,CAAC,EAC9C,OAAO,CACR,CAAC;CACH;AAED,MAAM,WAAW,wBAAwB,CAAC,QAAQ,GAAG,OAAO;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,wBAAwB,CAAC;IACvC,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9C,OAAO,EAAE,oBAAoB,CAAC;CAC/B;AA0BD,wBAAsB,iBAAiB,CAAC,QAAQ,GAAG,OAAO,EACxD,KAAK,EAAE,sBAAsB,CAAC,QAAQ,CAAC,GACtC,OAAO,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CA6BxC;AAiBD,wBAAsB,kBAAkB,CACtC,QAAQ,GAAG,OAAO,EAClB,WAAW,GAAG,OAAO,EAErB,KAAK,EAAE,uBAAuB,CAAC,QAAQ,EAAE,WAAW,CAAC,GACpD,OAAO,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC,CAmD7C"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runRegressionCase = runRegressionCase;
|
|
4
|
+
exports.runRegressionSuite = runRegressionSuite;
|
|
5
|
+
const run_regression_summary_1 = require("./run-regression-summary");
|
|
6
|
+
const run_record_utils_1 = require("./run-record-utils");
|
|
7
|
+
function toRegressionCaseName(input) {
|
|
8
|
+
return input.name ?? input.baseline.runId;
|
|
9
|
+
}
|
|
10
|
+
function toReplayRunOptions(input) {
|
|
11
|
+
return {
|
|
12
|
+
...(input.runOptions ?? {}),
|
|
13
|
+
record: input.runOptions?.record ?? {},
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function toReplayAgentFields(input) {
|
|
17
|
+
return {
|
|
18
|
+
...(input.agent ? { agent: input.agent } : {}),
|
|
19
|
+
...(input.agentFactory ? { agentFactory: input.agentFactory } : {}),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
async function runRegressionCase(input) {
|
|
23
|
+
const replay = await (0, run_record_utils_1.replayFromRunRecord)({
|
|
24
|
+
sourceRunRecord: input.baseline,
|
|
25
|
+
...toReplayAgentFields(input),
|
|
26
|
+
mode: input.mode,
|
|
27
|
+
runOptions: toReplayRunOptions(input),
|
|
28
|
+
inputMode: input.inputMode,
|
|
29
|
+
metadataOverrides: input.metadataOverrides,
|
|
30
|
+
onMissingToolCall: input.onMissingToolCall,
|
|
31
|
+
});
|
|
32
|
+
if (!replay.replayRunRecord) {
|
|
33
|
+
throw new Error("runRegressionCase expected replayFromRunRecord to produce a candidate RunRecord.");
|
|
34
|
+
}
|
|
35
|
+
const comparison = (0, run_record_utils_1.compareRunRecords)(input.baseline, replay.replayRunRecord, input.comparisonOptions);
|
|
36
|
+
return {
|
|
37
|
+
name: toRegressionCaseName(input),
|
|
38
|
+
baseline: input.baseline,
|
|
39
|
+
candidate: replay.replayRunRecord,
|
|
40
|
+
comparison,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function resolveSuiteCase(caseDefinition) {
|
|
44
|
+
const baseline = caseDefinition.baseline;
|
|
45
|
+
const name = caseDefinition.name ?? baseline.runId;
|
|
46
|
+
return {
|
|
47
|
+
name,
|
|
48
|
+
baseline,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
async function runRegressionSuite(input) {
|
|
52
|
+
const results = [];
|
|
53
|
+
const { suite, judge, baselineDescriptor, candidateDescriptor, summaryOptions, ...caseOptions } = input;
|
|
54
|
+
for (const caseDefinition of suite.cases) {
|
|
55
|
+
const regressionCase = resolveSuiteCase(caseDefinition);
|
|
56
|
+
const result = await runRegressionCase({
|
|
57
|
+
...caseOptions,
|
|
58
|
+
name: regressionCase.name,
|
|
59
|
+
baseline: regressionCase.baseline,
|
|
60
|
+
});
|
|
61
|
+
const judgeResult = judge
|
|
62
|
+
? await judge({
|
|
63
|
+
baseline: result.baseline,
|
|
64
|
+
candidate: result.candidate,
|
|
65
|
+
comparison: result.comparison,
|
|
66
|
+
...(typeof suite.expectation === "undefined"
|
|
67
|
+
? {}
|
|
68
|
+
: { expectation: suite.expectation }),
|
|
69
|
+
...(typeof baselineDescriptor === "undefined"
|
|
70
|
+
? {}
|
|
71
|
+
: { baselineDescriptor }),
|
|
72
|
+
...(typeof candidateDescriptor === "undefined"
|
|
73
|
+
? {}
|
|
74
|
+
: { candidateDescriptor }),
|
|
75
|
+
})
|
|
76
|
+
: undefined;
|
|
77
|
+
results.push(judgeResult ? { ...result, judge: judgeResult } : result);
|
|
78
|
+
}
|
|
79
|
+
const summary = (0, run_regression_summary_1.summarizeRunRegressionResults)(results, {
|
|
80
|
+
suite: suite.name,
|
|
81
|
+
...(summaryOptions ?? {}),
|
|
82
|
+
});
|
|
83
|
+
return {
|
|
84
|
+
...(typeof suite.name === "undefined" ? {} : { name: suite.name }),
|
|
85
|
+
...(typeof suite.expectation === "undefined"
|
|
86
|
+
? {}
|
|
87
|
+
: { expectation: suite.expectation }),
|
|
88
|
+
results,
|
|
89
|
+
summary,
|
|
90
|
+
};
|
|
91
|
+
}
|
package/dist/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAqB,MAAM,SAAS,CAAC;AAsCnD,OAAO,EACL,cAAc,EAEd,mBAAmB,EAGnB,SAAS,EACT,cAAc,EACd,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAUjB,KAAK,eAAe,CAAC,QAAQ,IAAI;IAC/B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CAC5B,CAAC;AAGF,YAAY,EACV,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,mBAAmB,CAAC;AAmxB3B,qBAAa,iBAAiB,CAAC,QAAQ,GAAG,OAAO;IAC/C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAA0C;IACxD,OAAO,CAAC,KAAK,CAA4B;gBAGvC,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,EAC/C,KAAK,EAAE,eAAe,CAAC,QAAQ,CAAC;IAMlC,QAAQ,IAAI,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAQnD,IAAI,OAAO,IAAI,cAAc,EAAE,CAE9B;IAED,IAAI,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,CAE/B;CACF;AAED,wBAAsB,GAAG,CAAC,QAAQ,GAAG,OAAO,EAC1C,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,EAC9B,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,EAChC,OAAO,EAAE,gBAAgB,CAAC,QAAQ,CAAC,GAClC,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAExC,wBAAsB,GAAG,CAAC,QAAQ,GAAG,OAAO,EAC1C,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,EAC9B,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,EAChC,OAAO,CAAC,EAAE,mBAAmB,CAAC,QAAQ,CAAC,GACtC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC"}
|
package/dist/run.js
CHANGED
|
@@ -62,11 +62,16 @@ function sanitizeToolSegment(input) {
|
|
|
62
62
|
.replace(/^_+|_+$/g, "");
|
|
63
63
|
return sanitized || "agent";
|
|
64
64
|
}
|
|
65
|
-
function buildTurnTools(agent) {
|
|
65
|
+
function buildTurnTools(agent, runContext) {
|
|
66
66
|
const handoffRegistry = new Map();
|
|
67
67
|
const handoffTools = [];
|
|
68
68
|
const reservedNames = new Set(agent.tools.map((tool) => tool.name));
|
|
69
|
-
|
|
69
|
+
const handoffRules = agent.handoffRules;
|
|
70
|
+
for (const handoffRule of handoffRules) {
|
|
71
|
+
if (handoffRule.enabled && !handoffRule.enabled(runContext)) {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
const handoffAgent = handoffRule.agent;
|
|
70
75
|
const baseName = `handoff_to_${sanitizeToolSegment(handoffAgent.name)}`;
|
|
71
76
|
let toolName = baseName;
|
|
72
77
|
let suffix = 2;
|
|
@@ -331,7 +336,7 @@ async function* runLoop(state, runId, provider, runContext, maxTurns, logger, po
|
|
|
331
336
|
activeTurn = turn + 1;
|
|
332
337
|
const currentAgent = state.lastAgent;
|
|
333
338
|
await logEmitter.turnStarted(currentAgent.name, activeTurn);
|
|
334
|
-
const { providerTools, handoffRegistry } = buildTurnTools(currentAgent);
|
|
339
|
+
const { providerTools, handoffRegistry } = buildTurnTools(currentAgent, runContext);
|
|
335
340
|
const model = resolveAgentModel(currentAgent);
|
|
336
341
|
const systemPrompt = await currentAgent.resolveInstructions(runContext);
|
|
337
342
|
onPromptSnapshot?.({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@axiastudio/aioc",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.6",
|
|
4
4
|
"main": "dist/index.js",
|
|
5
5
|
"types": "dist/index.d.ts",
|
|
6
6
|
"files": [
|
|
@@ -27,12 +27,14 @@
|
|
|
27
27
|
"example:tool-policy": "tsx src/examples/basic/tools.ts",
|
|
28
28
|
"example:policy-composition": "tsx src/examples/basic/policy-composition.ts",
|
|
29
29
|
"example:harness": "tsx src/examples/harness-descriptor/customer-support.ts",
|
|
30
|
+
"example:harness-rerun": "tsx src/examples/harness-descriptor/rerun-modified-harness.ts",
|
|
30
31
|
"example:run-record": "tsx src/examples/basic/run-record-sink.ts",
|
|
31
32
|
"example:rru:01-extract": "tsx src/examples/run-record-utils-minimal/01-extract-tool-calls.ts",
|
|
32
33
|
"example:rru:02-compare": "tsx src/examples/run-record-utils-minimal/02-compare-run-records.ts",
|
|
33
34
|
"example:rru:03-replay-strict": "tsx src/examples/run-record-utils-minimal/03-replay-strict.ts",
|
|
34
35
|
"example:rru:04-replay-hybrid": "tsx src/examples/run-record-utils-minimal/04-replay-hybrid.ts",
|
|
35
36
|
"example:non-regression": "tsx src/examples/non-regression/v1-v2-runrecord-diff.ts",
|
|
37
|
+
"example:run-regression": "tsx src/examples/run-regression/age-adapted-suite.ts",
|
|
36
38
|
"inspect:samples": "tsx scripts/generate-aioc-inspect-samples.ts",
|
|
37
39
|
"docs:sync": "node scripts/sync-aioc-docs.mjs",
|
|
38
40
|
"docs:dev": "npm run docs:sync && cd apps/aioc-docs && zsh -lc 'npm run dev'",
|