@axiastudio/aioc 0.2.5 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -39,6 +39,7 @@ AIOC `0.2.2` completes the descriptor instruction-composition surface with reusa
39
39
  AIOC `0.2.3` adds RFC-0010 policy composition helpers for exact-name tool and handoff policy dispatch without changing runtime enforcement semantics.
40
40
  AIOC `0.2.4` adds experimental governance-event packages and an OpenTelemetry Logs exporter for reduced, operational observability events derived from `RunRecord` values.
41
41
  AIOC `0.2.5` makes replay history-faithful by recording the initial input scope in `RunRecord` and replaying from it by default.
42
+ AIOC `0.2.6` adds descriptor-level conditional agent handoffs with boolean `where` gates that filter handoff tools before provider requests.
42
43
 
43
44
  ### Experimental Packages
44
45
 
@@ -146,7 +147,7 @@ console.log(result.finalOutput);
146
147
  - `loadAgentHarnessDescriptorFromFile(...)`
147
148
  - `AgentHarnessDescriptor` and related descriptor types
148
149
 
149
- The Agent Harness Descriptor is included in the supported `0.2.x` API surface as the `aioc.agent_graph.v0` descriptor contract. Use it for controlled configuration, examples, and evaluation harnesses; keep executable tools, policies, providers, persistence, approvals, and deployment configuration in application code.
150
+ The Agent Harness Descriptor is included in the supported `0.2.x` API surface as the `aioc.agent_graph.v0` descriptor contract. Use it for controlled configuration, examples, and evaluation harnesses; keep executable tools, policies, providers, persistence, approvals, and deployment configuration in application code. Descriptor handoffs may use boolean `where` gates to hide unavailable handoff tools before the provider call; `HandoffPolicy` still owns allow/deny/approval decisions for exposed handoffs.
150
151
 
151
152
  ## Policy Gate (Minimal)
152
153
 
@@ -263,6 +264,11 @@ This repository also contains `aioc-inspect`, a private reference example UI for
263
264
 
264
265
  ## Examples
265
266
 
267
+ Suggested reading path: start with `example:hello`, then policy, approval,
268
+ tool-policy, and policy-composition. Use run-record and harness examples after
269
+ the basics; `example:non-regression` and `example:run-regression` are advanced
270
+ workflows.
271
+
266
272
  | Command | Purpose | Needs API key |
267
273
  |---|---|---|
268
274
  | `npm run example:hello` | Minimal single-agent run | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
@@ -271,16 +277,20 @@ This repository also contains `aioc-inspect`, a private reference example UI for
271
277
  | `npm run example:approval-evidence` | Approval evidence passed through context and reevaluated by policy | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
272
278
  | `npm run example:tool-policy` | Straight tool + policy flow with allowed execution | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
273
279
  | `npm run example:policy-composition` | Exact-name tool policy dispatch with fallback deny | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
280
+ | `npm run example:harness-rerun` | Replay against a modified harness with a mocked new tool output | Yes (`OPENAI_API_KEY`) |
274
281
  | `npm run example:run-record` | Run-record persistence with redaction + audit | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
275
282
  | `npm run example:rru:01-extract` | Minimal `extractToolCalls(...)` | No |
276
283
  | `npm run example:rru:02-compare` | Minimal `compareRunRecords(...)` | No |
277
284
  | `npm run example:rru:03-replay-strict` | Minimal strict replay | No |
278
285
  | `npm run example:rru:04-replay-hybrid` | Minimal hybrid replay | No |
279
286
  | `npm run example:non-regression` | Advanced v1/v2 run-record diff | Yes (`AIOC_EXAMPLE_PROVIDER` + matching provider API key) |
287
+ | `npm run example:run-regression` | Local `runRegressionSuite(...)` over a modified harness | Yes (`OPENAI_API_KEY`) |
280
288
 
281
289
  Notes:
282
290
 
283
291
  - for live-provider examples, set `AIOC_EXAMPLE_PROVIDER` to `openai` or `mistral`; the matching API key must also be available
292
+ - `example:harness-rerun` and `example:run-regression` configure OpenAI directly from `OPENAI_API_KEY`; the model is declared in the YAML descriptor
293
+ - run-record utility examples are deterministic and do not need a provider
284
294
  - `example:non-regression` is educational and can be non-deterministic because it uses a live provider.
285
295
  - canonical examples guide: `docs/CANONICAL-EXAMPLES.md`.
286
296
 
package/dist/agent.d.ts CHANGED
@@ -3,6 +3,12 @@ import type { OutputGuardrail } from "./guardrails";
3
3
  import type { Tool } from "./tool";
4
4
  import type { ModelSettings } from "./types";
5
5
  export type AgentInstructions<TContext = unknown> = string | ((runContext: RunContext<TContext>) => string | Promise<string>);
6
+ export type AgentHandoffCondition<TContext = unknown> = (runContext: RunContext<TContext>) => boolean;
7
+ export interface AgentHandoff<TContext = unknown> {
8
+ agent: Agent<TContext>;
9
+ enabled?: AgentHandoffCondition<TContext>;
10
+ }
11
+ export type AgentHandoffEntry<TContext = unknown> = Agent<TContext> | AgentHandoff<TContext>;
6
12
  export interface AgentConfiguration<TContext = unknown> {
7
13
  name: string;
8
14
  handoffDescription?: string;
@@ -11,7 +17,7 @@ export interface AgentConfiguration<TContext = unknown> {
11
17
  model?: string;
12
18
  modelSettings?: ModelSettings;
13
19
  tools?: Tool<TContext>[];
14
- handoffs?: Agent<TContext>[];
20
+ handoffs?: AgentHandoffEntry<TContext>[];
15
21
  outputGuardrails?: OutputGuardrail<TContext>[];
16
22
  }
17
23
  export declare class Agent<TContext = unknown> {
@@ -23,6 +29,7 @@ export declare class Agent<TContext = unknown> {
23
29
  modelSettings?: ModelSettings;
24
30
  tools: Tool<TContext>[];
25
31
  handoffs: Agent<TContext>[];
32
+ handoffRules: AgentHandoff<TContext>[];
26
33
  outputGuardrails: OutputGuardrail<TContext>[];
27
34
  constructor(config: AgentConfiguration<TContext>);
28
35
  resolveInstructions(runContext: RunContext<TContext>): Promise<string | undefined>;
@@ -1 +1 @@
1
- {"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,MAAM,MAAM,iBAAiB,CAAC,QAAQ,GAAG,OAAO,IAC5C,MAAM,GACN,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AAErE,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,OAAO;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC7B,gBAAgB,CAAC,EAAE,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;CAChD;AAED,qBAAa,KAAK,CAAC,QAAQ,GAAG,OAAO;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC5B,gBAAgB,EAAE,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAElC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC;IAY1C,mBAAmB,CACvB,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,GAC/B,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;CAS/B"}
1
+ {"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,MAAM,MAAM,iBAAiB,CAAC,QAAQ,GAAG,OAAO,IAC5C,MAAM,GACN,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AAErE,MAAM,MAAM,qBAAqB,CAAC,QAAQ,GAAG,OAAO,IAAI,CACtD,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,KAC7B,OAAO,CAAC;AAEb,MAAM,WAAW,YAAY,CAAC,QAAQ,GAAG,OAAO;IAC9C,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,iBAAiB,CAAC,QAAQ,GAAG,OAAO,IAC5C,KAAK,CAAC,QAAQ,CAAC,GACf,YAAY,CAAC,QAAQ,CAAC,CAAC;AAE3B,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,OAAO;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzC,gBAAgB,CAAC,EAAE,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;CAChD;AAED,qBAAa,KAAK,CAAC,QAAQ,GAAG,OAAO;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvC,gBAAgB,EAAE,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAElC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC;IAe1C,mBAAmB,CACvB,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,GAC/B,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;CAS/B"}
package/dist/agent.js CHANGED
@@ -10,6 +10,7 @@ class Agent {
10
10
  modelSettings;
11
11
  tools;
12
12
  handoffs;
13
+ handoffRules;
13
14
  outputGuardrails;
14
15
  constructor(config) {
15
16
  this.name = config.name;
@@ -19,7 +20,8 @@ class Agent {
19
20
  this.model = config.model;
20
21
  this.modelSettings = config.modelSettings;
21
22
  this.tools = config.tools ?? [];
22
- this.handoffs = config.handoffs ?? [];
23
+ this.handoffRules = (config.handoffs ?? []).map((handoff) => handoff instanceof Agent ? { agent: handoff } : handoff);
24
+ this.handoffs = this.handoffRules.map((rule) => rule.agent);
23
25
  this.outputGuardrails = config.outputGuardrails ?? [];
24
26
  }
25
27
  async resolveInstructions(runContext) {
@@ -20,11 +20,17 @@ export interface HarnessAgentDescriptor {
20
20
  model?: string;
21
21
  modelSettings?: Record<string, unknown>;
22
22
  tools?: string[];
23
- handoffs?: string[];
23
+ handoffs?: HarnessHandoffEntryDescriptor[];
24
24
  }
25
- export interface HarnessInstructionWhereDescriptor {
25
+ export interface HarnessWhereDescriptor {
26
26
  context: string;
27
27
  }
28
+ export type HarnessInstructionWhereDescriptor = HarnessWhereDescriptor;
29
+ export interface HarnessHandoffDescriptor {
30
+ agent: string;
31
+ where?: HarnessWhereDescriptor;
32
+ }
33
+ export type HarnessHandoffEntryDescriptor = string | HarnessHandoffDescriptor;
28
34
  export interface HarnessInstructionPartDescriptor {
29
35
  text: string;
30
36
  where?: HarnessInstructionWhereDescriptor;
@@ -1 +1 @@
1
- {"version":3,"file":"harness-descriptor.d.ts","sourceRoot":"","sources":["../src/harness-descriptor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAA0B,MAAM,SAAS,CAAC;AAExD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEnD,MAAM,WAAW,yBAAyB;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,6BAA6B,CAAC;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,iCAAiC,CAAC;CAC3C;AAED,MAAM,MAAM,6BAA6B,GACrC,MAAM,GACN,gCAAgC,EAAE,CAAC;AAEvC,MAAM,WAAW,iCAAiC;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,4BAA4B,GACpC,OAAO,GACP,iCAAiC,CAAC;AAEtC,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;IACvD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,4BAA4B,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,sBAAsB;IACrC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,EAAE,yBAAyB,CAAC;IACrC,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAC9C,cAAc,CAAC,EAAE,IAAI,CACnB,sBAAsB,EACtB,OAAO,GAAG,eAAe,GAAG,cAAc,CAC3C,CAAC;IACF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,OAAO;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY,CAAC,QAAQ,GAAG,OAAO;IAC9C,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC1D,aAAa,CAAC,KAAK,CAAC,EAAE,yBAAyB,GAAG,QAAQ,CAAC;CAC5D;AAsiBD,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,sBAAsB,GACjC,MAAM,CAER;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,GAAG,OAAO,EAClD,UAAU,EAAE,sBAAsB,EAClC,QAAQ,GAAE,oBAAoB,CAAC,QAAQ,CAAM,GAC5C,YAAY,CAAC,QAAQ,CAAC,CAqExB"}
1
+ {"version":3,"file":"harness-descriptor.d.ts","sourceRoot":"","sources":["../src/harness-descriptor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAA6C,MAAM,SAAS,CAAC;AAG3E,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEnD,MAAM,WAAW,yBAAyB;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,6BAA6B,CAAC;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,6BAA6B,EAAE,CAAC;CAC5C;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,iCAAiC,GAAG,sBAAsB,CAAC;AAEvE,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,sBAAsB,CAAC;CAChC;AAED,MAAM,MAAM,6BAA6B,GAAG,MAAM,GAAG,wBAAwB,CAAC;AAE9E,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,iCAAiC,CAAC;CAC3C;AAED,MAAM,MAAM,6BAA6B,GACrC,MAAM,GACN,gCAAgC,EAAE,CAAC;AAEvC,MAAM,WAAW,iCAAiC;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,4BAA4B,GACpC,OAAO,GACP,iCAAiC,CAAC;AAEtC,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;IACvD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,4BAA4B,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,sBAAsB;IACrC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,EAAE,yBAAyB,CAAC;IACrC,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAC9C,cAAc,CAAC,EAAE,IAAI,CACnB,sBAAsB,EACtB,OAAO,GAAG,eAAe,GAAG,cAAc,CAC3C,CAAC;IACF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,OAAO;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY,CAAC,QAAQ,GAAG,OAAO;IAC9C,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC1D,aAAa,CAAC,KAAK,CAAC,EAAE,yBAAyB,GAAG,QAAQ,CAAC;CAC5D;AAqoBD,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,sBAAsB,GACjC,MAAM,CAER;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,GAAG,OAAO,EAClD,UAAU,EAAE,sBAAsB,EAClC,QAAQ,GAAE,oBAAoB,CAAC,QAAQ,CAAM,GAC5C,YAAY,CAAC,QAAQ,CAAC,CAiFxB"}
@@ -30,6 +30,28 @@ function assertNonEmptyString(value, label) {
30
30
  throw new Error(`${label} must be a non-empty string.`);
31
31
  }
32
32
  }
33
+ function assertHandoffEntry(value, label) {
34
+ if (typeof value === "string") {
35
+ assertNonEmptyString(value, label);
36
+ return;
37
+ }
38
+ if (!isPlainObject(value)) {
39
+ throw new Error(`${label} must be a non-empty string or an object.`);
40
+ }
41
+ assertNonEmptyString(value.agent, `${label}.agent`);
42
+ if (typeof value.where !== "undefined") {
43
+ assertInstructionWhere(value.where, label);
44
+ }
45
+ }
46
+ function assertHandoffEntries(value, label) {
47
+ if (typeof value === "undefined") {
48
+ return;
49
+ }
50
+ if (!Array.isArray(value)) {
51
+ throw new Error(`${label} must be an array.`);
52
+ }
53
+ value.forEach((entry, index) => assertHandoffEntry(entry, `${label}[${index}]`));
54
+ }
33
55
  function validateDescriptorShape(descriptor) {
34
56
  assertPlainObject(descriptor, "Harness descriptor");
35
57
  assertPlainObject(descriptor.runtime, "Harness descriptor runtime");
@@ -49,6 +71,7 @@ function validateDescriptorShape(descriptor) {
49
71
  for (const [agentId, agentDescriptor] of Object.entries(descriptor.agents)) {
50
72
  assertPlainObject(agentDescriptor, `Harness descriptor agent "${agentId}"`);
51
73
  assertNoUnresolvedInstructionFile(agentDescriptor, `Harness descriptor agent "${agentId}"`);
74
+ assertHandoffEntries(agentDescriptor.handoffs, `Harness descriptor agent "${agentId}".handoffs`);
52
75
  }
53
76
  for (const [toolId, toolDescriptor] of Object.entries(descriptor.tools ?? {})) {
54
77
  assertPlainObject(toolDescriptor, `Harness descriptor tool "${toolId}"`);
@@ -202,31 +225,37 @@ function assertDeclaredPromptPaths(agentId, paths, promptAccessRules) {
202
225
  }
203
226
  }
204
227
  }
228
+ function assertDeclaredBooleanContextPath(label, path, promptAccessRules) {
229
+ const rule = promptAccessRules.get(path);
230
+ if (!rule) {
231
+ throw new Error(`${label} references undeclared context path "${path}".`);
232
+ }
233
+ if (rule.type !== "boolean") {
234
+ throw new Error(`${label} context path "${path}" must be declared with type "boolean".`);
235
+ }
236
+ }
205
237
  function assertDeclaredBooleanWherePaths(agentId, paths, promptAccessRules) {
206
238
  for (const path of paths) {
207
- const rule = promptAccessRules.get(path);
208
- if (!rule) {
209
- throw new Error(`Agent "${agentId}" instruction where references undeclared context path "${path}".`);
210
- }
211
- if (rule.type !== "boolean") {
212
- throw new Error(`Agent "${agentId}" instruction where context path "${path}" must be declared with type "boolean".`);
213
- }
239
+ assertDeclaredBooleanContextPath(`Agent "${agentId}" instruction where`, path, promptAccessRules);
214
240
  }
215
241
  }
216
- function shouldIncludeInstructionPart(part, context) {
217
- if (!part.where) {
242
+ function shouldIncludeWhere(where, context, label) {
243
+ if (!where) {
218
244
  return true;
219
245
  }
220
- const path = normalizePromptPath(part.where.context, "Instruction where.context path");
246
+ const path = normalizePromptPath(where.context, "Instruction where.context path");
221
247
  const resolved = readWhereContextValue(context, path);
222
248
  if (!resolved.exists || typeof resolved.value === "undefined") {
223
- throw new Error(`Harness descriptor instruction where could not resolve context path "${path}".`);
249
+ throw new Error(`${label} could not resolve context path "${path}".`);
224
250
  }
225
251
  if (typeof resolved.value !== "boolean") {
226
- throw new Error(`Harness descriptor instruction where context path "${path}" must resolve to a boolean.`);
252
+ throw new Error(`${label} context path "${path}" must resolve to a boolean.`);
227
253
  }
228
254
  return resolved.value === true;
229
255
  }
256
+ function shouldIncludeInstructionPart(part, context) {
257
+ return shouldIncludeWhere(part.where, context, "Harness descriptor instruction where");
258
+ }
230
259
  function renderInstructionParts(instructions, context, promptAccessRules) {
231
260
  const parts = [];
232
261
  for (const part of instructions) {
@@ -323,6 +352,22 @@ function resolveTool(toolId, descriptor, registry) {
323
352
  }
324
353
  return toolDefinition;
325
354
  }
355
+ function readHandoffEntry(entry) {
356
+ if (typeof entry === "string") {
357
+ return {
358
+ agent: entry,
359
+ };
360
+ }
361
+ return entry;
362
+ }
363
+ function compileHandoffCondition(fromAgentId, toAgentId, where, promptAccessRules) {
364
+ if (!where) {
365
+ return undefined;
366
+ }
367
+ const path = normalizePromptPath(where.context, `Agent "${fromAgentId}" handoff "${toAgentId}" where.context path`);
368
+ assertDeclaredBooleanContextPath(`Agent "${fromAgentId}" handoff "${toAgentId}" where`, path, promptAccessRules);
369
+ return (runContext) => shouldIncludeWhere(where, runContext.context, `Harness descriptor handoff "${fromAgentId}" to "${toAgentId}" where`);
370
+ }
326
371
  function hashAgentHarnessDescriptor(descriptor) {
327
372
  return `sha256:${(0, canonical_json_1.hashCanonicalJsonValue)(descriptor)}`;
328
373
  }
@@ -350,13 +395,20 @@ function buildAgentHarness(descriptor, registry = {}) {
350
395
  if (!agent) {
351
396
  continue;
352
397
  }
353
- agent.handoffs = (agentDescriptor.handoffs ?? []).map((handoffId) => {
398
+ const handoffRules = (agentDescriptor.handoffs ?? []).map((entry) => {
399
+ const handoff = readHandoffEntry(entry);
400
+ const handoffId = handoff.agent;
354
401
  const handoffAgent = agents.get(handoffId);
355
402
  if (!handoffAgent) {
356
403
  throw new Error(`Harness descriptor references unknown handoff agent "${handoffId}" from "${agentId}".`);
357
404
  }
358
- return handoffAgent;
405
+ return {
406
+ agent: handoffAgent,
407
+ enabled: compileHandoffCondition(agentId, handoffId, handoff.where, promptAccessRules),
408
+ };
359
409
  });
410
+ agent.handoffRules = handoffRules;
411
+ agent.handoffs = handoffRules.map((handoff) => handoff.agent);
360
412
  }
361
413
  const entryAgent = agents.get(descriptor.runtime.entry_agent);
362
414
  if (!entryAgent) {
package/dist/index.d.ts CHANGED
@@ -18,6 +18,8 @@ export * from "./providers/mistral";
18
18
  export * from "./providers/openai";
19
19
  export * from "./run";
20
20
  export * from "./run-output-events";
21
+ export * from "./run-regression";
22
+ export * from "./run-regression-summary";
21
23
  export * from "./run-record";
22
24
  export * from "./run-record-utils";
23
25
  export * from "./run-context";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,OAAO,CAAC;AACtB,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,QAAQ,CAAC;AACvB,cAAc,SAAS,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,OAAO,CAAC;AACtB,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,QAAQ,CAAC;AACvB,cAAc,SAAS,CAAC"}
package/dist/index.js CHANGED
@@ -34,6 +34,8 @@ __exportStar(require("./providers/mistral"), exports);
34
34
  __exportStar(require("./providers/openai"), exports);
35
35
  __exportStar(require("./run"), exports);
36
36
  __exportStar(require("./run-output-events"), exports);
37
+ __exportStar(require("./run-regression"), exports);
38
+ __exportStar(require("./run-regression-summary"), exports);
37
39
  __exportStar(require("./run-record"), exports);
38
40
  __exportStar(require("./run-record-utils"), exports);
39
41
  __exportStar(require("./run-context"), exports);
@@ -0,0 +1,34 @@
1
+ import type { RunRegressionResult, RunRegressionStatus } from "./run-regression";
2
+ export interface RunRegressionCaseSummary {
3
+ name: string;
4
+ status: RunRegressionStatus;
5
+ baselineRunId: string;
6
+ candidateRunId: string;
7
+ signals: {
8
+ statusChanged: boolean;
9
+ toolsChanged: boolean;
10
+ policyChanged: boolean;
11
+ finalOutputChanged: boolean;
12
+ };
13
+ judge?: {
14
+ verdict: RunRegressionStatus;
15
+ summary: string;
16
+ };
17
+ }
18
+ export interface RunRegressionSummary {
19
+ suite?: string;
20
+ status: RunRegressionStatus;
21
+ totals: {
22
+ cases: number;
23
+ passed: number;
24
+ warned: number;
25
+ failed: number;
26
+ };
27
+ cases: RunRegressionCaseSummary[];
28
+ }
29
+ export interface SummarizeRunRegressionResultsOptions<TContext = unknown> {
30
+ suite?: string;
31
+ classifyCase?: (result: RunRegressionResult<TContext>) => RunRegressionStatus;
32
+ }
33
+ export declare function summarizeRunRegressionResults<TContext = unknown>(results: Array<RunRegressionResult<TContext>>, options?: SummarizeRunRegressionResultsOptions<TContext>): RunRegressionSummary;
34
+ //# sourceMappingURL=run-regression-summary.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run-regression-summary.d.ts","sourceRoot":"","sources":["../src/run-regression-summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EACnB,mBAAmB,EACpB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,mBAAmB,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE;QACP,aAAa,EAAE,OAAO,CAAC;QACvB,YAAY,EAAE,OAAO,CAAC;QACtB,aAAa,EAAE,OAAO,CAAC;QACvB,kBAAkB,EAAE,OAAO,CAAC;KAC7B,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,mBAAmB,CAAC;QAC7B,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,mBAAmB,CAAC;IAC5B,MAAM,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,KAAK,EAAE,wBAAwB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,oCAAoC,CAAC,QAAQ,GAAG,OAAO;IACtE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,mBAAmB,CAAC,QAAQ,CAAC,KAAK,mBAAmB,CAAC;CAC/E;AA2DD,wBAAgB,6BAA6B,CAAC,QAAQ,GAAG,OAAO,EAC9D,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,EAC7C,OAAO,GAAE,oCAAoC,CAAC,QAAQ,CAAM,GAC3D,oBAAoB,CA2BtB"}
@@ -0,0 +1,71 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.summarizeRunRegressionResults = summarizeRunRegressionResults;
4
+ function defaultClassifyRunRegressionCase(result) {
5
+ if (result.candidate.status === "failed" ||
6
+ result.judge?.verdict === "fail") {
7
+ return "fail";
8
+ }
9
+ if (!result.comparison.equal || result.judge?.verdict === "warn") {
10
+ return "warn";
11
+ }
12
+ return "pass";
13
+ }
14
+ function summarizeCase(result, status) {
15
+ const judge = result.judge
16
+ ? {
17
+ verdict: result.judge.verdict,
18
+ summary: result.judge.summary,
19
+ }
20
+ : undefined;
21
+ return {
22
+ name: result.name,
23
+ status,
24
+ baselineRunId: result.baseline.runId,
25
+ candidateRunId: result.candidate.runId,
26
+ signals: {
27
+ statusChanged: result.baseline.status !== result.candidate.status,
28
+ toolsChanged: !result.comparison.summary.sameToolCallShape,
29
+ policyChanged: !result.comparison.summary.samePolicyDecisions,
30
+ finalOutputChanged: !result.comparison.summary.sameFinalResponse,
31
+ },
32
+ ...(judge ? { judge } : {}),
33
+ };
34
+ }
35
+ function toSuiteStatus(totals) {
36
+ if (totals.failed > 0) {
37
+ return "fail";
38
+ }
39
+ if (totals.warned > 0) {
40
+ return "warn";
41
+ }
42
+ return "pass";
43
+ }
44
+ function summarizeRunRegressionResults(results, options = {}) {
45
+ const classifyCase = options.classifyCase ?? (defaultClassifyRunRegressionCase);
46
+ const totals = {
47
+ cases: results.length,
48
+ passed: 0,
49
+ warned: 0,
50
+ failed: 0,
51
+ };
52
+ const cases = results.map((result) => {
53
+ const status = classifyCase(result);
54
+ if (status === "pass") {
55
+ totals.passed += 1;
56
+ }
57
+ else if (status === "warn") {
58
+ totals.warned += 1;
59
+ }
60
+ else {
61
+ totals.failed += 1;
62
+ }
63
+ return summarizeCase(result, status);
64
+ });
65
+ return {
66
+ ...(typeof options.suite === "undefined" ? {} : { suite: options.suite }),
67
+ status: toSuiteStatus(totals),
68
+ totals,
69
+ cases,
70
+ };
71
+ }
@@ -0,0 +1,71 @@
1
+ import type { RunRecord } from "./run-record";
2
+ import { type RunRegressionSummary, type SummarizeRunRegressionResultsOptions } from "./run-regression-summary";
3
+ import { type CompareRunRecordsOptions, type ReplayFromRunRecordInput, type RunRecordComparison } from "./run-record-utils";
4
+ export type RunRegressionStatus = "pass" | "warn" | "fail";
5
+ export interface RunRegressionExpectation {
6
+ intent?: string;
7
+ shouldUseTools?: string[];
8
+ shouldAvoidTools?: string[];
9
+ shouldPreserve?: string[];
10
+ shouldImprove?: string[];
11
+ notes?: string;
12
+ }
13
+ export interface RunJudgeFinding {
14
+ severity: "info" | "warn" | "error";
15
+ reason: string;
16
+ evidence?: string;
17
+ }
18
+ export interface RunJudgeResult {
19
+ verdict: RunRegressionStatus;
20
+ summary: string;
21
+ findings: RunJudgeFinding[];
22
+ score?: number;
23
+ judgeModel?: string;
24
+ judgePromptVersion?: string;
25
+ }
26
+ export interface RunJudgeInput<TContext = unknown, TDescriptor = unknown> {
27
+ baseline: RunRecord<TContext>;
28
+ candidate: RunRecord<TContext>;
29
+ comparison: RunRecordComparison;
30
+ expectation?: RunRegressionExpectation;
31
+ baselineDescriptor?: TDescriptor;
32
+ candidateDescriptor?: TDescriptor;
33
+ }
34
+ export type RunJudge<TContext = unknown, TDescriptor = unknown> = (input: RunJudgeInput<TContext, TDescriptor>) => Promise<RunJudgeResult> | RunJudgeResult;
35
+ export interface RunRegressionResult<TContext = unknown> {
36
+ name: string;
37
+ baseline: RunRecord<TContext>;
38
+ candidate: RunRecord<TContext>;
39
+ comparison: RunRecordComparison;
40
+ judge?: RunJudgeResult;
41
+ }
42
+ export interface RunRegressionSuiteCase<TContext = unknown> {
43
+ name?: string;
44
+ baseline: RunRecord<TContext>;
45
+ }
46
+ export interface RunRegressionSuite<TContext = unknown> {
47
+ name?: string;
48
+ expectation?: RunRegressionExpectation;
49
+ cases: Array<RunRegressionSuiteCase<TContext>>;
50
+ }
51
+ export interface RunRegressionCaseInput<TContext = unknown> extends Omit<ReplayFromRunRecordInput<TContext>, "sourceRunRecord"> {
52
+ name?: string;
53
+ baseline: RunRecord<TContext>;
54
+ comparisonOptions?: CompareRunRecordsOptions;
55
+ }
56
+ export interface RunRegressionSuiteInput<TContext = unknown, TDescriptor = unknown> extends Omit<RunRegressionCaseInput<TContext>, "baseline" | "name"> {
57
+ suite: RunRegressionSuite<TContext>;
58
+ judge?: RunJudge<TContext, TDescriptor>;
59
+ baselineDescriptor?: TDescriptor;
60
+ candidateDescriptor?: TDescriptor;
61
+ summaryOptions?: Omit<SummarizeRunRegressionResultsOptions<TContext>, "suite">;
62
+ }
63
+ export interface RunRegressionSuiteResult<TContext = unknown> {
64
+ name?: string;
65
+ expectation?: RunRegressionExpectation;
66
+ results: Array<RunRegressionResult<TContext>>;
67
+ summary: RunRegressionSummary;
68
+ }
69
+ export declare function runRegressionCase<TContext = unknown>(input: RunRegressionCaseInput<TContext>): Promise<RunRegressionResult<TContext>>;
70
+ export declare function runRegressionSuite<TContext = unknown, TDescriptor = unknown>(input: RunRegressionSuiteInput<TContext, TDescriptor>): Promise<RunRegressionSuiteResult<TContext>>;
71
+ //# sourceMappingURL=run-regression.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run-regression.d.ts","sourceRoot":"","sources":["../src/run-regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,oCAAoC,EAC1C,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAGL,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAC7B,KAAK,mBAAmB,EACzB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,mBAAmB,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAE3D,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,mBAAmB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa,CAAC,QAAQ,GAAG,OAAO,EAAE,WAAW,GAAG,OAAO;IACtE,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC/B,UAAU,EAAE,mBAAmB,CAAC;IAChC,WAAW,CAAC,EAAE,wBAAwB,CAAC;IACvC,kBAAkB,CAAC,EAAE,WAAW,CAAC;IACjC,mBAAmB,CAAC,EAAE,WAAW,CAAC;CACnC;AAED,MAAM,MAAM,QAAQ,CAAC,QAAQ,GAAG,OAAO,EAAE,WAAW,GAAG,OAAO,IAAI,CAChE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,WAAW,CAAC,KACxC,OAAO,CAAC,cAAc,CAAC,GAAG,cAAc,CAAC;AAE9C,MAAM,WAAW,mBAAmB,CAAC,QAAQ,GAAG,OAAO;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC/B,UAAU,EAAE,mBAAmB,CAAC;IAChC,KAAK,CAAC,EAAE,cAAc,CAAC;CACxB;AAED,MAAM,WAAW,sBAAsB,CAAC,QAAQ,GAAG,OAAO;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,OAAO;IACpD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,wBAAwB,CAAC;IACvC,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,sBAAsB,CAAC,QAAQ,GAAG,OAAO,CAAE,SAAQ,IAAI,CACtE,wBAAwB,CAAC,QAAQ,CAAC,EAClC,iBAAiB,CAClB;IACC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,iBAAiB,CAAC,EAAE,wBAAwB,CAAC;CAC9C;AAED,MAAM,WAAW,uBAAuB,CACtC,QAAQ,GAAG,OAAO,EAClB,WAAW,GAAG,OAAO,CACrB,SAAQ,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;IACnE,KAAK,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACpC,KAAK,CAAC,EAAE,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACxC,kBAAkB,CAAC,EAAE,WAAW,CAAC;IACjC,mBAAmB,CAAC,EAAE,WAAW,CAAC;IAClC,cAAc,CAAC,EAAE,IAAI,CACnB,oCAAoC,CAAC,QAAQ,CAAC,EAC9C,OAAO,CACR,CAAC;CACH;AAED,MAAM,WAAW,wBAAwB,CAAC,QAAQ,GAAG,OAAO;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,wBAAwB,CAAC;IACvC,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9C,OAAO,EAAE,oBAAoB,CAAC;CAC/B;AA0BD,wBAAsB,iBAAiB,CAAC,QAAQ,GAAG,OAAO,EACxD,KAAK,EAAE,sBAAsB,CAAC,QAAQ,CAAC,GACtC,OAAO,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CA6BxC;AAiBD,wBAAsB,kBAAkB,CACtC,QAAQ,GAAG,OAAO,EAClB,WAAW,GAAG,OAAO,EAErB,KAAK,EAAE,uBAAuB,CAAC,QAAQ,EAAE,WAAW,CAAC,GACpD,OAAO,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC,CAmD7C"}
@@ -0,0 +1,91 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.runRegressionCase = runRegressionCase;
4
+ exports.runRegressionSuite = runRegressionSuite;
5
+ const run_regression_summary_1 = require("./run-regression-summary");
6
+ const run_record_utils_1 = require("./run-record-utils");
7
+ function toRegressionCaseName(input) {
8
+ return input.name ?? input.baseline.runId;
9
+ }
10
+ function toReplayRunOptions(input) {
11
+ return {
12
+ ...(input.runOptions ?? {}),
13
+ record: input.runOptions?.record ?? {},
14
+ };
15
+ }
16
+ function toReplayAgentFields(input) {
17
+ return {
18
+ ...(input.agent ? { agent: input.agent } : {}),
19
+ ...(input.agentFactory ? { agentFactory: input.agentFactory } : {}),
20
+ };
21
+ }
22
+ async function runRegressionCase(input) {
23
+ const replay = await (0, run_record_utils_1.replayFromRunRecord)({
24
+ sourceRunRecord: input.baseline,
25
+ ...toReplayAgentFields(input),
26
+ mode: input.mode,
27
+ runOptions: toReplayRunOptions(input),
28
+ inputMode: input.inputMode,
29
+ metadataOverrides: input.metadataOverrides,
30
+ onMissingToolCall: input.onMissingToolCall,
31
+ });
32
+ if (!replay.replayRunRecord) {
33
+ throw new Error("runRegressionCase expected replayFromRunRecord to produce a candidate RunRecord.");
34
+ }
35
+ const comparison = (0, run_record_utils_1.compareRunRecords)(input.baseline, replay.replayRunRecord, input.comparisonOptions);
36
+ return {
37
+ name: toRegressionCaseName(input),
38
+ baseline: input.baseline,
39
+ candidate: replay.replayRunRecord,
40
+ comparison,
41
+ };
42
+ }
43
+ function resolveSuiteCase(caseDefinition) {
44
+ const baseline = caseDefinition.baseline;
45
+ const name = caseDefinition.name ?? baseline.runId;
46
+ return {
47
+ name,
48
+ baseline,
49
+ };
50
+ }
51
+ async function runRegressionSuite(input) {
52
+ const results = [];
53
+ const { suite, judge, baselineDescriptor, candidateDescriptor, summaryOptions, ...caseOptions } = input;
54
+ for (const caseDefinition of suite.cases) {
55
+ const regressionCase = resolveSuiteCase(caseDefinition);
56
+ const result = await runRegressionCase({
57
+ ...caseOptions,
58
+ name: regressionCase.name,
59
+ baseline: regressionCase.baseline,
60
+ });
61
+ const judgeResult = judge
62
+ ? await judge({
63
+ baseline: result.baseline,
64
+ candidate: result.candidate,
65
+ comparison: result.comparison,
66
+ ...(typeof suite.expectation === "undefined"
67
+ ? {}
68
+ : { expectation: suite.expectation }),
69
+ ...(typeof baselineDescriptor === "undefined"
70
+ ? {}
71
+ : { baselineDescriptor }),
72
+ ...(typeof candidateDescriptor === "undefined"
73
+ ? {}
74
+ : { candidateDescriptor }),
75
+ })
76
+ : undefined;
77
+ results.push(judgeResult ? { ...result, judge: judgeResult } : result);
78
+ }
79
+ const summary = (0, run_regression_summary_1.summarizeRunRegressionResults)(results, {
80
+ suite: suite.name,
81
+ ...(summaryOptions ?? {}),
82
+ });
83
+ return {
84
+ ...(typeof suite.name === "undefined" ? {} : { name: suite.name }),
85
+ ...(typeof suite.expectation === "undefined"
86
+ ? {}
87
+ : { expectation: suite.expectation }),
88
+ results,
89
+ summary,
90
+ };
91
+ }
package/dist/run.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAsChC,OAAO,EACL,cAAc,EAEd,mBAAmB,EAGnB,SAAS,EACT,cAAc,EACd,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAUjB,KAAK,eAAe,CAAC,QAAQ,IAAI;IAC/B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CAC5B,CAAC;AAGF,YAAY,EACV,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,mBAAmB,CAAC;AAuwB3B,qBAAa,iBAAiB,CAAC,QAAQ,GAAG,OAAO;IAC/C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAA0C;IACxD,OAAO,CAAC,KAAK,CAA4B;gBAGvC,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,EAC/C,KAAK,EAAE,eAAe,CAAC,QAAQ,CAAC;IAMlC,QAAQ,IAAI,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAQnD,IAAI,OAAO,IAAI,cAAc,EAAE,CAE9B;IAED,IAAI,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,CAE/B;CACF;AAED,wBAAsB,GAAG,CAAC,QAAQ,GAAG,OAAO,EAC1C,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,EAC9B,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,EAChC,OAAO,EAAE,gBAAgB,CAAC,QAAQ,CAAC,GAClC,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAExC,wBAAsB,GAAG,CAAC,QAAQ,GAAG,OAAO,EAC1C,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,EAC9B,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,EAChC,OAAO,CAAC,EAAE,mBAAmB,CAAC,QAAQ,CAAC,GACtC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAqB,MAAM,SAAS,CAAC;AAsCnD,OAAO,EACL,cAAc,EAEd,mBAAmB,EAGnB,SAAS,EACT,cAAc,EACd,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAUjB,KAAK,eAAe,CAAC,QAAQ,IAAI;IAC/B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CAC5B,CAAC;AAGF,YAAY,EACV,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,mBAAmB,CAAC;AAmxB3B,qBAAa,iBAAiB,CAAC,QAAQ,GAAG,OAAO;IAC/C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAA0C;IACxD,OAAO,CAAC,KAAK,CAA4B;gBAGvC,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,EAC/C,KAAK,EAAE,eAAe,CAAC,QAAQ,CAAC;IAMlC,QAAQ,IAAI,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAQnD,IAAI,OAAO,IAAI,cAAc,EAAE,CAE9B;IAED,IAAI,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,CAE/B;CACF;AAED,wBAAsB,GAAG,CAAC,QAAQ,GAAG,OAAO,EAC1C,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,EAC9B,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,EAChC,OAAO,EAAE,gBAAgB,CAAC,QAAQ,CAAC,GAClC,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAExC,wBAAsB,GAAG,CAAC,QAAQ,GAAG,OAAO,EAC1C,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,EAC9B,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,EAChC,OAAO,CAAC,EAAE,mBAAmB,CAAC,QAAQ,CAAC,GACtC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC"}
package/dist/run.js CHANGED
@@ -62,11 +62,16 @@ function sanitizeToolSegment(input) {
62
62
  .replace(/^_+|_+$/g, "");
63
63
  return sanitized || "agent";
64
64
  }
65
- function buildTurnTools(agent) {
65
+ function buildTurnTools(agent, runContext) {
66
66
  const handoffRegistry = new Map();
67
67
  const handoffTools = [];
68
68
  const reservedNames = new Set(agent.tools.map((tool) => tool.name));
69
- for (const handoffAgent of agent.handoffs) {
69
+ const handoffRules = agent.handoffRules;
70
+ for (const handoffRule of handoffRules) {
71
+ if (handoffRule.enabled && !handoffRule.enabled(runContext)) {
72
+ continue;
73
+ }
74
+ const handoffAgent = handoffRule.agent;
70
75
  const baseName = `handoff_to_${sanitizeToolSegment(handoffAgent.name)}`;
71
76
  let toolName = baseName;
72
77
  let suffix = 2;
@@ -331,7 +336,7 @@ async function* runLoop(state, runId, provider, runContext, maxTurns, logger, po
331
336
  activeTurn = turn + 1;
332
337
  const currentAgent = state.lastAgent;
333
338
  await logEmitter.turnStarted(currentAgent.name, activeTurn);
334
- const { providerTools, handoffRegistry } = buildTurnTools(currentAgent);
339
+ const { providerTools, handoffRegistry } = buildTurnTools(currentAgent, runContext);
335
340
  const model = resolveAgentModel(currentAgent);
336
341
  const systemPrompt = await currentAgent.resolveInstructions(runContext);
337
342
  onPromptSnapshot?.({
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@axiastudio/aioc",
3
- "version": "0.2.5",
3
+ "version": "0.2.6",
4
4
  "main": "dist/index.js",
5
5
  "types": "dist/index.d.ts",
6
6
  "files": [
@@ -27,12 +27,14 @@
27
27
  "example:tool-policy": "tsx src/examples/basic/tools.ts",
28
28
  "example:policy-composition": "tsx src/examples/basic/policy-composition.ts",
29
29
  "example:harness": "tsx src/examples/harness-descriptor/customer-support.ts",
30
+ "example:harness-rerun": "tsx src/examples/harness-descriptor/rerun-modified-harness.ts",
30
31
  "example:run-record": "tsx src/examples/basic/run-record-sink.ts",
31
32
  "example:rru:01-extract": "tsx src/examples/run-record-utils-minimal/01-extract-tool-calls.ts",
32
33
  "example:rru:02-compare": "tsx src/examples/run-record-utils-minimal/02-compare-run-records.ts",
33
34
  "example:rru:03-replay-strict": "tsx src/examples/run-record-utils-minimal/03-replay-strict.ts",
34
35
  "example:rru:04-replay-hybrid": "tsx src/examples/run-record-utils-minimal/04-replay-hybrid.ts",
35
36
  "example:non-regression": "tsx src/examples/non-regression/v1-v2-runrecord-diff.ts",
37
+ "example:run-regression": "tsx src/examples/run-regression/age-adapted-suite.ts",
36
38
  "inspect:samples": "tsx scripts/generate-aioc-inspect-samples.ts",
37
39
  "docs:sync": "node scripts/sync-aioc-docs.mjs",
38
40
  "docs:dev": "npm run docs:sync && cd apps/aioc-docs && zsh -lc 'npm run dev'",