@sweny-ai/core 0.1.162 → 0.1.163

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,6 +12,7 @@ export declare function workflowRunAction(file: string | undefined, options: Rec
12
12
  maxTokens?: string;
13
13
  maxCost?: string;
14
14
  yes?: boolean;
15
+ decider?: string;
15
16
  }): Promise<void>;
16
17
  export declare function workflowExportAction(name: string): void;
17
18
  export declare function workflowValidateAction(file: string, options: {
package/dist/cli/main.js CHANGED
@@ -731,6 +731,10 @@ export function loadWorkflowFile(filePath, knownSkills) {
731
731
  return result.workflow;
732
732
  }
733
733
  export async function workflowRunAction(file, options) {
734
+ if (options.decider !== undefined && options.decider !== "off" && options.decider !== "shadow") {
735
+ console.error(chalk.red(`\n --decider must be off or shadow, got "${options.decider}"\n`));
736
+ process.exit(1);
737
+ }
734
738
  // Reject junk --timeout/--max-steps up front (both paths) instead of
735
739
  // silently falling back to a default.
736
740
  let budget;
@@ -1005,6 +1009,7 @@ export async function workflowRunAction(file, options) {
1005
1009
  fileRoot: config.fileRoot || undefined,
1006
1010
  signal,
1007
1011
  max_steps: wfMaxSteps,
1012
+ ...(options.decider ? { decider: options.decider } : {}),
1008
1013
  stageOutputs: options.stage === true,
1009
1014
  ...(spendBudget ? { budget: spendBudget } : {}),
1010
1015
  harnessPolicy: resolveHarnessPolicy(process.env, typeof options.harnessPolicy === "string" ? options.harnessPolicy : undefined, runLogger),
@@ -1024,7 +1029,7 @@ export async function workflowRunAction(file, options) {
1024
1029
  // PR billboard markdown (metadata only). Written before any early exit so
1025
1030
  // --json runs and failed runs still get a comment.
1026
1031
  if (options.commentFile) {
1027
- writeRunComment(options.commentFile, workflow, results, summarizeRun(results, wfDurationMs), {
1032
+ writeRunComment(options.commentFile, workflow, results, summarizeRun(results, wfDurationMs, false, trace), {
1028
1033
  trace,
1029
1034
  durationsMs: Object.fromEntries(nodeTimer.durations),
1030
1035
  });
@@ -1051,7 +1056,7 @@ export async function workflowRunAction(file, options) {
1051
1056
  }
1052
1057
  // Run receipt (metadata only) + optional $GITHUB_STEP_SUMMARY.
1053
1058
  runLogger.flush();
1054
- const receipt = summarizeRun(results, wfDurationMs);
1059
+ const receipt = summarizeRun(results, wfDurationMs, false, trace);
1055
1060
  writeStepSummary(workflow, results, receipt, trace);
1056
1061
  // The answer, above the receipt: the terminal node's result, or the failed node's error.
1057
1062
  // Also saved to .sweny/runs/<id>/output.md (local only; --no-history skips the file).
@@ -55,6 +55,12 @@ export interface RunRecord {
55
55
  };
56
56
  /** Opinions the harness could not honor natively on some node, deduped. Absent when none. */
57
57
  degraded?: string[];
58
+ /** Shadow-mode decider agreement counts (#357). Absent when the decider was off. */
59
+ decider?: {
60
+ compared: number;
61
+ agreed: number;
62
+ fell_through: number;
63
+ };
58
64
  }
59
65
  /** sha256 (hex) of the workflow as canonical JSON: keys sorted, undefined dropped. */
60
66
  export declare function hashWorkflow(workflow: Workflow): string;
@@ -69,7 +69,7 @@ export function createNodeTimer(now = Date.now) {
69
69
  // ── Build the record ────────────────────────────────────────────
70
70
  const isNum = (v) => typeof v === "number" && Number.isFinite(v);
71
71
  export function buildRunRecord(i) {
72
- const summary = summarizeRun(i.results, i.durationMs, i.crashed);
72
+ const summary = summarizeRun(i.results, i.durationMs, i.crashed, i.trace);
73
73
  const retries = new Map();
74
74
  for (const s of i.trace?.steps ?? []) {
75
75
  if (s.retryAttempt !== undefined)
@@ -113,6 +113,15 @@ export function buildRunRecord(i) {
113
113
  },
114
114
  ...(harness ? { harness: { id: harness.id, version: harness.version } } : {}),
115
115
  ...(degraded.length > 0 ? { degraded } : {}),
116
+ ...(summary.decider
117
+ ? {
118
+ decider: {
119
+ compared: summary.decider.compared,
120
+ agreed: summary.decider.agreed,
121
+ fell_through: summary.decider.fellThrough,
122
+ },
123
+ }
124
+ : {}),
116
125
  };
117
126
  }
118
127
  // ── Write, prune, read ──────────────────────────────────────────
@@ -39,6 +39,16 @@ export interface RunSummary {
39
39
  limit: number;
40
40
  spent: number;
41
41
  };
42
+ /**
43
+ * Shadow-mode decider agreement (#357), counts only. `compared` decisions
44
+ * passed every gate; `agreed` matched the agent; the rest fell through.
45
+ * Absent when the decider was off or never consulted.
46
+ */
47
+ decider?: {
48
+ compared: number;
49
+ agreed: number;
50
+ fellThrough: number;
51
+ };
42
52
  }
43
53
  export interface RunPolicy {
44
54
  /** Agent env narrowed to the allowlist. Absent when no node reported it. */
@@ -57,7 +67,7 @@ export interface RunPolicy {
57
67
  /** The short key of a `degraded` entry: the text before its first colon. */
58
68
  export declare function degradedKey(entry: string): string;
59
69
  /** Sum the per-node results into one run summary. */
60
- export declare function summarizeRun(results: Map<string, NodeResult>, durationMs: number, crashed?: boolean): RunSummary;
70
+ export declare function summarizeRun(results: Map<string, NodeResult>, durationMs: number, crashed?: boolean, trace?: ExecutionTrace): RunSummary;
61
71
  /**
62
72
  * `policy: env scoped, sandbox on, 1 output staged`. Only what the run knew:
63
73
  * no segment for a fact no node reported. Undefined when there is nothing to say.
@@ -8,6 +8,7 @@
8
8
  import fs from "node:fs";
9
9
  import chalk from "chalk";
10
10
  import { consoleLogger } from "../types.js";
11
+ import { summarizeDecisions } from "../decider.js";
11
12
  import { toMermaidBlock } from "../mermaid.js";
12
13
  /** The short key of a `degraded` entry: the text before its first colon. */
13
14
  export function degradedKey(entry) {
@@ -15,7 +16,7 @@ export function degradedKey(entry) {
15
16
  return (i === -1 ? entry : entry.slice(0, i)).trim();
16
17
  }
17
18
  /** Sum the per-node results into one run summary. */
18
- export function summarizeRun(results, durationMs, crashed = false) {
19
+ export function summarizeRun(results, durationMs, crashed = false, trace) {
19
20
  let nodesOk = 0;
20
21
  let nodesSkipped = 0;
21
22
  let failed = crashed;
@@ -81,8 +82,13 @@ export function summarizeRun(results, durationMs, crashed = false) {
81
82
  ...(degraded.size > 0 ? { degraded: [...degraded] } : {}),
82
83
  ...(Object.keys(policy).length > 0 ? { policy } : {}),
83
84
  ...(budget ? { budget } : {}),
85
+ ...(trace?.decisions && trace.decisions.length > 0 ? { decider: decisionCounts(trace) } : {}),
84
86
  };
85
87
  }
88
+ function decisionCounts(trace) {
89
+ const d = summarizeDecisions(trace.decisions ?? []);
90
+ return { compared: d.compared, agreed: d.agreed, fellThrough: d.fell_through };
91
+ }
86
92
  /**
87
93
  * `policy: env scoped, sandbox on, 1 output staged`. Only what the run knew:
88
94
  * no segment for a fact no node reported. Undefined when there is nothing to say.
@@ -156,6 +162,13 @@ export function formatReceipt(s) {
156
162
  ...(s.degraded && s.degraded.length > 0 ? [`degraded: ${s.degraded.join(", ")}`] : []),
157
163
  ...(formatPolicySegment(s.policy) ? [formatPolicySegment(s.policy)] : []),
158
164
  ...(s.budget ? [formatBudgetOverrun(s.budget)] : []),
165
+ ...(s.decider
166
+ ? [
167
+ s.decider.compared > 0
168
+ ? `decider agreed ${s.decider.agreed}/${s.decider.compared}`
169
+ : `decider fell through ${s.decider.fellThrough}/${s.decider.fellThrough}`,
170
+ ]
171
+ : []),
159
172
  ];
160
173
  return parts.join(" · ");
161
174
  }
@@ -286,4 +299,8 @@ export const WORKFLOW_RUN_OPTIONS = [
286
299
  "--harness-policy <mode>",
287
300
  "strict: refuse a node whose policy the agent cannot enforce; warn: run it and report what was not enforced (default: strict under GitHub Actions, warn elsewhere; env SWENY_HARNESS_POLICY)",
288
301
  ],
302
+ [
303
+ "--decider <mode>",
304
+ "Decision model for route choices: off (default) or shadow. Shadow asks the workflow's decider.provider alongside the agent and reports agreement; the route is always the agent's. Needs decider.provider in the workflow (no default URL)",
305
+ ],
289
306
  ];
@@ -0,0 +1,144 @@
1
+ /**
2
+ * Decision models (#357), shadow mode.
3
+ *
4
+ * A System One decision model (Ollama `/v1/systemone`, TypeSafe Jev) answers
5
+ * one typed Choice question over a state in milliseconds. In shadow mode the
6
+ * executor asks it the same route question the agent answers, in parallel,
7
+ * and records whether they agree. The route is ALWAYS the agent's.
8
+ *
9
+ * Privacy: the state goes only to the provider the workflow configured, never
10
+ * to a default or fallback URL. What this module logs is metadata only
11
+ * (labels are edge target node ids, plus numbers and a hash).
12
+ */
13
+ /** Accept a decider answer only at or above this confidence. */
14
+ export declare const DECIDER_MIN_CONFIDENCE = 0.85;
15
+ /** ... and only when the top label leads the runner-up by at least this much. */
16
+ export declare const DECIDER_MIN_MARGIN = 0.2;
17
+ /** Per-call timeout when the provider does not set one. */
18
+ export declare const DECIDER_DEFAULT_TIMEOUT_MS = 2000;
19
+ export type DeciderMode = "off" | "shadow";
20
+ /** Workflow-level `decider` block. */
21
+ export interface DeciderConfig {
22
+ mode: DeciderMode;
23
+ provider?: {
24
+ /** Server root, e.g. `http://localhost:11434` or `https://api.typesafe.ai`. */
25
+ base_url: string;
26
+ model: string;
27
+ /** Name of the env var holding the bearer key. Never the key itself. */
28
+ api_key_env?: string;
29
+ };
30
+ }
31
+ /** One Choice question: pick a key of `criteria`. */
32
+ export interface ChoiceAsk {
33
+ kind: "choice";
34
+ instructions: string;
35
+ criteria: Record<string, string>;
36
+ }
37
+ /** Only Choice exists today. Noul and Score arrive as further union members. */
38
+ export type Ask = ChoiceAsk;
39
+ export interface ChoiceVerdict {
40
+ kind: "choice";
41
+ label: string;
42
+ probabilities: Record<string, number>;
43
+ confidence: number;
44
+ }
45
+ export type Verdict = ChoiceVerdict;
46
+ export type DecideErrorCode = "timeout" | "aborted" | "network" | "unauthorized" | "invalid_request" | "rate_limited" | "overloaded" | "http_error" | "malformed" | "unknown_label";
47
+ export declare class DecideError extends Error {
48
+ readonly code: DecideErrorCode;
49
+ constructor(code: DecideErrorCode, message: string);
50
+ }
51
+ export interface DecideRequest {
52
+ state: unknown;
53
+ asks: Record<string, Ask>;
54
+ timeoutMs?: number;
55
+ signal?: AbortSignal;
56
+ }
57
+ export interface DecisionProvider {
58
+ /** Short provider family label for logs, e.g. `systemone`. */
59
+ id: string;
60
+ model: string;
61
+ /** Throws {@link DecideError}; never returns a partial answer. */
62
+ decide(req: DecideRequest): Promise<Record<string, Verdict>>;
63
+ }
64
+ export interface SystemOneOptions {
65
+ baseUrl: string;
66
+ model: string;
67
+ apiKey?: string;
68
+ /** Default timeout when `decide()` is not given one. Default 2000 ms. */
69
+ timeoutMs?: number;
70
+ }
71
+ /**
72
+ * `POST {baseUrl}/v1/systemone`. Ollama (local) and TypeSafe Jev (hosted)
73
+ * share this wire shape; only base URL, key and model differ. Response
74
+ * validation is strict: any deviation is a `malformed` or `unknown_label`
75
+ * error, never a guess.
76
+ */
77
+ export declare function systemOneProvider(opts: SystemOneOptions): DecisionProvider;
78
+ export type FallThroughReason = DecideErrorCode | "low_confidence" | "low_margin" | "label_outside_edge_set";
79
+ export interface GateResult {
80
+ accepted: boolean;
81
+ reason?: FallThroughReason;
82
+ /** Top probability minus the runner-up (0 when there is no runner-up). */
83
+ margin: number;
84
+ }
85
+ /** All gates must pass: label in the edge set, confidence, then top-two margin. */
86
+ export declare function gateVerdict(v: ChoiceVerdict, edgeSet: ReadonlySet<string>): GateResult;
87
+ /** One logged shadow decision. Metadata only: no state, no prompts, no prose. */
88
+ export interface DeciderRecord {
89
+ /** Node the route was decided at. */
90
+ node: string;
91
+ /** Edge target the decider picked (a node id). Null when it gave no answer. */
92
+ decider_label: string | null;
93
+ /** Edge target the agent picked. Null when the agent's evaluation failed. */
94
+ agent_label: string | null;
95
+ confidence: number | null;
96
+ margin: number | null;
97
+ /** `compared`: the decider passed every gate. `fell_through`: it did not, see `reason`. */
98
+ outcome: "compared" | "fell_through";
99
+ reason?: FallThroughReason | "agent_failed";
100
+ /** Decider label equals agent label. Null unless `outcome` is `compared`. */
101
+ agree: boolean | null;
102
+ latency_ms: number;
103
+ model: string;
104
+ /** First 16 hex chars of sha256 over the state and criteria sent. */
105
+ input_hash: string;
106
+ }
107
+ /** What the executor holds for one run in shadow mode. */
108
+ export interface ShadowDecider {
109
+ provider: DecisionProvider;
110
+ records: DeciderRecord[];
111
+ }
112
+ export declare function hashDecisionInput(state: unknown, criteria: Record<string, string>): string;
113
+ export interface PendingShadow {
114
+ /** Never rejects. */
115
+ settled: Promise<Omit<DeciderRecord, "agent_label" | "agree" | "node">>;
116
+ edgeSet: ReadonlySet<string>;
117
+ }
118
+ /**
119
+ * Start the decider call. Returns at once; the promise never rejects, so the
120
+ * agent's own evaluation is never delayed or failed by it.
121
+ */
122
+ export declare function startShadowDecision(provider: DecisionProvider, req: {
123
+ question: string;
124
+ state: unknown;
125
+ choices: {
126
+ id: string;
127
+ description: string;
128
+ }[];
129
+ signal?: AbortSignal;
130
+ }): PendingShadow;
131
+ /** Join the decider's settled result with the agent's choice. */
132
+ export declare function finishShadowDecision(pending: PendingShadow, node: string, agentLabel: string | null): Promise<DeciderRecord>;
133
+ /** `{ compared, agreed, fell_through }` over a run's records. */
134
+ export declare function summarizeDecisions(records: readonly DeciderRecord[]): {
135
+ compared: number;
136
+ agreed: number;
137
+ fell_through: number;
138
+ };
139
+ /**
140
+ * Resolve the run's shadow decider, or null (mode off, nothing configured, or
141
+ * a declared key env var is unset). No hosted fallback exists: null means no
142
+ * HTTP at all.
143
+ */
144
+ export declare function createShadowDecider(config: DeciderConfig | undefined, modeOverride: DeciderMode | undefined, env: NodeJS.ProcessEnv, warn: (msg: string) => void): ShadowDecider | null;
@@ -0,0 +1,260 @@
1
+ /**
2
+ * Decision models (#357), shadow mode.
3
+ *
4
+ * A System One decision model (Ollama `/v1/systemone`, TypeSafe Jev) answers
5
+ * one typed Choice question over a state in milliseconds. In shadow mode the
6
+ * executor asks it the same route question the agent answers, in parallel,
7
+ * and records whether they agree. The route is ALWAYS the agent's.
8
+ *
9
+ * Privacy: the state goes only to the provider the workflow configured, never
10
+ * to a default or fallback URL. What this module logs is metadata only
11
+ * (labels are edge target node ids, plus numbers and a hash).
12
+ */
13
+ import crypto from "node:crypto";
14
+ /** Accept a decider answer only at or above this confidence. */
15
+ export const DECIDER_MIN_CONFIDENCE = 0.85;
16
+ /** ... and only when the top label leads the runner-up by at least this much. */
17
+ export const DECIDER_MIN_MARGIN = 0.2;
18
+ /** Per-call timeout when the provider does not set one. */
19
+ export const DECIDER_DEFAULT_TIMEOUT_MS = 2000;
20
+ export class DecideError extends Error {
21
+ code;
22
+ constructor(code, message) {
23
+ super(message);
24
+ this.code = code;
25
+ this.name = "DecideError";
26
+ }
27
+ }
28
+ const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
29
+ const isProb = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
30
+ /**
31
+ * `POST {baseUrl}/v1/systemone`. Ollama (local) and TypeSafe Jev (hosted)
32
+ * share this wire shape; only base URL, key and model differ. Response
33
+ * validation is strict: any deviation is a `malformed` or `unknown_label`
34
+ * error, never a guess.
35
+ */
36
+ export function systemOneProvider(opts) {
37
+ const url = `${opts.baseUrl.replace(/\/+$/, "")}/v1/systemone`;
38
+ return {
39
+ id: "systemone",
40
+ model: opts.model,
41
+ async decide(req) {
42
+ const timeoutMs = req.timeoutMs ?? opts.timeoutMs ?? DECIDER_DEFAULT_TIMEOUT_MS;
43
+ const questions = {};
44
+ for (const [name, ask] of Object.entries(req.asks)) {
45
+ questions[name] = { type: ask.kind, instructions: ask.instructions, criteria: ask.criteria };
46
+ }
47
+ const ctl = new AbortController();
48
+ let timedOut = false;
49
+ const timer = setTimeout(() => {
50
+ timedOut = true;
51
+ ctl.abort();
52
+ }, timeoutMs);
53
+ const onAbort = () => ctl.abort();
54
+ if (req.signal?.aborted)
55
+ ctl.abort();
56
+ req.signal?.addEventListener("abort", onAbort, { once: true });
57
+ try {
58
+ let res;
59
+ try {
60
+ res = await fetch(url, {
61
+ method: "POST",
62
+ headers: {
63
+ "content-type": "application/json",
64
+ ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}),
65
+ },
66
+ body: JSON.stringify({ model: opts.model, state: req.state, questions }),
67
+ signal: ctl.signal,
68
+ });
69
+ }
70
+ catch (err) {
71
+ if (timedOut)
72
+ throw new DecideError("timeout", `no answer within ${timeoutMs}ms`);
73
+ if (ctl.signal.aborted)
74
+ throw new DecideError("aborted", "request aborted");
75
+ throw new DecideError("network", err instanceof Error ? err.name : "fetch failed");
76
+ }
77
+ if (!res.ok) {
78
+ const s = res.status;
79
+ const code = s === 401 || s === 403
80
+ ? "unauthorized"
81
+ : s === 400 || s === 422
82
+ ? "invalid_request"
83
+ : s === 429
84
+ ? "rate_limited"
85
+ : s === 529 || s === 503
86
+ ? "overloaded"
87
+ : "http_error";
88
+ throw new DecideError(code, `HTTP ${s}`);
89
+ }
90
+ let body;
91
+ try {
92
+ body = await res.json();
93
+ }
94
+ catch {
95
+ if (timedOut)
96
+ throw new DecideError("timeout", `no answer within ${timeoutMs}ms`);
97
+ throw new DecideError("malformed", "response is not JSON");
98
+ }
99
+ return parseAnswers(body, req.asks);
100
+ }
101
+ finally {
102
+ clearTimeout(timer);
103
+ req.signal?.removeEventListener("abort", onAbort);
104
+ }
105
+ },
106
+ };
107
+ }
108
+ function parseAnswers(body, asks) {
109
+ if (!isRecord(body) || !isRecord(body.answers))
110
+ throw new DecideError("malformed", "missing answers object");
111
+ const out = {};
112
+ for (const [name, ask] of Object.entries(asks)) {
113
+ const a = body.answers[name];
114
+ if (!isRecord(a))
115
+ throw new DecideError("malformed", `no answer for question '${name}'`);
116
+ if (a.type !== undefined && a.type !== ask.kind)
117
+ throw new DecideError("malformed", "answer type mismatch");
118
+ if (typeof a.choice !== "string")
119
+ throw new DecideError("malformed", "choice is not a string");
120
+ if (!isProb(a.confidence))
121
+ throw new DecideError("malformed", "confidence is not a number in 0..1");
122
+ if (!isRecord(a.probabilities))
123
+ throw new DecideError("malformed", "probabilities is not an object");
124
+ const probabilities = {};
125
+ for (const [k, v] of Object.entries(a.probabilities)) {
126
+ if (!isProb(v))
127
+ throw new DecideError("malformed", "probability is not a number in 0..1");
128
+ probabilities[k] = v;
129
+ }
130
+ const known = Object.keys(ask.criteria);
131
+ if (!known.includes(a.choice) ||
132
+ !(a.choice in probabilities) ||
133
+ Object.keys(probabilities).some((k) => !known.includes(k))) {
134
+ throw new DecideError("unknown_label", "answer names a label outside the criteria");
135
+ }
136
+ out[name] = { kind: "choice", label: a.choice, probabilities, confidence: a.confidence };
137
+ }
138
+ return out;
139
+ }
140
+ /** All gates must pass: label in the edge set, confidence, then top-two margin. */
141
+ export function gateVerdict(v, edgeSet) {
142
+ const top = v.probabilities[v.label] ?? 0;
143
+ const second = Math.max(0, ...Object.entries(v.probabilities)
144
+ .filter(([k]) => k !== v.label)
145
+ .map(([, p]) => p));
146
+ const margin = top - second;
147
+ if (!edgeSet.has(v.label))
148
+ return { accepted: false, reason: "label_outside_edge_set", margin };
149
+ if (!(v.confidence >= DECIDER_MIN_CONFIDENCE))
150
+ return { accepted: false, reason: "low_confidence", margin };
151
+ if (!(margin >= DECIDER_MIN_MARGIN))
152
+ return { accepted: false, reason: "low_margin", margin };
153
+ return { accepted: true, margin };
154
+ }
155
+ export function hashDecisionInput(state, criteria) {
156
+ let json;
157
+ try {
158
+ json = JSON.stringify({ state, criteria }) ?? "";
159
+ }
160
+ catch {
161
+ json = "unserializable";
162
+ }
163
+ return crypto.createHash("sha256").update(json).digest("hex").slice(0, 16);
164
+ }
165
+ /**
166
+ * Start the decider call. Returns at once; the promise never rejects, so the
167
+ * agent's own evaluation is never delayed or failed by it.
168
+ */
169
+ export function startShadowDecision(provider, req) {
170
+ const criteria = {};
171
+ for (const c of req.choices)
172
+ if (!(c.id in criteria))
173
+ criteria[c.id] = c.description;
174
+ const edgeSet = new Set(Object.keys(criteria));
175
+ const input_hash = hashDecisionInput(req.state, criteria);
176
+ const started = Date.now();
177
+ const base = { model: provider.model, input_hash };
178
+ const settled = (async () => {
179
+ try {
180
+ const out = await provider.decide({
181
+ state: req.state,
182
+ asks: { route: { kind: "choice", instructions: req.question, criteria } },
183
+ timeoutMs: DECIDER_DEFAULT_TIMEOUT_MS,
184
+ signal: req.signal,
185
+ });
186
+ const v = out.route;
187
+ if (!v || v.kind !== "choice")
188
+ throw new DecideError("malformed", "no route answer");
189
+ const gate = gateVerdict(v, edgeSet);
190
+ return {
191
+ ...base,
192
+ decider_label: v.label,
193
+ confidence: v.confidence,
194
+ margin: gate.margin,
195
+ outcome: gate.accepted ? "compared" : "fell_through",
196
+ ...(gate.reason ? { reason: gate.reason } : {}),
197
+ latency_ms: Date.now() - started,
198
+ };
199
+ }
200
+ catch (err) {
201
+ return {
202
+ ...base,
203
+ decider_label: null,
204
+ confidence: null,
205
+ margin: null,
206
+ outcome: "fell_through",
207
+ reason: err instanceof DecideError ? err.code : "malformed",
208
+ latency_ms: Date.now() - started,
209
+ };
210
+ }
211
+ })();
212
+ return { settled, edgeSet };
213
+ }
214
+ /** Join the decider's settled result with the agent's choice. */
215
+ export async function finishShadowDecision(pending, node, agentLabel) {
216
+ const s = await pending.settled;
217
+ if (s.outcome === "fell_through")
218
+ return { node, agent_label: agentLabel, agree: null, ...s };
219
+ if (agentLabel === null) {
220
+ return { node, agent_label: null, agree: null, ...s, outcome: "fell_through", reason: "agent_failed" };
221
+ }
222
+ return { node, agent_label: agentLabel, agree: s.decider_label === agentLabel, ...s };
223
+ }
224
+ /** `{ compared, agreed, fell_through }` over a run's records. */
225
+ export function summarizeDecisions(records) {
226
+ let compared = 0;
227
+ let agreed = 0;
228
+ for (const r of records) {
229
+ if (r.outcome === "compared") {
230
+ compared++;
231
+ if (r.agree)
232
+ agreed++;
233
+ }
234
+ }
235
+ return { compared, agreed, fell_through: records.length - compared };
236
+ }
237
+ /**
238
+ * Resolve the run's shadow decider, or null (mode off, nothing configured, or
239
+ * a declared key env var is unset). No hosted fallback exists: null means no
240
+ * HTTP at all.
241
+ */
242
+ export function createShadowDecider(config, modeOverride, env, warn) {
243
+ const mode = modeOverride ?? config?.mode ?? "off";
244
+ if (mode !== "shadow")
245
+ return null;
246
+ const p = config?.provider;
247
+ if (!p) {
248
+ warn(" decider: shadow mode needs decider.provider in the workflow; running without it.");
249
+ return null;
250
+ }
251
+ let apiKey;
252
+ if (p.api_key_env) {
253
+ apiKey = env[p.api_key_env];
254
+ if (!apiKey) {
255
+ warn(` decider: env var ${p.api_key_env} is not set; running without the decider.`);
256
+ return null;
257
+ }
258
+ }
259
+ return { provider: systemOneProvider({ baseUrl: p.base_url, model: p.model, apiKey }), records: [] };
260
+ }
@@ -15,6 +15,7 @@
15
15
  import type { Workflow, Skill, Tool, Claude, Observer, Logger, ExecutionResult } from "./types.js";
16
16
  import type { HarnessPolicyMode } from "./harness/policy.js";
17
17
  import type { AgentHarness } from "./harness/types.js";
18
+ import type { DeciderMode } from "./decider.js";
18
19
  import type { Budget } from "./budget.js";
19
20
  import { type ActorInfo } from "./safe-outputs.js";
20
21
  export interface ExecuteOptions {
@@ -97,6 +98,12 @@ export interface ExecuteOptions {
97
98
  * all (cost on Codex). Default: {@link resolveHarnessPolicy} from the run env.
98
99
  */
99
100
  harnessPolicy?: HarnessPolicyMode;
101
+ /**
102
+ * Override `workflow.decider.mode` (CLI `--decider`). `shadow` still needs
103
+ * `decider.provider` in the workflow; there is no default URL. Default: the
104
+ * workflow's own mode, else off (zero HTTP calls).
105
+ */
106
+ decider?: DeciderMode;
100
107
  }
101
108
  /**
102
109
  * Default workflow-level step budget. Counts total node executions across the
package/dist/executor.js CHANGED
@@ -26,6 +26,7 @@ import { fenceUntrusted } from "./untrusted.js";
26
26
  import { asClaude } from "./harness/compat.js";
27
27
  import { budgetGate, isToolClass, policyGate, resolveHarnessPolicy } from "./harness/policy.js";
28
28
  import { BudgetGuard, describeOverrun, minLimits, toLimits } from "./budget.js";
29
+ import { createShadowDecider, finishShadowDecision, startShadowDecision } from "./decider.js";
29
30
  import { buildNodePolicy, resolveNodePermissions } from "./node-policy.js";
30
31
  import { applySafeOutputs, createEmitOutputTool, createWriteStageState, resolveActor, safeOutputsInstruction, unresolvedOutputs, SAFE_OUTPUT_SCREEN_INSTRUCTION, } from "./safe-outputs.js";
31
32
  /**
@@ -88,6 +89,10 @@ export async function execute(workflow, input, options) {
88
89
  // `budget` and the caller's (`--max-tokens`, `--max-cost`).
89
90
  const budgetGuard = new BudgetGuard(minLimits(toLimits(workflow.budget), toLimits(options.budget)));
90
91
  const harnessPolicy = options.harnessPolicy ?? resolveHarnessPolicy(runEnv);
92
+ // Decision model (#357): shadow only. Null (the default) means no HTTP at all.
93
+ const shadow = createShadowDecider(workflow.decider, options.decider, runEnv, (m) => logger.warn(m));
94
+ if (shadow)
95
+ trace.decisions = shadow.records;
91
96
  // Build an eval-time alias table from the loaded skills. Each skill owns
92
97
  // its own mapping between skill-tool names and equivalent MCP names. Core
93
98
  // stays vendor-neutral; this call just unions what the skills declare.
@@ -254,7 +259,7 @@ export async function execute(workflow, input, options) {
254
259
  }
255
260
  // Apply normal routing rules (dry run gate + resolveNext).
256
261
  // TODO: dedupe with requires path — see advanceFromNode helper below
257
- const next = await advanceFromNode(workflow, currentId, results, input, claude, observer, edgeCounts, logger, trace, { signal, timeoutMs });
262
+ const next = await advanceFromNode(workflow, currentId, results, input, claude, observer, edgeCounts, logger, trace, { signal, timeoutMs, shadow });
258
263
  currentId = next;
259
264
  continue;
260
265
  }
@@ -591,6 +596,7 @@ export async function execute(workflow, input, options) {
591
596
  currentId = await advanceFromNode(workflow, currentId, results, input, claude, observer, edgeCounts, logger, trace, {
592
597
  signal,
593
598
  timeoutMs,
599
+ shadow,
594
600
  });
595
601
  }
596
602
  safeObserve(observer, {
@@ -1248,13 +1254,31 @@ async function resolveNext(workflow, current, results, input, claude, observer,
1248
1254
  if (defaultEdge) {
1249
1255
  choices.push({ id: defaultEdge.to, description: "None of the above / default path" });
1250
1256
  }
1257
+ const question = "Based on the results so far, which condition is true?";
1258
+ // Shadow mode (#357): the decider gets the same question in parallel. Its
1259
+ // promise never rejects and its answer is only logged, so the route below
1260
+ // is the agent's in every case.
1261
+ const pending = abort?.shadow
1262
+ ? startShadowDecision(abort.shadow.provider, { question, state: context, choices, signal: abort.signal })
1263
+ : undefined;
1251
1264
  const chosen = await claude.evaluate({
1252
- question: "Based on the results so far, which condition is true?",
1265
+ question,
1253
1266
  context,
1254
1267
  choices,
1255
1268
  signal: abort?.signal,
1256
1269
  timeoutMs: abort?.timeoutMs,
1257
1270
  });
1271
+ if (pending && abort?.shadow) {
1272
+ try {
1273
+ const rec = await finishShadowDecision(pending, current, chosen);
1274
+ abort.shadow.records.push(rec);
1275
+ const verdict = rec.outcome === "compared" ? (rec.agree ? "agreed" : "disagreed") : `fell through (${rec.reason})`;
1276
+ logger?.info(` decider (shadow): node '${current}' ${verdict}`, { ...rec });
1277
+ }
1278
+ catch {
1279
+ // shadow logging must never affect a route
1280
+ }
1281
+ }
1258
1282
  // Fail closed. `evaluate` returns null when the routing decision could not
1259
1283
  // be made (SDK error, timeout, non-success subtype, or an unparseable
1260
1284
  // answer). We must NOT fall through to a conditional edge: on a node with a
package/dist/index.d.ts CHANGED
@@ -27,6 +27,8 @@ export { EVALUATOR_KINDS, EVAL_POLICIES, REQUIRES_ON_FAIL, NODE_ON_FAIL, MCP_TRA
27
27
  export { consoleLogger } from "./types.js";
28
28
  export { execute, RouteEvaluationError } from "./executor.js";
29
29
  export type { ExecuteOptions } from "./executor.js";
30
+ export { systemOneProvider, DecideError, gateVerdict, DECIDER_MIN_CONFIDENCE, DECIDER_MIN_MARGIN } from "./decider.js";
31
+ export type { DecisionProvider, DeciderConfig, DeciderMode, DeciderRecord, SystemOneOptions } from "./decider.js";
30
32
  export { BudgetGuard, describeOverrun, minLimits, toLimits } from "./budget.js";
31
33
  export type { Budget, BudgetOverrun, BudgetUnit, SpendLimits } from "./budget.js";
32
34
  export { resolveNodePermissions, buildNodePolicy } from "./node-policy.js";