@sweny-ai/core 0.1.162 → 0.1.164
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/main.d.ts +1 -0
- package/dist/cli/main.js +7 -2
- package/dist/cli/run-history.d.ts +6 -0
- package/dist/cli/run-history.js +10 -1
- package/dist/cli/run-output.d.ts +11 -1
- package/dist/cli/run-output.js +18 -1
- package/dist/decider.d.ts +144 -0
- package/dist/decider.js +260 -0
- package/dist/executor.d.ts +7 -0
- package/dist/executor.js +26 -2
- package/dist/harness/codex.d.ts +7 -0
- package/dist/harness/codex.js +14 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/schema.d.ts +190 -26
- package/dist/schema.js +51 -0
- package/dist/types.d.ts +8 -0
- package/package.json +1 -1
package/dist/cli/main.d.ts
CHANGED
|
@@ -12,6 +12,7 @@ export declare function workflowRunAction(file: string | undefined, options: Rec
|
|
|
12
12
|
maxTokens?: string;
|
|
13
13
|
maxCost?: string;
|
|
14
14
|
yes?: boolean;
|
|
15
|
+
decider?: string;
|
|
15
16
|
}): Promise<void>;
|
|
16
17
|
export declare function workflowExportAction(name: string): void;
|
|
17
18
|
export declare function workflowValidateAction(file: string, options: {
|
package/dist/cli/main.js
CHANGED
|
@@ -731,6 +731,10 @@ export function loadWorkflowFile(filePath, knownSkills) {
|
|
|
731
731
|
return result.workflow;
|
|
732
732
|
}
|
|
733
733
|
export async function workflowRunAction(file, options) {
|
|
734
|
+
if (options.decider !== undefined && options.decider !== "off" && options.decider !== "shadow") {
|
|
735
|
+
console.error(chalk.red(`\n --decider must be off or shadow, got "${options.decider}"\n`));
|
|
736
|
+
process.exit(1);
|
|
737
|
+
}
|
|
734
738
|
// Reject junk --timeout/--max-steps up front (both paths) instead of
|
|
735
739
|
// silently falling back to a default.
|
|
736
740
|
let budget;
|
|
@@ -1005,6 +1009,7 @@ export async function workflowRunAction(file, options) {
|
|
|
1005
1009
|
fileRoot: config.fileRoot || undefined,
|
|
1006
1010
|
signal,
|
|
1007
1011
|
max_steps: wfMaxSteps,
|
|
1012
|
+
...(options.decider ? { decider: options.decider } : {}),
|
|
1008
1013
|
stageOutputs: options.stage === true,
|
|
1009
1014
|
...(spendBudget ? { budget: spendBudget } : {}),
|
|
1010
1015
|
harnessPolicy: resolveHarnessPolicy(process.env, typeof options.harnessPolicy === "string" ? options.harnessPolicy : undefined, runLogger),
|
|
@@ -1024,7 +1029,7 @@ export async function workflowRunAction(file, options) {
|
|
|
1024
1029
|
// PR billboard markdown (metadata only). Written before any early exit so
|
|
1025
1030
|
// --json runs and failed runs still get a comment.
|
|
1026
1031
|
if (options.commentFile) {
|
|
1027
|
-
writeRunComment(options.commentFile, workflow, results, summarizeRun(results, wfDurationMs), {
|
|
1032
|
+
writeRunComment(options.commentFile, workflow, results, summarizeRun(results, wfDurationMs, false, trace), {
|
|
1028
1033
|
trace,
|
|
1029
1034
|
durationsMs: Object.fromEntries(nodeTimer.durations),
|
|
1030
1035
|
});
|
|
@@ -1051,7 +1056,7 @@ export async function workflowRunAction(file, options) {
|
|
|
1051
1056
|
}
|
|
1052
1057
|
// Run receipt (metadata only) + optional $GITHUB_STEP_SUMMARY.
|
|
1053
1058
|
runLogger.flush();
|
|
1054
|
-
const receipt = summarizeRun(results, wfDurationMs);
|
|
1059
|
+
const receipt = summarizeRun(results, wfDurationMs, false, trace);
|
|
1055
1060
|
writeStepSummary(workflow, results, receipt, trace);
|
|
1056
1061
|
// The answer, above the receipt: the terminal node's result, or the failed node's error.
|
|
1057
1062
|
// Also saved to .sweny/runs/<id>/output.md (local only; --no-history skips the file).
|
|
@@ -55,6 +55,12 @@ export interface RunRecord {
|
|
|
55
55
|
};
|
|
56
56
|
/** Opinions the harness could not honor natively on some node, deduped. Absent when none. */
|
|
57
57
|
degraded?: string[];
|
|
58
|
+
/** Shadow-mode decider agreement counts (#357). Absent when the decider was off. */
|
|
59
|
+
decider?: {
|
|
60
|
+
compared: number;
|
|
61
|
+
agreed: number;
|
|
62
|
+
fell_through: number;
|
|
63
|
+
};
|
|
58
64
|
}
|
|
59
65
|
/** sha256 (hex) of the workflow as canonical JSON: keys sorted, undefined dropped. */
|
|
60
66
|
export declare function hashWorkflow(workflow: Workflow): string;
|
package/dist/cli/run-history.js
CHANGED
|
@@ -69,7 +69,7 @@ export function createNodeTimer(now = Date.now) {
|
|
|
69
69
|
// ── Build the record ────────────────────────────────────────────
|
|
70
70
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
71
71
|
export function buildRunRecord(i) {
|
|
72
|
-
const summary = summarizeRun(i.results, i.durationMs, i.crashed);
|
|
72
|
+
const summary = summarizeRun(i.results, i.durationMs, i.crashed, i.trace);
|
|
73
73
|
const retries = new Map();
|
|
74
74
|
for (const s of i.trace?.steps ?? []) {
|
|
75
75
|
if (s.retryAttempt !== undefined)
|
|
@@ -113,6 +113,15 @@ export function buildRunRecord(i) {
|
|
|
113
113
|
},
|
|
114
114
|
...(harness ? { harness: { id: harness.id, version: harness.version } } : {}),
|
|
115
115
|
...(degraded.length > 0 ? { degraded } : {}),
|
|
116
|
+
...(summary.decider
|
|
117
|
+
? {
|
|
118
|
+
decider: {
|
|
119
|
+
compared: summary.decider.compared,
|
|
120
|
+
agreed: summary.decider.agreed,
|
|
121
|
+
fell_through: summary.decider.fellThrough,
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
: {}),
|
|
116
125
|
};
|
|
117
126
|
}
|
|
118
127
|
// ── Write, prune, read ──────────────────────────────────────────
|
package/dist/cli/run-output.d.ts
CHANGED
|
@@ -39,6 +39,16 @@ export interface RunSummary {
|
|
|
39
39
|
limit: number;
|
|
40
40
|
spent: number;
|
|
41
41
|
};
|
|
42
|
+
/**
|
|
43
|
+
* Shadow-mode decider agreement (#357), counts only. `compared` decisions
|
|
44
|
+
* passed every gate; `agreed` matched the agent; the rest fell through.
|
|
45
|
+
* Absent when the decider was off or never consulted.
|
|
46
|
+
*/
|
|
47
|
+
decider?: {
|
|
48
|
+
compared: number;
|
|
49
|
+
agreed: number;
|
|
50
|
+
fellThrough: number;
|
|
51
|
+
};
|
|
42
52
|
}
|
|
43
53
|
export interface RunPolicy {
|
|
44
54
|
/** Agent env narrowed to the allowlist. Absent when no node reported it. */
|
|
@@ -57,7 +67,7 @@ export interface RunPolicy {
|
|
|
57
67
|
/** The short key of a `degraded` entry: the text before its first colon. */
|
|
58
68
|
export declare function degradedKey(entry: string): string;
|
|
59
69
|
/** Sum the per-node results into one run summary. */
|
|
60
|
-
export declare function summarizeRun(results: Map<string, NodeResult>, durationMs: number, crashed?: boolean): RunSummary;
|
|
70
|
+
export declare function summarizeRun(results: Map<string, NodeResult>, durationMs: number, crashed?: boolean, trace?: ExecutionTrace): RunSummary;
|
|
61
71
|
/**
|
|
62
72
|
* `policy: env scoped, sandbox on, 1 output staged`. Only what the run knew:
|
|
63
73
|
* no segment for a fact no node reported. Undefined when there is nothing to say.
|
package/dist/cli/run-output.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import fs from "node:fs";
|
|
9
9
|
import chalk from "chalk";
|
|
10
10
|
import { consoleLogger } from "../types.js";
|
|
11
|
+
import { summarizeDecisions } from "../decider.js";
|
|
11
12
|
import { toMermaidBlock } from "../mermaid.js";
|
|
12
13
|
/** The short key of a `degraded` entry: the text before its first colon. */
|
|
13
14
|
export function degradedKey(entry) {
|
|
@@ -15,7 +16,7 @@ export function degradedKey(entry) {
|
|
|
15
16
|
return (i === -1 ? entry : entry.slice(0, i)).trim();
|
|
16
17
|
}
|
|
17
18
|
/** Sum the per-node results into one run summary. */
|
|
18
|
-
export function summarizeRun(results, durationMs, crashed = false) {
|
|
19
|
+
export function summarizeRun(results, durationMs, crashed = false, trace) {
|
|
19
20
|
let nodesOk = 0;
|
|
20
21
|
let nodesSkipped = 0;
|
|
21
22
|
let failed = crashed;
|
|
@@ -81,8 +82,13 @@ export function summarizeRun(results, durationMs, crashed = false) {
|
|
|
81
82
|
...(degraded.size > 0 ? { degraded: [...degraded] } : {}),
|
|
82
83
|
...(Object.keys(policy).length > 0 ? { policy } : {}),
|
|
83
84
|
...(budget ? { budget } : {}),
|
|
85
|
+
...(trace?.decisions && trace.decisions.length > 0 ? { decider: decisionCounts(trace) } : {}),
|
|
84
86
|
};
|
|
85
87
|
}
|
|
88
|
+
function decisionCounts(trace) {
|
|
89
|
+
const d = summarizeDecisions(trace.decisions ?? []);
|
|
90
|
+
return { compared: d.compared, agreed: d.agreed, fellThrough: d.fell_through };
|
|
91
|
+
}
|
|
86
92
|
/**
|
|
87
93
|
* `policy: env scoped, sandbox on, 1 output staged`. Only what the run knew:
|
|
88
94
|
* no segment for a fact no node reported. Undefined when there is nothing to say.
|
|
@@ -156,6 +162,13 @@ export function formatReceipt(s) {
|
|
|
156
162
|
...(s.degraded && s.degraded.length > 0 ? [`degraded: ${s.degraded.join(", ")}`] : []),
|
|
157
163
|
...(formatPolicySegment(s.policy) ? [formatPolicySegment(s.policy)] : []),
|
|
158
164
|
...(s.budget ? [formatBudgetOverrun(s.budget)] : []),
|
|
165
|
+
...(s.decider
|
|
166
|
+
? [
|
|
167
|
+
s.decider.compared > 0
|
|
168
|
+
? `decider agreed ${s.decider.agreed}/${s.decider.compared}`
|
|
169
|
+
: `decider fell through ${s.decider.fellThrough}/${s.decider.fellThrough}`,
|
|
170
|
+
]
|
|
171
|
+
: []),
|
|
159
172
|
];
|
|
160
173
|
return parts.join(" · ");
|
|
161
174
|
}
|
|
@@ -286,4 +299,8 @@ export const WORKFLOW_RUN_OPTIONS = [
|
|
|
286
299
|
"--harness-policy <mode>",
|
|
287
300
|
"strict: refuse a node whose policy the agent cannot enforce; warn: run it and report what was not enforced (default: strict under GitHub Actions, warn elsewhere; env SWENY_HARNESS_POLICY)",
|
|
288
301
|
],
|
|
302
|
+
[
|
|
303
|
+
"--decider <mode>",
|
|
304
|
+
"Decision model for route choices: off (default) or shadow. Shadow asks the workflow's decider.provider alongside the agent and reports agreement; the route is always the agent's. Needs decider.provider in the workflow (no default URL)",
|
|
305
|
+
],
|
|
289
306
|
];
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decision models (#357), shadow mode.
|
|
3
|
+
*
|
|
4
|
+
* A System One decision model (Ollama `/v1/systemone`, TypeSafe Jev) answers
|
|
5
|
+
* one typed Choice question over a state in milliseconds. In shadow mode the
|
|
6
|
+
* executor asks it the same route question the agent answers, in parallel,
|
|
7
|
+
* and records whether they agree. The route is ALWAYS the agent's.
|
|
8
|
+
*
|
|
9
|
+
* Privacy: the state goes only to the provider the workflow configured, never
|
|
10
|
+
* to a default or fallback URL. What this module logs is metadata only
|
|
11
|
+
* (labels are edge target node ids, plus numbers and a hash).
|
|
12
|
+
*/
|
|
13
|
+
/** Accept a decider answer only at or above this confidence. */
|
|
14
|
+
export declare const DECIDER_MIN_CONFIDENCE = 0.85;
|
|
15
|
+
/** ... and only when the top label leads the runner-up by at least this much. */
|
|
16
|
+
export declare const DECIDER_MIN_MARGIN = 0.2;
|
|
17
|
+
/** Per-call timeout when the provider does not set one. */
|
|
18
|
+
export declare const DECIDER_DEFAULT_TIMEOUT_MS = 2000;
|
|
19
|
+
export type DeciderMode = "off" | "shadow";
|
|
20
|
+
/** Workflow-level `decider` block. */
|
|
21
|
+
export interface DeciderConfig {
|
|
22
|
+
mode: DeciderMode;
|
|
23
|
+
provider?: {
|
|
24
|
+
/** Server root, e.g. `http://localhost:11434` or `https://api.typesafe.ai`. */
|
|
25
|
+
base_url: string;
|
|
26
|
+
model: string;
|
|
27
|
+
/** Name of the env var holding the bearer key. Never the key itself. */
|
|
28
|
+
api_key_env?: string;
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/** One Choice question: pick a key of `criteria`. */
|
|
32
|
+
export interface ChoiceAsk {
|
|
33
|
+
kind: "choice";
|
|
34
|
+
instructions: string;
|
|
35
|
+
criteria: Record<string, string>;
|
|
36
|
+
}
|
|
37
|
+
/** Only Choice exists today. Noul and Score arrive as further union members. */
|
|
38
|
+
export type Ask = ChoiceAsk;
|
|
39
|
+
export interface ChoiceVerdict {
|
|
40
|
+
kind: "choice";
|
|
41
|
+
label: string;
|
|
42
|
+
probabilities: Record<string, number>;
|
|
43
|
+
confidence: number;
|
|
44
|
+
}
|
|
45
|
+
export type Verdict = ChoiceVerdict;
|
|
46
|
+
export type DecideErrorCode = "timeout" | "aborted" | "network" | "unauthorized" | "invalid_request" | "rate_limited" | "overloaded" | "http_error" | "malformed" | "unknown_label";
|
|
47
|
+
export declare class DecideError extends Error {
|
|
48
|
+
readonly code: DecideErrorCode;
|
|
49
|
+
constructor(code: DecideErrorCode, message: string);
|
|
50
|
+
}
|
|
51
|
+
export interface DecideRequest {
|
|
52
|
+
state: unknown;
|
|
53
|
+
asks: Record<string, Ask>;
|
|
54
|
+
timeoutMs?: number;
|
|
55
|
+
signal?: AbortSignal;
|
|
56
|
+
}
|
|
57
|
+
export interface DecisionProvider {
|
|
58
|
+
/** Short provider family label for logs, e.g. `systemone`. */
|
|
59
|
+
id: string;
|
|
60
|
+
model: string;
|
|
61
|
+
/** Throws {@link DecideError}; never returns a partial answer. */
|
|
62
|
+
decide(req: DecideRequest): Promise<Record<string, Verdict>>;
|
|
63
|
+
}
|
|
64
|
+
export interface SystemOneOptions {
|
|
65
|
+
baseUrl: string;
|
|
66
|
+
model: string;
|
|
67
|
+
apiKey?: string;
|
|
68
|
+
/** Default timeout when `decide()` is not given one. Default 2000 ms. */
|
|
69
|
+
timeoutMs?: number;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* `POST {baseUrl}/v1/systemone`. Ollama (local) and TypeSafe Jev (hosted)
|
|
73
|
+
* share this wire shape; only base URL, key and model differ. Response
|
|
74
|
+
* validation is strict: any deviation is a `malformed` or `unknown_label`
|
|
75
|
+
* error, never a guess.
|
|
76
|
+
*/
|
|
77
|
+
export declare function systemOneProvider(opts: SystemOneOptions): DecisionProvider;
|
|
78
|
+
export type FallThroughReason = DecideErrorCode | "low_confidence" | "low_margin" | "label_outside_edge_set";
|
|
79
|
+
export interface GateResult {
|
|
80
|
+
accepted: boolean;
|
|
81
|
+
reason?: FallThroughReason;
|
|
82
|
+
/** Top probability minus the runner-up (0 when there is no runner-up). */
|
|
83
|
+
margin: number;
|
|
84
|
+
}
|
|
85
|
+
/** All gates must pass: label in the edge set, confidence, then top-two margin. */
|
|
86
|
+
export declare function gateVerdict(v: ChoiceVerdict, edgeSet: ReadonlySet<string>): GateResult;
|
|
87
|
+
/** One logged shadow decision. Metadata only: no state, no prompts, no prose. */
|
|
88
|
+
export interface DeciderRecord {
|
|
89
|
+
/** Node the route was decided at. */
|
|
90
|
+
node: string;
|
|
91
|
+
/** Edge target the decider picked (a node id). Null when it gave no answer. */
|
|
92
|
+
decider_label: string | null;
|
|
93
|
+
/** Edge target the agent picked. Null when the agent's evaluation failed. */
|
|
94
|
+
agent_label: string | null;
|
|
95
|
+
confidence: number | null;
|
|
96
|
+
margin: number | null;
|
|
97
|
+
/** `compared`: the decider passed every gate. `fell_through`: it did not, see `reason`. */
|
|
98
|
+
outcome: "compared" | "fell_through";
|
|
99
|
+
reason?: FallThroughReason | "agent_failed";
|
|
100
|
+
/** Decider label equals agent label. Null unless `outcome` is `compared`. */
|
|
101
|
+
agree: boolean | null;
|
|
102
|
+
latency_ms: number;
|
|
103
|
+
model: string;
|
|
104
|
+
/** First 16 hex chars of sha256 over the state and criteria sent. */
|
|
105
|
+
input_hash: string;
|
|
106
|
+
}
|
|
107
|
+
/** What the executor holds for one run in shadow mode. */
|
|
108
|
+
export interface ShadowDecider {
|
|
109
|
+
provider: DecisionProvider;
|
|
110
|
+
records: DeciderRecord[];
|
|
111
|
+
}
|
|
112
|
+
export declare function hashDecisionInput(state: unknown, criteria: Record<string, string>): string;
|
|
113
|
+
export interface PendingShadow {
|
|
114
|
+
/** Never rejects. */
|
|
115
|
+
settled: Promise<Omit<DeciderRecord, "agent_label" | "agree" | "node">>;
|
|
116
|
+
edgeSet: ReadonlySet<string>;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Start the decider call. Returns at once; the promise never rejects, so the
|
|
120
|
+
* agent's own evaluation is never delayed or failed by it.
|
|
121
|
+
*/
|
|
122
|
+
export declare function startShadowDecision(provider: DecisionProvider, req: {
|
|
123
|
+
question: string;
|
|
124
|
+
state: unknown;
|
|
125
|
+
choices: {
|
|
126
|
+
id: string;
|
|
127
|
+
description: string;
|
|
128
|
+
}[];
|
|
129
|
+
signal?: AbortSignal;
|
|
130
|
+
}): PendingShadow;
|
|
131
|
+
/** Join the decider's settled result with the agent's choice. */
|
|
132
|
+
export declare function finishShadowDecision(pending: PendingShadow, node: string, agentLabel: string | null): Promise<DeciderRecord>;
|
|
133
|
+
/** `{ compared, agreed, fell_through }` over a run's records. */
|
|
134
|
+
export declare function summarizeDecisions(records: readonly DeciderRecord[]): {
|
|
135
|
+
compared: number;
|
|
136
|
+
agreed: number;
|
|
137
|
+
fell_through: number;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* Resolve the run's shadow decider, or null (mode off, nothing configured, or
|
|
141
|
+
* a declared key env var is unset). No hosted fallback exists: null means no
|
|
142
|
+
* HTTP at all.
|
|
143
|
+
*/
|
|
144
|
+
export declare function createShadowDecider(config: DeciderConfig | undefined, modeOverride: DeciderMode | undefined, env: NodeJS.ProcessEnv, warn: (msg: string) => void): ShadowDecider | null;
|
package/dist/decider.js
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decision models (#357), shadow mode.
|
|
3
|
+
*
|
|
4
|
+
* A System One decision model (Ollama `/v1/systemone`, TypeSafe Jev) answers
|
|
5
|
+
* one typed Choice question over a state in milliseconds. In shadow mode the
|
|
6
|
+
* executor asks it the same route question the agent answers, in parallel,
|
|
7
|
+
* and records whether they agree. The route is ALWAYS the agent's.
|
|
8
|
+
*
|
|
9
|
+
* Privacy: the state goes only to the provider the workflow configured, never
|
|
10
|
+
* to a default or fallback URL. What this module logs is metadata only
|
|
11
|
+
* (labels are edge target node ids, plus numbers and a hash).
|
|
12
|
+
*/
|
|
13
|
+
import crypto from "node:crypto";
|
|
14
|
+
/** Accept a decider answer only at or above this confidence. */
|
|
15
|
+
export const DECIDER_MIN_CONFIDENCE = 0.85;
|
|
16
|
+
/** ... and only when the top label leads the runner-up by at least this much. */
|
|
17
|
+
export const DECIDER_MIN_MARGIN = 0.2;
|
|
18
|
+
/** Per-call timeout when the provider does not set one. */
|
|
19
|
+
export const DECIDER_DEFAULT_TIMEOUT_MS = 2000;
|
|
20
|
+
export class DecideError extends Error {
|
|
21
|
+
code;
|
|
22
|
+
constructor(code, message) {
|
|
23
|
+
super(message);
|
|
24
|
+
this.code = code;
|
|
25
|
+
this.name = "DecideError";
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
29
|
+
const isProb = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
|
|
30
|
+
/**
|
|
31
|
+
* `POST {baseUrl}/v1/systemone`. Ollama (local) and TypeSafe Jev (hosted)
|
|
32
|
+
* share this wire shape; only base URL, key and model differ. Response
|
|
33
|
+
* validation is strict: any deviation is a `malformed` or `unknown_label`
|
|
34
|
+
* error, never a guess.
|
|
35
|
+
*/
|
|
36
|
+
export function systemOneProvider(opts) {
|
|
37
|
+
const url = `${opts.baseUrl.replace(/\/+$/, "")}/v1/systemone`;
|
|
38
|
+
return {
|
|
39
|
+
id: "systemone",
|
|
40
|
+
model: opts.model,
|
|
41
|
+
async decide(req) {
|
|
42
|
+
const timeoutMs = req.timeoutMs ?? opts.timeoutMs ?? DECIDER_DEFAULT_TIMEOUT_MS;
|
|
43
|
+
const questions = {};
|
|
44
|
+
for (const [name, ask] of Object.entries(req.asks)) {
|
|
45
|
+
questions[name] = { type: ask.kind, instructions: ask.instructions, criteria: ask.criteria };
|
|
46
|
+
}
|
|
47
|
+
const ctl = new AbortController();
|
|
48
|
+
let timedOut = false;
|
|
49
|
+
const timer = setTimeout(() => {
|
|
50
|
+
timedOut = true;
|
|
51
|
+
ctl.abort();
|
|
52
|
+
}, timeoutMs);
|
|
53
|
+
const onAbort = () => ctl.abort();
|
|
54
|
+
if (req.signal?.aborted)
|
|
55
|
+
ctl.abort();
|
|
56
|
+
req.signal?.addEventListener("abort", onAbort, { once: true });
|
|
57
|
+
try {
|
|
58
|
+
let res;
|
|
59
|
+
try {
|
|
60
|
+
res = await fetch(url, {
|
|
61
|
+
method: "POST",
|
|
62
|
+
headers: {
|
|
63
|
+
"content-type": "application/json",
|
|
64
|
+
...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}),
|
|
65
|
+
},
|
|
66
|
+
body: JSON.stringify({ model: opts.model, state: req.state, questions }),
|
|
67
|
+
signal: ctl.signal,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
if (timedOut)
|
|
72
|
+
throw new DecideError("timeout", `no answer within ${timeoutMs}ms`);
|
|
73
|
+
if (ctl.signal.aborted)
|
|
74
|
+
throw new DecideError("aborted", "request aborted");
|
|
75
|
+
throw new DecideError("network", err instanceof Error ? err.name : "fetch failed");
|
|
76
|
+
}
|
|
77
|
+
if (!res.ok) {
|
|
78
|
+
const s = res.status;
|
|
79
|
+
const code = s === 401 || s === 403
|
|
80
|
+
? "unauthorized"
|
|
81
|
+
: s === 400 || s === 422
|
|
82
|
+
? "invalid_request"
|
|
83
|
+
: s === 429
|
|
84
|
+
? "rate_limited"
|
|
85
|
+
: s === 529 || s === 503
|
|
86
|
+
? "overloaded"
|
|
87
|
+
: "http_error";
|
|
88
|
+
throw new DecideError(code, `HTTP ${s}`);
|
|
89
|
+
}
|
|
90
|
+
let body;
|
|
91
|
+
try {
|
|
92
|
+
body = await res.json();
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
if (timedOut)
|
|
96
|
+
throw new DecideError("timeout", `no answer within ${timeoutMs}ms`);
|
|
97
|
+
throw new DecideError("malformed", "response is not JSON");
|
|
98
|
+
}
|
|
99
|
+
return parseAnswers(body, req.asks);
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
clearTimeout(timer);
|
|
103
|
+
req.signal?.removeEventListener("abort", onAbort);
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function parseAnswers(body, asks) {
|
|
109
|
+
if (!isRecord(body) || !isRecord(body.answers))
|
|
110
|
+
throw new DecideError("malformed", "missing answers object");
|
|
111
|
+
const out = {};
|
|
112
|
+
for (const [name, ask] of Object.entries(asks)) {
|
|
113
|
+
const a = body.answers[name];
|
|
114
|
+
if (!isRecord(a))
|
|
115
|
+
throw new DecideError("malformed", `no answer for question '${name}'`);
|
|
116
|
+
if (a.type !== undefined && a.type !== ask.kind)
|
|
117
|
+
throw new DecideError("malformed", "answer type mismatch");
|
|
118
|
+
if (typeof a.choice !== "string")
|
|
119
|
+
throw new DecideError("malformed", "choice is not a string");
|
|
120
|
+
if (!isProb(a.confidence))
|
|
121
|
+
throw new DecideError("malformed", "confidence is not a number in 0..1");
|
|
122
|
+
if (!isRecord(a.probabilities))
|
|
123
|
+
throw new DecideError("malformed", "probabilities is not an object");
|
|
124
|
+
const probabilities = {};
|
|
125
|
+
for (const [k, v] of Object.entries(a.probabilities)) {
|
|
126
|
+
if (!isProb(v))
|
|
127
|
+
throw new DecideError("malformed", "probability is not a number in 0..1");
|
|
128
|
+
probabilities[k] = v;
|
|
129
|
+
}
|
|
130
|
+
const known = Object.keys(ask.criteria);
|
|
131
|
+
if (!known.includes(a.choice) ||
|
|
132
|
+
!(a.choice in probabilities) ||
|
|
133
|
+
Object.keys(probabilities).some((k) => !known.includes(k))) {
|
|
134
|
+
throw new DecideError("unknown_label", "answer names a label outside the criteria");
|
|
135
|
+
}
|
|
136
|
+
out[name] = { kind: "choice", label: a.choice, probabilities, confidence: a.confidence };
|
|
137
|
+
}
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
/** All gates must pass: label in the edge set, confidence, then top-two margin. */
|
|
141
|
+
export function gateVerdict(v, edgeSet) {
|
|
142
|
+
const top = v.probabilities[v.label] ?? 0;
|
|
143
|
+
const second = Math.max(0, ...Object.entries(v.probabilities)
|
|
144
|
+
.filter(([k]) => k !== v.label)
|
|
145
|
+
.map(([, p]) => p));
|
|
146
|
+
const margin = top - second;
|
|
147
|
+
if (!edgeSet.has(v.label))
|
|
148
|
+
return { accepted: false, reason: "label_outside_edge_set", margin };
|
|
149
|
+
if (!(v.confidence >= DECIDER_MIN_CONFIDENCE))
|
|
150
|
+
return { accepted: false, reason: "low_confidence", margin };
|
|
151
|
+
if (!(margin >= DECIDER_MIN_MARGIN))
|
|
152
|
+
return { accepted: false, reason: "low_margin", margin };
|
|
153
|
+
return { accepted: true, margin };
|
|
154
|
+
}
|
|
155
|
+
export function hashDecisionInput(state, criteria) {
|
|
156
|
+
let json;
|
|
157
|
+
try {
|
|
158
|
+
json = JSON.stringify({ state, criteria }) ?? "";
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
json = "unserializable";
|
|
162
|
+
}
|
|
163
|
+
return crypto.createHash("sha256").update(json).digest("hex").slice(0, 16);
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Start the decider call. Returns at once; the promise never rejects, so the
|
|
167
|
+
* agent's own evaluation is never delayed or failed by it.
|
|
168
|
+
*/
|
|
169
|
+
export function startShadowDecision(provider, req) {
|
|
170
|
+
const criteria = {};
|
|
171
|
+
for (const c of req.choices)
|
|
172
|
+
if (!(c.id in criteria))
|
|
173
|
+
criteria[c.id] = c.description;
|
|
174
|
+
const edgeSet = new Set(Object.keys(criteria));
|
|
175
|
+
const input_hash = hashDecisionInput(req.state, criteria);
|
|
176
|
+
const started = Date.now();
|
|
177
|
+
const base = { model: provider.model, input_hash };
|
|
178
|
+
const settled = (async () => {
|
|
179
|
+
try {
|
|
180
|
+
const out = await provider.decide({
|
|
181
|
+
state: req.state,
|
|
182
|
+
asks: { route: { kind: "choice", instructions: req.question, criteria } },
|
|
183
|
+
timeoutMs: DECIDER_DEFAULT_TIMEOUT_MS,
|
|
184
|
+
signal: req.signal,
|
|
185
|
+
});
|
|
186
|
+
const v = out.route;
|
|
187
|
+
if (!v || v.kind !== "choice")
|
|
188
|
+
throw new DecideError("malformed", "no route answer");
|
|
189
|
+
const gate = gateVerdict(v, edgeSet);
|
|
190
|
+
return {
|
|
191
|
+
...base,
|
|
192
|
+
decider_label: v.label,
|
|
193
|
+
confidence: v.confidence,
|
|
194
|
+
margin: gate.margin,
|
|
195
|
+
outcome: gate.accepted ? "compared" : "fell_through",
|
|
196
|
+
...(gate.reason ? { reason: gate.reason } : {}),
|
|
197
|
+
latency_ms: Date.now() - started,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
catch (err) {
|
|
201
|
+
return {
|
|
202
|
+
...base,
|
|
203
|
+
decider_label: null,
|
|
204
|
+
confidence: null,
|
|
205
|
+
margin: null,
|
|
206
|
+
outcome: "fell_through",
|
|
207
|
+
reason: err instanceof DecideError ? err.code : "malformed",
|
|
208
|
+
latency_ms: Date.now() - started,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
})();
|
|
212
|
+
return { settled, edgeSet };
|
|
213
|
+
}
|
|
214
|
+
/** Join the decider's settled result with the agent's choice. */
|
|
215
|
+
export async function finishShadowDecision(pending, node, agentLabel) {
|
|
216
|
+
const s = await pending.settled;
|
|
217
|
+
if (s.outcome === "fell_through")
|
|
218
|
+
return { node, agent_label: agentLabel, agree: null, ...s };
|
|
219
|
+
if (agentLabel === null) {
|
|
220
|
+
return { node, agent_label: null, agree: null, ...s, outcome: "fell_through", reason: "agent_failed" };
|
|
221
|
+
}
|
|
222
|
+
return { node, agent_label: agentLabel, agree: s.decider_label === agentLabel, ...s };
|
|
223
|
+
}
|
|
224
|
+
/** `{ compared, agreed, fell_through }` over a run's records. */
|
|
225
|
+
export function summarizeDecisions(records) {
|
|
226
|
+
let compared = 0;
|
|
227
|
+
let agreed = 0;
|
|
228
|
+
for (const r of records) {
|
|
229
|
+
if (r.outcome === "compared") {
|
|
230
|
+
compared++;
|
|
231
|
+
if (r.agree)
|
|
232
|
+
agreed++;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
return { compared, agreed, fell_through: records.length - compared };
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Resolve the run's shadow decider, or null (mode off, nothing configured, or
|
|
239
|
+
* a declared key env var is unset). No hosted fallback exists: null means no
|
|
240
|
+
* HTTP at all.
|
|
241
|
+
*/
|
|
242
|
+
export function createShadowDecider(config, modeOverride, env, warn) {
|
|
243
|
+
const mode = modeOverride ?? config?.mode ?? "off";
|
|
244
|
+
if (mode !== "shadow")
|
|
245
|
+
return null;
|
|
246
|
+
const p = config?.provider;
|
|
247
|
+
if (!p) {
|
|
248
|
+
warn(" decider: shadow mode needs decider.provider in the workflow; running without it.");
|
|
249
|
+
return null;
|
|
250
|
+
}
|
|
251
|
+
let apiKey;
|
|
252
|
+
if (p.api_key_env) {
|
|
253
|
+
apiKey = env[p.api_key_env];
|
|
254
|
+
if (!apiKey) {
|
|
255
|
+
warn(` decider: env var ${p.api_key_env} is not set; running without the decider.`);
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return { provider: systemOneProvider({ baseUrl: p.base_url, model: p.model, apiKey }), records: [] };
|
|
260
|
+
}
|
package/dist/executor.d.ts
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import type { Workflow, Skill, Tool, Claude, Observer, Logger, ExecutionResult } from "./types.js";
|
|
16
16
|
import type { HarnessPolicyMode } from "./harness/policy.js";
|
|
17
17
|
import type { AgentHarness } from "./harness/types.js";
|
|
18
|
+
import type { DeciderMode } from "./decider.js";
|
|
18
19
|
import type { Budget } from "./budget.js";
|
|
19
20
|
import { type ActorInfo } from "./safe-outputs.js";
|
|
20
21
|
export interface ExecuteOptions {
|
|
@@ -97,6 +98,12 @@ export interface ExecuteOptions {
|
|
|
97
98
|
* all (cost on Codex). Default: {@link resolveHarnessPolicy} from the run env.
|
|
98
99
|
*/
|
|
99
100
|
harnessPolicy?: HarnessPolicyMode;
|
|
101
|
+
/**
|
|
102
|
+
* Override `workflow.decider.mode` (CLI `--decider`). `shadow` still needs
|
|
103
|
+
* `decider.provider` in the workflow; there is no default URL. Default: the
|
|
104
|
+
* workflow's own mode, else off (zero HTTP calls).
|
|
105
|
+
*/
|
|
106
|
+
decider?: DeciderMode;
|
|
100
107
|
}
|
|
101
108
|
/**
|
|
102
109
|
* Default workflow-level step budget. Counts total node executions across the
|
package/dist/executor.js
CHANGED
|
@@ -26,6 +26,7 @@ import { fenceUntrusted } from "./untrusted.js";
|
|
|
26
26
|
import { asClaude } from "./harness/compat.js";
|
|
27
27
|
import { budgetGate, isToolClass, policyGate, resolveHarnessPolicy } from "./harness/policy.js";
|
|
28
28
|
import { BudgetGuard, describeOverrun, minLimits, toLimits } from "./budget.js";
|
|
29
|
+
import { createShadowDecider, finishShadowDecision, startShadowDecision } from "./decider.js";
|
|
29
30
|
import { buildNodePolicy, resolveNodePermissions } from "./node-policy.js";
|
|
30
31
|
import { applySafeOutputs, createEmitOutputTool, createWriteStageState, resolveActor, safeOutputsInstruction, unresolvedOutputs, SAFE_OUTPUT_SCREEN_INSTRUCTION, } from "./safe-outputs.js";
|
|
31
32
|
/**
|
|
@@ -88,6 +89,10 @@ export async function execute(workflow, input, options) {
|
|
|
88
89
|
// `budget` and the caller's (`--max-tokens`, `--max-cost`).
|
|
89
90
|
const budgetGuard = new BudgetGuard(minLimits(toLimits(workflow.budget), toLimits(options.budget)));
|
|
90
91
|
const harnessPolicy = options.harnessPolicy ?? resolveHarnessPolicy(runEnv);
|
|
92
|
+
// Decision model (#357): shadow only. Null (the default) means no HTTP at all.
|
|
93
|
+
const shadow = createShadowDecider(workflow.decider, options.decider, runEnv, (m) => logger.warn(m));
|
|
94
|
+
if (shadow)
|
|
95
|
+
trace.decisions = shadow.records;
|
|
91
96
|
// Build an eval-time alias table from the loaded skills. Each skill owns
|
|
92
97
|
// its own mapping between skill-tool names and equivalent MCP names. Core
|
|
93
98
|
// stays vendor-neutral; this call just unions what the skills declare.
|
|
@@ -254,7 +259,7 @@ export async function execute(workflow, input, options) {
|
|
|
254
259
|
}
|
|
255
260
|
// Apply normal routing rules (dry run gate + resolveNext).
|
|
256
261
|
// TODO: dedupe with requires path — see advanceFromNode helper below
|
|
257
|
-
const next = await advanceFromNode(workflow, currentId, results, input, claude, observer, edgeCounts, logger, trace, { signal, timeoutMs });
|
|
262
|
+
const next = await advanceFromNode(workflow, currentId, results, input, claude, observer, edgeCounts, logger, trace, { signal, timeoutMs, shadow });
|
|
258
263
|
currentId = next;
|
|
259
264
|
continue;
|
|
260
265
|
}
|
|
@@ -591,6 +596,7 @@ export async function execute(workflow, input, options) {
|
|
|
591
596
|
currentId = await advanceFromNode(workflow, currentId, results, input, claude, observer, edgeCounts, logger, trace, {
|
|
592
597
|
signal,
|
|
593
598
|
timeoutMs,
|
|
599
|
+
shadow,
|
|
594
600
|
});
|
|
595
601
|
}
|
|
596
602
|
safeObserve(observer, {
|
|
@@ -1248,13 +1254,31 @@ async function resolveNext(workflow, current, results, input, claude, observer,
|
|
|
1248
1254
|
if (defaultEdge) {
|
|
1249
1255
|
choices.push({ id: defaultEdge.to, description: "None of the above / default path" });
|
|
1250
1256
|
}
|
|
1257
|
+
const question = "Based on the results so far, which condition is true?";
|
|
1258
|
+
// Shadow mode (#357): the decider gets the same question in parallel. Its
|
|
1259
|
+
// promise never rejects and its answer is only logged, so the route below
|
|
1260
|
+
// is the agent's in every case.
|
|
1261
|
+
const pending = abort?.shadow
|
|
1262
|
+
? startShadowDecision(abort.shadow.provider, { question, state: context, choices, signal: abort.signal })
|
|
1263
|
+
: undefined;
|
|
1251
1264
|
const chosen = await claude.evaluate({
|
|
1252
|
-
question
|
|
1265
|
+
question,
|
|
1253
1266
|
context,
|
|
1254
1267
|
choices,
|
|
1255
1268
|
signal: abort?.signal,
|
|
1256
1269
|
timeoutMs: abort?.timeoutMs,
|
|
1257
1270
|
});
|
|
1271
|
+
if (pending && abort?.shadow) {
|
|
1272
|
+
try {
|
|
1273
|
+
const rec = await finishShadowDecision(pending, current, chosen);
|
|
1274
|
+
abort.shadow.records.push(rec);
|
|
1275
|
+
const verdict = rec.outcome === "compared" ? (rec.agree ? "agreed" : "disagreed") : `fell through (${rec.reason})`;
|
|
1276
|
+
logger?.info(` decider (shadow): node '${current}' ${verdict}`, { ...rec });
|
|
1277
|
+
}
|
|
1278
|
+
catch {
|
|
1279
|
+
// shadow logging must never affect a route
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1258
1282
|
// Fail closed. `evaluate` returns null when the routing decision could not
|
|
1259
1283
|
// be made (SDK error, timeout, non-success subtype, or an unparseable
|
|
1260
1284
|
// answer). We must NOT fall through to a conditional edge: on a node with a
|
package/dist/harness/codex.d.ts
CHANGED
|
@@ -33,6 +33,13 @@ import { type AuthProbe } from "./auth.js";
|
|
|
33
33
|
export { CODEX_CAPABILITIES };
|
|
34
34
|
/** Oldest Codex CLI whose flags this adapter uses (`--ignore-user-config`, `--ignore-rules`). */
|
|
35
35
|
export declare const MIN_CODEX_VERSION = "0.159.0";
|
|
36
|
+
/**
|
|
37
|
+
* The Codex CLI the capability claims in capabilities.ts were read from and the contract suite
|
|
38
|
+
* was written against (openai/codex rust-v0.159.2). action.yml installs this version by default.
|
|
39
|
+
* A newer CLI is not refused (it is only a warning, see `preflight`): the claims are about a
|
|
40
|
+
* version, not a range, so a newer one is unverified rather than known-bad.
|
|
41
|
+
*/
|
|
42
|
+
export declare const TESTED_CODEX_VERSION = "0.159.2";
|
|
36
43
|
/**
|
|
37
44
|
* Codex features sweny always turns off, so a node sees only the tools sweny
|
|
38
45
|
* gave it: ChatGPT apps and plugins (their own MCP servers), tool suggestions,
|