okengine 0.23.2 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -1
- package/manifest.v1.schema.json +53 -3
- package/package.json +2 -1
- package/site/content/docs/ai/mcp.mdx +1 -1
- package/site/content/docs/ai/skills.mdx +1 -1
- package/site/content/docs/elements/ai/decide.mdx +95 -417
- package/site/content/docs/elements/ai/deciders.mdx +154 -0
- package/site/content/docs/elements/ai/index.mdx +2 -1
- package/site/content/docs/elements/ai/meta.json +1 -1
- package/site/content/docs/reference/cli.mdx +2 -2
- package/site/content/docs/reference/configuration.mdx +2 -1
- package/site/content/docs/reference/fx.mdx +11 -9
- package/src/cli/agents-md.test.ts +8 -2
- package/src/cli/decide-certify.test.ts +166 -0
- package/src/cli/decide.ts +170 -3
- package/src/cli/decision-lock-watch.test.ts +4 -4
- package/src/cli/eval.ts +38 -59
- package/src/cli/mcp-from-console.ts +1 -1
- package/src/cli/registry.ts +45 -9
- package/src/compiler/decisions.extract.test.ts +86 -31
- package/src/compiler/effects-infer.ts +1 -0
- package/src/compiler/extract.ts +239 -24
- package/src/config/index.ts +6 -0
- package/src/console/server/decisions.test.ts +6 -1
- package/src/console/server/decisions.ts +32 -1
- package/src/console/ui-next/dist/assets/{access-page-zBMhZnFm.js → access-page-DX2dKLmq.js} +1 -1
- package/src/console/ui-next/dist/assets/{agent-disclosure-CBj9f7Ju.js → agent-disclosure-CiRvA3M9.js} +1 -1
- package/src/console/ui-next/dist/assets/{cache-glyph-D3MY49h3.js → cache-glyph-CC1yFJ8A.js} +1 -1
- package/src/console/ui-next/dist/assets/{call-pii-button-L9JxdBhp.js → call-pii-button-CLXQgIDD.js} +1 -1
- package/src/console/ui-next/dist/assets/{collapsible-BvFaX-Zt.js → collapsible-C0AThBZg.js} +1 -1
- package/src/console/ui-next/dist/assets/{decisions-page-DgB76m-X.js → decisions-page-Bf3AhTAa.js} +1 -1
- package/src/console/ui-next/dist/assets/{duration-tone-l1DSJ2Vz.js → duration-tone-Bptur2gX.js} +1 -1
- package/src/console/ui-next/dist/assets/{flows-page-Bb9t1lu6.js → flows-page-DRZVeTZU.js} +1 -1
- package/src/console/ui-next/dist/assets/{highlighted-json-DMvsi3Ti.js → highlighted-json-BKf0PjBK.js} +1 -1
- package/src/console/ui-next/dist/assets/{http-method-BwhJBTcQ.js → http-method-uhrnWrGC.js} +1 -1
- package/src/console/ui-next/dist/assets/{index-RhV2jT_7.js → index-Duxus_sG.js} +3 -3
- package/src/console/ui-next/dist/assets/{observability-page-BRfUILBq.js → observability-page-B-QM8X0O.js} +1 -1
- package/src/console/ui-next/dist/assets/{replica-lag-BsceAzIG.js → replica-lag-DuqHCANi.js} +1 -1
- package/src/console/ui-next/dist/assets/{request-meta-C6lTyCXp.js → request-meta-BTBZ3Ulc.js} +1 -1
- package/src/console/ui-next/dist/assets/{store-page-YmE806bJ.js → store-page-fpo7VxYb.js} +1 -1
- package/src/console/ui-next/dist/assets/{trace-detail-sheet-Di4irS49.js → trace-detail-sheet-DKtr1-K1.js} +1 -1
- package/src/console/ui-next/dist/assets/{tree-expand-toggle-DYeL6Rye.js → tree-expand-toggle-QeTceKKI.js} +1 -1
- package/src/console/ui-next/dist/assets/{units-page-07agasRA.js → units-page-ChCp8mOy.js} +1 -1
- package/src/console/ui-next/dist/assets/{vault-page-CHBck4n_.js → vault-page-CEYLXhUX.js} +1 -1
- package/src/console/ui-next/dist/index.html +1 -1
- package/src/console/ui-next/src/client.ts +8 -0
- package/src/console/ui-next/src/features/flows/decisions/decisions-page.tsx +16 -0
- package/src/elements/ai/deciders/presets.ts +182 -0
- package/src/elements/ai/decisions/bind.ts +6 -5
- package/src/elements/ai/decisions/catalog.ts +93 -0
- package/src/elements/ai/decisions/certificate.test.ts +47 -19
- package/src/elements/ai/decisions/certificate.ts +107 -24
- package/src/elements/ai/decisions/certify.ts +104 -9
- package/src/elements/ai/decisions/codec.test.ts +142 -0
- package/src/elements/ai/decisions/codec.ts +302 -0
- package/src/elements/ai/decisions/decide.test.ts +108 -31
- package/src/elements/ai/decisions/decider-runtime.test.ts +415 -0
- package/src/elements/ai/decisions/decider.types.test.ts +62 -0
- package/src/elements/ai/decisions/e2e.test.ts +20 -16
- package/src/elements/ai/decisions/http.ts +15 -7
- package/src/elements/ai/decisions/labels.test.ts +6 -1
- package/src/elements/ai/decisions/labels.ts +4 -3
- package/src/elements/ai/decisions/openai.live.test.ts +42 -0
- package/src/elements/ai/decisions/openrouter.live.test.ts +5 -1
- package/src/elements/ai/decisions/provider.ts +22 -1
- package/src/elements/ai/declare.ts +208 -51
- package/src/elements/ai.ts +1 -0
- package/src/index.ts +1 -0
- package/src/kernel/app.ts +8 -0
- package/src/kernel/decision-budget-entry.ts +11 -0
- package/src/kernel/element-registries.ts +3 -0
- package/src/kernel/fx-decide.ts +276 -88
- package/src/manifest/types.ts +38 -2
- package/src/release/measure.ts +47 -8
- package/src/test/create-test-app.ts +21 -1
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backup, breaker, lock version, expiry, and scripted answers.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
6
|
+
import { mkdtemp } from "node:fs/promises";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import { ai, resetAiDecls } from "../../ai.ts";
|
|
10
|
+
import { createFx } from "../../../kernel/fx.ts";
|
|
11
|
+
import { flow } from "../../../kernel/flow.ts";
|
|
12
|
+
import { oke } from "../../../kernel/app.ts";
|
|
13
|
+
import { resetBindings } from "../../../kernel/on.ts";
|
|
14
|
+
import { http } from "../../../kernel/triggers.ts";
|
|
15
|
+
import { createTestApp } from "../../../test/create-test-app.ts";
|
|
16
|
+
import {
|
|
17
|
+
DecisionLockVersionError,
|
|
18
|
+
parseDecisionLockfile,
|
|
19
|
+
questionHash,
|
|
20
|
+
resetDecisionCertificates,
|
|
21
|
+
setDecisionLock,
|
|
22
|
+
} from "./certificate.ts";
|
|
23
|
+
import { resetDecisionBreakers, decisionBreakerOpen } from "./http.ts";
|
|
24
|
+
import { DecisionInputTooLarge, DecisionOutageError, DecisionRequestError } from "./provider.ts";
|
|
25
|
+
import { resetDecisionProvider, setDecisionProvider } from "../../../kernel/fx-decide.ts";
|
|
26
|
+
|
|
27
|
+
afterEach(() => {
|
|
28
|
+
resetBindings();
|
|
29
|
+
resetAiDecls();
|
|
30
|
+
resetDecisionCertificates();
|
|
31
|
+
resetDecisionProvider();
|
|
32
|
+
resetDecisionBreakers();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
function choice() {
|
|
36
|
+
return ai.choice("which team", { billing: "Billing", technical: "Technical" });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function cert(model: string, question: ReturnType<typeof choice>) {
|
|
40
|
+
return {
|
|
41
|
+
model,
|
|
42
|
+
pinned: true as const,
|
|
43
|
+
questions: {
|
|
44
|
+
team: {
|
|
45
|
+
"": {
|
|
46
|
+
hash: questionHash(question),
|
|
47
|
+
calibrator: { kind: "temperature" as const, t: 1 },
|
|
48
|
+
threshold: 0.5,
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function answer(model: string) {
|
|
56
|
+
return {
|
|
57
|
+
model,
|
|
58
|
+
answers: {
|
|
59
|
+
team: {
|
|
60
|
+
type: "choice",
|
|
61
|
+
choice: "technical",
|
|
62
|
+
probabilities: { billing: 0.05, technical: 0.93, none_of_these: 0.02 },
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
usage: {},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
describe("decider runtime", () => {
|
|
70
|
+
test("a certified backup answers as auto and names itself", async () => {
|
|
71
|
+
const question = choice();
|
|
72
|
+
const primary = ai.decider("primary", { provider: "openrouter", model: "primary-model" });
|
|
73
|
+
const spare = ai.decider("spare", { provider: "openrouter", model: "spare-model" });
|
|
74
|
+
const decl = ai.decision("triage", {
|
|
75
|
+
decider: primary,
|
|
76
|
+
backup: [spare],
|
|
77
|
+
otherwise: "abstain",
|
|
78
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
79
|
+
ask: { team: question },
|
|
80
|
+
});
|
|
81
|
+
setDecisionLock({
|
|
82
|
+
version: 2,
|
|
83
|
+
decisions: { triage: { deciders: { spare: cert("spare-model", question) } } },
|
|
84
|
+
});
|
|
85
|
+
setDecisionProvider(async (_decl, _input, _signal, decider) => {
|
|
86
|
+
if (decider?.name === "primary") throw new DecisionOutageError("down");
|
|
87
|
+
return answer("spare-model");
|
|
88
|
+
});
|
|
89
|
+
const fx = createFx({ flow: "run", effects: { decides: ["triage"] } });
|
|
90
|
+
const result = (await fx.decide(decl, {})) as {
|
|
91
|
+
team: string;
|
|
92
|
+
$: { team: { how: string; by: string; why?: string } };
|
|
93
|
+
};
|
|
94
|
+
expect(result.team).toBe("technical");
|
|
95
|
+
expect(result.$.team).toMatchObject({ how: "auto", by: "spare" });
|
|
96
|
+
expect(result.$.team.why).toBeUndefined();
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("an uncertified backup takes otherwise", async () => {
|
|
100
|
+
const primary = ai.decider("primary", { provider: "openrouter", model: "primary-model" });
|
|
101
|
+
const spare = ai.decider("spare", { provider: "openrouter", model: "spare-model" });
|
|
102
|
+
const decl = ai.decision("triage", {
|
|
103
|
+
decider: primary,
|
|
104
|
+
backup: [spare],
|
|
105
|
+
otherwise: "abstain",
|
|
106
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
107
|
+
ask: { team: choice() },
|
|
108
|
+
});
|
|
109
|
+
setDecisionProvider(async (_decl, _input, _signal, decider) => {
|
|
110
|
+
if (decider?.name === "primary") throw new DecisionOutageError("down");
|
|
111
|
+
return answer("spare-model");
|
|
112
|
+
});
|
|
113
|
+
const fx = createFx({ flow: "run", effects: { decides: ["triage"] } });
|
|
114
|
+
const result = (await fx.decide(decl, {})) as {
|
|
115
|
+
team: null;
|
|
116
|
+
$: { team: { why: string; by: string } };
|
|
117
|
+
};
|
|
118
|
+
expect(result.team).toBeNull();
|
|
119
|
+
expect(result.$.team).toMatchObject({ why: "uncertified", by: "spare" });
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("a 4xx does not open the breaker or try the backup", async () => {
|
|
123
|
+
const primary = ai.decider("primary", { provider: "openrouter", model: "primary-model" });
|
|
124
|
+
const spare = ai.decider("spare", { provider: "openrouter", model: "spare-model" });
|
|
125
|
+
const decl = ai.decision("triage", {
|
|
126
|
+
decider: primary,
|
|
127
|
+
backup: [spare],
|
|
128
|
+
otherwise: "abstain",
|
|
129
|
+
ask: { team: choice() },
|
|
130
|
+
});
|
|
131
|
+
const original = globalThis.fetch;
|
|
132
|
+
let spareHits = 0;
|
|
133
|
+
globalThis.fetch = (async (_input, init) => {
|
|
134
|
+
const body = JSON.parse(typeof init?.body === "string" ? init.body : "{}") as {
|
|
135
|
+
model?: string;
|
|
136
|
+
};
|
|
137
|
+
if (body.model === "spare-model") spareHits += 1;
|
|
138
|
+
return new Response("no", { status: 422 });
|
|
139
|
+
}) as typeof fetch;
|
|
140
|
+
try {
|
|
141
|
+
const fx = createFx({
|
|
142
|
+
flow: "run",
|
|
143
|
+
effects: { decides: ["triage"], secrets: ["OPENROUTER_API_KEY"] },
|
|
144
|
+
secrets: { OPENROUTER_API_KEY: "test-key" },
|
|
145
|
+
});
|
|
146
|
+
await expect(fx.decide(decl, {})).rejects.toBeInstanceOf(DecisionRequestError);
|
|
147
|
+
expect(spareHits).toBe(0);
|
|
148
|
+
expect(decisionBreakerOpen("primary")).toBe(false);
|
|
149
|
+
} finally {
|
|
150
|
+
globalThis.fetch = original;
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
test("the breaker opens on the primary and the next call skips it", async () => {
|
|
155
|
+
const question = choice();
|
|
156
|
+
const primary = ai.decider("primary", { provider: "openrouter", model: "primary-model" });
|
|
157
|
+
const spare = ai.decider("spare", { provider: "openrouter", model: "spare-model" });
|
|
158
|
+
const decl = ai.decision("triage", {
|
|
159
|
+
decider: primary,
|
|
160
|
+
backup: [spare],
|
|
161
|
+
otherwise: "abstain",
|
|
162
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
163
|
+
ask: { team: question },
|
|
164
|
+
});
|
|
165
|
+
setDecisionLock({
|
|
166
|
+
version: 2,
|
|
167
|
+
decisions: { triage: { deciders: { spare: cert("spare-model", question) } } },
|
|
168
|
+
});
|
|
169
|
+
const original = globalThis.fetch;
|
|
170
|
+
let primaryHits = 0;
|
|
171
|
+
globalThis.fetch = (async (_input, init) => {
|
|
172
|
+
const body = JSON.parse(typeof init?.body === "string" ? init.body : "{}") as {
|
|
173
|
+
model?: string;
|
|
174
|
+
};
|
|
175
|
+
if (body.model === "primary-model") {
|
|
176
|
+
primaryHits += 1;
|
|
177
|
+
return new Response("down", { status: 500 });
|
|
178
|
+
}
|
|
179
|
+
return new Response(JSON.stringify(answer("spare-model")), {
|
|
180
|
+
status: 200,
|
|
181
|
+
headers: { "content-type": "application/json" },
|
|
182
|
+
});
|
|
183
|
+
}) as typeof fetch;
|
|
184
|
+
try {
|
|
185
|
+
const fx = createFx({
|
|
186
|
+
flow: "run",
|
|
187
|
+
effects: { decides: ["triage"], secrets: ["OPENROUTER_API_KEY"] },
|
|
188
|
+
secrets: { OPENROUTER_API_KEY: "test-key" },
|
|
189
|
+
});
|
|
190
|
+
for (let i = 0; i < 3; i += 1) {
|
|
191
|
+
const result = (await fx.decide(decl, {})) as { $: { team: { by: string } } };
|
|
192
|
+
expect(result.$.team.by).toBe("spare");
|
|
193
|
+
}
|
|
194
|
+
expect(decisionBreakerOpen("primary")).toBe(true);
|
|
195
|
+
const skipped = (await fx.decide(decl, {})) as { team: string; $: { team: { by: string } } };
|
|
196
|
+
expect(primaryHits).toBe(3);
|
|
197
|
+
expect(skipped.team).toBe("technical");
|
|
198
|
+
expect(skipped.$.team.by).toBe("spare");
|
|
199
|
+
} finally {
|
|
200
|
+
globalThis.fetch = original;
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test("version 1 lockfiles ask for recertify", () => {
|
|
205
|
+
expect(() =>
|
|
206
|
+
parseDecisionLockfile({
|
|
207
|
+
version: 1,
|
|
208
|
+
decisions: { triage: { model: "m", questions: {} } },
|
|
209
|
+
}),
|
|
210
|
+
).toThrow(DecisionLockVersionError);
|
|
211
|
+
expect(() =>
|
|
212
|
+
parseDecisionLockfile({
|
|
213
|
+
version: 1,
|
|
214
|
+
decisions: { triage: { model: "m", questions: {} } },
|
|
215
|
+
}),
|
|
216
|
+
).toThrow(/recertify/);
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
test("a different echoed model is uncertified", async () => {
|
|
220
|
+
const question = choice();
|
|
221
|
+
const jev = ai.decider("jev", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
222
|
+
const decl = ai.decision("triage", {
|
|
223
|
+
decider: jev,
|
|
224
|
+
otherwise: "abstain",
|
|
225
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
226
|
+
ask: { team: question },
|
|
227
|
+
});
|
|
228
|
+
setDecisionLock({
|
|
229
|
+
version: 2,
|
|
230
|
+
decisions: { triage: { deciders: { jev: cert("typesafe/jev-1.13.0", question) } } },
|
|
231
|
+
});
|
|
232
|
+
setDecisionProvider(async () => answer("typesafe/jev-1.13-20260917"));
|
|
233
|
+
const fx = createFx({ flow: "run", effects: { decides: ["triage"] } });
|
|
234
|
+
const result = (await fx.decide(decl, {})) as { $: { team: { why: string } } };
|
|
235
|
+
expect(result.$.team.why).toBe("uncertified");
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
test("an expired alias certificate is uncertified", async () => {
|
|
239
|
+
const question = choice();
|
|
240
|
+
const luna = ai.decider("luna", { provider: "openai", model: "gpt-6-luna" });
|
|
241
|
+
const decl = ai.decision("triage", {
|
|
242
|
+
decider: luna,
|
|
243
|
+
otherwise: "abstain",
|
|
244
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
245
|
+
ask: { team: question },
|
|
246
|
+
});
|
|
247
|
+
const slice = cert("gpt-6-luna", question);
|
|
248
|
+
setDecisionLock({
|
|
249
|
+
version: 2,
|
|
250
|
+
decisions: {
|
|
251
|
+
triage: {
|
|
252
|
+
deciders: {
|
|
253
|
+
luna: { ...slice, pinned: false, expiresAt: 1_000 },
|
|
254
|
+
},
|
|
255
|
+
},
|
|
256
|
+
},
|
|
257
|
+
});
|
|
258
|
+
setDecisionProvider(async () => answer("gpt-6-luna"));
|
|
259
|
+
const fx = createFx({ flow: "run", effects: { decides: ["triage"] }, now: () => 1_000 });
|
|
260
|
+
const result = (await fx.decide(decl, {})) as { $: { team: { why: string; by: string } } };
|
|
261
|
+
expect(result.$.team).toMatchObject({ why: "uncertified", by: "luna" });
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test("switching to an already certified decider does not need another certificate", async () => {
|
|
265
|
+
const question = choice();
|
|
266
|
+
const spare = ai.decider("spare", { provider: "openrouter", model: "spare-model" });
|
|
267
|
+
const decl = ai.decision("triage", {
|
|
268
|
+
decider: spare,
|
|
269
|
+
otherwise: "abstain",
|
|
270
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
271
|
+
ask: { team: question },
|
|
272
|
+
});
|
|
273
|
+
setDecisionLock({
|
|
274
|
+
version: 2,
|
|
275
|
+
decisions: { triage: { deciders: { spare: cert("spare-model", question) } } },
|
|
276
|
+
});
|
|
277
|
+
setDecisionProvider(async () => answer("spare-model"));
|
|
278
|
+
const fx = createFx({ flow: "run", effects: { decides: ["triage"] } });
|
|
279
|
+
const result = (await fx.decide(decl, {})) as {
|
|
280
|
+
team: string;
|
|
281
|
+
$: { team: { how: string; by: string } };
|
|
282
|
+
};
|
|
283
|
+
expect(result.team).toBe("technical");
|
|
284
|
+
expect(result.$.team).toMatchObject({ how: "auto", by: "spare" });
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
test("state past max context throws before any call", async () => {
|
|
288
|
+
const small = ai.decider("small", {
|
|
289
|
+
driverId: "systemone",
|
|
290
|
+
baseUrl: "https://example.test/decisions",
|
|
291
|
+
model: "m",
|
|
292
|
+
secret: "KEY",
|
|
293
|
+
capabilities: {
|
|
294
|
+
boolean: true,
|
|
295
|
+
choice: true,
|
|
296
|
+
score: false,
|
|
297
|
+
refusal: false,
|
|
298
|
+
maxContext: 1,
|
|
299
|
+
},
|
|
300
|
+
});
|
|
301
|
+
const decl = ai.decision("triage", {
|
|
302
|
+
decider: small,
|
|
303
|
+
otherwise: "abstain",
|
|
304
|
+
ask: { team: choice() },
|
|
305
|
+
});
|
|
306
|
+
let called = false;
|
|
307
|
+
setDecisionProvider(async () => {
|
|
308
|
+
called = true;
|
|
309
|
+
return answer("m");
|
|
310
|
+
});
|
|
311
|
+
const fx = createFx({ flow: "run", effects: { decides: ["triage"] } });
|
|
312
|
+
await expect(fx.decide(decl, { ticket: "1234567890" })).rejects.toBeInstanceOf(
|
|
313
|
+
DecisionInputTooLarge,
|
|
314
|
+
);
|
|
315
|
+
expect(called).toBe(false);
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
test("refusal and none_of_these take otherwise and store no refusal text", async () => {
|
|
319
|
+
const question = choice();
|
|
320
|
+
const jev = ai.decider("jev", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
321
|
+
const decl = ai.decision("triage", {
|
|
322
|
+
decider: jev,
|
|
323
|
+
otherwise: "abstain",
|
|
324
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
325
|
+
ask: { team: question },
|
|
326
|
+
});
|
|
327
|
+
setDecisionProvider(async () => ({
|
|
328
|
+
model: "typesafe/jev-1.13.0",
|
|
329
|
+
answers: { team: { type: "refusal", text: "nope" } },
|
|
330
|
+
usage: {},
|
|
331
|
+
}));
|
|
332
|
+
const fx = createFx({ flow: "run", effects: { decides: ["triage"] } });
|
|
333
|
+
const refused = (await fx.decide(decl, {})) as {
|
|
334
|
+
team: null;
|
|
335
|
+
$: { team: { why: string; by: string; raw: unknown } };
|
|
336
|
+
};
|
|
337
|
+
expect(refused.team).toBeNull();
|
|
338
|
+
expect(refused.$.team).toMatchObject({ why: "refused", by: "jev", raw: { type: "refusal" } });
|
|
339
|
+
expect(JSON.stringify(refused)).not.toContain("nope");
|
|
340
|
+
|
|
341
|
+
setDecisionLock({
|
|
342
|
+
version: 2,
|
|
343
|
+
decisions: {
|
|
344
|
+
triage: { deciders: { jev: cert("typesafe/jev-1.13.0", question) } },
|
|
345
|
+
},
|
|
346
|
+
});
|
|
347
|
+
setDecisionProvider(async () => ({
|
|
348
|
+
model: "typesafe/jev-1.13.0",
|
|
349
|
+
answers: {
|
|
350
|
+
team: {
|
|
351
|
+
type: "choice",
|
|
352
|
+
choice: "none_of_these",
|
|
353
|
+
probabilities: { billing: 0.1, technical: 0.1, none_of_these: 0.8 },
|
|
354
|
+
},
|
|
355
|
+
},
|
|
356
|
+
usage: {},
|
|
357
|
+
}));
|
|
358
|
+
const missed = (await fx.decide(decl, {})) as { team: null; $: { team: { why: string } } };
|
|
359
|
+
expect(missed.team).toBeNull();
|
|
360
|
+
expect(missed.$.team.why).toBe("none_of_these");
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
test("t.ai.decide scripts probabilities and a refusal", async () => {
|
|
364
|
+
const question = choice();
|
|
365
|
+
const jev = ai.decider("jev", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
366
|
+
const triage = ai.decision("scripted", {
|
|
367
|
+
decider: jev,
|
|
368
|
+
otherwise: "abstain",
|
|
369
|
+
autonomy: { maxError: 0.05, audit: 0 },
|
|
370
|
+
ask: { team: question },
|
|
371
|
+
});
|
|
372
|
+
const work = flow("work", {
|
|
373
|
+
effects: { decides: ["scripted"] },
|
|
374
|
+
do: async (_input: unknown, fx) => fx.decide(triage, { ticket: "1" }),
|
|
375
|
+
});
|
|
376
|
+
const root = await mkdtemp(join(tmpdir(), "oke-script-"));
|
|
377
|
+
const app = oke({
|
|
378
|
+
name: "scripted",
|
|
379
|
+
env: "test",
|
|
380
|
+
startScheduler: false,
|
|
381
|
+
registry: "ignore",
|
|
382
|
+
rootDir: root,
|
|
383
|
+
bindings: [{ trigger: http.post("/work"), flow: work }],
|
|
384
|
+
});
|
|
385
|
+
const t = await createTestApp(app, {
|
|
386
|
+
capability: "open",
|
|
387
|
+
boot: { rootDir: root },
|
|
388
|
+
});
|
|
389
|
+
try {
|
|
390
|
+
setDecisionLock({
|
|
391
|
+
version: 2,
|
|
392
|
+
decisions: {
|
|
393
|
+
scripted: { deciders: { jev: cert("typesafe/jev-1.13.0", question) } },
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
t.ai.decide(triage, {
|
|
397
|
+
team: { probabilities: { billing: 0.05, technical: 0.93, none_of_these: 0.02 } },
|
|
398
|
+
});
|
|
399
|
+
const auto = (await t.app.call(work, {})) as { team: string; $: { team: { how: string } } };
|
|
400
|
+
expect(auto.team).toBe("technical");
|
|
401
|
+
expect(auto.$.team.how).toBe("auto");
|
|
402
|
+
|
|
403
|
+
t.ai.decide(triage, { team: { refusal: "not stored" } });
|
|
404
|
+
const refused = (await t.app.call(work, {})) as {
|
|
405
|
+
team: null;
|
|
406
|
+
$: { team: { why: string; raw: unknown } };
|
|
407
|
+
};
|
|
408
|
+
expect(refused.team).toBeNull();
|
|
409
|
+
expect(refused.$.team.why).toBe("refused");
|
|
410
|
+
expect(JSON.stringify(refused)).not.toContain("not stored");
|
|
411
|
+
} finally {
|
|
412
|
+
await t.close();
|
|
413
|
+
}
|
|
414
|
+
});
|
|
415
|
+
});
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Type checks for decider capabilities and reserved fields.
|
|
3
|
+
* The invalid calls are not executed.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { describe, expect, test } from "bun:test";
|
|
7
|
+
import { ai } from "../../ai.ts";
|
|
8
|
+
|
|
9
|
+
function typeChecks(): void {
|
|
10
|
+
const jev = ai.decider("jev", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
11
|
+
const narrow = ai.decider("narrow", {
|
|
12
|
+
driverId: "systemone",
|
|
13
|
+
baseUrl: "https://example.test/decisions",
|
|
14
|
+
model: "m",
|
|
15
|
+
secret: "KEY",
|
|
16
|
+
capabilities: { boolean: true, choice: false, score: false, refusal: false },
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
ai.decision("missing-decider", {
|
|
20
|
+
otherwise: "abstain",
|
|
21
|
+
// @ts-expect-error decider is required, so the questions do not typecheck
|
|
22
|
+
ask: { team: ai.choice("which", { a: "A" }) },
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// @ts-expect-error otherwise is required
|
|
26
|
+
ai.decision("missing-otherwise", {
|
|
27
|
+
decider: jev,
|
|
28
|
+
ask: { team: ai.choice("which", { a: "A" }) },
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
ai.decision("shadowed", {
|
|
32
|
+
decider: jev,
|
|
33
|
+
otherwise: "abstain",
|
|
34
|
+
ask: { team: ai.choice("which", { a: "A" }) },
|
|
35
|
+
// @ts-expect-error shadow is planned
|
|
36
|
+
shadow: [],
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
ai.decision("too-wide", {
|
|
40
|
+
decider: narrow,
|
|
41
|
+
otherwise: "abstain",
|
|
42
|
+
ask: {
|
|
43
|
+
// @ts-expect-error this decider cannot answer a choice
|
|
44
|
+
team: ai.choice("which", { a: "A" }),
|
|
45
|
+
},
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
void typeChecks;
|
|
50
|
+
|
|
51
|
+
describe("decider types", () => {
|
|
52
|
+
test("a preset decider accepts a choice", () => {
|
|
53
|
+
const jev = ai.decider("jev-ok", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
54
|
+
const decl = ai.decision("ok", {
|
|
55
|
+
decider: jev,
|
|
56
|
+
otherwise: "abstain",
|
|
57
|
+
ask: { team: ai.choice("which", { a: "A" }) },
|
|
58
|
+
});
|
|
59
|
+
expect(decl.decider).toBe("jev-ok");
|
|
60
|
+
expect(decl.otherwise).toBe("abstain");
|
|
61
|
+
});
|
|
62
|
+
});
|
|
@@ -70,8 +70,10 @@ describe("fx.decide end to end", () => {
|
|
|
70
70
|
delete process.env["OKE_ROOT_DIR"];
|
|
71
71
|
const manifest = await extractFromSources({
|
|
72
72
|
"src/flows/run.ts": `
|
|
73
|
+
const jev = ai.decider("jev", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
73
74
|
ai.decision("triage", {
|
|
74
|
-
|
|
75
|
+
decider: jev,
|
|
76
|
+
otherwise: "abstain",
|
|
75
77
|
ask: { team: ai.choice("which", { a: "A" }) },
|
|
76
78
|
});
|
|
77
79
|
`,
|
|
@@ -94,23 +96,24 @@ describe("fx.decide end to end", () => {
|
|
|
94
96
|
test("certify, review, drift, and promote use the real routes", async () => {
|
|
95
97
|
const root = await mkdtemp(join(tmpdir(), "oke-decide-e2e-"));
|
|
96
98
|
const question = ai.choice("which team", { billing: "Billing", technical: "Technical" });
|
|
99
|
+
const jev = ai.decider("jev", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
97
100
|
const ship = ai.decision("ship", {
|
|
98
|
-
|
|
99
|
-
|
|
101
|
+
decider: jev,
|
|
102
|
+
otherwise: "abstain",
|
|
100
103
|
autonomy: { maxError: 0.05, audit: 0 },
|
|
101
104
|
evals: join(root, "ship.jsonl"),
|
|
102
105
|
ask: { team: question },
|
|
103
106
|
});
|
|
104
107
|
const ops = gate.policy("ops", (ctx) => ctx.operator.id === "reviewer");
|
|
105
108
|
const triage = ai.decision("triage", {
|
|
106
|
-
|
|
107
|
-
|
|
109
|
+
decider: jev,
|
|
110
|
+
otherwise: "ops",
|
|
108
111
|
autonomy: { maxError: 0.05, audit: 0 },
|
|
109
112
|
ask: { team: question },
|
|
110
113
|
});
|
|
111
114
|
const route = ai.decision("route", {
|
|
112
|
-
|
|
113
|
-
|
|
115
|
+
decider: jev,
|
|
116
|
+
otherwise: "ops",
|
|
114
117
|
autonomy: { maxError: 0.05, audit: 0 },
|
|
115
118
|
ask: { team: question },
|
|
116
119
|
});
|
|
@@ -125,23 +128,24 @@ describe("fx.decide end to end", () => {
|
|
|
125
128
|
const evals = JSON.stringify(join(root, "ship.jsonl"));
|
|
126
129
|
const manifest = (await extractFromSources({
|
|
127
130
|
"src/flows/support/route.ts": `
|
|
131
|
+
const jev = ai.decider("jev", { provider: "openrouter", model: "typesafe/jev-1.13.0" });
|
|
128
132
|
const ops = gate.policy("ops", () => true);
|
|
129
133
|
ai.decision("ship", {
|
|
130
|
-
|
|
131
|
-
|
|
134
|
+
decider: jev,
|
|
135
|
+
otherwise: "abstain",
|
|
132
136
|
autonomy: { maxError: 0.05, audit: 0 },
|
|
133
137
|
evals: ${evals},
|
|
134
138
|
ask: { team: ai.choice("which team", { billing: "Billing", technical: "Technical" }) },
|
|
135
139
|
});
|
|
136
140
|
ai.decision("triage", {
|
|
137
|
-
|
|
138
|
-
|
|
141
|
+
decider: jev,
|
|
142
|
+
otherwise: ops,
|
|
139
143
|
autonomy: { maxError: 0.05, audit: 0 },
|
|
140
144
|
ask: { team: ai.choice("which team", { billing: "Billing", technical: "Technical" }) },
|
|
141
145
|
});
|
|
142
146
|
ai.decision("route", {
|
|
143
|
-
|
|
144
|
-
|
|
147
|
+
decider: jev,
|
|
148
|
+
otherwise: ops,
|
|
145
149
|
autonomy: { maxError: 0.05, audit: 0 },
|
|
146
150
|
ask: { team: ai.choice("which team", { billing: "Billing", technical: "Technical" }) },
|
|
147
151
|
});
|
|
@@ -214,7 +218,7 @@ describe("fx.decide end to end", () => {
|
|
|
214
218
|
};
|
|
215
219
|
expect(auto.team).toBe("technical");
|
|
216
220
|
expect(auto.$.team.how).toBe("auto");
|
|
217
|
-
expect(getDecisionLock()?.decisions.ship?.model).toBe("typesafe/jev-1.13.0");
|
|
221
|
+
expect(getDecisionLock()?.decisions.ship?.deciders.jev?.model).toBe("typesafe/jev-1.13.0");
|
|
218
222
|
|
|
219
223
|
await locked.call(work, { which: "triage" });
|
|
220
224
|
const run = (await store.list()).find((item) =>
|
|
@@ -343,7 +347,7 @@ describe("fx.decide end to end", () => {
|
|
|
343
347
|
},
|
|
344
348
|
});
|
|
345
349
|
expect(promoted.decisions.ship).toBeDefined();
|
|
346
|
-
expect(promoted.decisions.route?.questions.team?.[""]).toBeDefined();
|
|
350
|
+
expect(promoted.decisions.route?.deciders.jev?.questions.team?.[""]).toBeDefined();
|
|
347
351
|
expect(decisionDriftSuspended("route")).toBe(false);
|
|
348
352
|
expect(decisionDriftSuspended("ship")).toBe(true);
|
|
349
353
|
await loadDecisionLockfile(root);
|
|
@@ -386,7 +390,7 @@ describe("fx.decide end to end", () => {
|
|
|
386
390
|
});
|
|
387
391
|
await restarted.boot({ env: "test" });
|
|
388
392
|
expect((await loadDecisionLabels("route")).length).toBeGreaterThan(0);
|
|
389
|
-
expect(getDecisionLock()?.decisions.route?.questions.team?.[""]).toBeDefined();
|
|
393
|
+
expect(getDecisionLock()?.decisions.route?.deciders.jev?.questions.team?.[""]).toBeDefined();
|
|
390
394
|
expect(decisionDriftSuspended("ship")).toBe(true);
|
|
391
395
|
expect(decisionDriftSuspended("route")).toBe(false);
|
|
392
396
|
const restartedAuto = (await restarted.call(work, { which: "route" })) as {
|
|
@@ -24,6 +24,8 @@ export interface DecisionHttpOptions {
|
|
|
24
24
|
readonly attempts?: number;
|
|
25
25
|
/** Breaker key. Calls that share it share the open/closed state. */
|
|
26
26
|
readonly breakerKey: string;
|
|
27
|
+
/** Encoded body. When omitted, the System One `{ model, state, questions }` body is sent. */
|
|
28
|
+
readonly body?: unknown;
|
|
27
29
|
}
|
|
28
30
|
|
|
29
31
|
interface BreakerState {
|
|
@@ -138,13 +140,17 @@ export function parseDecisionResponse(body: unknown): DecisionResponse {
|
|
|
138
140
|
answers?: unknown;
|
|
139
141
|
usage?: unknown;
|
|
140
142
|
};
|
|
141
|
-
if (
|
|
143
|
+
if (
|
|
144
|
+
typeof record.model !== "string" ||
|
|
145
|
+
record.answers == null ||
|
|
146
|
+
typeof record.answers !== "object"
|
|
147
|
+
) {
|
|
142
148
|
throw new DecisionRequestError(422, "decision response missing model or answers");
|
|
143
149
|
}
|
|
144
150
|
return {
|
|
145
151
|
model: record.model,
|
|
146
152
|
...(typeof record.provider === "string" ? { provider: record.provider } : {}),
|
|
147
|
-
answers: record.answers as
|
|
153
|
+
answers: record.answers as DecisionResponse["answers"],
|
|
148
154
|
usage: usageFrom(record.usage),
|
|
149
155
|
};
|
|
150
156
|
}
|
|
@@ -175,11 +181,13 @@ export async function decisionHttp(options: DecisionHttpOptions): Promise<Decisi
|
|
|
175
181
|
authorization: `Bearer ${options.apiKey}`,
|
|
176
182
|
"content-type": "application/json",
|
|
177
183
|
},
|
|
178
|
-
body: JSON.stringify(
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
184
|
+
body: JSON.stringify(
|
|
185
|
+
options.body ?? {
|
|
186
|
+
model: options.request.model,
|
|
187
|
+
state: options.request.state,
|
|
188
|
+
questions: options.request.questions,
|
|
189
|
+
},
|
|
190
|
+
),
|
|
183
191
|
signal: local.signal,
|
|
184
192
|
});
|
|
185
193
|
if (REQUEST_STATUSES.has(res.status)) {
|
|
@@ -147,8 +147,13 @@ describe("decision labels", () => {
|
|
|
147
147
|
closeDecisionLabelStore();
|
|
148
148
|
setDecisionDrift("triage", false);
|
|
149
149
|
setDecisionLock({
|
|
150
|
+
version: 2,
|
|
150
151
|
decisions: {
|
|
151
|
-
triage: {
|
|
152
|
+
triage: {
|
|
153
|
+
deciders: {
|
|
154
|
+
jev: { model: "typesafe/jev-1.13.0", pinned: true, certifiedAt: 200, questions: {} },
|
|
155
|
+
},
|
|
156
|
+
},
|
|
152
157
|
},
|
|
153
158
|
});
|
|
154
159
|
await openDecisionLabelStore(journal, setDecisionDrift);
|
|
@@ -251,7 +251,8 @@ export function persistDecisionDrift(name: string, suspended: boolean, certified
|
|
|
251
251
|
export function pinnedDecision(
|
|
252
252
|
name: string,
|
|
253
253
|
): { readonly model: string; readonly since: number } | undefined {
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
254
|
+
const deciders = getDecisionLock()?.decisions[name]?.deciders;
|
|
255
|
+
const first = deciders ? Object.values(deciders)[0] : undefined;
|
|
256
|
+
if (!first) return undefined;
|
|
257
|
+
return { model: first.model, since: first.certifiedAt ?? 0 };
|
|
257
258
|
}
|