okengine 0.23.2 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -1
- package/manifest.v1.schema.json +53 -3
- package/package.json +2 -1
- package/site/content/docs/ai/mcp.mdx +1 -1
- package/site/content/docs/ai/skills.mdx +1 -1
- package/site/content/docs/elements/ai/decide.mdx +95 -417
- package/site/content/docs/elements/ai/deciders.mdx +154 -0
- package/site/content/docs/elements/ai/index.mdx +2 -1
- package/site/content/docs/elements/ai/meta.json +1 -1
- package/site/content/docs/reference/cli.mdx +2 -2
- package/site/content/docs/reference/configuration.mdx +2 -1
- package/site/content/docs/reference/fx.mdx +11 -9
- package/src/cli/agents-md.test.ts +8 -2
- package/src/cli/decide-certify.test.ts +166 -0
- package/src/cli/decide.ts +170 -3
- package/src/cli/decision-lock-watch.test.ts +4 -4
- package/src/cli/eval.ts +38 -59
- package/src/cli/mcp-from-console.ts +1 -1
- package/src/cli/registry.ts +45 -9
- package/src/compiler/decisions.extract.test.ts +86 -31
- package/src/compiler/effects-infer.ts +1 -0
- package/src/compiler/extract.ts +239 -24
- package/src/config/index.ts +6 -0
- package/src/console/server/decisions.test.ts +6 -1
- package/src/console/server/decisions.ts +32 -1
- package/src/console/ui-next/dist/assets/{access-page-zBMhZnFm.js → access-page-DX2dKLmq.js} +1 -1
- package/src/console/ui-next/dist/assets/{agent-disclosure-CBj9f7Ju.js → agent-disclosure-CiRvA3M9.js} +1 -1
- package/src/console/ui-next/dist/assets/{cache-glyph-D3MY49h3.js → cache-glyph-CC1yFJ8A.js} +1 -1
- package/src/console/ui-next/dist/assets/{call-pii-button-L9JxdBhp.js → call-pii-button-CLXQgIDD.js} +1 -1
- package/src/console/ui-next/dist/assets/{collapsible-BvFaX-Zt.js → collapsible-C0AThBZg.js} +1 -1
- package/src/console/ui-next/dist/assets/{decisions-page-DgB76m-X.js → decisions-page-Bf3AhTAa.js} +1 -1
- package/src/console/ui-next/dist/assets/{duration-tone-l1DSJ2Vz.js → duration-tone-Bptur2gX.js} +1 -1
- package/src/console/ui-next/dist/assets/{flows-page-Bb9t1lu6.js → flows-page-DRZVeTZU.js} +1 -1
- package/src/console/ui-next/dist/assets/{highlighted-json-DMvsi3Ti.js → highlighted-json-BKf0PjBK.js} +1 -1
- package/src/console/ui-next/dist/assets/{http-method-BwhJBTcQ.js → http-method-uhrnWrGC.js} +1 -1
- package/src/console/ui-next/dist/assets/{index-RhV2jT_7.js → index-Duxus_sG.js} +3 -3
- package/src/console/ui-next/dist/assets/{observability-page-BRfUILBq.js → observability-page-B-QM8X0O.js} +1 -1
- package/src/console/ui-next/dist/assets/{replica-lag-BsceAzIG.js → replica-lag-DuqHCANi.js} +1 -1
- package/src/console/ui-next/dist/assets/{request-meta-C6lTyCXp.js → request-meta-BTBZ3Ulc.js} +1 -1
- package/src/console/ui-next/dist/assets/{store-page-YmE806bJ.js → store-page-fpo7VxYb.js} +1 -1
- package/src/console/ui-next/dist/assets/{trace-detail-sheet-Di4irS49.js → trace-detail-sheet-DKtr1-K1.js} +1 -1
- package/src/console/ui-next/dist/assets/{tree-expand-toggle-DYeL6Rye.js → tree-expand-toggle-QeTceKKI.js} +1 -1
- package/src/console/ui-next/dist/assets/{units-page-07agasRA.js → units-page-ChCp8mOy.js} +1 -1
- package/src/console/ui-next/dist/assets/{vault-page-CHBck4n_.js → vault-page-CEYLXhUX.js} +1 -1
- package/src/console/ui-next/dist/index.html +1 -1
- package/src/console/ui-next/src/client.ts +8 -0
- package/src/console/ui-next/src/features/flows/decisions/decisions-page.tsx +16 -0
- package/src/elements/ai/deciders/presets.ts +182 -0
- package/src/elements/ai/decisions/bind.ts +6 -5
- package/src/elements/ai/decisions/catalog.ts +93 -0
- package/src/elements/ai/decisions/certificate.test.ts +47 -19
- package/src/elements/ai/decisions/certificate.ts +107 -24
- package/src/elements/ai/decisions/certify.ts +104 -9
- package/src/elements/ai/decisions/codec.test.ts +142 -0
- package/src/elements/ai/decisions/codec.ts +302 -0
- package/src/elements/ai/decisions/decide.test.ts +108 -31
- package/src/elements/ai/decisions/decider-runtime.test.ts +415 -0
- package/src/elements/ai/decisions/decider.types.test.ts +62 -0
- package/src/elements/ai/decisions/e2e.test.ts +20 -16
- package/src/elements/ai/decisions/http.ts +15 -7
- package/src/elements/ai/decisions/labels.test.ts +6 -1
- package/src/elements/ai/decisions/labels.ts +4 -3
- package/src/elements/ai/decisions/openai.live.test.ts +42 -0
- package/src/elements/ai/decisions/openrouter.live.test.ts +5 -1
- package/src/elements/ai/decisions/provider.ts +22 -1
- package/src/elements/ai/declare.ts +208 -51
- package/src/elements/ai.ts +1 -0
- package/src/index.ts +1 -0
- package/src/kernel/app.ts +8 -0
- package/src/kernel/decision-budget-entry.ts +11 -0
- package/src/kernel/element-registries.ts +3 -0
- package/src/kernel/fx-decide.ts +276 -88
- package/src/manifest/types.ts +38 -2
- package/src/release/measure.ts +47 -8
- package/src/test/create-test-app.ts +21 -1
|
@@ -1,20 +1,15 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: "Decisions"
|
|
3
|
-
description: "One
|
|
3
|
+
description: "One call that labels a ticket, and a person when the answering decider is not allowed to decide alone."
|
|
4
4
|
icon: "Scale"
|
|
5
5
|
source: "docs/spec/unified-theory.md"
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
A decision
|
|
9
|
-
|
|
10
|
-
<Callout title="OpenRouter">
|
|
11
|
-
The OpenRouter decisions endpoint is alpha on the provider side. Score levels and a whole question
|
|
12
|
-
held in a same-file const are checked at compile time.
|
|
13
|
-
</Callout>
|
|
8
|
+
A decision labels one piece of work: which queue a ticket belongs in, and whether it needs a person today. The decider is the model that answers. Autonomy is that decider's certificate, not a property of the decision.
|
|
14
9
|
|
|
15
10
|
<Callout title="The one rule">
|
|
16
|
-
|
|
17
|
-
`oke-decisions.lock.json` can
|
|
11
|
+
Name a decider and an `otherwise`. `otherwise` is a Gate, or `"abstain"`. Only that decider's
|
|
12
|
+
certificate in `oke-decisions.lock.json` can return `auto`.
|
|
18
13
|
</Callout>
|
|
19
14
|
|
|
20
15
|
## Quick start
|
|
@@ -22,515 +17,198 @@ A decision picks labels for one piece of work: which queue a ticket belongs in,
|
|
|
22
17
|
<Steps>
|
|
23
18
|
|
|
24
19
|
<Step>
|
|
25
|
-
### Declare the questions
|
|
20
|
+
### Declare the decider and the questions
|
|
26
21
|
|
|
27
|
-
`
|
|
28
|
-
model is not allowed to.
|
|
22
|
+
`otherwise` is a real gate. The people who hold it answer when the model is not allowed to.
|
|
29
23
|
|
|
30
24
|
```typescript title="src/core/ai.ts"
|
|
31
25
|
import { ai, gate } from "okengine";
|
|
32
26
|
|
|
33
27
|
export const ops = gate.policy("ops", ({ operator }) => operator.id !== null);
|
|
34
28
|
|
|
29
|
+
export const jev = ai.decider("jev", {
|
|
30
|
+
provider: "openrouter",
|
|
31
|
+
model: "typesafe/jev-1.13-20260917",
|
|
32
|
+
});
|
|
33
|
+
|
|
35
34
|
export const triage = ai.decision("triage", {
|
|
36
|
-
|
|
35
|
+
decider: jev,
|
|
36
|
+
otherwise: ops,
|
|
37
37
|
ask: {
|
|
38
38
|
team: ai.choice("Which team owns this ticket?", {
|
|
39
39
|
billing: "Billing",
|
|
40
40
|
technical: "Technical",
|
|
41
41
|
}),
|
|
42
|
-
urgent: ai.boolean("Does this need a person today?",
|
|
43
|
-
true: "A person should see it today",
|
|
44
|
-
false: "It can wait",
|
|
45
|
-
}),
|
|
42
|
+
urgent: ai.boolean("Does this need a person today?"),
|
|
46
43
|
},
|
|
47
44
|
});
|
|
48
45
|
```
|
|
49
46
|
|
|
50
|
-
|
|
47
|
+
Every choice also accepts `none_of_these`. That answer is never `auto`. Question ids `meta` and `$` are reserved.
|
|
51
48
|
|
|
52
49
|
</Step>
|
|
53
50
|
|
|
54
51
|
<Step>
|
|
55
|
-
###
|
|
52
|
+
### Call it from a durable Flow
|
|
56
53
|
|
|
57
|
-
A
|
|
58
|
-
be 204 and the caller would see nothing. Accept the ticket on HTTP, emit a signal, and decide
|
|
59
|
-
in a durable consumer.
|
|
54
|
+
A gate parks the run. An HTTP trigger cannot park. Emit a signal and decide in a consumer.
|
|
60
55
|
|
|
61
|
-
```typescript title="src/flows/support
|
|
62
|
-
import {
|
|
63
|
-
import {
|
|
56
|
+
```typescript title="src/flows/support.ts"
|
|
57
|
+
import { flow, on, signal } from "okengine";
|
|
58
|
+
import { triage } from "../core/ai.ts";
|
|
64
59
|
|
|
65
|
-
export const
|
|
66
|
-
|
|
67
|
-
export const open = on(
|
|
68
|
-
http.post({ in: z.object({ text: z.string().min(1) }) }).public(),
|
|
69
|
-
flow({
|
|
70
|
-
do: async ({ text }, fx) => {
|
|
71
|
-
await fx.emit(ticketOpened, { text });
|
|
72
|
-
return { accepted: true };
|
|
73
|
-
},
|
|
74
|
-
}),
|
|
75
|
-
);
|
|
76
|
-
```
|
|
77
|
-
|
|
78
|
-
```typescript title="src/flows/support/route.ts"
|
|
79
|
-
import { on, flow } from "okengine";
|
|
80
|
-
import { triage } from "@/core/ai";
|
|
81
|
-
import { ticketOpened } from "./open";
|
|
60
|
+
export const routed = signal.once("support.routed");
|
|
82
61
|
|
|
83
62
|
export const route = on(
|
|
84
|
-
|
|
85
|
-
flow({
|
|
63
|
+
signal.once("support.opened"),
|
|
64
|
+
flow("support.route", {
|
|
86
65
|
durable: true,
|
|
87
|
-
do: async (input, fx) =>
|
|
66
|
+
do: async (input, fx) => {
|
|
67
|
+
const result = await fx.decide(triage, input);
|
|
68
|
+
await fx.emit(routed, result);
|
|
69
|
+
return result;
|
|
70
|
+
},
|
|
88
71
|
}),
|
|
89
72
|
);
|
|
90
73
|
```
|
|
91
74
|
|
|
92
|
-
`durable: true` is required
|
|
75
|
+
`durable: true` is required when `otherwise` is a gate. `"abstain"` can run without it.
|
|
93
76
|
|
|
94
77
|
</Step>
|
|
95
78
|
|
|
96
79
|
<Step>
|
|
97
|
-
###
|
|
98
|
-
|
|
99
|
-
The park waits until someone who passes the `ops` gate submits every open question. Console on port 6533 does that as an operator: the route sends the signed-in operator and that session's auth scopes into the review gate. The gate must allow that operator. An operator may resolve any tenant.
|
|
100
|
-
|
|
101
|
-
Open `/flows/decisions`, or POST the review id from the queue:
|
|
102
|
-
|
|
103
|
-
```bash
|
|
104
|
-
curl -X POST http://127.0.0.1:6533/console/decisions/resolve \
|
|
105
|
-
-H "Authorization: $OKE_OPERATOR_AUTHORIZATION" \
|
|
106
|
-
-H "content-type: application/json" \
|
|
107
|
-
-d '{"id":"<review id>","values":{"team":"billing","urgent":false}}'
|
|
108
|
-
```
|
|
80
|
+
### Read `how`, `by`, and `why`
|
|
109
81
|
|
|
110
|
-
|
|
82
|
+
`auto` is the decider's value. Anything else takes `otherwise`.
|
|
111
83
|
|
|
112
84
|
```json
|
|
113
85
|
{
|
|
114
|
-
"team": "
|
|
115
|
-
"urgent": false,
|
|
86
|
+
"team": "technical",
|
|
116
87
|
"$": {
|
|
117
|
-
"
|
|
118
|
-
|
|
119
|
-
"provider": "openrouter",
|
|
120
|
-
"usage": { "inputTokens": 120, "outputTokens": 40 }
|
|
121
|
-
},
|
|
122
|
-
"team": {
|
|
123
|
-
"how": "reviewed",
|
|
124
|
-
"p": 0.42,
|
|
125
|
-
"raw": { "billing": 0.2, "technical": 0.2, "none_of_these": 0.6 }
|
|
126
|
-
},
|
|
127
|
-
"urgent": { "how": "reviewed", "p": 0.51, "raw": { "noul": 0.51 } }
|
|
88
|
+
"team": { "how": "auto", "by": "jev", "p": 0.91, "raw": { "technical": 0.91 } },
|
|
89
|
+
"urgent": { "how": "auto", "by": "jev", "p": 0.54, "raw": { "noul": 0.54 } }
|
|
128
90
|
}
|
|
129
91
|
}
|
|
130
92
|
```
|
|
131
93
|
|
|
132
|
-
`how: "reviewed"` is the value the gate submitted, including a choice of `none_of_these`. `p` stays the model's confidence. `raw.noul` is the provider's yes/no score for a boolean question.
|
|
133
|
-
|
|
134
|
-
| `how` | Value the Flow sees |
|
|
135
|
-
| ----------- | ----------------------------------------------------------------------------- |
|
|
136
|
-
| `reviewed` | The value the review gate submitted. |
|
|
137
|
-
| `abstained` | `null` on that question only. Certain questions in the same call stay `auto`. |
|
|
138
|
-
| `auto` | The calibrated answer, after a lockfile exists. See Certify and promote. |
|
|
139
|
-
|
|
140
94
|
</Step>
|
|
141
95
|
|
|
142
96
|
</Steps>
|
|
143
97
|
|
|
144
|
-
##
|
|
98
|
+
## Result
|
|
145
99
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
description:
|
|
155
|
-
"Gate that may resolve a park. Exclusive with onUncertain. The Flow must be durable. No timeout.",
|
|
156
|
-
type: "Gate | string",
|
|
157
|
-
},
|
|
158
|
-
onUncertain: {
|
|
159
|
-
description: "Only abstain. Returns null and does not park. Exclusive with review.",
|
|
160
|
-
type: '"abstain"',
|
|
161
|
-
},
|
|
162
|
-
autonomy: {
|
|
163
|
-
description: "maxError is required. audit is required, from 0 to 1. risk defaults to 0.1.",
|
|
164
|
-
type: "{ maxError: number; audit: number; risk?: number }",
|
|
165
|
-
},
|
|
166
|
-
model: {
|
|
167
|
-
description:
|
|
168
|
-
"Model id. Default is typesafe/jev-1.13 on OpenRouter, or jev-1.13.0 on TypeSafe.",
|
|
169
|
-
type: "AiModel | string",
|
|
170
|
-
},
|
|
171
|
-
driverId: {
|
|
172
|
-
description: "Which provider to call.",
|
|
173
|
-
type: '"openrouter" | "typesafe"',
|
|
174
|
-
default: "openrouter",
|
|
175
|
-
},
|
|
176
|
-
locale: {
|
|
177
|
-
description: "Slice the certificate by a string from the input. Absent means one slice.",
|
|
178
|
-
type: "(input) => string | undefined",
|
|
179
|
-
},
|
|
180
|
-
evals: { description: "Seed JSONL path. Same role as a prompt evals file.", type: "string" },
|
|
181
|
-
timeout: {
|
|
182
|
-
description: "Provider deadline. A duration string or milliseconds.",
|
|
183
|
-
type: "string | number",
|
|
184
|
-
default: "30s",
|
|
185
|
-
},
|
|
186
|
-
}}
|
|
187
|
-
/>
|
|
188
|
-
|
|
189
|
-
## Lifecycle
|
|
190
|
-
|
|
191
|
-
A decision is in one list state. Drift is one flag per decision. Suspending `triage` leaves `route` alone.
|
|
192
|
-
|
|
193
|
-
```text
|
|
194
|
-
learning --labels--> candidate ready --promote--> certified
|
|
195
|
-
\ /
|
|
196
|
-
\---- drift ----> suspended
|
|
197
|
-
```
|
|
100
|
+
| Field | When |
|
|
101
|
+
| --------- | --------------------------------------------- |
|
|
102
|
+
| `how` | `auto`, `reviewed`, or `abstained`. |
|
|
103
|
+
| `by` | The decider that answered. |
|
|
104
|
+
| `why` | Only when the question is not `auto`. |
|
|
105
|
+
| `p` | Calibrated confidence. |
|
|
106
|
+
| `raw` | The distribution. A boolean keeps `{ noul }`. |
|
|
107
|
+
| `audited` | The auto value was sampled for a label. |
|
|
198
108
|
|
|
199
|
-
|
|
200
|
-
| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- |
|
|
201
|
-
| `learning` | No lockfile entry and no candidate. | Reviews and audit labels accumulate. |
|
|
202
|
-
| `candidate ready` | The clock has built a candidate from those labels. The page shows the fit and `oke decide promote <name>`. Counts are not the grant. | Promote writes that slot. |
|
|
203
|
-
| `certified` | `oke-decisions.lock.json` has this decision, and the declaration sets `autonomy`. A lock entry is ignored without `autonomy`. | The runtime may return `auto` under that certificate. |
|
|
204
|
-
| `suspended` | This decision's audit labels since its certificate, inside 7 days, for the pinned model, failed the error cap. The monitor emitted `oke/decision/drift` with that name. | Promote or recertify clears only this decision. A newer `certifiedAt` on the next load ignores the flag. |
|
|
109
|
+
`why` is one of `uncertain`, `refused`, `none_of_these`, `uncertified`, `drift`, or `outage`.
|
|
205
110
|
|
|
206
|
-
`
|
|
111
|
+
`otherwise: "abstain"` returns `null` and does not park. `how` is `abstained`. A Gate parks until someone who passes it submits every open question. Console on port 6533 resolves that park. There is no timeout.
|
|
207
112
|
|
|
208
|
-
|
|
209
|
-
question, a model version mismatch, an uncertified locale, low confidence, drift, and an
|
|
210
|
-
outage all take the non-auto path.
|
|
113
|
+
A refusal is not a label. The value is `null`, `why` is `refused`, and the review row reads `refused by <decider>`.
|
|
211
114
|
|
|
212
|
-
##
|
|
213
|
-
|
|
214
|
-
Open Console at `/flows/decisions` on port 6533. Each row is `learning`, `candidate ready`, `certified`, or `suspended` for that decision. The queue shows how long each row has been waiting. The resolve form only accepts a choice option, `none_of_these`, or a score level. An audit row submits `labelOnly: true`. A second resolve is `Conflict`. A lease collision retries. A failed label write is listed on the page.
|
|
215
|
-
|
|
216
|
-
An audit row is a label only. The original Flow already returned. A review row is the park:
|
|
217
|
-
submitting it wakes the consumer.
|
|
218
|
-
|
|
219
|
-
The review gate must allow the caller. Console resolve runs on the operator plane: it passes the signed-in operator id and the session auth into the gate, and it may resolve any tenant. A gate that checks `operator.id` matches that call.
|
|
220
|
-
|
|
221
|
-
A review body must include every question that was not `auto`. An audit row (`labelOnly: true`) must include every question, including ones that were already `auto`. A choice may be `none_of_these`. A partial body is 422: `review values do not match the open questions`.
|
|
222
|
-
|
|
223
|
-
```bash
|
|
224
|
-
curl -X POST http://127.0.0.1:6533/console/decisions/resolve \
|
|
225
|
-
-H "Authorization: $OKE_OPERATOR_AUTHORIZATION" \
|
|
226
|
-
-H "content-type: application/json" \
|
|
227
|
-
-d '{"id":"<audit id>","labelOnly":true,"values":{"team":"billing","urgent":true}}'
|
|
228
|
-
```
|
|
229
|
-
|
|
230
|
-
| Result | HTTP | What you do |
|
|
231
|
-
| ---------------------------- | --------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- |
|
|
232
|
-
| `{ ok: true }` | 200 | The consumer resumes with `how: "reviewed"`. |
|
|
233
|
-
| Already resolved | 409 `Conflict`. No `Retry-After`. `That value is already in use.` | Stop. |
|
|
234
|
-
| Another worker holds the run | 409 `JournalLeaseBusy` with `Retry-After`. `This run is locked by another worker. Retry after the given delay.` | Retry after the delay. A wait over 60 seconds is 503. |
|
|
235
|
-
|
|
236
|
-
The same 409 split applies to tool approval. Retry only `JournalLeaseBusy`.
|
|
237
|
-
|
|
238
|
-
## Certify and promote
|
|
239
|
-
|
|
240
|
-
**maxError** is the highest fraction of wrong answers you will accept among the ones the
|
|
241
|
-
model is allowed to return alone. `0.05` means at most 5 in 100 of those automatic answers.
|
|
242
|
-
|
|
243
|
-
**audit** is how often an automatic answer is also queued for a person, from 0 to 1.
|
|
244
|
-
`0.1` queues about one in ten. Those labels are how the next certificate is fit. Omitting
|
|
245
|
-
`audit` when `autonomy` is set fails to compile.
|
|
246
|
-
|
|
247
|
-
**risk** is how sure the statistical test must be before it accepts a threshold. Default
|
|
248
|
-
`0.1`. A smaller risk demands more labels.
|
|
249
|
-
|
|
250
|
-
**Calibration** rescales the provider's raw scores so `p` means a probability. Choice and
|
|
251
|
-
score use a temperature. Yes/no uses a curve fit on the provider score. The lockfile stores
|
|
252
|
-
that fit next to the question.
|
|
253
|
-
|
|
254
|
-
**The test**, once: for each confidence cutoff from 0.50 to 0.99 (50 cutoffs), count mistakes on labels that cleared it.
|
|
255
|
-
A binomial test passes only when those mistakes are few enough to show the error rate is under `maxError`.
|
|
256
|
-
The loosest cutoff that passes is the threshold. If none pass, that question cannot return `auto`.
|
|
257
|
-
|
|
258
|
-
With `risk` at 0.1 and no mistakes, the label count is about `log(0.1 / 50) / log(1 − maxError)`:
|
|
259
|
-
|
|
260
|
-
| `maxError` | Error-free labels |
|
|
261
|
-
| ---------- | ----------------- |
|
|
262
|
-
| `0.1` | ≈ 59 |
|
|
263
|
-
| `0.05` | ≈ 122 |
|
|
264
|
-
| `0.01` | ≈ 619 |
|
|
265
|
-
|
|
266
|
-
A seed file is JSONL. `expect` is the label for each question. `locale` is optional.
|
|
115
|
+
## Options
|
|
267
116
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
117
|
+
| Option | Required | Meaning |
|
|
118
|
+
| ----------- | -------- | --------------------------------------------------------------- |
|
|
119
|
+
| `decider` | yes | An `ai.decider` handle. There is no default. |
|
|
120
|
+
| `otherwise` | yes | A Gate, or `"abstain"`. |
|
|
121
|
+
| `backup` | no | Tried only after an outage: open breaker, HTTP 5xx, or timeout. |
|
|
122
|
+
| `ask` | yes | `ai.choice`, `ai.score`, or `ai.boolean`. |
|
|
123
|
+
| `autonomy` | no | `{ maxError, audit }`. `audit` is required when this is set. |
|
|
124
|
+
| `locale` | no | `(input) => string`. Each locale is its own certificate slice. |
|
|
125
|
+
| `evals` | no | Seed JSONL for `oke decide certify`. |
|
|
126
|
+
| `in` | no | Fields kept when labels are exported. |
|
|
272
127
|
|
|
273
|
-
|
|
274
|
-
It writes `oke-decisions.lock.json` in that root (`OKE_ROOT_DIR`, or `rootDir`), not in the shell's current directory. Boot reads that same file. Declared decisions without a project root fail at boot.
|
|
128
|
+
`shadow` is reserved. Using it fails with `shadow is planned`.
|
|
275
129
|
|
|
276
|
-
`
|
|
130
|
+
`backup` does not run on HTTP 4xx. That is `DecisionRequestError`, and the breaker stays closed. State larger than `maxContext` throws `DecisionInputTooLarge` before any call. It does not take `otherwise`.
|
|
277
131
|
|
|
278
|
-
|
|
279
|
-
{ "team": "technical", "$": { "team": { "how": "auto", "p": 0.91, "audited": true } } }
|
|
280
|
-
```
|
|
132
|
+
Auto requires the answering decider's own certificate: same echoed model, unexpired, same question hash, and a calibrated score at or above the threshold. A backup that already has a certificate does not need another one.
|
|
281
133
|
|
|
282
|
-
|
|
283
|
-
OKE_ROOT_DIR=/srv/app oke eval --certify
|
|
284
|
-
```
|
|
134
|
+
## Certificates
|
|
285
135
|
|
|
286
|
-
|
|
287
|
-
own key. `hash` is the question text and options. Edit the question and the hash no longer
|
|
288
|
-
matches: the runtime stops returning `auto` until you certify again.
|
|
136
|
+
`oke decide certify <name> --deciders a,b` calls each decider on the seed file and writes lockfile version 2. One certificate per decider. The table prints accuracy, ECE, certified coverage, p50, p95, and cost.
|
|
289
137
|
|
|
290
138
|
```json
|
|
291
139
|
{
|
|
140
|
+
"version": 2,
|
|
292
141
|
"decisions": {
|
|
293
142
|
"triage": {
|
|
294
|
-
"
|
|
295
|
-
|
|
296
|
-
"questions": {
|
|
297
|
-
"team": {
|
|
298
|
-
"": {
|
|
299
|
-
"hash": "…",
|
|
300
|
-
"calibrator": { "kind": "temperature", "t": 1.2 },
|
|
301
|
-
"threshold": 0.8,
|
|
302
|
-
"metrics": {
|
|
303
|
-
"labels": 122,
|
|
304
|
-
"accepted": 122,
|
|
305
|
-
"errors": 0,
|
|
306
|
-
"maxError": 0.05,
|
|
307
|
-
"delta": 0.1
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
}
|
|
143
|
+
"deciders": {
|
|
144
|
+
"jev": { "model": "typesafe/jev-1.13-20260917", "pinned": true, "questions": {} }
|
|
311
145
|
}
|
|
312
146
|
}
|
|
313
147
|
}
|
|
314
148
|
}
|
|
315
149
|
```
|
|
316
150
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
```bash
|
|
320
|
-
OKE_ORIGIN=http://127.0.0.1:6530 \
|
|
321
|
-
OKE_OPERATOR_AUTHORIZATION='Bearer …' \
|
|
322
|
-
oke decide promote triage \
|
|
323
|
-
--origin "$OKE_ORIGIN" \
|
|
324
|
-
--authorization "$OKE_OPERATOR_AUTHORIZATION" \
|
|
325
|
-
--lock "$OKE_ROOT_DIR/oke-decisions.lock.json"
|
|
326
|
-
```
|
|
327
|
-
|
|
328
|
-
That GET is `/_oke/decisions/triage/candidate`. The caller must be an operator. A missing candidate is 404. The route exists only when a decision declares `autonomy`.
|
|
329
|
-
|
|
330
|
-
`oke decide labels <name> --export` writes reviewed labels as seed JSONL (`input`, `expect`, `locale`). It is operator-only and limited to the caller's tenant. The file keeps the decision's declared `in` fields. Secret and redacted fields are masked. The command prints that the file contains production data.
|
|
331
|
-
|
|
332
|
-
```bash
|
|
333
|
-
oke decide labels triage --export --out triage.labels.jsonl
|
|
334
|
-
```
|
|
335
|
-
|
|
336
|
-
## Patterns
|
|
337
|
-
|
|
338
|
-
<Tabs items={["Review", "Abstain", "Autonomy", "Locale"]}>
|
|
339
|
-
|
|
340
|
-
<Tab value="Review">
|
|
341
|
-
|
|
342
|
-
Non-auto questions park the consumer. The result is returned only after the gate submits
|
|
343
|
-
every open question. `how` on those questions is `reviewed`.
|
|
344
|
-
|
|
345
|
-
```typescript
|
|
346
|
-
export const triage = ai.decision("triage", {
|
|
347
|
-
review: ops,
|
|
348
|
-
ask: {
|
|
349
|
-
team: ai.choice("Which team owns this ticket?", {
|
|
350
|
-
billing: "Billing",
|
|
351
|
-
technical: "Technical",
|
|
352
|
-
}),
|
|
353
|
-
},
|
|
354
|
-
});
|
|
355
|
-
```
|
|
356
|
-
|
|
357
|
-
</Tab>
|
|
358
|
-
|
|
359
|
-
<Tab value="Abstain">
|
|
360
|
-
|
|
361
|
-
Only the uncertain questions return `null` and `how: "abstained"`. A question that cleared its threshold stays `auto`. The Flow is not parked, so it can run on HTTP and without `durable`.
|
|
362
|
-
|
|
363
|
-
```typescript
|
|
364
|
-
export const triage = ai.decision("triage", {
|
|
365
|
-
onUncertain: "abstain",
|
|
366
|
-
ask: {
|
|
367
|
-
urgent: ai.boolean("Does this need a person today?"),
|
|
368
|
-
},
|
|
369
|
-
});
|
|
370
|
-
```
|
|
371
|
-
|
|
372
|
-
</Tab>
|
|
151
|
+
A version 1 file fails with: oke-decisions.lock.json is not version 2. Run `oke decide certify` to recertify.
|
|
373
152
|
|
|
374
|
-
|
|
153
|
+
OpenRouter certificates bind to the dated slug. If the catalog lists a dated id, certify requires that id. OpenAI binds to the echoed alias, stores `pinned: false`, and sets `expiresAt` to certify time plus 30 days. Certify prints that the model is unpinned. After expiry, `why` is `uncertified`.
|
|
375
154
|
|
|
376
|
-
`
|
|
377
|
-
to spend. Without a matching certificate, every question still takes the non-auto path.
|
|
155
|
+
`oke decide promote <name>` installs the operator candidate for one decision. `oke decide labels <name> --export` writes reviewed labels as seed JSONL. The file contains production data. `oke decide models` lists catalog id, canonical slug, context, and input modalities.
|
|
378
156
|
|
|
379
|
-
|
|
380
|
-
export const triage = ai.decision("triage", {
|
|
381
|
-
review: ops,
|
|
382
|
-
autonomy: { maxError: 0.05, audit: 0.1, risk: 0.1 },
|
|
383
|
-
evals: "evals/triage.jsonl",
|
|
384
|
-
ask: {
|
|
385
|
-
team: ai.choice("Which team owns this ticket?", {
|
|
386
|
-
billing: "Billing",
|
|
387
|
-
technical: "Technical",
|
|
388
|
-
}),
|
|
389
|
-
},
|
|
390
|
-
});
|
|
391
|
-
```
|
|
157
|
+
`t.ai.decide(decision, answers)` scripts probabilities, or `{ refusal: "…" }`. The refusal string is not stored.
|
|
392
158
|
|
|
393
|
-
|
|
159
|
+
## Drivers
|
|
394
160
|
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
Return a string to keep a separate certificate slice. `undefined` uses the default slice.
|
|
398
|
-
An input whose locale has no slice is not `auto`.
|
|
399
|
-
|
|
400
|
-
```typescript
|
|
401
|
-
export const triage = ai.decision("triage", {
|
|
402
|
-
review: ops,
|
|
403
|
-
locale: (input) => {
|
|
404
|
-
const row = input as { locale?: string };
|
|
405
|
-
return row.locale;
|
|
406
|
-
},
|
|
407
|
-
ask: {
|
|
408
|
-
team: ai.choice("Which team owns this ticket?", {
|
|
409
|
-
billing: "Billing",
|
|
410
|
-
technical: "Technical",
|
|
411
|
-
}),
|
|
412
|
-
},
|
|
413
|
-
});
|
|
414
|
-
```
|
|
415
|
-
|
|
416
|
-
</Tab>
|
|
417
|
-
|
|
418
|
-
</Tabs>
|
|
419
|
-
|
|
420
|
-
## Provider
|
|
421
|
-
|
|
422
|
-
The key is a Vault secret. `fx.decide` reads `OPENROUTER_API_KEY`, or `TYPESAFE_API_KEY` when
|
|
423
|
-
`driverId` is `"typesafe"`. Declare that secret on the Flow when you write an `effects` block.
|
|
424
|
-
An empty value throws `DecisionConfigError`.
|
|
425
|
-
|
|
426
|
-
| `driverId` | Default model | Secret |
|
|
427
|
-
| ------------ | ------------------------------------------------------------------------------------------------- | -------------------- |
|
|
428
|
-
| `openrouter` | `typesafe/jev-1.13` (the response `model` is a dated pin, currently `typesafe/jev-1.13-20260917`) | `OPENROUTER_API_KEY` |
|
|
429
|
-
| `typesafe` | `jev-1.13.0` | `TYPESAFE_API_KEY` |
|
|
430
|
-
|
|
431
|
-
The deadline defaults to 30 seconds. `400`, `401`, `402`, `403`, `404`, and `422` throw `DecisionRequestError` and leave the breaker closed.
|
|
432
|
-
`429` and `529` retry up to 3 attempts and honor `Retry-After`. A wait over 60 seconds is an outage.
|
|
433
|
-
|
|
434
|
-
Network errors, timeouts, and HTTP 5xx count toward the breaker. Three failures open it for 30 seconds.
|
|
435
|
-
An open breaker is an outage: review parks, abstain returns `null`.
|
|
161
|
+
`drivers.decide` defaults to `{ test: "mock" }`. Unset `dev` and `prod` call the decider host. Pin `mock` when a local boot must stay off the network.
|
|
436
162
|
|
|
437
163
|
## Troubleshooting
|
|
438
164
|
|
|
439
165
|
<Accordions>
|
|
440
166
|
|
|
441
|
-
<Accordion title=
|
|
442
|
-
|
|
167
|
+
<Accordion title="otherwise is required">
|
|
168
|
+
`ai.decision("triage"): otherwise is required`. Pass a Gate or `"abstain"`.
|
|
443
169
|
</Accordion>
|
|
444
170
|
|
|
445
|
-
<Accordion title="
|
|
446
|
-
`ai.decision("triage")
|
|
447
|
-
a signal and decide in a consumer.` Move `fx.decide` to a signal or clock Flow with `durable:
|
|
448
|
-
true`.
|
|
171
|
+
<Accordion title="shadow is planned">
|
|
172
|
+
`ai.decision("triage"): shadow is planned`. Remove `shadow`.
|
|
449
173
|
</Accordion>
|
|
450
174
|
|
|
451
|
-
<Accordion title="
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
that can skip `durable`.
|
|
455
|
-
</Accordion>
|
|
456
|
-
|
|
457
|
-
<Accordion title="choice, score, and duplicate names">
|
|
458
|
-
`choice "team" has 255 options; max 254`. `choice "team" must not set none_of_these` — that option
|
|
459
|
-
is added for you. `score "urgency" needs 2–10 levels`. `question id "$" is reserved`. `duplicate
|
|
460
|
-
decision name` when two declarations share a name. `ask is empty` when `ask` has no questions.
|
|
461
|
-
</Accordion>
|
|
462
|
-
|
|
463
|
-
<Accordion title="OKE1010 UNDECLARED_DECIDE">
|
|
464
|
-
Cause: `Flow "support.route" decides "triage" without declaring it.` Add `triage` to
|
|
465
|
-
`effects.decides`, or drop the hand-written `effects` block so inference can see `fx.decide`.
|
|
175
|
+
<Accordion title="review cannot run in HTTP flow">
|
|
176
|
+
`ai.decision("triage") review cannot run in HTTP flow "…" — a park answers 204.` Emit a signal and
|
|
177
|
+
decide in a consumer. A gate also needs `durable: true`: `must set durable: true to review`.
|
|
466
178
|
</Accordion>
|
|
467
179
|
|
|
468
180
|
<Accordion title='fx.decide: secret "OPENROUTER_API_KEY" is not configured'>
|
|
469
|
-
|
|
470
|
-
`TYPESAFE_API_KEY` instead. This is not an outage, and it does not park.
|
|
471
|
-
</Accordion>
|
|
472
|
-
|
|
473
|
-
<Accordion title="400, 401, 402, 403, 404, or 422 from the provider">
|
|
474
|
-
`DecisionRequestError`. 400 and 422 are a body the provider rejected. 401 is the key. 402 is
|
|
475
|
-
billing. 403 is forbidden. 404 is an unknown model or path. The breaker stays closed. Fix the
|
|
476
|
-
request. Do not retry as if the provider were down.
|
|
477
|
-
</Accordion>
|
|
478
|
-
|
|
479
|
-
<Accordion title="oke boot: decisions are declared but rootDir and OKE_ROOT_DIR are unset">
|
|
480
|
-
`oke boot: decisions are declared but rootDir and OKE_ROOT_DIR are unset, so the decision lockfile
|
|
481
|
-
cannot load.` Boot with `rootDir`, or set `OKE_ROOT_DIR` to the directory that holds
|
|
482
|
-
`oke-decisions.lock.json`.
|
|
181
|
+
The decider's secret is not in the vault. OpenAI's preset reads `OPENAI_API_KEY`.
|
|
483
182
|
</Accordion>
|
|
484
183
|
|
|
485
|
-
<Accordion title="
|
|
486
|
-
|
|
487
|
-
|
|
184
|
+
<Accordion title="why is uncertified">
|
|
185
|
+
The lock is missing, the echoed model differs, the question hash moved, the locale has no slice,
|
|
186
|
+
or an alias certificate expired. Run `oke decide certify`.
|
|
488
187
|
</Accordion>
|
|
489
188
|
|
|
490
|
-
<Accordion title="
|
|
491
|
-
`
|
|
492
|
-
|
|
493
|
-
be resolved: `ai.decision: review gate name could not be resolved`.
|
|
494
|
-
</Accordion>
|
|
495
|
-
|
|
496
|
-
<Accordion title="The candidate route is missing">
|
|
497
|
-
The candidate route is registered only when some decision declares `autonomy`. Without that block
|
|
498
|
-
there is no lockfile grant to promote, and `GET /_oke/decisions/:name/candidate` is not mounted.
|
|
499
|
-
</Accordion>
|
|
500
|
-
|
|
501
|
-
<Accordion title="Conflict or JournalLeaseBusy">
|
|
502
|
-
A second resolve after the review is stored is `Conflict`. Do not retry it. A lost race is
|
|
503
|
-
`JournalLeaseBusy` with `Retry-After`. Retry only that code. A suggested wait over 60 seconds
|
|
504
|
-
comes back as 503.
|
|
505
|
-
</Accordion>
|
|
506
|
-
|
|
507
|
-
<Accordion title="The certificate went stale after an edit">
|
|
508
|
-
Changing the question text or its options changes the hash. The runtime then refuses `auto`
|
|
509
|
-
(`stale-hash`) until `oke eval --certify` or `oke decide promote` writes a new lock entry. The old
|
|
510
|
-
threshold is not reused.
|
|
189
|
+
<Accordion title="state is N tokens; max context is M">
|
|
190
|
+
`fx.decide: decider "…" state is N tokens; max context is M`. Shrink the input. This throws. It
|
|
191
|
+
does not take `otherwise`.
|
|
511
192
|
</Accordion>
|
|
512
193
|
|
|
513
194
|
</Accordions>
|
|
514
195
|
|
|
515
196
|
## Learn more
|
|
516
197
|
|
|
517
|
-
- [
|
|
518
|
-
- [Gate](/docs/elements/gate) — the policy on `
|
|
519
|
-
- [fx](/docs/reference/fx) — `fx.decide`
|
|
520
|
-
- [CLI](/docs/reference/cli) — `oke decide promote
|
|
198
|
+
- [Deciders](/docs/elements/ai/deciders) — presets, protocols, and pinning
|
|
199
|
+
- [Gate](/docs/elements/gate) — the policy on `otherwise`
|
|
200
|
+
- [fx](/docs/reference/fx) — `fx.decide`
|
|
201
|
+
- [CLI](/docs/reference/cli) — `oke decide certify`, `promote`, `labels`, `models`
|
|
202
|
+
- [Configuration](/docs/reference/configuration) — `drivers.decide`
|
|
521
203
|
|
|
522
204
|
## Next
|
|
523
205
|
|
|
524
206
|
<Cards>
|
|
525
207
|
<Card
|
|
526
|
-
title="
|
|
527
|
-
description="
|
|
528
|
-
href="/docs/elements/ai/
|
|
529
|
-
/>
|
|
530
|
-
<Card
|
|
531
|
-
title="Agents"
|
|
532
|
-
description="Bounded tool loops with fx.run."
|
|
533
|
-
href="/docs/elements/ai/agents"
|
|
208
|
+
title="Deciders"
|
|
209
|
+
description="The model that answers a decision."
|
|
210
|
+
href="/docs/elements/ai/deciders"
|
|
534
211
|
/>
|
|
535
|
-
<Card title="
|
|
212
|
+
<Card title="Agents" description="Tools, steps, and approval." href="/docs/elements/ai/agents" />
|
|
213
|
+
<Card title="fx" description="What a Flow is allowed to touch." href="/docs/reference/fx" />
|
|
536
214
|
</Cards>
|