@corbits/system-one 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -224
- package/dist/adapter.js +11 -10
- package/dist/client.d.ts +11 -6
- package/dist/client.js +32 -18
- package/dist/config.d.ts +11 -4
- package/dist/config.js +11 -12
- package/dist/errors.d.ts +12 -25
- package/dist/errors.js +20 -34
- package/dist/evaluate.d.ts +22 -7
- package/dist/evaluate.js +36 -27
- package/dist/index.d.ts +6 -6
- package/dist/index.js +4 -5
- package/dist/schemas.d.ts +0 -23
- package/dist/schemas.js +0 -3
- package/dist/telemetry.d.ts +0 -14
- package/dist/telemetry.js +4 -35
- package/package.json +40 -34
package/README.md
CHANGED
|
@@ -1,279 +1,184 @@
|
|
|
1
1
|
# @corbits/system-one
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
[](#development)
|
|
5
|
-
|
|
6
|
-
**Typed decisions from System One (Jev-class) models — as an Interchange
|
|
7
|
-
inference provider, or as a direct `evaluate()` call.**
|
|
8
|
-
|
|
9
|
-
Ask choice, score, and boolean questions over a state payload. You get
|
|
10
|
-
back per-kind decisions, or a typed fallback when the backend is
|
|
11
|
-
unreachable. Official, gateway, and custom endpoints are config, not
|
|
12
|
-
forked code.
|
|
13
|
-
|
|
14
|
-
## Contents
|
|
15
|
-
|
|
16
|
-
- [Why](#why)
|
|
17
|
-
- [Quickstart](#quickstart)
|
|
18
|
-
- [How it works](#how-it-works)
|
|
19
|
-
- [Development](#development)
|
|
20
|
-
- [License](#license)
|
|
21
|
-
|
|
22
|
-
## Why
|
|
23
|
-
|
|
24
|
-
Interchange agents and workflows already speak providers. This package
|
|
25
|
-
is the System One adapter for that surface:
|
|
26
|
-
|
|
27
|
-
- **One provider id.** `SYSTEM_ONE_PROVIDER` is `"corbits-system-one"`
|
|
28
|
-
— same naming as the other Corbits libraries, not a generic
|
|
29
|
-
`"system-one"`. Use it in `defineAgent` / workflow inference sources
|
|
30
|
-
the way you would `"anthropic"`.
|
|
31
|
-
- **Questions on the call.** Pass Jev questions (and optional state)
|
|
32
|
-
through `providerOptions.systemOne`. The transcript is the default
|
|
33
|
-
state when you omit it.
|
|
34
|
-
- **Two output shapes.** Direct `evaluate()` returns an
|
|
35
|
-
`EvaluateResult` or `FallbackResult`. The adapter unwraps the same
|
|
36
|
-
decisions into Interchange `inference.text.delta` / `inference.usage`
|
|
37
|
-
events so an agent step can consume them.
|
|
38
|
-
- **Host-owned credentials.** The adapter sends Interchange's bearer
|
|
39
|
-
sentinel; the harness injects the real key. Peers are
|
|
40
|
-
`@intx/inference` and `@intx/types` — one copy, the host's.
|
|
3
|
+
Typed choice, score and yes/no decisions about a JSON state from TypeSafe's Jev model, returned as a validated result or a typed fallback. An inference provider for Corbits and Interchange agents: registers as an `@intx/inference` adapter, and also works standalone through `evaluate()`.
|
|
41
4
|
|
|
42
|
-
##
|
|
5
|
+
## Why @corbits/system-one?
|
|
43
6
|
|
|
44
|
-
|
|
45
|
-
|
|
7
|
+
1. **Typed decisions, not free text.** You send questions with ids; you get back one validated decision per id, with probabilities and confidence. No prompt parsing.
|
|
8
|
+
2. **Failures are data.** A missing key, timeout, HTTP error or bad body returns a `FallbackResult` with a `reason`, so a gate can fail closed without a `try`. Only invalid caller input throws.
|
|
9
|
+
3. **One call, many questions.** Every question in a list goes out in one POST, bounded by a 1.5 s default timeout.
|
|
46
10
|
|
|
47
|
-
|
|
48
|
-
bun add @corbits/system-one
|
|
49
|
-
```
|
|
11
|
+
It evaluates structured state against questions. For open-ended chat completions, use a general inference provider such as [`@corbits/openai-responses`](https://github.com/corbitsdev/corbits-openai-responses).
|
|
50
12
|
|
|
51
|
-
|
|
13
|
+
## Install
|
|
52
14
|
|
|
53
|
-
```
|
|
54
|
-
|
|
55
|
-
{
|
|
56
|
-
id: "route",
|
|
57
|
-
type: "choice",
|
|
58
|
-
instructions: "What permission decision does this request warrant?",
|
|
59
|
-
criteria: { allow: "Low-risk request", deny: "Refuse the request" },
|
|
60
|
-
},
|
|
61
|
-
{
|
|
62
|
-
id: "escalate",
|
|
63
|
-
type: "boolean",
|
|
64
|
-
instructions: "Must this request be escalated for human approval?",
|
|
65
|
-
},
|
|
66
|
-
];
|
|
15
|
+
```bash
|
|
16
|
+
bun add @corbits/system-one @intx/inference@^0.4.0 @intx/types@^0.4.0
|
|
67
17
|
```
|
|
68
18
|
|
|
69
|
-
|
|
19
|
+
Runs on Bun >= 1.2 or Node >= 24.
|
|
70
20
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
21
|
+
## Quickstart
|
|
22
|
+
|
|
23
|
+
Needs `TYPESAFE_API_KEY` set.
|
|
74
24
|
|
|
75
25
|
```ts
|
|
76
26
|
import { evaluate } from "@corbits/system-one";
|
|
77
27
|
|
|
78
28
|
const result = await evaluate({
|
|
79
29
|
state: { action: "deploy", env: "production" },
|
|
80
|
-
questions
|
|
81
|
-
config: { apiKey: process.env.TYPESAFE_API_KEY },
|
|
82
|
-
});
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
Success looks like:
|
|
86
|
-
|
|
87
|
-
```ts
|
|
88
|
-
{
|
|
89
|
-
fallback: false,
|
|
90
|
-
backend: "corbits-system-one", // official endpoint; "gateway" | "custom" otherwise
|
|
91
|
-
modelId: "jev-1.13.0",
|
|
92
|
-
latencyMs: 412,
|
|
93
|
-
usage: { inputTokens: 296, outputTokens: 20 },
|
|
94
|
-
decisions: [
|
|
95
|
-
{
|
|
96
|
-
id: "route",
|
|
97
|
-
type: "choice",
|
|
98
|
-
choice: "deny",
|
|
99
|
-
confidence: 0.86,
|
|
100
|
-
probabilities: { allow: 0.14, deny: 0.86 },
|
|
101
|
-
},
|
|
30
|
+
questions: [
|
|
102
31
|
{
|
|
103
32
|
id: "escalate",
|
|
104
|
-
type: "
|
|
105
|
-
|
|
33
|
+
type: "boolean",
|
|
34
|
+
instructions: "Must this request be escalated for human approval?",
|
|
106
35
|
},
|
|
107
36
|
],
|
|
108
|
-
}
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
console.log(result.fallback ? result.reason : result.decisions);
|
|
109
40
|
```
|
|
110
41
|
|
|
111
|
-
|
|
42
|
+
On success it prints one decision per question, or the fallback reason on failure:
|
|
112
43
|
|
|
113
44
|
```ts
|
|
114
|
-
{
|
|
115
|
-
fallback: true,
|
|
116
|
-
reason: "no-key", // or "timeout" | "network" | "http-error" | "parse-error" | "backend-unreachable"
|
|
117
|
-
latencyMs: 2,
|
|
118
|
-
backendAttempted: "https://api.typesafe.ai/...",
|
|
119
|
-
detail: "no API key in config.apiKey, TYPESAFE_API_KEY, or SYSTEM_ONE_API_KEY",
|
|
120
|
-
}
|
|
45
|
+
[{ id: "escalate", type: "noul", noul: 0.91 }];
|
|
121
46
|
```
|
|
122
47
|
|
|
123
|
-
|
|
124
|
-
still **throws** a `SystemOneError` — that is not a fallback.
|
|
48
|
+
Boolean questions answer as `noul`, a probability from 0 to 1 that the answer is yes.
|
|
125
49
|
|
|
126
|
-
|
|
127
|
-
`AI_GATEWAY_API_KEY` / `VERCEL_OIDC_TOKEN` (gateway), else
|
|
128
|
-
`SYSTEM_ONE_API_KEY`. Endpoint: omit for official (`jev-latest`);
|
|
129
|
-
`{ kind: "gateway" }`; `{ kind: "custom", url }`.
|
|
50
|
+
## Where it fits
|
|
130
51
|
|
|
131
|
-
|
|
52
|
+
[Interchange](https://github.com/faremeter/interchange) runs AI agents as principals (accounts that hold their own identity, permissions and credentials). Corbits packages add what an agent product needs around it.
|
|
132
53
|
|
|
133
|
-
`
|
|
134
|
-
`
|
|
135
|
-
|
|
54
|
+
- **Runs in:** the agent sidecar (the runtime next to each agent), or any process. `evaluate()` needs no Interchange runtime.
|
|
55
|
+
- **Plugs into:** the [`@intx/inference`](https://github.com/faremeter/interchange/tree/main/packages/inference) adapter registry, as the adapter for the `corbits-system-one` provider id.
|
|
56
|
+
- **Pairs with:** [`@corbits/openai-responses`](https://github.com/corbitsdev/corbits-openai-responses) and [`@corbits/ollama-adapter`](https://github.com/corbitsdev/corbits-ollama-adapter), the other Corbits inference providers.
|
|
136
57
|
|
|
137
|
-
|
|
138
|
-
import { createAgent, defineAgent } from "@intx/agent";
|
|
139
|
-
import {
|
|
140
|
-
createSystemOneAdapter,
|
|
141
|
-
SYSTEM_ONE_PROVIDER, // "corbits-system-one"
|
|
142
|
-
} from "@corbits/system-one";
|
|
58
|
+
## Reference
|
|
143
59
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
}
|
|
60
|
+
| Export | Description |
|
|
61
|
+
| --------------------------------- | -------------------------------------------------------------------------------- |
|
|
62
|
+
| `evaluate(input, options?)` | Runs one evaluation. Returns `EvaluateResult`: decisions or a `FallbackResult`. |
|
|
63
|
+
| `createSystemOneAdapter(config?)` | Returns a `ProviderAdapter` for `runInference`. `config` is an `EvaluateConfig`. |
|
|
64
|
+
| `SYSTEM_ONE_PROVIDER` | The `"corbits-system-one"` provider id. |
|
|
65
|
+
| `DEFAULT_TIMEOUT_MS` | `1500`, the default `timeoutMs`. |
|
|
66
|
+
| `SystemOneError` | Thrown for invalid input or a `custom` endpoint without a `url`. |
|
|
67
|
+
| `SystemOneTelemetryEvent` | Schema and type for events passed to `onTelemetry`. |
|
|
68
|
+
| `EvaluateOptions`, `EvaluateDeps` | Types for `options`: `onTelemetry`, and `deps: { fetch, scheduler }`. |
|
|
153
69
|
|
|
154
|
-
|
|
155
|
-
source: {
|
|
156
|
-
id: SYSTEM_ONE_PROVIDER,
|
|
157
|
-
provider: SYSTEM_ONE_PROVIDER,
|
|
158
|
-
model: "jev-latest",
|
|
159
|
-
apiKey: process.env.TYPESAFE_API_KEY,
|
|
160
|
-
adapter: createSystemOneAdapter(),
|
|
161
|
-
},
|
|
162
|
-
});
|
|
70
|
+
`Question`, `QuestionList`, `Decision`, `EvaluateInput`, `EvaluateConfig`, `EndpointConfig`, `EvaluateResult` and `FallbackResult` are exported as arktype schemas and types.
|
|
163
71
|
|
|
164
|
-
|
|
165
|
-
providerOptions: { systemOne: { questions } },
|
|
166
|
-
});
|
|
167
|
-
```
|
|
72
|
+
### Questions
|
|
168
73
|
|
|
169
|
-
|
|
74
|
+
| `type` | Fields | Decision fields |
|
|
75
|
+
| --------- | ------------------------------------------------------------------------------- | ------------------------------------------------ |
|
|
76
|
+
| `choice` | `instructions`, `criteria` (option name → description or `null`, 1–255 options) | `choice`, `probabilities`, `confidence` |
|
|
77
|
+
| `score` | `instructions`, `criteria` (2–10 descriptions) | `score`, `legend`, `probabilities`, `confidence` |
|
|
78
|
+
| `boolean` | `instructions`, optional `criteria: { true?, false? }` | `noul` |
|
|
79
|
+
| `noul` | Same as `boolean` | `noul` |
|
|
170
80
|
|
|
171
|
-
|
|
172
|
-
[
|
|
173
|
-
{
|
|
174
|
-
type: "inference.text.delta",
|
|
175
|
-
data: {
|
|
176
|
-
index: 0,
|
|
177
|
-
token:
|
|
178
|
-
'{"id":"route","type":"choice","choice":"deny","confidence":0.86,"probabilities":{"allow":0.14,"deny":0.86}}',
|
|
179
|
-
},
|
|
180
|
-
},
|
|
181
|
-
{
|
|
182
|
-
type: "inference.text.delta",
|
|
183
|
-
data: {
|
|
184
|
-
index: 1,
|
|
185
|
-
token: '{"id":"escalate","type":"noul","noul":0.91}',
|
|
186
|
-
},
|
|
187
|
-
},
|
|
188
|
-
{
|
|
189
|
-
type: "inference.usage",
|
|
190
|
-
data: {
|
|
191
|
-
usage: {
|
|
192
|
-
input: 296,
|
|
193
|
-
output: 20,
|
|
194
|
-
cacheRead: 0,
|
|
195
|
-
cacheWrite: 0,
|
|
196
|
-
thinking: 0,
|
|
197
|
-
},
|
|
198
|
-
},
|
|
199
|
-
},
|
|
200
|
-
];
|
|
201
|
-
```
|
|
81
|
+
Every question has an `id`, unique within the list.
|
|
202
82
|
|
|
203
|
-
|
|
204
|
-
`result.decisions`. There is no `fallback` flag on this path: a bad
|
|
205
|
-
wire body is a `ProtocolMismatchError`; auth/retry stay with the
|
|
206
|
-
harness (`retry-after` is surfaced for 429/529).
|
|
83
|
+
### Config
|
|
207
84
|
|
|
208
|
-
|
|
85
|
+
`input.config` for `evaluate`, or the argument to `createSystemOneAdapter`:
|
|
209
86
|
|
|
210
|
-
|
|
87
|
+
| Field | Type | Default | Description |
|
|
88
|
+
| ----------- | ---------------- | ---------------------- | -------------------------------------------------------------------------------- |
|
|
89
|
+
| `endpoint` | `EndpointConfig` | `{ kind: "official" }` | `official`, `gateway`, or `custom` with a `url`. Each takes an optional `model`. |
|
|
90
|
+
| `timeoutMs` | `number` | `1500` | Bound for the whole round trip. |
|
|
91
|
+
| `apiKey` | `string` | from the environment | Overrides the environment key. |
|
|
92
|
+
|
|
93
|
+
### Endpoints and keys
|
|
94
|
+
|
|
95
|
+
| `kind` | URL | Default model | Key from the environment |
|
|
96
|
+
| ---------- | ---------------------------------------------------- | ----------------- | ---------------------------------------------- |
|
|
97
|
+
| `official` | `https://api.typesafe.ai/v1/systemone` | `jev-latest` | `TYPESAFE_API_KEY` |
|
|
98
|
+
| `gateway` | `https://ai-gateway.vercel.sh/typesafe/v1/systemone` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN` |
|
|
99
|
+
| `custom` | `url` | `typesafe-ai/jev` | `SYSTEM_ONE_API_KEY` |
|
|
100
|
+
|
|
101
|
+
`SYSTEM_ONE_API_KEY` is also a supported alias on every endpoint, read after the endpoint's own key.
|
|
102
|
+
|
|
103
|
+
### Fallback reasons
|
|
104
|
+
|
|
105
|
+
| `reason` | When |
|
|
106
|
+
| ------------- | ------------------------------------------------------------- |
|
|
107
|
+
| `no-key` | No key in `apiKey` or the environment. No request is sent. |
|
|
108
|
+
| `timeout` | The round trip, including the body read, exceeds `timeoutMs`. |
|
|
109
|
+
| `network` | The request fails before a response. |
|
|
110
|
+
| `http-error` | Non-2xx response. `httpStatus` holds the status. |
|
|
111
|
+
| `parse-error` | The body is not JSON or fails validation. |
|
|
112
|
+
|
|
113
|
+
`detail` carries the underlying message.
|
|
114
|
+
|
|
115
|
+
## Using with Interchange
|
|
116
|
+
|
|
117
|
+
Register the adapter under `SYSTEM_ONE_PROVIDER` and pass questions per call in `providerOptions.systemOne` (`{ state?, questions? }`). Without `state`, the adapter sends the transcript text as state. Without `questions`, it asks one boolean question with id `response`.
|
|
211
118
|
|
|
212
119
|
```ts
|
|
213
|
-
import {
|
|
120
|
+
import { createDependencies, runInference } from "@intx/inference";
|
|
214
121
|
import {
|
|
215
122
|
createSystemOneAdapter,
|
|
216
123
|
SYSTEM_ONE_PROVIDER,
|
|
124
|
+
type QuestionList,
|
|
217
125
|
} from "@corbits/system-one";
|
|
218
126
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
127
|
+
const deps = createDependencies({
|
|
128
|
+
has: (provider) => provider === SYSTEM_ONE_PROVIDER,
|
|
129
|
+
resolve: () => createSystemOneAdapter(),
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
const questions = [
|
|
133
|
+
{
|
|
134
|
+
id: "escalate",
|
|
135
|
+
type: "boolean",
|
|
136
|
+
instructions: "Must this request be escalated for human approval?",
|
|
137
|
+
},
|
|
138
|
+
] satisfies QuestionList;
|
|
139
|
+
|
|
140
|
+
let seq = 0;
|
|
141
|
+
for await (const event of runInference({
|
|
142
|
+
deps,
|
|
143
|
+
source: {
|
|
144
|
+
id: "system-one",
|
|
145
|
+
provider: SYSTEM_ONE_PROVIDER,
|
|
146
|
+
baseURL: "https://api.typesafe.ai/v1/systemone",
|
|
147
|
+
credentialId: "TYPESAFE_API_KEY",
|
|
148
|
+
model: "jev-latest",
|
|
149
|
+
},
|
|
150
|
+
turns: [
|
|
222
151
|
{
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
provider: SYSTEM_ONE_PROVIDER,
|
|
227
|
-
model: "jev-latest",
|
|
228
|
-
adapter: createSystemOneAdapter(),
|
|
229
|
-
},
|
|
230
|
-
providerOptions: {
|
|
231
|
-
systemOne: {
|
|
232
|
-
state: { amount: 48, priorRefunds: 1 },
|
|
233
|
-
questions: [
|
|
234
|
-
{
|
|
235
|
-
id: "approve",
|
|
236
|
-
type: "boolean",
|
|
237
|
-
instructions: "Approve this refund without review?",
|
|
238
|
-
},
|
|
239
|
-
],
|
|
240
|
-
},
|
|
241
|
-
},
|
|
242
|
-
},
|
|
152
|
+
role: "user",
|
|
153
|
+
timestamp: Date.now(),
|
|
154
|
+
content: [{ type: "text", text: "Deploy to production?" }],
|
|
243
155
|
},
|
|
244
156
|
],
|
|
245
|
-
}
|
|
157
|
+
inferenceOptions: { providerOptions: { systemOne: { questions } } },
|
|
158
|
+
nextSeq: () => seq++,
|
|
159
|
+
readMaterial: (id) => {
|
|
160
|
+
const secret = process.env[id];
|
|
161
|
+
if (secret === undefined) throw new Error(`${id} is not set`);
|
|
162
|
+
return { secret };
|
|
163
|
+
},
|
|
164
|
+
})) {
|
|
165
|
+
if (event.type === "inference.text.delta")
|
|
166
|
+
console.log(JSON.parse(event.data.token));
|
|
167
|
+
}
|
|
246
168
|
```
|
|
247
169
|
|
|
248
|
-
The
|
|
249
|
-
whose token is `{"id":"approve","type":"noul","noul":0.2}`, then
|
|
250
|
-
usage.
|
|
251
|
-
|
|
252
|
-
## How it works
|
|
170
|
+
The adapter emits one `inference.text.delta` per decision, whose `token` is the decision as JSON, then an `inference.usage` event. The harness supplies the credential through `readMaterial` and owns retries. A malformed response body is a `ProtocolMismatchError`, not a fallback.
|
|
253
171
|
|
|
254
|
-
|
|
255
|
-
`state` is string | object | array; questions go out as an id-keyed
|
|
256
|
-
map; answers come back the same way.
|
|
257
|
-
- **Adapter.** Transcript → evaluation state; each decision →
|
|
258
|
-
`inference.text.delta`; token counts → `inference.usage`.
|
|
259
|
-
- **Schemas.** Every trust boundary is arktype (`src/schemas.ts`).
|
|
260
|
-
Confidence is narrowed, not clamped. Public surface is
|
|
261
|
-
`src/index.ts` only.
|
|
262
|
-
- **Telemetry.** Direct `evaluate()` records `evaluate.start` /
|
|
263
|
-
`.success` / `.fallback` in-memory (`drainTelemetryEvents`). No keys
|
|
264
|
-
in events.
|
|
265
|
-
|
|
266
|
-
## Development
|
|
267
|
-
|
|
268
|
-
```
|
|
269
|
-
bun install
|
|
270
|
-
bun run check # typecheck + lint + format:check + test
|
|
271
|
-
```
|
|
172
|
+
## Upgrading from 0.1
|
|
272
173
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
174
|
+
- `HttpError`, `NetworkError` and `TimeoutError` are removed. Transport failures return a `FallbackResult`.
|
|
175
|
+
- A 2xx body that is not JSON is `parse-error` (was `network`). A timeout during the body read is `timeout`.
|
|
176
|
+
- `recordTelemetryEvent` and `drainTelemetryEvents` are removed. Pass `evaluate(input, { onTelemetry })`.
|
|
177
|
+
- Wire helpers, quirks presets, `JsonRecord`, `Confidence` and the env-var name constants are no longer exported.
|
|
178
|
+
- A `custom` endpoint without a `model` sends `typesafe-ai/jev`.
|
|
179
|
+
- Peers are `@intx/inference` and `@intx/types` `^0.4.0`.
|
|
180
|
+
- Environment variables are unchanged, including `SYSTEM_ONE_API_KEY`.
|
|
276
181
|
|
|
277
182
|
## License
|
|
278
183
|
|
|
279
|
-
LGPL-2.1-only.
|
|
184
|
+
[LGPL-2.1-only](https://github.com/corbitsdev/corbits-system-one/blob/main/LICENSE)
|
package/dist/adapter.js
CHANGED
|
@@ -100,7 +100,7 @@ function parseDecisions(body, provider) {
|
|
|
100
100
|
throw new ProtocolMismatchError(`${provider} parseJSONResponse: ${cause instanceof Error ? cause.message : String(cause)}`, parsed);
|
|
101
101
|
}
|
|
102
102
|
});
|
|
103
|
-
return { decisions, usage: envelope.usage };
|
|
103
|
+
return { decisions, model: envelope.model, usage: envelope.usage };
|
|
104
104
|
}
|
|
105
105
|
/**
|
|
106
106
|
* Builds the Interchange `ProviderAdapter` for System One against the
|
|
@@ -111,11 +111,6 @@ function parseDecisions(body, provider) {
|
|
|
111
111
|
*/
|
|
112
112
|
export function createSystemOneAdapter(config) {
|
|
113
113
|
const endpoint = resolveEndpoint(config?.endpoint);
|
|
114
|
-
const source = {
|
|
115
|
-
sourceId: SYSTEM_ONE_PROVIDER,
|
|
116
|
-
provider: SYSTEM_ONE_PROVIDER,
|
|
117
|
-
model: endpoint.model ?? "unknown",
|
|
118
|
-
};
|
|
119
114
|
const buildRequest = (messages, model, options) => {
|
|
120
115
|
let state = { transcript: transcriptText(messages) };
|
|
121
116
|
let questions = DEFAULT_QUESTIONS;
|
|
@@ -130,8 +125,7 @@ export function createSystemOneAdapter(config) {
|
|
|
130
125
|
if (overrides.questions !== undefined)
|
|
131
126
|
questions = overrides.questions;
|
|
132
127
|
}
|
|
133
|
-
const requestModel = model || endpoint.model
|
|
134
|
-
source.model = requestModel;
|
|
128
|
+
const requestModel = model || endpoint.model;
|
|
135
129
|
const headers = {
|
|
136
130
|
"content-type": "application/json",
|
|
137
131
|
accept: "application/json",
|
|
@@ -148,7 +142,7 @@ export function createSystemOneAdapter(config) {
|
|
|
148
142
|
};
|
|
149
143
|
};
|
|
150
144
|
const parseJSONResponse = (responseBody) => {
|
|
151
|
-
const { decisions, usage } = parseDecisions(responseBody, SYSTEM_ONE_PROVIDER);
|
|
145
|
+
const { decisions, model, usage } = parseDecisions(responseBody, SYSTEM_ONE_PROVIDER);
|
|
152
146
|
const events = [];
|
|
153
147
|
let index = 0;
|
|
154
148
|
for (const decision of decisions) {
|
|
@@ -163,7 +157,14 @@ export function createSystemOneAdapter(config) {
|
|
|
163
157
|
events.push({
|
|
164
158
|
type: "inference.usage",
|
|
165
159
|
seq: 0,
|
|
166
|
-
data: {
|
|
160
|
+
data: {
|
|
161
|
+
usage: toTokenUsage(usage),
|
|
162
|
+
source: {
|
|
163
|
+
sourceId: SYSTEM_ONE_PROVIDER,
|
|
164
|
+
provider: SYSTEM_ONE_PROVIDER,
|
|
165
|
+
model: model ?? endpoint.model,
|
|
166
|
+
},
|
|
167
|
+
},
|
|
167
168
|
});
|
|
168
169
|
return events;
|
|
169
170
|
};
|
package/dist/client.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
+
import { type Dependencies } from "@intx/inference";
|
|
1
2
|
/** Options for one evaluation POST: timeout and auth, resolved by `evaluate`. */
|
|
2
3
|
export type PostEvaluateOptions = {
|
|
3
4
|
timeoutMs?: number;
|
|
4
5
|
apiKey?: string;
|
|
5
6
|
};
|
|
7
|
+
/** Transport dependencies: the `fetch` to POST with and the timeout scheduler. */
|
|
8
|
+
export type EvaluateDeps = Pick<Dependencies, "fetch" | "scheduler">;
|
|
6
9
|
/** The decoded JSON body of a 2xx evaluation response, with its latency. */
|
|
7
10
|
export type PostEvaluateResponse = {
|
|
8
11
|
data: unknown;
|
|
@@ -12,10 +15,12 @@ export type PostEvaluateResponse = {
|
|
|
12
15
|
/**
|
|
13
16
|
* POSTs an evaluation payload and returns the decoded JSON body. Sends
|
|
14
17
|
* `Authorization: Bearer <apiKey>` only when a key is present; aborts the
|
|
15
|
-
* exchange when `timeoutMs` elapses. A `timeoutMs`
|
|
16
|
-
* number >= 0 falls back to `DEFAULT_TIMEOUT_MS`.
|
|
17
|
-
*
|
|
18
|
-
* `
|
|
19
|
-
*
|
|
18
|
+
* exchange when `timeoutMs` elapses on `deps.scheduler`. A `timeoutMs`
|
|
19
|
+
* that is not a finite number >= 0 falls back to `DEFAULT_TIMEOUT_MS`.
|
|
20
|
+
* Throws a `TransportError` carrying a `timeout` classification on
|
|
21
|
+
* timeout, a `classifyHTTPError` classification (status only, never the
|
|
22
|
+
* body) on non-2xx, a `classifyNetworkError` classification when no HTTP
|
|
23
|
+
* exchange completes, and a `classifyProtocolMismatch` classification when
|
|
24
|
+
* the 2xx body is not valid JSON.
|
|
20
25
|
*/
|
|
21
|
-
export declare function postEvaluate(url: string, body: unknown, options: PostEvaluateOptions): Promise<PostEvaluateResponse>;
|
|
26
|
+
export declare function postEvaluate(url: string, body: unknown, options: PostEvaluateOptions, deps: EvaluateDeps): Promise<PostEvaluateResponse>;
|
package/dist/client.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
|
+
import { classifyHTTPError, classifyNetworkError, classifyProtocolMismatch, } from "@intx/inference";
|
|
1
2
|
import { DEFAULT_TIMEOUT_MS } from "./config.js";
|
|
2
|
-
import {
|
|
3
|
+
import { SystemOneError, TransportError } from "./errors.js";
|
|
3
4
|
/**
|
|
4
5
|
* POSTs an evaluation payload and returns the decoded JSON body. Sends
|
|
5
6
|
* `Authorization: Bearer <apiKey>` only when a key is present; aborts the
|
|
6
|
-
* exchange when `timeoutMs` elapses. A `timeoutMs`
|
|
7
|
-
* number >= 0 falls back to `DEFAULT_TIMEOUT_MS`.
|
|
8
|
-
*
|
|
9
|
-
* `
|
|
10
|
-
*
|
|
7
|
+
* exchange when `timeoutMs` elapses on `deps.scheduler`. A `timeoutMs`
|
|
8
|
+
* that is not a finite number >= 0 falls back to `DEFAULT_TIMEOUT_MS`.
|
|
9
|
+
* Throws a `TransportError` carrying a `timeout` classification on
|
|
10
|
+
* timeout, a `classifyHTTPError` classification (status only, never the
|
|
11
|
+
* body) on non-2xx, a `classifyNetworkError` classification when no HTTP
|
|
12
|
+
* exchange completes, and a `classifyProtocolMismatch` classification when
|
|
13
|
+
* the 2xx body is not valid JSON.
|
|
11
14
|
*/
|
|
12
|
-
export async function postEvaluate(url, body, options) {
|
|
15
|
+
export async function postEvaluate(url, body, options, deps) {
|
|
13
16
|
let serialized;
|
|
14
17
|
try {
|
|
15
18
|
serialized = JSON.stringify(body);
|
|
@@ -24,10 +27,16 @@ export async function postEvaluate(url, body, options) {
|
|
|
24
27
|
? rawTimeoutMs
|
|
25
28
|
: DEFAULT_TIMEOUT_MS;
|
|
26
29
|
const controller = new AbortController();
|
|
27
|
-
const
|
|
30
|
+
const cancelTimeout = deps.scheduler.setTimeout(() => {
|
|
28
31
|
controller.abort();
|
|
29
32
|
}, timeoutMs);
|
|
30
|
-
const start =
|
|
33
|
+
const start = deps.scheduler.now();
|
|
34
|
+
// The timeout can fire while awaiting headers or while reading the body.
|
|
35
|
+
// Built by hand: @intx/inference does not export classifyTimeoutError.
|
|
36
|
+
const timedOut = {
|
|
37
|
+
category: "timeout",
|
|
38
|
+
message: `system-one request timed out after ${timeoutMs}ms`,
|
|
39
|
+
};
|
|
31
40
|
try {
|
|
32
41
|
const headers = {
|
|
33
42
|
"content-type": "application/json",
|
|
@@ -38,7 +47,7 @@ export async function postEvaluate(url, body, options) {
|
|
|
38
47
|
}
|
|
39
48
|
let res;
|
|
40
49
|
try {
|
|
41
|
-
res = await fetch(url, {
|
|
50
|
+
res = await deps.fetch(url, {
|
|
42
51
|
method: "POST",
|
|
43
52
|
headers,
|
|
44
53
|
body: serialized,
|
|
@@ -46,22 +55,27 @@ export async function postEvaluate(url, body, options) {
|
|
|
46
55
|
});
|
|
47
56
|
}
|
|
48
57
|
catch (cause) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
58
|
+
throw new TransportError(controller.signal.aborted ? timedOut : classifyNetworkError(cause));
|
|
59
|
+
}
|
|
60
|
+
if (!res.ok) {
|
|
61
|
+
throw new TransportError(classifyHTTPError(res.status, `system-one request failed with HTTP ${res.status}`));
|
|
52
62
|
}
|
|
53
|
-
if (!res.ok)
|
|
54
|
-
throw new HttpError(res.status);
|
|
55
63
|
let data;
|
|
56
64
|
try {
|
|
57
65
|
data = await res.json();
|
|
58
66
|
}
|
|
59
67
|
catch {
|
|
60
|
-
throw new
|
|
68
|
+
throw new TransportError(controller.signal.aborted
|
|
69
|
+
? timedOut
|
|
70
|
+
: classifyProtocolMismatch("system-one response body could not be read as JSON"));
|
|
61
71
|
}
|
|
62
|
-
return {
|
|
72
|
+
return {
|
|
73
|
+
data,
|
|
74
|
+
latencyMs: deps.scheduler.now() - start,
|
|
75
|
+
httpStatus: res.status,
|
|
76
|
+
};
|
|
63
77
|
}
|
|
64
78
|
finally {
|
|
65
|
-
|
|
79
|
+
cancelTimeout();
|
|
66
80
|
}
|
|
67
81
|
}
|
package/dist/config.d.ts
CHANGED
|
@@ -4,8 +4,14 @@ export declare const SystemOneQuirks: import("arktype/internal/variants/object.t
|
|
|
4
4
|
model?: string;
|
|
5
5
|
}, {}>;
|
|
6
6
|
export type SystemOneQuirks = typeof SystemOneQuirks.infer;
|
|
7
|
-
export declare const SYSTEM_ONE_DEFAULT_QUIRKS:
|
|
8
|
-
|
|
7
|
+
export declare const SYSTEM_ONE_DEFAULT_QUIRKS: {
|
|
8
|
+
baseUrl: string;
|
|
9
|
+
model: string;
|
|
10
|
+
};
|
|
11
|
+
export declare const GATEWAY_QUIRKS: {
|
|
12
|
+
baseUrl: string;
|
|
13
|
+
model: string;
|
|
14
|
+
};
|
|
9
15
|
export declare const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
10
16
|
export declare const GATEWAY_API_KEY_ENV = "AI_GATEWAY_API_KEY";
|
|
11
17
|
export declare const GATEWAY_OIDC_ENV = "VERCEL_OIDC_TOKEN";
|
|
@@ -23,7 +29,7 @@ export declare const DEFAULT_TIMEOUT_MS = 1500;
|
|
|
23
29
|
export type ResolvedEndpoint = {
|
|
24
30
|
url: string;
|
|
25
31
|
backend: "corbits-system-one" | "gateway" | "custom";
|
|
26
|
-
model
|
|
32
|
+
model: string;
|
|
27
33
|
};
|
|
28
34
|
/**
|
|
29
35
|
* Resolves an evaluation endpoint to a concrete POST target. A per-call
|
|
@@ -31,6 +37,7 @@ export type ResolvedEndpoint = {
|
|
|
31
37
|
* carry its own `url` and throws a typed `SystemOneError` when it does
|
|
32
38
|
* not. A `custom` target keeps the `'custom'` backend label so telemetry
|
|
33
39
|
* distinguishes caller-routed traffic from the official endpoint —
|
|
34
|
-
* `'gateway'` is reserved for the proxy quirks path.
|
|
40
|
+
* `'gateway'` is reserved for the proxy quirks path. The model always
|
|
41
|
+
* resolves here: a `custom` target without one sends `typesafe-ai/jev`.
|
|
35
42
|
*/
|
|
36
43
|
export declare function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint;
|
package/dist/config.js
CHANGED
|
@@ -32,8 +32,8 @@ export const GATEWAY_QUIRKS = {
|
|
|
32
32
|
};
|
|
33
33
|
// Credential sources, one per endpoint: a TypeSafe key never authenticates
|
|
34
34
|
// to the gateway and a gateway key never authenticates to TypeSafe direct,
|
|
35
|
-
// so each endpoint reads its own names. `SYSTEM_ONE_API_KEY`
|
|
36
|
-
//
|
|
35
|
+
// so each endpoint reads its own names. `SYSTEM_ONE_API_KEY` is a supported
|
|
36
|
+
// alias for every endpoint (and the key for `custom` endpoints).
|
|
37
37
|
export const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
38
38
|
export const GATEWAY_API_KEY_ENV = "AI_GATEWAY_API_KEY";
|
|
39
39
|
export const GATEWAY_OIDC_ENV = "VERCEL_OIDC_TOKEN";
|
|
@@ -59,17 +59,19 @@ export const DEFAULT_TIMEOUT_MS = 1500;
|
|
|
59
59
|
* carry its own `url` and throws a typed `SystemOneError` when it does
|
|
60
60
|
* not. A `custom` target keeps the `'custom'` backend label so telemetry
|
|
61
61
|
* distinguishes caller-routed traffic from the official endpoint —
|
|
62
|
-
* `'gateway'` is reserved for the proxy quirks path.
|
|
62
|
+
* `'gateway'` is reserved for the proxy quirks path. The model always
|
|
63
|
+
* resolves here: a `custom` target without one sends `typesafe-ai/jev`.
|
|
63
64
|
*/
|
|
64
65
|
export function resolveEndpoint(endpoint) {
|
|
65
66
|
if (endpoint?.kind === "custom") {
|
|
66
67
|
if (endpoint.url === "") {
|
|
67
68
|
throw new SystemOneError("config-error", 'evaluate: endpoint kind "custom" requires an explicit url');
|
|
68
69
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
return {
|
|
71
|
+
url: endpoint.url,
|
|
72
|
+
backend: "custom",
|
|
73
|
+
model: endpoint.model ?? GATEWAY_QUIRKS.model,
|
|
74
|
+
};
|
|
73
75
|
}
|
|
74
76
|
const kind = endpoint?.kind ?? "official";
|
|
75
77
|
const quirks = kind === "gateway" ? GATEWAY_QUIRKS : SYSTEM_ONE_DEFAULT_QUIRKS;
|
|
@@ -77,12 +79,9 @@ export function resolveEndpoint(endpoint) {
|
|
|
77
79
|
if (url === undefined || url === "") {
|
|
78
80
|
throw new SystemOneError("backend-unreachable", `evaluate: no baseUrl configured for endpoint kind "${kind}"`);
|
|
79
81
|
}
|
|
80
|
-
|
|
82
|
+
return {
|
|
81
83
|
url,
|
|
82
84
|
backend: kind === "gateway" ? "gateway" : "corbits-system-one",
|
|
85
|
+
model: endpoint?.model ?? quirks.model,
|
|
83
86
|
};
|
|
84
|
-
const model = endpoint?.model ?? quirks.model;
|
|
85
|
-
if (model !== undefined)
|
|
86
|
-
resolved.model = model;
|
|
87
|
-
return resolved;
|
|
88
87
|
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -1,35 +1,22 @@
|
|
|
1
|
+
import type { InferenceError } from "@intx/types/runtime";
|
|
1
2
|
import type { FallbackReason } from "./schemas.js";
|
|
2
3
|
/** Every error code `evaluate` can throw. Thrown-only codes (`"not-implemented"`, `"config-error"`) never appear as fallback reasons. */
|
|
3
4
|
export type SystemOneErrorCode = FallbackReason | "not-implemented" | "config-error";
|
|
4
5
|
/** Typed error thrown when `evaluate` cannot return a result at all. */
|
|
5
6
|
export declare class SystemOneError extends Error {
|
|
6
7
|
readonly code: SystemOneErrorCode;
|
|
7
|
-
|
|
8
|
+
readonly reason?: InferenceError;
|
|
9
|
+
constructor(code: SystemOneErrorCode, message: string, reason?: InferenceError);
|
|
8
10
|
}
|
|
9
|
-
/**
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
export declare class TimeoutError extends SystemOneError {
|
|
14
|
-
readonly timeoutMs: number;
|
|
15
|
-
constructor(timeoutMs: number);
|
|
16
|
-
}
|
|
17
|
-
/**
|
|
18
|
-
* The evaluation POST never completed an HTTP exchange (DNS/TLS failure,
|
|
19
|
-
* refused connection, reset stream) or the 200 body was not valid JSON.
|
|
20
|
-
* Thrown by `postEvaluate`; `evaluate` reports it as a `FallbackResult`
|
|
21
|
-
* with reason `'network'`.
|
|
22
|
-
*/
|
|
23
|
-
export declare class NetworkError extends SystemOneError {
|
|
24
|
-
constructor(message: string);
|
|
11
|
+
/** A transport failure: always carries its `InferenceError` classification. */
|
|
12
|
+
export declare class TransportError extends SystemOneError {
|
|
13
|
+
readonly reason: InferenceError;
|
|
14
|
+
constructor(reason: InferenceError);
|
|
25
15
|
}
|
|
26
16
|
/**
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
17
|
+
* Maps a transport classification to its fallback reason: a timeout stays
|
|
18
|
+
* `'timeout'`, an unreadable body is `'parse-error'`, anything with an HTTP
|
|
19
|
+
* status is `'http-error'`, and the rest never completed an exchange
|
|
20
|
+
* (`'network'`).
|
|
31
21
|
*/
|
|
32
|
-
export declare
|
|
33
|
-
readonly httpStatus: number;
|
|
34
|
-
constructor(httpStatus: number);
|
|
35
|
-
}
|
|
22
|
+
export declare function transportFallbackReason(reason: InferenceError): FallbackReason;
|
package/dist/errors.js
CHANGED
|
@@ -1,47 +1,33 @@
|
|
|
1
1
|
/** Typed error thrown when `evaluate` cannot return a result at all. */
|
|
2
2
|
export class SystemOneError extends Error {
|
|
3
3
|
code;
|
|
4
|
-
|
|
4
|
+
reason;
|
|
5
|
+
constructor(code, message, reason) {
|
|
5
6
|
super(message);
|
|
6
7
|
this.name = "SystemOneError";
|
|
7
8
|
this.code = code;
|
|
9
|
+
if (reason !== undefined)
|
|
10
|
+
this.reason = reason;
|
|
8
11
|
}
|
|
9
12
|
}
|
|
10
|
-
/**
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
export class TimeoutError extends SystemOneError {
|
|
15
|
-
timeoutMs;
|
|
16
|
-
constructor(timeoutMs) {
|
|
17
|
-
super("timeout", `system-one request timed out after ${timeoutMs}ms`);
|
|
18
|
-
this.name = "TimeoutError";
|
|
19
|
-
this.timeoutMs = timeoutMs;
|
|
13
|
+
/** A transport failure: always carries its `InferenceError` classification. */
|
|
14
|
+
export class TransportError extends SystemOneError {
|
|
15
|
+
constructor(reason) {
|
|
16
|
+
super(transportFallbackReason(reason), reason.message, reason);
|
|
20
17
|
}
|
|
21
18
|
}
|
|
22
19
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
20
|
+
* Maps a transport classification to its fallback reason: a timeout stays
|
|
21
|
+
* `'timeout'`, an unreadable body is `'parse-error'`, anything with an HTTP
|
|
22
|
+
* status is `'http-error'`, and the rest never completed an exchange
|
|
23
|
+
* (`'network'`).
|
|
27
24
|
*/
|
|
28
|
-
export
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
* only — never the response body, which is untrusted vendor output.
|
|
37
|
-
* Thrown by `postEvaluate`; `evaluate` reports it as a `FallbackResult`
|
|
38
|
-
* with reason `'http-error'`.
|
|
39
|
-
*/
|
|
40
|
-
export class HttpError extends SystemOneError {
|
|
41
|
-
httpStatus;
|
|
42
|
-
constructor(httpStatus) {
|
|
43
|
-
super("http-error", `system-one request failed with HTTP ${httpStatus}`);
|
|
44
|
-
this.name = "HttpError";
|
|
45
|
-
this.httpStatus = httpStatus;
|
|
46
|
-
}
|
|
25
|
+
export function transportFallbackReason(reason) {
|
|
26
|
+
if (reason.category === "timeout")
|
|
27
|
+
return "timeout";
|
|
28
|
+
if (reason.category === "protocol_mismatch")
|
|
29
|
+
return "parse-error";
|
|
30
|
+
if (reason.statusCode !== undefined)
|
|
31
|
+
return "http-error";
|
|
32
|
+
return "network";
|
|
47
33
|
}
|
package/dist/evaluate.d.ts
CHANGED
|
@@ -1,4 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type EvaluateDeps } from "./client.js";
|
|
2
|
+
import { EvaluateInput, type EvaluateResult, type FallbackResult, type Question, type WireAnswer } from "./schemas.js";
|
|
3
|
+
import type { SystemOneTelemetryEvent } from "./telemetry.js";
|
|
4
|
+
export declare function checkAnswers(questions: Question[], answers: Record<string, WireAnswer>): string | undefined;
|
|
5
|
+
/**
|
|
6
|
+
* Per-call hooks for `evaluate`: `onTelemetry` receives each lifecycle
|
|
7
|
+
* event and must not throw, since a throw propagates out of `evaluate`;
|
|
8
|
+
* `deps` replaces the global `fetch` and `createDefaultScheduler()`.
|
|
9
|
+
*/
|
|
10
|
+
export type EvaluateOptions = {
|
|
11
|
+
onTelemetry?: (event: SystemOneTelemetryEvent) => void;
|
|
12
|
+
deps?: EvaluateDeps;
|
|
13
|
+
};
|
|
2
14
|
/**
|
|
3
15
|
* Runs one typed-decision evaluation against System One (or its gateway)
|
|
4
16
|
* and returns either the backend's decisions or a typed fallback. One POST
|
|
@@ -6,13 +18,16 @@ import { EvaluateInput, type EvaluateResult, type FallbackResult } from "./schem
|
|
|
6
18
|
* a non-empty per-call `apiKey` (an empty string falls through to the
|
|
7
19
|
* environment, never shadowing it), then the endpoint's environment key —
|
|
8
20
|
* `TYPESAFE_API_KEY` for official, `AI_GATEWAY_API_KEY` then
|
|
9
|
-
* `VERCEL_OIDC_TOKEN` for gateway — then the
|
|
10
|
-
*
|
|
21
|
+
* `VERCEL_OIDC_TOKEN` for gateway — then the `SYSTEM_ONE_API_KEY` alias,
|
|
22
|
+
* then a fail-closed `'no-key'` fallback with zero fetch calls.
|
|
11
23
|
* A `timeoutMs` that is not a finite number >= 0 falls back to
|
|
12
24
|
* `DEFAULT_TIMEOUT_MS`. Transport failures map to matching fallback reasons
|
|
13
|
-
* (`'timeout'`, `'network'`, `'http-error'`
|
|
14
|
-
*
|
|
25
|
+
* (`'timeout'`, `'network'`, `'http-error'`, or `'parse-error'` for a 2xx
|
|
26
|
+
* body that is not JSON) with the transport message in `detail`; backend
|
|
27
|
+
* output that fails strict validation maps to `'parse-error'` with the
|
|
28
|
+
* violation in `detail`.
|
|
15
29
|
* Only caller-side misuse (invalid input, a `custom` endpoint without a
|
|
16
|
-
* `url`) throws, as a typed `SystemOneError`.
|
|
30
|
+
* `url`) throws, as a typed `SystemOneError`. Lifecycle events go to
|
|
31
|
+
* `options.onTelemetry` when supplied.
|
|
17
32
|
*/
|
|
18
|
-
export declare function evaluate(input: EvaluateInput): Promise<EvaluateResult | FallbackResult>;
|
|
33
|
+
export declare function evaluate(input: EvaluateInput, options?: EvaluateOptions): Promise<EvaluateResult | FallbackResult>;
|
package/dist/evaluate.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { type } from "arktype";
|
|
2
|
+
import { createDefaultScheduler } from "@intx/inference";
|
|
2
3
|
import { postEvaluate } from "./client.js";
|
|
3
4
|
import { DEFAULT_TIMEOUT_MS, GATEWAY_API_KEY_ENV, GATEWAY_OIDC_ENV, resolveEndpoint, SYSTEM_ONE_API_KEY_ENV, TYPESAFE_API_KEY_ENV, } from "./config.js";
|
|
4
|
-
import {
|
|
5
|
+
import { SystemOneError, TransportError, transportFallbackReason, } from "./errors.js";
|
|
5
6
|
import { APPROXIMATE_SUM_TOLERANCE as SUM_TOLERANCE, EvaluateInput, toDecision, toWireQuestions, WireRequest, WireResponseBody, } from "./schemas.js";
|
|
6
|
-
import { recordTelemetryEvent, } from "./telemetry.js";
|
|
7
7
|
// ---------------------------------------------------------------------------
|
|
8
8
|
// evaluate
|
|
9
9
|
// ---------------------------------------------------------------------------
|
|
@@ -100,7 +100,7 @@ function checkAnswer(question, id, answer) {
|
|
|
100
100
|
if (question.type === "choice" && answer.type === "choice") {
|
|
101
101
|
if (answer.choice === undefined)
|
|
102
102
|
return `answer "${id}" is missing choice`;
|
|
103
|
-
if (!(answer.choice
|
|
103
|
+
if (!Object.hasOwn(question.criteria, answer.choice)) {
|
|
104
104
|
return `answer "${id}" illegal option ${JSON.stringify(answer.choice)}`;
|
|
105
105
|
}
|
|
106
106
|
return checkProbabilities(id, Object.keys(question.criteria), answer.probabilities);
|
|
@@ -149,7 +149,7 @@ function checkProbabilities(id, requiredKeys, probabilities) {
|
|
|
149
149
|
// questions: the answer key set must equal the submitted id set exactly (no
|
|
150
150
|
// unknowns, no drops — map keys are unique by construction), and each answer
|
|
151
151
|
// must satisfy its question. Returns the first violation, if any.
|
|
152
|
-
function checkAnswers(questions, answers) {
|
|
152
|
+
export function checkAnswers(questions, answers) {
|
|
153
153
|
const byId = new Map();
|
|
154
154
|
for (const question of questions)
|
|
155
155
|
byId.set(question.id, question);
|
|
@@ -181,17 +181,27 @@ function envKey(name) {
|
|
|
181
181
|
* a non-empty per-call `apiKey` (an empty string falls through to the
|
|
182
182
|
* environment, never shadowing it), then the endpoint's environment key —
|
|
183
183
|
* `TYPESAFE_API_KEY` for official, `AI_GATEWAY_API_KEY` then
|
|
184
|
-
* `VERCEL_OIDC_TOKEN` for gateway — then the
|
|
185
|
-
*
|
|
184
|
+
* `VERCEL_OIDC_TOKEN` for gateway — then the `SYSTEM_ONE_API_KEY` alias,
|
|
185
|
+
* then a fail-closed `'no-key'` fallback with zero fetch calls.
|
|
186
186
|
* A `timeoutMs` that is not a finite number >= 0 falls back to
|
|
187
187
|
* `DEFAULT_TIMEOUT_MS`. Transport failures map to matching fallback reasons
|
|
188
|
-
* (`'timeout'`, `'network'`, `'http-error'`
|
|
189
|
-
*
|
|
188
|
+
* (`'timeout'`, `'network'`, `'http-error'`, or `'parse-error'` for a 2xx
|
|
189
|
+
* body that is not JSON) with the transport message in `detail`; backend
|
|
190
|
+
* output that fails strict validation maps to `'parse-error'` with the
|
|
191
|
+
* violation in `detail`.
|
|
190
192
|
* Only caller-side misuse (invalid input, a `custom` endpoint without a
|
|
191
|
-
* `url`) throws, as a typed `SystemOneError`.
|
|
193
|
+
* `url`) throws, as a typed `SystemOneError`. Lifecycle events go to
|
|
194
|
+
* `options.onTelemetry` when supplied.
|
|
192
195
|
*/
|
|
193
|
-
export async function evaluate(input) {
|
|
194
|
-
const
|
|
196
|
+
export async function evaluate(input, options) {
|
|
197
|
+
const deps = options?.deps ?? {
|
|
198
|
+
fetch: globalThis.fetch,
|
|
199
|
+
scheduler: createDefaultScheduler(),
|
|
200
|
+
};
|
|
201
|
+
const start = deps.scheduler.now();
|
|
202
|
+
const emit = (event) => {
|
|
203
|
+
options?.onTelemetry?.(event);
|
|
204
|
+
};
|
|
195
205
|
const parsed = EvaluateInput(input);
|
|
196
206
|
if (parsed instanceof type.errors) {
|
|
197
207
|
let summary = parsed.summary;
|
|
@@ -230,12 +240,12 @@ export async function evaluate(input) {
|
|
|
230
240
|
apiKey = envKey(SYSTEM_ONE_API_KEY_ENV);
|
|
231
241
|
keySources = SYSTEM_ONE_API_KEY_ENV;
|
|
232
242
|
}
|
|
233
|
-
const model = endpoint.model
|
|
243
|
+
const model = endpoint.model;
|
|
234
244
|
const fallback = (reason, extra) => {
|
|
235
245
|
const result = {
|
|
236
246
|
fallback: true,
|
|
237
247
|
reason,
|
|
238
|
-
latencyMs:
|
|
248
|
+
latencyMs: deps.scheduler.now() - start,
|
|
239
249
|
backendAttempted,
|
|
240
250
|
};
|
|
241
251
|
if (extra?.httpStatus !== undefined)
|
|
@@ -251,7 +261,7 @@ export async function evaluate(input) {
|
|
|
251
261
|
};
|
|
252
262
|
if (extra?.detail !== undefined)
|
|
253
263
|
event.detail = extra.detail;
|
|
254
|
-
|
|
264
|
+
emit(event);
|
|
255
265
|
return result;
|
|
256
266
|
};
|
|
257
267
|
const startEvent = {
|
|
@@ -260,7 +270,7 @@ export async function evaluate(input) {
|
|
|
260
270
|
latencyMs: 0,
|
|
261
271
|
modelId: model,
|
|
262
272
|
};
|
|
263
|
-
|
|
273
|
+
emit(startEvent);
|
|
264
274
|
if (apiKey === undefined || apiKey === "") {
|
|
265
275
|
return fallback("no-key", {
|
|
266
276
|
detail: `no API key supplied; set config.apiKey or ${keySources}`,
|
|
@@ -277,22 +287,21 @@ export async function evaluate(input) {
|
|
|
277
287
|
let data;
|
|
278
288
|
let transportLatencyMs;
|
|
279
289
|
try {
|
|
280
|
-
const response = await postEvaluate(endpoint.url, wire, {
|
|
281
|
-
timeoutMs,
|
|
282
|
-
apiKey,
|
|
283
|
-
});
|
|
290
|
+
const response = await postEvaluate(endpoint.url, wire, { timeoutMs, apiKey }, deps);
|
|
284
291
|
data = response.data;
|
|
285
292
|
transportLatencyMs = response.latencyMs;
|
|
286
293
|
}
|
|
287
294
|
catch (cause) {
|
|
288
|
-
if (cause instanceof
|
|
289
|
-
|
|
295
|
+
if (!(cause instanceof TransportError))
|
|
296
|
+
throw cause;
|
|
297
|
+
const reason = transportFallbackReason(cause.reason);
|
|
298
|
+
if (cause.reason.statusCode === undefined) {
|
|
299
|
+
return fallback(reason, { detail: cause.message });
|
|
290
300
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
throw cause;
|
|
301
|
+
return fallback(reason, {
|
|
302
|
+
httpStatus: cause.reason.statusCode,
|
|
303
|
+
detail: cause.message,
|
|
304
|
+
});
|
|
296
305
|
}
|
|
297
306
|
const envelope = WireResponseBody(data);
|
|
298
307
|
if (envelope instanceof type.errors) {
|
|
@@ -341,6 +350,6 @@ export async function evaluate(input) {
|
|
|
341
350
|
latencyMs: result.latencyMs,
|
|
342
351
|
modelId,
|
|
343
352
|
};
|
|
344
|
-
|
|
353
|
+
emit(successEvent);
|
|
345
354
|
return result;
|
|
346
355
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export { DEFAULT_TIMEOUT_MS
|
|
2
|
-
export {
|
|
3
|
-
export type { PostEvaluateOptions, PostEvaluateResponse } from "./client.js";
|
|
1
|
+
export { DEFAULT_TIMEOUT_MS } from "./config.js";
|
|
2
|
+
export type { EvaluateDeps } from "./client.js";
|
|
4
3
|
export { evaluate } from "./evaluate.js";
|
|
4
|
+
export type { EvaluateOptions } from "./evaluate.js";
|
|
5
5
|
export { createSystemOneAdapter, SYSTEM_ONE_PROVIDER } from "./adapter.js";
|
|
6
|
-
export {
|
|
6
|
+
export { SystemOneError } from "./errors.js";
|
|
7
7
|
export type { SystemOneErrorCode } from "./errors.js";
|
|
8
|
-
export {
|
|
9
|
-
export {
|
|
8
|
+
export { SystemOneTelemetryEvent } from "./telemetry.js";
|
|
9
|
+
export { BooleanQuestion, ChoiceQuestion, Decision, Description, EndpointConfig, EvaluateConfig, EvaluateInput, EvaluateResult, FallbackReason, FallbackResult, NoulQuestion, ProbabilityMap, Question, QuestionList, ScoreQuestion, State, Usage, } from "./schemas.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
export { DEFAULT_TIMEOUT_MS
|
|
2
|
-
export { postEvaluate } from "./client.js";
|
|
1
|
+
export { DEFAULT_TIMEOUT_MS } from "./config.js";
|
|
3
2
|
export { evaluate } from "./evaluate.js";
|
|
4
3
|
export { createSystemOneAdapter, SYSTEM_ONE_PROVIDER } from "./adapter.js";
|
|
5
|
-
export {
|
|
6
|
-
export {
|
|
7
|
-
export {
|
|
4
|
+
export { SystemOneError } from "./errors.js";
|
|
5
|
+
export { SystemOneTelemetryEvent } from "./telemetry.js";
|
|
6
|
+
export { BooleanQuestion, ChoiceQuestion, Decision, Description, EndpointConfig, EvaluateConfig, EvaluateInput, EvaluateResult, FallbackReason, FallbackResult, NoulQuestion, ProbabilityMap, Question, QuestionList, ScoreQuestion, State, Usage, } from "./schemas.js";
|
package/dist/schemas.d.ts
CHANGED
|
@@ -373,29 +373,6 @@ export declare const Usage: import("arktype/internal/variants/object.ts").Object
|
|
|
373
373
|
outputTokens: number;
|
|
374
374
|
}, {}>;
|
|
375
375
|
export type Usage = typeof Usage.infer;
|
|
376
|
-
export declare const EvaluateResponseBody: import("arktype/internal/variants/object.ts").ObjectType<{
|
|
377
|
-
answers: {
|
|
378
|
-
[x: string]: {
|
|
379
|
-
type: "choice" | "score" | "noul";
|
|
380
|
-
choice?: string;
|
|
381
|
-
score?: number;
|
|
382
|
-
noul?: number;
|
|
383
|
-
probabilities?: {
|
|
384
|
-
[x: string]: number;
|
|
385
|
-
};
|
|
386
|
-
confidence?: number;
|
|
387
|
-
legend?: {
|
|
388
|
-
[x: string]: string;
|
|
389
|
-
};
|
|
390
|
-
};
|
|
391
|
-
};
|
|
392
|
-
model?: string;
|
|
393
|
-
usage?: {
|
|
394
|
-
input_tokens: number;
|
|
395
|
-
output_tokens: number;
|
|
396
|
-
};
|
|
397
|
-
}, {}>;
|
|
398
|
-
export type EvaluateResponseBody = typeof EvaluateResponseBody.infer;
|
|
399
376
|
export declare const EvaluateResult: import("arktype/internal/variants/object.ts").ObjectType<{
|
|
400
377
|
decisions: ({
|
|
401
378
|
id: string;
|
package/dist/schemas.js
CHANGED
|
@@ -285,9 +285,6 @@ export const Usage = type({
|
|
|
285
285
|
outputTokens: "number",
|
|
286
286
|
"+": "reject",
|
|
287
287
|
});
|
|
288
|
-
// Backwards-compatible alias: the envelope shared by `evaluate()` and the
|
|
289
|
-
// provider adapter.
|
|
290
|
-
export const EvaluateResponseBody = WireResponseBody;
|
|
291
288
|
export const EvaluateResult = type({
|
|
292
289
|
decisions: Decision.array(),
|
|
293
290
|
modelId: "string",
|
package/dist/telemetry.d.ts
CHANGED
|
@@ -8,17 +8,3 @@ export declare const SystemOneTelemetryEvent: import("arktype/internal/variants/
|
|
|
8
8
|
detail?: string;
|
|
9
9
|
}, {}>;
|
|
10
10
|
export type SystemOneTelemetryEvent = typeof SystemOneTelemetryEvent.infer;
|
|
11
|
-
/** Maximum buffered events; beyond this the oldest drop first. */
|
|
12
|
-
export declare const MAX_BUFFERED_TELEMETRY_EVENTS = 256;
|
|
13
|
-
/**
|
|
14
|
-
* Records one telemetry event to the in-memory ring. Validates the shape
|
|
15
|
-
* (a malformed event is caller misuse and throws a typed `SystemOneError`)
|
|
16
|
-
* and never touches the network, disk, or console.
|
|
17
|
-
*/
|
|
18
|
-
export declare function recordTelemetryEvent(event: SystemOneTelemetryEvent): void;
|
|
19
|
-
/**
|
|
20
|
-
* Drains the buffered telemetry events in arrival order and clears the
|
|
21
|
-
* buffer. The host calls this after each evaluation (or on its own cadence)
|
|
22
|
-
* to forward events to its own sink.
|
|
23
|
-
*/
|
|
24
|
-
export declare function drainTelemetryEvents(): SystemOneTelemetryEvent[];
|
package/dist/telemetry.js
CHANGED
|
@@ -1,18 +1,13 @@
|
|
|
1
1
|
import { type } from "arktype";
|
|
2
|
-
import { SystemOneError } from "./errors.js";
|
|
3
2
|
import { FallbackReason } from "./schemas.js";
|
|
4
3
|
// ---------------------------------------------------------------------------
|
|
5
4
|
// Telemetry
|
|
6
5
|
// ---------------------------------------------------------------------------
|
|
7
6
|
//
|
|
8
|
-
// Event
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
// second public surface to version for no load-bearing reason: the host
|
|
13
|
-
// already owns every destination, it only needs lossless-per-call access to
|
|
14
|
-
// the events. The cap keeps a chatty loop from growing memory without bound
|
|
15
|
-
// (oldest events drop first). No event field ever carries key material.
|
|
7
|
+
// Event shape for evaluation observability. `evaluate` hands each event to
|
|
8
|
+
// the caller's `onTelemetry` sink, which forwards it wherever the host wants
|
|
9
|
+
// (stdout, OTLP, a host hook); no state is kept between calls. No event
|
|
10
|
+
// field ever carries key material.
|
|
16
11
|
/** One evaluation lifecycle event. */
|
|
17
12
|
export const SystemOneTelemetryEvent = type({
|
|
18
13
|
event: "'evaluate.start' | 'evaluate.success' | 'evaluate.fallback'",
|
|
@@ -23,29 +18,3 @@ export const SystemOneTelemetryEvent = type({
|
|
|
23
18
|
"detail?": "string",
|
|
24
19
|
"+": "reject",
|
|
25
20
|
});
|
|
26
|
-
/** Maximum buffered events; beyond this the oldest drop first. */
|
|
27
|
-
export const MAX_BUFFERED_TELEMETRY_EVENTS = 256;
|
|
28
|
-
const bufferedEvents = [];
|
|
29
|
-
/**
|
|
30
|
-
* Records one telemetry event to the in-memory ring. Validates the shape
|
|
31
|
-
* (a malformed event is caller misuse and throws a typed `SystemOneError`)
|
|
32
|
-
* and never touches the network, disk, or console.
|
|
33
|
-
*/
|
|
34
|
-
export function recordTelemetryEvent(event) {
|
|
35
|
-
const parsed = SystemOneTelemetryEvent(event);
|
|
36
|
-
if (parsed instanceof type.errors) {
|
|
37
|
-
throw new SystemOneError("parse-error", `recordTelemetryEvent: invalid event: ${parsed.summary}`);
|
|
38
|
-
}
|
|
39
|
-
bufferedEvents.push(parsed);
|
|
40
|
-
if (bufferedEvents.length > MAX_BUFFERED_TELEMETRY_EVENTS) {
|
|
41
|
-
bufferedEvents.splice(0, bufferedEvents.length - MAX_BUFFERED_TELEMETRY_EVENTS);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* Drains the buffered telemetry events in arrival order and clears the
|
|
46
|
-
* buffer. The host calls this after each evaluation (or on its own cadence)
|
|
47
|
-
* to forward events to its own sink.
|
|
48
|
-
*/
|
|
49
|
-
export function drainTelemetryEvents() {
|
|
50
|
-
return bufferedEvents.splice(0, bufferedEvents.length);
|
|
51
|
-
}
|
package/package.json
CHANGED
|
@@ -1,64 +1,70 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@corbits/system-one",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Typed-decision evaluation client for System One (Jev-class) models: arktype question/decision schemas, endpoint configuration with gateway override, and timeout/fallback semantics.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"corbits",
|
|
7
|
+
"evaluation",
|
|
8
|
+
"inference",
|
|
9
|
+
"interchange",
|
|
10
|
+
"system-one",
|
|
11
|
+
"typesafe"
|
|
12
|
+
],
|
|
13
|
+
"homepage": "https://github.com/corbitsdev/corbits-system-one#readme",
|
|
14
|
+
"bugs": {
|
|
15
|
+
"url": "https://github.com/corbitsdev/corbits-system-one/issues"
|
|
16
|
+
},
|
|
5
17
|
"license": "LGPL-2.1-only",
|
|
6
18
|
"author": "Sawyer Cutler <sawyer@dirtroad.dev>",
|
|
7
|
-
"
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/corbitsdev/corbits-system-one.git"
|
|
22
|
+
},
|
|
11
23
|
"files": [
|
|
12
24
|
"dist",
|
|
13
25
|
"README.md",
|
|
14
26
|
"LICENSE"
|
|
15
27
|
],
|
|
16
|
-
"
|
|
17
|
-
|
|
18
|
-
},
|
|
28
|
+
"type": "module",
|
|
29
|
+
"sideEffects": false,
|
|
19
30
|
"exports": {
|
|
20
31
|
".": {
|
|
21
32
|
"types": "./dist/index.d.ts",
|
|
22
33
|
"default": "./dist/index.js"
|
|
23
34
|
}
|
|
24
35
|
},
|
|
36
|
+
"publishConfig": {
|
|
37
|
+
"access": "public"
|
|
38
|
+
},
|
|
25
39
|
"scripts": {
|
|
26
|
-
"build": "tsc -p tsconfig.build.json",
|
|
27
|
-
"typecheck": "tsc --noEmit",
|
|
28
|
-
"lint": "eslint .",
|
|
29
|
-
"format": "prettier --write .",
|
|
30
|
-
"format:check": "prettier --check .",
|
|
31
|
-
"test": "bun test",
|
|
40
|
+
"build": "rm -rf dist && tsc -p tsconfig.build.json",
|
|
32
41
|
"prepack": "bun run build",
|
|
42
|
+
"typecheck": "tsc --noEmit",
|
|
43
|
+
"lint": "oxlint",
|
|
44
|
+
"format": "oxfmt",
|
|
45
|
+
"format:check": "oxfmt --check",
|
|
46
|
+
"test": "bun test src --pass-with-no-tests",
|
|
47
|
+
"test:e2e": "bun test e2e",
|
|
33
48
|
"check": "bun run typecheck && bun run lint && bun run format:check && bun run test"
|
|
34
49
|
},
|
|
35
50
|
"dependencies": {
|
|
36
51
|
"arktype": "2.2.3"
|
|
37
52
|
},
|
|
38
|
-
"peerDependencies": {
|
|
39
|
-
"@intx/inference": ">=0.4.0",
|
|
40
|
-
"@intx/types": ">=0.4.0"
|
|
41
|
-
},
|
|
42
53
|
"devDependencies": {
|
|
43
|
-
"@eslint/js": "9.34.0",
|
|
44
54
|
"@intx/inference": "0.4.0",
|
|
55
|
+
"@intx/inference-testing": "0.4.0",
|
|
45
56
|
"@intx/types": "0.4.0",
|
|
46
|
-
"@types/bun": "1.3.
|
|
47
|
-
"
|
|
48
|
-
"
|
|
49
|
-
"
|
|
50
|
-
"typescript": "5.9.3",
|
|
51
|
-
"typescript-eslint": "8.42.0"
|
|
52
|
-
},
|
|
53
|
-
"engines": {
|
|
54
|
-
"bun": ">=1.2.0"
|
|
57
|
+
"@types/bun": "1.3.14",
|
|
58
|
+
"oxfmt": "0.70.0",
|
|
59
|
+
"oxlint": "1.85.0",
|
|
60
|
+
"typescript": "6.0.3"
|
|
55
61
|
},
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
62
|
+
"peerDependencies": {
|
|
63
|
+
"@intx/inference": "^0.4.0",
|
|
64
|
+
"@intx/types": "^0.4.0"
|
|
59
65
|
},
|
|
60
|
-
"
|
|
61
|
-
|
|
62
|
-
"
|
|
66
|
+
"engines": {
|
|
67
|
+
"bun": ">=1.2.0",
|
|
68
|
+
"node": ">=24"
|
|
63
69
|
}
|
|
64
70
|
}
|