@corbits/system-one 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +162 -220
- package/dist/adapter.d.ts +6 -7
- package/dist/adapter.js +68 -9
- package/dist/client.d.ts +11 -6
- package/dist/client.js +32 -18
- package/dist/config.d.ts +11 -4
- package/dist/config.js +11 -12
- package/dist/errors.d.ts +12 -25
- package/dist/errors.js +20 -34
- package/dist/evaluate.d.ts +22 -7
- package/dist/evaluate.js +36 -27
- package/dist/index.d.ts +7 -7
- package/dist/index.js +5 -6
- package/dist/schemas.d.ts +0 -23
- package/dist/schemas.js +0 -3
- package/dist/telemetry.d.ts +0 -14
- package/dist/telemetry.js +4 -35
- package/package.json +40 -34
package/dist/client.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
|
+
import { classifyHTTPError, classifyNetworkError, classifyProtocolMismatch, } from "@intx/inference";
|
|
1
2
|
import { DEFAULT_TIMEOUT_MS } from "./config.js";
|
|
2
|
-
import {
|
|
3
|
+
import { SystemOneError, TransportError } from "./errors.js";
|
|
3
4
|
/**
|
|
4
5
|
* POSTs an evaluation payload and returns the decoded JSON body. Sends
|
|
5
6
|
* `Authorization: Bearer <apiKey>` only when a key is present; aborts the
|
|
6
|
-
* exchange when `timeoutMs` elapses. A `timeoutMs`
|
|
7
|
-
* number >= 0 falls back to `DEFAULT_TIMEOUT_MS`.
|
|
8
|
-
*
|
|
9
|
-
* `
|
|
10
|
-
*
|
|
7
|
+
* exchange when `timeoutMs` elapses on `deps.scheduler`. A `timeoutMs`
|
|
8
|
+
* that is not a finite number >= 0 falls back to `DEFAULT_TIMEOUT_MS`.
|
|
9
|
+
* Throws a `TransportError` carrying a `timeout` classification on
|
|
10
|
+
* timeout, a `classifyHTTPError` classification (status only, never the
|
|
11
|
+
* body) on non-2xx, a `classifyNetworkError` classification when no HTTP
|
|
12
|
+
* exchange completes, and a `classifyProtocolMismatch` classification when
|
|
13
|
+
* the 2xx body is not valid JSON.
|
|
11
14
|
*/
|
|
12
|
-
export async function postEvaluate(url, body, options) {
|
|
15
|
+
export async function postEvaluate(url, body, options, deps) {
|
|
13
16
|
let serialized;
|
|
14
17
|
try {
|
|
15
18
|
serialized = JSON.stringify(body);
|
|
@@ -24,10 +27,16 @@ export async function postEvaluate(url, body, options) {
|
|
|
24
27
|
? rawTimeoutMs
|
|
25
28
|
: DEFAULT_TIMEOUT_MS;
|
|
26
29
|
const controller = new AbortController();
|
|
27
|
-
const
|
|
30
|
+
const cancelTimeout = deps.scheduler.setTimeout(() => {
|
|
28
31
|
controller.abort();
|
|
29
32
|
}, timeoutMs);
|
|
30
|
-
const start =
|
|
33
|
+
const start = deps.scheduler.now();
|
|
34
|
+
// The timeout can fire while awaiting headers or while reading the body.
|
|
35
|
+
// Built by hand: @intx/inference does not export classifyTimeoutError.
|
|
36
|
+
const timedOut = {
|
|
37
|
+
category: "timeout",
|
|
38
|
+
message: `system-one request timed out after ${timeoutMs}ms`,
|
|
39
|
+
};
|
|
31
40
|
try {
|
|
32
41
|
const headers = {
|
|
33
42
|
"content-type": "application/json",
|
|
@@ -38,7 +47,7 @@ export async function postEvaluate(url, body, options) {
|
|
|
38
47
|
}
|
|
39
48
|
let res;
|
|
40
49
|
try {
|
|
41
|
-
res = await fetch(url, {
|
|
50
|
+
res = await deps.fetch(url, {
|
|
42
51
|
method: "POST",
|
|
43
52
|
headers,
|
|
44
53
|
body: serialized,
|
|
@@ -46,22 +55,27 @@ export async function postEvaluate(url, body, options) {
|
|
|
46
55
|
});
|
|
47
56
|
}
|
|
48
57
|
catch (cause) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
58
|
+
throw new TransportError(controller.signal.aborted ? timedOut : classifyNetworkError(cause));
|
|
59
|
+
}
|
|
60
|
+
if (!res.ok) {
|
|
61
|
+
throw new TransportError(classifyHTTPError(res.status, `system-one request failed with HTTP ${res.status}`));
|
|
52
62
|
}
|
|
53
|
-
if (!res.ok)
|
|
54
|
-
throw new HttpError(res.status);
|
|
55
63
|
let data;
|
|
56
64
|
try {
|
|
57
65
|
data = await res.json();
|
|
58
66
|
}
|
|
59
67
|
catch {
|
|
60
|
-
throw new
|
|
68
|
+
throw new TransportError(controller.signal.aborted
|
|
69
|
+
? timedOut
|
|
70
|
+
: classifyProtocolMismatch("system-one response body could not be read as JSON"));
|
|
61
71
|
}
|
|
62
|
-
return {
|
|
72
|
+
return {
|
|
73
|
+
data,
|
|
74
|
+
latencyMs: deps.scheduler.now() - start,
|
|
75
|
+
httpStatus: res.status,
|
|
76
|
+
};
|
|
63
77
|
}
|
|
64
78
|
finally {
|
|
65
|
-
|
|
79
|
+
cancelTimeout();
|
|
66
80
|
}
|
|
67
81
|
}
|
package/dist/config.d.ts
CHANGED
|
@@ -4,8 +4,14 @@ export declare const SystemOneQuirks: import("arktype/internal/variants/object.t
|
|
|
4
4
|
model?: string;
|
|
5
5
|
}, {}>;
|
|
6
6
|
export type SystemOneQuirks = typeof SystemOneQuirks.infer;
|
|
7
|
-
export declare const SYSTEM_ONE_DEFAULT_QUIRKS:
|
|
8
|
-
|
|
7
|
+
export declare const SYSTEM_ONE_DEFAULT_QUIRKS: {
|
|
8
|
+
baseUrl: string;
|
|
9
|
+
model: string;
|
|
10
|
+
};
|
|
11
|
+
export declare const GATEWAY_QUIRKS: {
|
|
12
|
+
baseUrl: string;
|
|
13
|
+
model: string;
|
|
14
|
+
};
|
|
9
15
|
export declare const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
10
16
|
export declare const GATEWAY_API_KEY_ENV = "AI_GATEWAY_API_KEY";
|
|
11
17
|
export declare const GATEWAY_OIDC_ENV = "VERCEL_OIDC_TOKEN";
|
|
@@ -23,7 +29,7 @@ export declare const DEFAULT_TIMEOUT_MS = 1500;
|
|
|
23
29
|
export type ResolvedEndpoint = {
|
|
24
30
|
url: string;
|
|
25
31
|
backend: "corbits-system-one" | "gateway" | "custom";
|
|
26
|
-
model
|
|
32
|
+
model: string;
|
|
27
33
|
};
|
|
28
34
|
/**
|
|
29
35
|
* Resolves an evaluation endpoint to a concrete POST target. A per-call
|
|
@@ -31,6 +37,7 @@ export type ResolvedEndpoint = {
|
|
|
31
37
|
* carry its own `url` and throws a typed `SystemOneError` when it does
|
|
32
38
|
* not. A `custom` target keeps the `'custom'` backend label so telemetry
|
|
33
39
|
* distinguishes caller-routed traffic from the official endpoint —
|
|
34
|
-
* `'gateway'` is reserved for the proxy quirks path.
|
|
40
|
+
* `'gateway'` is reserved for the proxy quirks path. The model always
|
|
41
|
+
* resolves here: a `custom` target without one sends `typesafe-ai/jev`.
|
|
35
42
|
*/
|
|
36
43
|
export declare function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint;
|
package/dist/config.js
CHANGED
|
@@ -32,8 +32,8 @@ export const GATEWAY_QUIRKS = {
|
|
|
32
32
|
};
|
|
33
33
|
// Credential sources, one per endpoint: a TypeSafe key never authenticates
|
|
34
34
|
// to the gateway and a gateway key never authenticates to TypeSafe direct,
|
|
35
|
-
// so each endpoint reads its own names. `SYSTEM_ONE_API_KEY`
|
|
36
|
-
//
|
|
35
|
+
// so each endpoint reads its own names. `SYSTEM_ONE_API_KEY` is a supported
|
|
36
|
+
// alias for every endpoint (and the key for `custom` endpoints).
|
|
37
37
|
export const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
38
38
|
export const GATEWAY_API_KEY_ENV = "AI_GATEWAY_API_KEY";
|
|
39
39
|
export const GATEWAY_OIDC_ENV = "VERCEL_OIDC_TOKEN";
|
|
@@ -59,17 +59,19 @@ export const DEFAULT_TIMEOUT_MS = 1500;
|
|
|
59
59
|
* carry its own `url` and throws a typed `SystemOneError` when it does
|
|
60
60
|
* not. A `custom` target keeps the `'custom'` backend label so telemetry
|
|
61
61
|
* distinguishes caller-routed traffic from the official endpoint —
|
|
62
|
-
* `'gateway'` is reserved for the proxy quirks path.
|
|
62
|
+
* `'gateway'` is reserved for the proxy quirks path. The model always
|
|
63
|
+
* resolves here: a `custom` target without one sends `typesafe-ai/jev`.
|
|
63
64
|
*/
|
|
64
65
|
export function resolveEndpoint(endpoint) {
|
|
65
66
|
if (endpoint?.kind === "custom") {
|
|
66
67
|
if (endpoint.url === "") {
|
|
67
68
|
throw new SystemOneError("config-error", 'evaluate: endpoint kind "custom" requires an explicit url');
|
|
68
69
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
return {
|
|
71
|
+
url: endpoint.url,
|
|
72
|
+
backend: "custom",
|
|
73
|
+
model: endpoint.model ?? GATEWAY_QUIRKS.model,
|
|
74
|
+
};
|
|
73
75
|
}
|
|
74
76
|
const kind = endpoint?.kind ?? "official";
|
|
75
77
|
const quirks = kind === "gateway" ? GATEWAY_QUIRKS : SYSTEM_ONE_DEFAULT_QUIRKS;
|
|
@@ -77,12 +79,9 @@ export function resolveEndpoint(endpoint) {
|
|
|
77
79
|
if (url === undefined || url === "") {
|
|
78
80
|
throw new SystemOneError("backend-unreachable", `evaluate: no baseUrl configured for endpoint kind "${kind}"`);
|
|
79
81
|
}
|
|
80
|
-
|
|
82
|
+
return {
|
|
81
83
|
url,
|
|
82
84
|
backend: kind === "gateway" ? "gateway" : "corbits-system-one",
|
|
85
|
+
model: endpoint?.model ?? quirks.model,
|
|
83
86
|
};
|
|
84
|
-
const model = endpoint?.model ?? quirks.model;
|
|
85
|
-
if (model !== undefined)
|
|
86
|
-
resolved.model = model;
|
|
87
|
-
return resolved;
|
|
88
87
|
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -1,35 +1,22 @@
|
|
|
1
|
+
import type { InferenceError } from "@intx/types/runtime";
|
|
1
2
|
import type { FallbackReason } from "./schemas.js";
|
|
2
3
|
/** Every error code `evaluate` can throw. Thrown-only codes (`"not-implemented"`, `"config-error"`) never appear as fallback reasons. */
|
|
3
4
|
export type SystemOneErrorCode = FallbackReason | "not-implemented" | "config-error";
|
|
4
5
|
/** Typed error thrown when `evaluate` cannot return a result at all. */
|
|
5
6
|
export declare class SystemOneError extends Error {
|
|
6
7
|
readonly code: SystemOneErrorCode;
|
|
7
|
-
|
|
8
|
+
readonly reason?: InferenceError;
|
|
9
|
+
constructor(code: SystemOneErrorCode, message: string, reason?: InferenceError);
|
|
8
10
|
}
|
|
9
|
-
/**
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
export declare class TimeoutError extends SystemOneError {
|
|
14
|
-
readonly timeoutMs: number;
|
|
15
|
-
constructor(timeoutMs: number);
|
|
16
|
-
}
|
|
17
|
-
/**
|
|
18
|
-
* The evaluation POST never completed an HTTP exchange (DNS/TLS failure,
|
|
19
|
-
* refused connection, reset stream) or the 200 body was not valid JSON.
|
|
20
|
-
* Thrown by `postEvaluate`; `evaluate` reports it as a `FallbackResult`
|
|
21
|
-
* with reason `'network'`.
|
|
22
|
-
*/
|
|
23
|
-
export declare class NetworkError extends SystemOneError {
|
|
24
|
-
constructor(message: string);
|
|
11
|
+
/** A transport failure: always carries its `InferenceError` classification. */
|
|
12
|
+
export declare class TransportError extends SystemOneError {
|
|
13
|
+
readonly reason: InferenceError;
|
|
14
|
+
constructor(reason: InferenceError);
|
|
25
15
|
}
|
|
26
16
|
/**
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
17
|
+
* Maps a transport classification to its fallback reason: a timeout stays
|
|
18
|
+
* `'timeout'`, an unreadable body is `'parse-error'`, anything with an HTTP
|
|
19
|
+
* status is `'http-error'`, and the rest never completed an exchange
|
|
20
|
+
* (`'network'`).
|
|
31
21
|
*/
|
|
32
|
-
export declare
|
|
33
|
-
readonly httpStatus: number;
|
|
34
|
-
constructor(httpStatus: number);
|
|
35
|
-
}
|
|
22
|
+
export declare function transportFallbackReason(reason: InferenceError): FallbackReason;
|
package/dist/errors.js
CHANGED
|
@@ -1,47 +1,33 @@
|
|
|
1
1
|
/** Typed error thrown when `evaluate` cannot return a result at all. */
|
|
2
2
|
export class SystemOneError extends Error {
|
|
3
3
|
code;
|
|
4
|
-
|
|
4
|
+
reason;
|
|
5
|
+
constructor(code, message, reason) {
|
|
5
6
|
super(message);
|
|
6
7
|
this.name = "SystemOneError";
|
|
7
8
|
this.code = code;
|
|
9
|
+
if (reason !== undefined)
|
|
10
|
+
this.reason = reason;
|
|
8
11
|
}
|
|
9
12
|
}
|
|
10
|
-
/**
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
export class TimeoutError extends SystemOneError {
|
|
15
|
-
timeoutMs;
|
|
16
|
-
constructor(timeoutMs) {
|
|
17
|
-
super("timeout", `system-one request timed out after ${timeoutMs}ms`);
|
|
18
|
-
this.name = "TimeoutError";
|
|
19
|
-
this.timeoutMs = timeoutMs;
|
|
13
|
+
/** A transport failure: always carries its `InferenceError` classification. */
|
|
14
|
+
export class TransportError extends SystemOneError {
|
|
15
|
+
constructor(reason) {
|
|
16
|
+
super(transportFallbackReason(reason), reason.message, reason);
|
|
20
17
|
}
|
|
21
18
|
}
|
|
22
19
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
20
|
+
* Maps a transport classification to its fallback reason: a timeout stays
|
|
21
|
+
* `'timeout'`, an unreadable body is `'parse-error'`, anything with an HTTP
|
|
22
|
+
* status is `'http-error'`, and the rest never completed an exchange
|
|
23
|
+
* (`'network'`).
|
|
27
24
|
*/
|
|
28
|
-
export
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
* only — never the response body, which is untrusted vendor output.
|
|
37
|
-
* Thrown by `postEvaluate`; `evaluate` reports it as a `FallbackResult`
|
|
38
|
-
* with reason `'http-error'`.
|
|
39
|
-
*/
|
|
40
|
-
export class HttpError extends SystemOneError {
|
|
41
|
-
httpStatus;
|
|
42
|
-
constructor(httpStatus) {
|
|
43
|
-
super("http-error", `system-one request failed with HTTP ${httpStatus}`);
|
|
44
|
-
this.name = "HttpError";
|
|
45
|
-
this.httpStatus = httpStatus;
|
|
46
|
-
}
|
|
25
|
+
export function transportFallbackReason(reason) {
|
|
26
|
+
if (reason.category === "timeout")
|
|
27
|
+
return "timeout";
|
|
28
|
+
if (reason.category === "protocol_mismatch")
|
|
29
|
+
return "parse-error";
|
|
30
|
+
if (reason.statusCode !== undefined)
|
|
31
|
+
return "http-error";
|
|
32
|
+
return "network";
|
|
47
33
|
}
|
package/dist/evaluate.d.ts
CHANGED
|
@@ -1,4 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type EvaluateDeps } from "./client.js";
|
|
2
|
+
import { EvaluateInput, type EvaluateResult, type FallbackResult, type Question, type WireAnswer } from "./schemas.js";
|
|
3
|
+
import type { SystemOneTelemetryEvent } from "./telemetry.js";
|
|
4
|
+
export declare function checkAnswers(questions: Question[], answers: Record<string, WireAnswer>): string | undefined;
|
|
5
|
+
/**
|
|
6
|
+
* Per-call hooks for `evaluate`: `onTelemetry` receives each lifecycle
|
|
7
|
+
* event and must not throw, since a throw propagates out of `evaluate`;
|
|
8
|
+
* `deps` replaces the global `fetch` and `createDefaultScheduler()`.
|
|
9
|
+
*/
|
|
10
|
+
export type EvaluateOptions = {
|
|
11
|
+
onTelemetry?: (event: SystemOneTelemetryEvent) => void;
|
|
12
|
+
deps?: EvaluateDeps;
|
|
13
|
+
};
|
|
2
14
|
/**
|
|
3
15
|
* Runs one typed-decision evaluation against System One (or its gateway)
|
|
4
16
|
* and returns either the backend's decisions or a typed fallback. One POST
|
|
@@ -6,13 +18,16 @@ import { EvaluateInput, type EvaluateResult, type FallbackResult } from "./schem
|
|
|
6
18
|
* a non-empty per-call `apiKey` (an empty string falls through to the
|
|
7
19
|
* environment, never shadowing it), then the endpoint's environment key —
|
|
8
20
|
* `TYPESAFE_API_KEY` for official, `AI_GATEWAY_API_KEY` then
|
|
9
|
-
* `VERCEL_OIDC_TOKEN` for gateway — then the
|
|
10
|
-
*
|
|
21
|
+
* `VERCEL_OIDC_TOKEN` for gateway — then the `SYSTEM_ONE_API_KEY` alias,
|
|
22
|
+
* then a fail-closed `'no-key'` fallback with zero fetch calls.
|
|
11
23
|
* A `timeoutMs` that is not a finite number >= 0 falls back to
|
|
12
24
|
* `DEFAULT_TIMEOUT_MS`. Transport failures map to matching fallback reasons
|
|
13
|
-
* (`'timeout'`, `'network'`, `'http-error'`
|
|
14
|
-
*
|
|
25
|
+
* (`'timeout'`, `'network'`, `'http-error'`, or `'parse-error'` for a 2xx
|
|
26
|
+
* body that is not JSON) with the transport message in `detail`; backend
|
|
27
|
+
* output that fails strict validation maps to `'parse-error'` with the
|
|
28
|
+
* violation in `detail`.
|
|
15
29
|
* Only caller-side misuse (invalid input, a `custom` endpoint without a
|
|
16
|
-
* `url`) throws, as a typed `SystemOneError`.
|
|
30
|
+
* `url`) throws, as a typed `SystemOneError`. Lifecycle events go to
|
|
31
|
+
* `options.onTelemetry` when supplied.
|
|
17
32
|
*/
|
|
18
|
-
export declare function evaluate(input: EvaluateInput): Promise<EvaluateResult | FallbackResult>;
|
|
33
|
+
export declare function evaluate(input: EvaluateInput, options?: EvaluateOptions): Promise<EvaluateResult | FallbackResult>;
|
package/dist/evaluate.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { type } from "arktype";
|
|
2
|
+
import { createDefaultScheduler } from "@intx/inference";
|
|
2
3
|
import { postEvaluate } from "./client.js";
|
|
3
4
|
import { DEFAULT_TIMEOUT_MS, GATEWAY_API_KEY_ENV, GATEWAY_OIDC_ENV, resolveEndpoint, SYSTEM_ONE_API_KEY_ENV, TYPESAFE_API_KEY_ENV, } from "./config.js";
|
|
4
|
-
import {
|
|
5
|
+
import { SystemOneError, TransportError, transportFallbackReason, } from "./errors.js";
|
|
5
6
|
import { APPROXIMATE_SUM_TOLERANCE as SUM_TOLERANCE, EvaluateInput, toDecision, toWireQuestions, WireRequest, WireResponseBody, } from "./schemas.js";
|
|
6
|
-
import { recordTelemetryEvent, } from "./telemetry.js";
|
|
7
7
|
// ---------------------------------------------------------------------------
|
|
8
8
|
// evaluate
|
|
9
9
|
// ---------------------------------------------------------------------------
|
|
@@ -100,7 +100,7 @@ function checkAnswer(question, id, answer) {
|
|
|
100
100
|
if (question.type === "choice" && answer.type === "choice") {
|
|
101
101
|
if (answer.choice === undefined)
|
|
102
102
|
return `answer "${id}" is missing choice`;
|
|
103
|
-
if (!(answer.choice
|
|
103
|
+
if (!Object.hasOwn(question.criteria, answer.choice)) {
|
|
104
104
|
return `answer "${id}" illegal option ${JSON.stringify(answer.choice)}`;
|
|
105
105
|
}
|
|
106
106
|
return checkProbabilities(id, Object.keys(question.criteria), answer.probabilities);
|
|
@@ -149,7 +149,7 @@ function checkProbabilities(id, requiredKeys, probabilities) {
|
|
|
149
149
|
// questions: the answer key set must equal the submitted id set exactly (no
|
|
150
150
|
// unknowns, no drops — map keys are unique by construction), and each answer
|
|
151
151
|
// must satisfy its question. Returns the first violation, if any.
|
|
152
|
-
function checkAnswers(questions, answers) {
|
|
152
|
+
export function checkAnswers(questions, answers) {
|
|
153
153
|
const byId = new Map();
|
|
154
154
|
for (const question of questions)
|
|
155
155
|
byId.set(question.id, question);
|
|
@@ -181,17 +181,27 @@ function envKey(name) {
|
|
|
181
181
|
* a non-empty per-call `apiKey` (an empty string falls through to the
|
|
182
182
|
* environment, never shadowing it), then the endpoint's environment key —
|
|
183
183
|
* `TYPESAFE_API_KEY` for official, `AI_GATEWAY_API_KEY` then
|
|
184
|
-
* `VERCEL_OIDC_TOKEN` for gateway — then the
|
|
185
|
-
*
|
|
184
|
+
* `VERCEL_OIDC_TOKEN` for gateway — then the `SYSTEM_ONE_API_KEY` alias,
|
|
185
|
+
* then a fail-closed `'no-key'` fallback with zero fetch calls.
|
|
186
186
|
* A `timeoutMs` that is not a finite number >= 0 falls back to
|
|
187
187
|
* `DEFAULT_TIMEOUT_MS`. Transport failures map to matching fallback reasons
|
|
188
|
-
* (`'timeout'`, `'network'`, `'http-error'`
|
|
189
|
-
*
|
|
188
|
+
* (`'timeout'`, `'network'`, `'http-error'`, or `'parse-error'` for a 2xx
|
|
189
|
+
* body that is not JSON) with the transport message in `detail`; backend
|
|
190
|
+
* output that fails strict validation maps to `'parse-error'` with the
|
|
191
|
+
* violation in `detail`.
|
|
190
192
|
* Only caller-side misuse (invalid input, a `custom` endpoint without a
|
|
191
|
-
* `url`) throws, as a typed `SystemOneError`.
|
|
193
|
+
* `url`) throws, as a typed `SystemOneError`. Lifecycle events go to
|
|
194
|
+
* `options.onTelemetry` when supplied.
|
|
192
195
|
*/
|
|
193
|
-
export async function evaluate(input) {
|
|
194
|
-
const
|
|
196
|
+
export async function evaluate(input, options) {
|
|
197
|
+
const deps = options?.deps ?? {
|
|
198
|
+
fetch: globalThis.fetch,
|
|
199
|
+
scheduler: createDefaultScheduler(),
|
|
200
|
+
};
|
|
201
|
+
const start = deps.scheduler.now();
|
|
202
|
+
const emit = (event) => {
|
|
203
|
+
options?.onTelemetry?.(event);
|
|
204
|
+
};
|
|
195
205
|
const parsed = EvaluateInput(input);
|
|
196
206
|
if (parsed instanceof type.errors) {
|
|
197
207
|
let summary = parsed.summary;
|
|
@@ -230,12 +240,12 @@ export async function evaluate(input) {
|
|
|
230
240
|
apiKey = envKey(SYSTEM_ONE_API_KEY_ENV);
|
|
231
241
|
keySources = SYSTEM_ONE_API_KEY_ENV;
|
|
232
242
|
}
|
|
233
|
-
const model = endpoint.model
|
|
243
|
+
const model = endpoint.model;
|
|
234
244
|
const fallback = (reason, extra) => {
|
|
235
245
|
const result = {
|
|
236
246
|
fallback: true,
|
|
237
247
|
reason,
|
|
238
|
-
latencyMs:
|
|
248
|
+
latencyMs: deps.scheduler.now() - start,
|
|
239
249
|
backendAttempted,
|
|
240
250
|
};
|
|
241
251
|
if (extra?.httpStatus !== undefined)
|
|
@@ -251,7 +261,7 @@ export async function evaluate(input) {
|
|
|
251
261
|
};
|
|
252
262
|
if (extra?.detail !== undefined)
|
|
253
263
|
event.detail = extra.detail;
|
|
254
|
-
|
|
264
|
+
emit(event);
|
|
255
265
|
return result;
|
|
256
266
|
};
|
|
257
267
|
const startEvent = {
|
|
@@ -260,7 +270,7 @@ export async function evaluate(input) {
|
|
|
260
270
|
latencyMs: 0,
|
|
261
271
|
modelId: model,
|
|
262
272
|
};
|
|
263
|
-
|
|
273
|
+
emit(startEvent);
|
|
264
274
|
if (apiKey === undefined || apiKey === "") {
|
|
265
275
|
return fallback("no-key", {
|
|
266
276
|
detail: `no API key supplied; set config.apiKey or ${keySources}`,
|
|
@@ -277,22 +287,21 @@ export async function evaluate(input) {
|
|
|
277
287
|
let data;
|
|
278
288
|
let transportLatencyMs;
|
|
279
289
|
try {
|
|
280
|
-
const response = await postEvaluate(endpoint.url, wire, {
|
|
281
|
-
timeoutMs,
|
|
282
|
-
apiKey,
|
|
283
|
-
});
|
|
290
|
+
const response = await postEvaluate(endpoint.url, wire, { timeoutMs, apiKey }, deps);
|
|
284
291
|
data = response.data;
|
|
285
292
|
transportLatencyMs = response.latencyMs;
|
|
286
293
|
}
|
|
287
294
|
catch (cause) {
|
|
288
|
-
if (cause instanceof
|
|
289
|
-
|
|
295
|
+
if (!(cause instanceof TransportError))
|
|
296
|
+
throw cause;
|
|
297
|
+
const reason = transportFallbackReason(cause.reason);
|
|
298
|
+
if (cause.reason.statusCode === undefined) {
|
|
299
|
+
return fallback(reason, { detail: cause.message });
|
|
290
300
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
throw cause;
|
|
301
|
+
return fallback(reason, {
|
|
302
|
+
httpStatus: cause.reason.statusCode,
|
|
303
|
+
detail: cause.message,
|
|
304
|
+
});
|
|
296
305
|
}
|
|
297
306
|
const envelope = WireResponseBody(data);
|
|
298
307
|
if (envelope instanceof type.errors) {
|
|
@@ -341,6 +350,6 @@ export async function evaluate(input) {
|
|
|
341
350
|
latencyMs: result.latencyMs,
|
|
342
351
|
modelId,
|
|
343
352
|
};
|
|
344
|
-
|
|
353
|
+
emit(successEvent);
|
|
345
354
|
return result;
|
|
346
355
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export { DEFAULT_TIMEOUT_MS
|
|
2
|
-
export {
|
|
3
|
-
export type { PostEvaluateOptions, PostEvaluateResponse } from "./client.js";
|
|
1
|
+
export { DEFAULT_TIMEOUT_MS } from "./config.js";
|
|
2
|
+
export type { EvaluateDeps } from "./client.js";
|
|
4
3
|
export { evaluate } from "./evaluate.js";
|
|
5
|
-
export {
|
|
6
|
-
export {
|
|
4
|
+
export type { EvaluateOptions } from "./evaluate.js";
|
|
5
|
+
export { createSystemOneAdapter, createSystemOneAdapterFactory, SYSTEM_ONE_PROVIDER, } from "./adapter.js";
|
|
6
|
+
export { SystemOneError } from "./errors.js";
|
|
7
7
|
export type { SystemOneErrorCode } from "./errors.js";
|
|
8
|
-
export {
|
|
9
|
-
export {
|
|
8
|
+
export { SystemOneTelemetryEvent } from "./telemetry.js";
|
|
9
|
+
export { BooleanQuestion, ChoiceQuestion, Decision, Description, EndpointConfig, EvaluateConfig, EvaluateInput, EvaluateResult, FallbackReason, FallbackResult, NoulQuestion, ProbabilityMap, Question, QuestionList, ScoreQuestion, State, Usage, } from "./schemas.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
export { DEFAULT_TIMEOUT_MS
|
|
2
|
-
export { postEvaluate } from "./client.js";
|
|
1
|
+
export { DEFAULT_TIMEOUT_MS } from "./config.js";
|
|
3
2
|
export { evaluate } from "./evaluate.js";
|
|
4
|
-
export { createSystemOneAdapter, SYSTEM_ONE_PROVIDER } from "./adapter.js";
|
|
5
|
-
export {
|
|
6
|
-
export {
|
|
7
|
-
export {
|
|
3
|
+
export { createSystemOneAdapter, createSystemOneAdapterFactory, SYSTEM_ONE_PROVIDER, } from "./adapter.js";
|
|
4
|
+
export { SystemOneError } from "./errors.js";
|
|
5
|
+
export { SystemOneTelemetryEvent } from "./telemetry.js";
|
|
6
|
+
export { BooleanQuestion, ChoiceQuestion, Decision, Description, EndpointConfig, EvaluateConfig, EvaluateInput, EvaluateResult, FallbackReason, FallbackResult, NoulQuestion, ProbabilityMap, Question, QuestionList, ScoreQuestion, State, Usage, } from "./schemas.js";
|
package/dist/schemas.d.ts
CHANGED
|
@@ -373,29 +373,6 @@ export declare const Usage: import("arktype/internal/variants/object.ts").Object
|
|
|
373
373
|
outputTokens: number;
|
|
374
374
|
}, {}>;
|
|
375
375
|
export type Usage = typeof Usage.infer;
|
|
376
|
-
export declare const EvaluateResponseBody: import("arktype/internal/variants/object.ts").ObjectType<{
|
|
377
|
-
answers: {
|
|
378
|
-
[x: string]: {
|
|
379
|
-
type: "choice" | "score" | "noul";
|
|
380
|
-
choice?: string;
|
|
381
|
-
score?: number;
|
|
382
|
-
noul?: number;
|
|
383
|
-
probabilities?: {
|
|
384
|
-
[x: string]: number;
|
|
385
|
-
};
|
|
386
|
-
confidence?: number;
|
|
387
|
-
legend?: {
|
|
388
|
-
[x: string]: string;
|
|
389
|
-
};
|
|
390
|
-
};
|
|
391
|
-
};
|
|
392
|
-
model?: string;
|
|
393
|
-
usage?: {
|
|
394
|
-
input_tokens: number;
|
|
395
|
-
output_tokens: number;
|
|
396
|
-
};
|
|
397
|
-
}, {}>;
|
|
398
|
-
export type EvaluateResponseBody = typeof EvaluateResponseBody.infer;
|
|
399
376
|
export declare const EvaluateResult: import("arktype/internal/variants/object.ts").ObjectType<{
|
|
400
377
|
decisions: ({
|
|
401
378
|
id: string;
|
package/dist/schemas.js
CHANGED
|
@@ -285,9 +285,6 @@ export const Usage = type({
|
|
|
285
285
|
outputTokens: "number",
|
|
286
286
|
"+": "reject",
|
|
287
287
|
});
|
|
288
|
-
// Backwards-compatible alias: the envelope shared by `evaluate()` and the
|
|
289
|
-
// provider adapter.
|
|
290
|
-
export const EvaluateResponseBody = WireResponseBody;
|
|
291
288
|
export const EvaluateResult = type({
|
|
292
289
|
decisions: Decision.array(),
|
|
293
290
|
modelId: "string",
|
package/dist/telemetry.d.ts
CHANGED
|
@@ -8,17 +8,3 @@ export declare const SystemOneTelemetryEvent: import("arktype/internal/variants/
|
|
|
8
8
|
detail?: string;
|
|
9
9
|
}, {}>;
|
|
10
10
|
export type SystemOneTelemetryEvent = typeof SystemOneTelemetryEvent.infer;
|
|
11
|
-
/** Maximum buffered events; beyond this the oldest drop first. */
|
|
12
|
-
export declare const MAX_BUFFERED_TELEMETRY_EVENTS = 256;
|
|
13
|
-
/**
|
|
14
|
-
* Records one telemetry event to the in-memory ring. Validates the shape
|
|
15
|
-
* (a malformed event is caller misuse and throws a typed `SystemOneError`)
|
|
16
|
-
* and never touches the network, disk, or console.
|
|
17
|
-
*/
|
|
18
|
-
export declare function recordTelemetryEvent(event: SystemOneTelemetryEvent): void;
|
|
19
|
-
/**
|
|
20
|
-
* Drains the buffered telemetry events in arrival order and clears the
|
|
21
|
-
* buffer. The host calls this after each evaluation (or on its own cadence)
|
|
22
|
-
* to forward events to its own sink.
|
|
23
|
-
*/
|
|
24
|
-
export declare function drainTelemetryEvents(): SystemOneTelemetryEvent[];
|
package/dist/telemetry.js
CHANGED
|
@@ -1,18 +1,13 @@
|
|
|
1
1
|
import { type } from "arktype";
|
|
2
|
-
import { SystemOneError } from "./errors.js";
|
|
3
2
|
import { FallbackReason } from "./schemas.js";
|
|
4
3
|
// ---------------------------------------------------------------------------
|
|
5
4
|
// Telemetry
|
|
6
5
|
// ---------------------------------------------------------------------------
|
|
7
6
|
//
|
|
8
|
-
// Event
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
// second public surface to version for no load-bearing reason: the host
|
|
13
|
-
// already owns every destination, it only needs lossless-per-call access to
|
|
14
|
-
// the events. The cap keeps a chatty loop from growing memory without bound
|
|
15
|
-
// (oldest events drop first). No event field ever carries key material.
|
|
7
|
+
// Event shape for evaluation observability. `evaluate` hands each event to
|
|
8
|
+
// the caller's `onTelemetry` sink, which forwards it wherever the host wants
|
|
9
|
+
// (stdout, OTLP, a host hook); no state is kept between calls. No event
|
|
10
|
+
// field ever carries key material.
|
|
16
11
|
/** One evaluation lifecycle event. */
|
|
17
12
|
export const SystemOneTelemetryEvent = type({
|
|
18
13
|
event: "'evaluate.start' | 'evaluate.success' | 'evaluate.fallback'",
|
|
@@ -23,29 +18,3 @@ export const SystemOneTelemetryEvent = type({
|
|
|
23
18
|
"detail?": "string",
|
|
24
19
|
"+": "reject",
|
|
25
20
|
});
|
|
26
|
-
/** Maximum buffered events; beyond this the oldest drop first. */
|
|
27
|
-
export const MAX_BUFFERED_TELEMETRY_EVENTS = 256;
|
|
28
|
-
const bufferedEvents = [];
|
|
29
|
-
/**
|
|
30
|
-
* Records one telemetry event to the in-memory ring. Validates the shape
|
|
31
|
-
* (a malformed event is caller misuse and throws a typed `SystemOneError`)
|
|
32
|
-
* and never touches the network, disk, or console.
|
|
33
|
-
*/
|
|
34
|
-
export function recordTelemetryEvent(event) {
|
|
35
|
-
const parsed = SystemOneTelemetryEvent(event);
|
|
36
|
-
if (parsed instanceof type.errors) {
|
|
37
|
-
throw new SystemOneError("parse-error", `recordTelemetryEvent: invalid event: ${parsed.summary}`);
|
|
38
|
-
}
|
|
39
|
-
bufferedEvents.push(parsed);
|
|
40
|
-
if (bufferedEvents.length > MAX_BUFFERED_TELEMETRY_EVENTS) {
|
|
41
|
-
bufferedEvents.splice(0, bufferedEvents.length - MAX_BUFFERED_TELEMETRY_EVENTS);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* Drains the buffered telemetry events in arrival order and clears the
|
|
46
|
-
* buffer. The host calls this after each evaluation (or on its own cadence)
|
|
47
|
-
* to forward events to its own sink.
|
|
48
|
-
*/
|
|
49
|
-
export function drainTelemetryEvents() {
|
|
50
|
-
return bufferedEvents.splice(0, bufferedEvents.length);
|
|
51
|
-
}
|