okengine 0.11.1 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/site/content/docs/elements/ai.mdx +22 -1
- package/site/content/docs/elements/store.mdx +3 -1
- package/site/content/docs/elements/vault.mdx +19 -11
- package/site/content/docs/get-started/installation.mdx +7 -1
- package/site/content/docs/recipes/llama-cpp.mdx +10 -9
- package/site/content/docs/reference/cli.md +6 -2
- package/site/content/docs/reference/environment-variables.mdx +9 -9
- package/src/cli/ai-setup/ai-setup.test.ts +3 -1
- package/src/cli/ai-setup/apply.ts +61 -1
- package/src/cli/ask-seed.test.ts +4 -3
- package/src/cli/ask-seed.ts +5 -6
- package/src/cli/client-add.test.ts +2 -1
- package/src/cli/dev.test.ts +116 -0
- package/src/cli/dev.ts +107 -18
- package/src/cli/project-state.test.ts +50 -0
- package/src/cli/project-state.ts +123 -0
- package/src/cli/vault-cmd.test.ts +47 -18
- package/src/cli/vault-cmd.ts +2 -1
- package/src/compiler/extract.ts +12 -1
- package/src/console/server/console.test.ts +3 -1
- package/src/console/server/operator-db.test.ts +48 -17
- package/src/console/server/operator-db.ts +5 -1
- package/src/docker/derive.ts +24 -3
- package/src/docker/docker.test.ts +4 -2
- package/src/docker/index.ts +1 -0
- package/src/docker/recipes/index.ts +1 -0
- package/src/docker/recipes/llama-cpp.ts +20 -4
- package/src/drivers/ai-openai-compatible.ts +15 -3
- package/src/drivers/vault-builtin.test.ts +50 -42
- package/src/elements/ai/declare.ts +73 -3
- package/src/elements/ai/errors.test.ts +35 -0
- package/src/elements/ai/errors.ts +139 -0
- package/src/elements/ai/eval.ts +26 -1
- package/src/elements/ai/runtime.ts +140 -80
- package/src/elements/ai/tools.test.ts +1 -1
- package/src/elements/ai.test.ts +99 -2
- package/src/elements/ai.ts +11 -1
- package/src/elements/index.ts +2 -0
- package/src/elements/store/index-boot.test.ts +23 -6
- package/src/elements/store/resource.test.ts +38 -19
- package/src/elements/store/sql-session.test.ts +55 -58
- package/src/elements/vault/builtin-adapter.test.ts +115 -58
- package/src/elements/vault/builtin-adapter.ts +241 -47
- package/src/elements/vault/chaos-child.ts +424 -0
- package/src/elements/vault/chaos.test.ts +651 -0
- package/src/elements/vault/resilience.ts +6 -1
- package/src/elements/vault/security-checklist.test.ts +10 -8
- package/src/elements/vault/storage.ts +130 -27
- package/src/elements/vault/test-helpers.ts +368 -0
- package/src/elements/vault.ts +6 -0
- package/src/index.ts +2 -0
- package/src/kernel/app.ts +83 -10
- package/src/kernel/auto-registry.test.ts +52 -1
- package/src/kernel/element-registries.ts +19 -4
- package/src/kernel/errors.ts +3 -3
- package/src/kernel/fx.test.ts +25 -0
- package/src/kernel/fx.ts +4 -1
- package/src/manifest/types.ts +4 -0
- package/src/test/create-test-app.ts +16 -11
- package/src/test/reset-element-registries.ts +17 -9
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI error classification + timeout parsing.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
import { aiHttpError, isRetryableAiError, outExpectsVia, resolveTimeoutMs } from "./errors.ts";
|
|
7
|
+
|
|
8
|
+
describe("resolveTimeoutMs", () => {
|
|
9
|
+
test("parses clock durations and ms numbers", () => {
|
|
10
|
+
expect(resolveTimeoutMs("30s")).toBe(30_000);
|
|
11
|
+
expect(resolveTimeoutMs(250)).toBe(250);
|
|
12
|
+
expect(resolveTimeoutMs(undefined)).toBeUndefined();
|
|
13
|
+
expect(resolveTimeoutMs("nope")).toBeUndefined();
|
|
14
|
+
});
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
describe("isRetryableAiError", () => {
|
|
18
|
+
test("classifies status and HTTP message", () => {
|
|
19
|
+
expect(isRetryableAiError(aiHttpError("rate", 429))).toBe(true);
|
|
20
|
+
expect(isRetryableAiError(aiHttpError("boom", 503))).toBe(true);
|
|
21
|
+
expect(isRetryableAiError(aiHttpError("auth", 401))).toBe(false);
|
|
22
|
+
expect(isRetryableAiError(new Error("openai-compatible HTTP 401"))).toBe(false);
|
|
23
|
+
const abort = new Error("aborted");
|
|
24
|
+
abort.name = "AbortError";
|
|
25
|
+
expect(isRetryableAiError(abort)).toBe(true);
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe("outExpectsVia", () => {
|
|
30
|
+
test("detects JSON-schema and Zod-like shape", () => {
|
|
31
|
+
expect(outExpectsVia({ properties: { via: { type: "string" } } })).toBe(true);
|
|
32
|
+
expect(outExpectsVia({ shape: { via: {} } })).toBe(true);
|
|
33
|
+
expect(outExpectsVia({ properties: { summary: { type: "string" } } })).toBe(false);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI provider / transport error classification for recovery chains.
|
|
3
|
+
*
|
|
4
|
+
* Retryable failures may retry once on the same model, then advance `via`.
|
|
5
|
+
* Permanent failures stop the chain immediately.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { parseDurationMs } from "../clock/duration.ts";
|
|
9
|
+
import type { AiTimeout } from "./declare.ts";
|
|
10
|
+
|
|
11
|
+
/** Optional structured fields drivers may attach to thrown errors. */
|
|
12
|
+
export interface AiErrorFields {
|
|
13
|
+
readonly status?: number;
|
|
14
|
+
readonly code?: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Resolve a prompt/ask timeout to milliseconds.
|
|
19
|
+
*
|
|
20
|
+
* @param timeout - Duration string (`"30s"`) or ms number
|
|
21
|
+
* @returns Milliseconds, or `undefined` when unset / invalid
|
|
22
|
+
*/
|
|
23
|
+
export function resolveTimeoutMs(timeout: AiTimeout | undefined): number | undefined {
|
|
24
|
+
if (timeout === undefined) return undefined;
|
|
25
|
+
if (typeof timeout === "number") {
|
|
26
|
+
return Number.isFinite(timeout) && timeout > 0 ? timeout : undefined;
|
|
27
|
+
}
|
|
28
|
+
const ms = parseDurationMs(timeout);
|
|
29
|
+
return ms > 0 ? ms : undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Merge an optional deadline into an ambient abort signal.
|
|
34
|
+
*
|
|
35
|
+
* @param timeoutMs - Deadline in ms
|
|
36
|
+
* @param ambient - Existing signal (e.g. request cancel)
|
|
37
|
+
*/
|
|
38
|
+
export function mergeAskAbortSignal(
|
|
39
|
+
timeoutMs: number | undefined,
|
|
40
|
+
ambient?: AbortSignal,
|
|
41
|
+
): AbortSignal | undefined {
|
|
42
|
+
if (timeoutMs === undefined) return ambient;
|
|
43
|
+
const deadline = AbortSignal.timeout(timeoutMs);
|
|
44
|
+
if (!ambient) return deadline;
|
|
45
|
+
if (typeof AbortSignal.any === "function") {
|
|
46
|
+
return AbortSignal.any([ambient, deadline]);
|
|
47
|
+
}
|
|
48
|
+
return deadline;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* HTTP / abort / network failures that may succeed on retry or another model.
|
|
53
|
+
*
|
|
54
|
+
* @param err - Thrown value from a model attempt
|
|
55
|
+
*/
|
|
56
|
+
export function isRetryableAiError(err: unknown): boolean {
|
|
57
|
+
if (err == null) return false;
|
|
58
|
+
|
|
59
|
+
const status = readStatus(err);
|
|
60
|
+
if (status !== undefined) {
|
|
61
|
+
if (status === 429) return true;
|
|
62
|
+
if (status >= 500 && status <= 599) return true;
|
|
63
|
+
if (status >= 400 && status <= 499) return false;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const name = err instanceof Error ? err.name : "";
|
|
67
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
68
|
+
const lower = message.toLowerCase();
|
|
69
|
+
|
|
70
|
+
const httpMatch = /\bHTTP\s*(\d{3})\b/i.exec(message);
|
|
71
|
+
if (httpMatch?.[1]) {
|
|
72
|
+
const code = Number(httpMatch[1]);
|
|
73
|
+
if (code === 429 || (code >= 500 && code <= 599)) return true;
|
|
74
|
+
if (code >= 400 && code <= 499) return false;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (name === "AbortError" || name === "TimeoutError") return true;
|
|
78
|
+
if (lower.includes("aborterror") || lower.includes("timeout")) return true;
|
|
79
|
+
if (
|
|
80
|
+
lower.includes("econnreset") ||
|
|
81
|
+
lower.includes("econnrefused") ||
|
|
82
|
+
lower.includes("enotfound") ||
|
|
83
|
+
lower.includes("socket hang up") ||
|
|
84
|
+
lower.includes("fetch failed") ||
|
|
85
|
+
lower.includes("network")
|
|
86
|
+
) {
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Unclassified provider/transport errors — allow recovery to the next model.
|
|
91
|
+
return err instanceof Error;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Attach an HTTP status onto an Error for {@link isRetryableAiError}.
|
|
96
|
+
*
|
|
97
|
+
* @param message - Error message
|
|
98
|
+
* @param status - HTTP status
|
|
99
|
+
*/
|
|
100
|
+
export function aiHttpError(message: string, status: number): Error {
|
|
101
|
+
const err = new Error(message) as Error & { status: number };
|
|
102
|
+
err.status = status;
|
|
103
|
+
return err;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function readStatus(err: unknown): number | undefined {
|
|
107
|
+
if (typeof err !== "object" || err === null) return undefined;
|
|
108
|
+
const status = (err as AiErrorFields).status;
|
|
109
|
+
return typeof status === "number" && Number.isFinite(status) ? status : undefined;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Whether a declared prompt `out` schema expects a `via` field
|
|
114
|
+
* (JSON-schema properties or Zod `.shape`).
|
|
115
|
+
*
|
|
116
|
+
* @param schema - Prompt `out` declaration
|
|
117
|
+
*/
|
|
118
|
+
export function outExpectsVia(schema: unknown): boolean {
|
|
119
|
+
if (schema == null || typeof schema !== "object") return false;
|
|
120
|
+
if (
|
|
121
|
+
"properties" in schema &&
|
|
122
|
+
schema.properties &&
|
|
123
|
+
typeof schema.properties === "object" &&
|
|
124
|
+
!Array.isArray(schema.properties) &&
|
|
125
|
+
"via" in (schema.properties as Record<string, unknown>)
|
|
126
|
+
) {
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
if (
|
|
130
|
+
"shape" in schema &&
|
|
131
|
+
schema.shape &&
|
|
132
|
+
typeof schema.shape === "object" &&
|
|
133
|
+
!Array.isArray(schema.shape) &&
|
|
134
|
+
"via" in (schema.shape as Record<string, unknown>)
|
|
135
|
+
) {
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
return false;
|
|
139
|
+
}
|
package/src/elements/ai/eval.ts
CHANGED
|
@@ -47,13 +47,38 @@ export interface RunPromptEvalsOptions {
|
|
|
47
47
|
readonly equals?: (actual: unknown, expect: unknown) => boolean;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Deep equality for eval expectations — ignore a runtime-stamped `via`
|
|
52
|
+
* when the case did not declare one.
|
|
53
|
+
*
|
|
54
|
+
* @param actual - Ask output
|
|
55
|
+
* @param expect - Case expectation
|
|
56
|
+
*/
|
|
57
|
+
function defaultEvalEquals(actual: unknown, expect: unknown): boolean {
|
|
58
|
+
if (JSON.stringify(actual) === JSON.stringify(expect)) return true;
|
|
59
|
+
if (
|
|
60
|
+
actual &&
|
|
61
|
+
typeof actual === "object" &&
|
|
62
|
+
!Array.isArray(actual) &&
|
|
63
|
+
expect &&
|
|
64
|
+
typeof expect === "object" &&
|
|
65
|
+
!Array.isArray(expect) &&
|
|
66
|
+
!("via" in expect) &&
|
|
67
|
+
"via" in actual
|
|
68
|
+
) {
|
|
69
|
+
const { via: _via, ...rest } = actual as Record<string, unknown>;
|
|
70
|
+
return JSON.stringify(rest) === JSON.stringify(expect);
|
|
71
|
+
}
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
|
|
50
75
|
/**
|
|
51
76
|
* Run a prompt eval set. Fails CI when any case fails (`ok === false`).
|
|
52
77
|
*
|
|
53
78
|
* @param options - Cases + ask fn
|
|
54
79
|
*/
|
|
55
80
|
export async function runPromptEvals(options: RunPromptEvalsOptions): Promise<EvalSuiteResult> {
|
|
56
|
-
const equals = options.equals ??
|
|
81
|
+
const equals = options.equals ?? defaultEvalEquals;
|
|
57
82
|
const results: EvalCaseResult[] = [];
|
|
58
83
|
|
|
59
84
|
for (let i = 0; i < options.cases.length; i++) {
|
|
@@ -12,7 +12,13 @@ import type { IndexStore } from "../../drivers/types.ts";
|
|
|
12
12
|
import { maskRedactedDeep } from "../../kernel/redacted.ts";
|
|
13
13
|
import type { GatePolicyContext } from "../gate/declare.ts";
|
|
14
14
|
import type { GateRuntime } from "../gate/runtime.ts";
|
|
15
|
-
import type { AiAgentDecl, AiEmbedDecl, AiModelDecl, AiPromptDecl } from "./declare.ts";
|
|
15
|
+
import type { AiAgentDecl, AiEmbedDecl, AiModelDecl, AiPromptDecl, AiTimeout } from "./declare.ts";
|
|
16
|
+
import {
|
|
17
|
+
isRetryableAiError,
|
|
18
|
+
mergeAskAbortSignal,
|
|
19
|
+
outExpectsVia,
|
|
20
|
+
resolveTimeoutMs,
|
|
21
|
+
} from "./errors.ts";
|
|
16
22
|
import {
|
|
17
23
|
AiSchemaValidationError,
|
|
18
24
|
coerceModelObject,
|
|
@@ -20,6 +26,9 @@ import {
|
|
|
20
26
|
type AiSchemaMismatch,
|
|
21
27
|
} from "./schema.ts";
|
|
22
28
|
|
|
29
|
+
/** Brief pause before the same-model retry on a retryable failure. */
|
|
30
|
+
const AI_SAME_MODEL_RETRY_BACKOFF_MS = 250;
|
|
31
|
+
|
|
23
32
|
/** Default bound for tool / agent loops. */
|
|
24
33
|
export const AI_DEFAULT_MAX_STEPS = 6;
|
|
25
34
|
|
|
@@ -146,6 +155,8 @@ export interface CreateAiRuntimeOptions {
|
|
|
146
155
|
/** Ask options. */
|
|
147
156
|
export interface AiAskOptions {
|
|
148
157
|
readonly via?: readonly string[];
|
|
158
|
+
/** Per-call deadline — overrides prompt `timeout` (`"30s"` or ms). */
|
|
159
|
+
readonly timeout?: AiTimeout;
|
|
149
160
|
readonly allowPii?: boolean;
|
|
150
161
|
/** Flow names offered as tools — each model call dispatches via `callTool`. */
|
|
151
162
|
readonly tools?: readonly string[];
|
|
@@ -275,6 +286,8 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
275
286
|
const model = models.get(name);
|
|
276
287
|
const opened = await options.defaultDriver.open({
|
|
277
288
|
model: model?.model ?? name,
|
|
289
|
+
...(model?.baseUrl !== undefined ? { baseUrl: model.baseUrl } : {}),
|
|
290
|
+
...(model?.apiKey !== undefined ? { apiKey: model.apiKey } : {}),
|
|
278
291
|
});
|
|
279
292
|
clients.set(name, opened);
|
|
280
293
|
return opened;
|
|
@@ -389,6 +402,7 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
389
402
|
readonly maxSteps: number;
|
|
390
403
|
readonly agentLabel: string;
|
|
391
404
|
readonly responseFormat?: unknown;
|
|
405
|
+
readonly signal?: AbortSignal;
|
|
392
406
|
readonly callTool?: (name: string, input: unknown) => Promise<unknown>;
|
|
393
407
|
readonly auth?: GatePolicyContext["auth"];
|
|
394
408
|
readonly operator?: GatePolicyContext["operator"];
|
|
@@ -421,6 +435,7 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
421
435
|
messages,
|
|
422
436
|
tools: defs.length > 0 ? defs : undefined,
|
|
423
437
|
responseFormat: opts.responseFormat,
|
|
438
|
+
...(opts.signal !== undefined ? { signal: opts.signal } : {}),
|
|
424
439
|
});
|
|
425
440
|
cost += result.usage?.cost ?? 0;
|
|
426
441
|
lastText = result.text;
|
|
@@ -498,6 +513,7 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
498
513
|
const version = decl.version;
|
|
499
514
|
const started = now();
|
|
500
515
|
const tools = opts?.tools ?? [];
|
|
516
|
+
const signal = mergeAskAbortSignal(resolveTimeoutMs(opts?.timeout ?? decl.timeout));
|
|
501
517
|
|
|
502
518
|
// Replay from journal when input matches (nondeterministic contract)
|
|
503
519
|
if (journalingForced && tools.length === 0) {
|
|
@@ -514,7 +530,8 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
514
530
|
}
|
|
515
531
|
}
|
|
516
532
|
|
|
517
|
-
const via =
|
|
533
|
+
const via =
|
|
534
|
+
opts?.via ?? decl.via ?? (decl.model ? [decl.model] : [...models.keys()].slice(0, 1));
|
|
518
535
|
const attempts: AiFallbackAttempt[] = [];
|
|
519
536
|
let lastError: string | undefined;
|
|
520
537
|
let lastSchema: AiSchemaMismatch | undefined;
|
|
@@ -522,72 +539,66 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
522
539
|
const userContent = promptContentFromInput(input);
|
|
523
540
|
|
|
524
541
|
for (const modelName of via) {
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
if (tools.length > 0) {
|
|
532
|
-
const loop = await toolLoop({
|
|
533
|
-
client,
|
|
534
|
-
modelName,
|
|
535
|
-
messages: [{ role: "user", content: userContent }],
|
|
536
|
-
tools,
|
|
537
|
-
maxSteps: opts?.maxSteps ?? AI_DEFAULT_MAX_STEPS,
|
|
538
|
-
agentLabel: prompt,
|
|
539
|
-
responseFormat: decl.out,
|
|
540
|
-
callTool: opts?.callTool,
|
|
541
|
-
});
|
|
542
|
-
raw = loop.lastToolResult !== undefined && !loop.text ? loop.lastToolResult : loop.raw;
|
|
543
|
-
attemptCost = loop.cost;
|
|
544
|
-
totalCost += attemptCost;
|
|
545
|
-
if (loop.denials.length > 0 && loop.trail.every((t) => t.status === "denied")) {
|
|
546
|
-
throw new Error(
|
|
547
|
-
`ai: all tool calls denied for prompt "${prompt}": ${loop.denials[0]?.reason}`,
|
|
548
|
-
);
|
|
549
|
-
}
|
|
550
|
-
} else {
|
|
551
|
-
const result = await client.complete({
|
|
552
|
-
model: wireModel(modelName, client),
|
|
553
|
-
messages: [{ role: "user", content: userContent }],
|
|
554
|
-
responseFormat: decl.out,
|
|
555
|
-
});
|
|
556
|
-
attemptCost = result.usage?.cost ?? 0;
|
|
557
|
-
totalCost += attemptCost;
|
|
558
|
-
raw = result.raw !== undefined ? result.raw : result.text;
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
const latencyMs = Math.max(0, now() - attemptStart);
|
|
562
|
-
|
|
542
|
+
let sameModelTries = 0;
|
|
543
|
+
let advance = true;
|
|
544
|
+
while (sameModelTries < 2 && advance) {
|
|
545
|
+
sameModelTries++;
|
|
546
|
+
const attemptStart = now();
|
|
563
547
|
try {
|
|
564
|
-
const
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
548
|
+
const client = await clientFor(modelName);
|
|
549
|
+
let raw: unknown;
|
|
550
|
+
let attemptCost = 0;
|
|
551
|
+
|
|
552
|
+
if (tools.length > 0) {
|
|
553
|
+
const loop = await toolLoop({
|
|
554
|
+
client,
|
|
555
|
+
modelName,
|
|
556
|
+
messages: [{ role: "user", content: userContent }],
|
|
557
|
+
tools,
|
|
558
|
+
maxSteps: opts?.maxSteps ?? AI_DEFAULT_MAX_STEPS,
|
|
559
|
+
agentLabel: prompt,
|
|
560
|
+
responseFormat: decl.out,
|
|
561
|
+
callTool: opts?.callTool,
|
|
562
|
+
...(signal !== undefined ? { signal } : {}),
|
|
563
|
+
});
|
|
564
|
+
raw =
|
|
565
|
+
loop.lastToolResult !== undefined && !loop.text ? loop.lastToolResult : loop.raw;
|
|
566
|
+
attemptCost = loop.cost;
|
|
567
|
+
totalCost += attemptCost;
|
|
568
|
+
if (loop.denials.length > 0 && loop.trail.every((t) => t.status === "denied")) {
|
|
569
|
+
throw new Error(
|
|
570
|
+
`ai: all tool calls denied for prompt "${prompt}": ${loop.denials[0]?.reason}`,
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
} else {
|
|
574
|
+
const result = await client.complete({
|
|
575
|
+
model: wireModel(modelName, client),
|
|
576
|
+
messages: [{ role: "user", content: userContent }],
|
|
577
|
+
responseFormat: decl.out,
|
|
578
|
+
...(signal !== undefined ? { signal } : {}),
|
|
585
579
|
});
|
|
580
|
+
attemptCost = result.usage?.cost ?? 0;
|
|
581
|
+
totalCost += attemptCost;
|
|
582
|
+
// Prefer assistant text — `raw` is often the transport envelope
|
|
583
|
+
// (OpenAI chat.completion object), which must not shadow the content.
|
|
584
|
+
raw =
|
|
585
|
+
typeof result.text === "string" && result.text.length > 0
|
|
586
|
+
? result.text
|
|
587
|
+
: result.raw !== undefined
|
|
588
|
+
? result.raw
|
|
589
|
+
: result.text;
|
|
586
590
|
}
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
+
|
|
592
|
+
const latencyMs = Math.max(0, now() - attemptStart);
|
|
593
|
+
|
|
594
|
+
try {
|
|
595
|
+
const coerced = coerceModelObject(raw);
|
|
596
|
+
const prepared = outExpectsVia(decl.out) ? { ...coerced, via: modelName } : coerced;
|
|
597
|
+
const validated = decl.out
|
|
598
|
+
? validatePromptOut(prompt, version, decl.out, prepared)
|
|
599
|
+
: prepared;
|
|
600
|
+
// Always report the winning logical model for recovery chains.
|
|
601
|
+
const output = { ...validated, via: modelName };
|
|
591
602
|
attempts.push({
|
|
592
603
|
model: modelName,
|
|
593
604
|
ok: true,
|
|
@@ -600,30 +611,79 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
600
611
|
prompt,
|
|
601
612
|
...(version !== undefined ? { version } : {}),
|
|
602
613
|
input,
|
|
603
|
-
output
|
|
614
|
+
output,
|
|
604
615
|
attempts: [...attempts],
|
|
605
|
-
outcome: "
|
|
616
|
+
outcome: "ok",
|
|
606
617
|
cost: totalCost,
|
|
607
618
|
latencyMs: Math.max(0, now() - started),
|
|
608
|
-
schemaMismatch: err.mismatch,
|
|
609
619
|
at: now(),
|
|
610
620
|
});
|
|
611
621
|
}
|
|
622
|
+
return output;
|
|
623
|
+
} catch (err) {
|
|
624
|
+
if (err instanceof AiSchemaValidationError) {
|
|
625
|
+
lastSchema = err.mismatch;
|
|
626
|
+
attempts.push({
|
|
627
|
+
model: modelName,
|
|
628
|
+
ok: true,
|
|
629
|
+
cost: attemptCost,
|
|
630
|
+
latencyMs,
|
|
631
|
+
at: now(),
|
|
632
|
+
});
|
|
633
|
+
if (journalingForced) {
|
|
634
|
+
journal.push({
|
|
635
|
+
prompt,
|
|
636
|
+
...(version !== undefined ? { version } : {}),
|
|
637
|
+
input,
|
|
638
|
+
output: coerceModelObject(raw),
|
|
639
|
+
attempts: [...attempts],
|
|
640
|
+
outcome: "schema_invalid",
|
|
641
|
+
cost: totalCost,
|
|
642
|
+
latencyMs: Math.max(0, now() - started),
|
|
643
|
+
schemaMismatch: err.mismatch,
|
|
644
|
+
at: now(),
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
throw err;
|
|
648
|
+
}
|
|
612
649
|
throw err;
|
|
613
650
|
}
|
|
614
|
-
|
|
651
|
+
} catch (err) {
|
|
652
|
+
if (err instanceof AiSchemaValidationError) throw err;
|
|
653
|
+
lastError = err instanceof Error ? err.message : String(err);
|
|
654
|
+
attempts.push({
|
|
655
|
+
model: modelName,
|
|
656
|
+
ok: false,
|
|
657
|
+
error: lastError,
|
|
658
|
+
cost: 0,
|
|
659
|
+
latencyMs: Math.max(0, now() - attemptStart),
|
|
660
|
+
at: now(),
|
|
661
|
+
});
|
|
662
|
+
|
|
663
|
+
if (!isRetryableAiError(err)) {
|
|
664
|
+
if (journalingForced) {
|
|
665
|
+
journal.push({
|
|
666
|
+
prompt,
|
|
667
|
+
...(version !== undefined ? { version } : {}),
|
|
668
|
+
input,
|
|
669
|
+
output: { error: lastError },
|
|
670
|
+
attempts,
|
|
671
|
+
outcome: "provider_error",
|
|
672
|
+
cost: totalCost,
|
|
673
|
+
latencyMs: Math.max(0, now() - started),
|
|
674
|
+
at: now(),
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
throw err instanceof Error ? err : new Error(String(err));
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
if (sameModelTries < 2) {
|
|
681
|
+
await new Promise((r) => setTimeout(r, AI_SAME_MODEL_RETRY_BACKOFF_MS));
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
advance = true;
|
|
685
|
+
break;
|
|
615
686
|
}
|
|
616
|
-
} catch (err) {
|
|
617
|
-
if (err instanceof AiSchemaValidationError) throw err;
|
|
618
|
-
lastError = err instanceof Error ? err.message : String(err);
|
|
619
|
-
attempts.push({
|
|
620
|
-
model: modelName,
|
|
621
|
-
ok: false,
|
|
622
|
-
error: lastError,
|
|
623
|
-
cost: 0,
|
|
624
|
-
latencyMs: Math.max(0, now() - attemptStart),
|
|
625
|
-
at: now(),
|
|
626
|
-
});
|
|
627
687
|
}
|
|
628
688
|
}
|
|
629
689
|
|
|
@@ -58,7 +58,7 @@ describe("fx.ask tools via fx.call", () => {
|
|
|
58
58
|
});
|
|
59
59
|
|
|
60
60
|
const out = await fx.ask(prompt, { q: "status?" }, { tools: ["lookup.booking"], maxSteps: 4 });
|
|
61
|
-
expect(out).toEqual({ answer: "found" });
|
|
61
|
+
expect(out).toEqual({ answer: "found", via: "smart" });
|
|
62
62
|
expect(calls).toEqual([{ name: "lookup.booking", input: { id: "B9" } }]);
|
|
63
63
|
expect(ledger.entries.some((e) => e.kind === "ask" && e.resource === "assistant")).toBe(true);
|
|
64
64
|
expect(
|
package/src/elements/ai.test.ts
CHANGED
|
@@ -29,10 +29,14 @@ describe("ai declaration", () => {
|
|
|
29
29
|
version: 3,
|
|
30
30
|
evals: "./evals/triage.jsonl",
|
|
31
31
|
budget: { maxCostPerCall: 0.02 },
|
|
32
|
+
via: ["smart", "fast"],
|
|
33
|
+
timeout: "30s",
|
|
32
34
|
});
|
|
33
35
|
expect(triage.name).toBe("ticket-triage");
|
|
34
36
|
expect(triage.version).toBe(3);
|
|
35
37
|
expect(triage.model).toBe("smart");
|
|
38
|
+
expect(triage.via).toEqual(["smart", "fast"]);
|
|
39
|
+
expect(triage.timeout).toBe("30s");
|
|
36
40
|
|
|
37
41
|
const agent = ai.agent("support", {
|
|
38
42
|
tools: [{ name: "bookings.getBooking" }, "bookings.refundBooking"],
|
|
@@ -334,12 +338,105 @@ describe("model fallback chain", () => {
|
|
|
334
338
|
},
|
|
335
339
|
);
|
|
336
340
|
expect(out.urgency).toBe("low");
|
|
341
|
+
expect(out.via).toBe("fast");
|
|
337
342
|
const entry = runtime.journal[0]!;
|
|
338
|
-
|
|
343
|
+
// Same-model retry (1) + second model success.
|
|
344
|
+
expect(entry.attempts.length).toBeGreaterThanOrEqual(2);
|
|
339
345
|
expect(entry.attempts[0]).toMatchObject({ model: "smart", ok: false });
|
|
340
|
-
expect(entry.attempts
|
|
346
|
+
expect(entry.attempts.at(-1)).toMatchObject({ model: "fast", ok: true });
|
|
341
347
|
expect(entry.outcome).toBe("ok");
|
|
342
348
|
});
|
|
349
|
+
|
|
350
|
+
test("prompt.via is used when ask omits via", async () => {
|
|
351
|
+
const failing = {
|
|
352
|
+
driverId: "mock" as const,
|
|
353
|
+
model: "smart",
|
|
354
|
+
async complete() {
|
|
355
|
+
throw new Error("smart down");
|
|
356
|
+
},
|
|
357
|
+
};
|
|
358
|
+
const ok = await createMockAiDriver({
|
|
359
|
+
"*": { summary: "ok" },
|
|
360
|
+
}).open({ model: "local" });
|
|
361
|
+
const smart = ai.model("smart", { provider: "mock" });
|
|
362
|
+
const local = ai.model("local", { provider: "mock" });
|
|
363
|
+
const summarize = smart.prompt("summarize-note", {
|
|
364
|
+
via: ["smart", "local"],
|
|
365
|
+
timeout: "30s",
|
|
366
|
+
});
|
|
367
|
+
const runtime = createAiRuntime({
|
|
368
|
+
models: [smart, local],
|
|
369
|
+
prompts: [summarize],
|
|
370
|
+
clients: { smart: failing, local: ok },
|
|
371
|
+
});
|
|
372
|
+
const out = await runtime.ask("summarize-note", { body: "x" });
|
|
373
|
+
expect(out.summary).toBe("ok");
|
|
374
|
+
expect(out.via).toBe("local");
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
test("permanent 401 does not advance via", async () => {
|
|
378
|
+
const unauthorized = {
|
|
379
|
+
driverId: "mock" as const,
|
|
380
|
+
model: "smart",
|
|
381
|
+
async complete() {
|
|
382
|
+
const err = new Error("openai-compatible HTTP 401") as Error & { status: number };
|
|
383
|
+
err.status = 401;
|
|
384
|
+
throw err;
|
|
385
|
+
},
|
|
386
|
+
};
|
|
387
|
+
let localCalls = 0;
|
|
388
|
+
const local = {
|
|
389
|
+
driverId: "mock" as const,
|
|
390
|
+
model: "local",
|
|
391
|
+
async complete() {
|
|
392
|
+
localCalls++;
|
|
393
|
+
return { text: "{}", raw: { summary: "nope" }, model: "local", driverId: "mock" as const };
|
|
394
|
+
},
|
|
395
|
+
};
|
|
396
|
+
const smart = ai.model("smart", { provider: "mock" });
|
|
397
|
+
const localModel = ai.model("local", { provider: "mock" });
|
|
398
|
+
const prompt = smart.prompt("summarize-note", { via: ["smart", "local"] });
|
|
399
|
+
const runtime = createAiRuntime({
|
|
400
|
+
models: [smart, localModel],
|
|
401
|
+
prompts: [prompt],
|
|
402
|
+
clients: { smart: unauthorized, local },
|
|
403
|
+
});
|
|
404
|
+
await expect(runtime.ask("summarize-note", {})).rejects.toThrow(/401/);
|
|
405
|
+
expect(localCalls).toBe(0);
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
test("timeout aborts a hanging complete", async () => {
|
|
409
|
+
const hanging = {
|
|
410
|
+
driverId: "mock" as const,
|
|
411
|
+
model: "smart",
|
|
412
|
+
async complete(opts: { signal?: AbortSignal }) {
|
|
413
|
+
const signal = opts.signal;
|
|
414
|
+
if (!signal) throw new Error("expected abort signal");
|
|
415
|
+
await new Promise<void>((_resolve, reject) => {
|
|
416
|
+
const onAbort = () => {
|
|
417
|
+
const err = new Error("aborted");
|
|
418
|
+
err.name = "AbortError";
|
|
419
|
+
reject(err);
|
|
420
|
+
};
|
|
421
|
+
if (signal.aborted) {
|
|
422
|
+
onAbort();
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
426
|
+
});
|
|
427
|
+
return { text: "late", raw: { text: "late" }, model: "smart", driverId: "mock" as const };
|
|
428
|
+
},
|
|
429
|
+
};
|
|
430
|
+
const smart = ai.model("smart", { provider: "mock" });
|
|
431
|
+
const prompt = smart.prompt("hang", { timeout: 40 });
|
|
432
|
+
const runtime = createAiRuntime({
|
|
433
|
+
models: [smart],
|
|
434
|
+
prompts: [prompt],
|
|
435
|
+
clients: { smart: hanging },
|
|
436
|
+
forceJournal: false,
|
|
437
|
+
});
|
|
438
|
+
await expect(runtime.ask("hang", {})).rejects.toThrow();
|
|
439
|
+
}, 2_000);
|
|
343
440
|
});
|
|
344
441
|
|
|
345
442
|
describe("schema-validation is its own class", () => {
|
package/src/elements/ai.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* @module
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
export { ai } from "./ai/declare.ts";
|
|
15
|
+
export { ai, listAiDecls, resetAiDecls } from "./ai/declare.ts";
|
|
16
16
|
export type {
|
|
17
17
|
AiAgentDecl,
|
|
18
18
|
AiAgentOptions,
|
|
@@ -23,8 +23,18 @@ export type {
|
|
|
23
23
|
AiModelOptions,
|
|
24
24
|
AiPromptDecl,
|
|
25
25
|
AiPromptOptions,
|
|
26
|
+
AiTimeout,
|
|
26
27
|
} from "./ai/declare.ts";
|
|
27
28
|
|
|
29
|
+
export {
|
|
30
|
+
aiHttpError,
|
|
31
|
+
isRetryableAiError,
|
|
32
|
+
mergeAskAbortSignal,
|
|
33
|
+
outExpectsVia,
|
|
34
|
+
resolveTimeoutMs,
|
|
35
|
+
} from "./ai/errors.ts";
|
|
36
|
+
export type { AiErrorFields } from "./ai/errors.ts";
|
|
37
|
+
|
|
28
38
|
export {
|
|
29
39
|
createAiRuntime,
|
|
30
40
|
AiSchemaValidationError,
|