okengine 0.11.1 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/package.json +1 -1
  2. package/site/content/docs/elements/ai.mdx +22 -1
  3. package/site/content/docs/elements/store.mdx +3 -1
  4. package/site/content/docs/elements/vault.mdx +19 -11
  5. package/site/content/docs/get-started/installation.mdx +7 -1
  6. package/site/content/docs/recipes/llama-cpp.mdx +10 -9
  7. package/site/content/docs/reference/cli.md +6 -2
  8. package/site/content/docs/reference/environment-variables.mdx +9 -9
  9. package/src/cli/ai-setup/ai-setup.test.ts +3 -1
  10. package/src/cli/ai-setup/apply.ts +61 -1
  11. package/src/cli/ask-seed.test.ts +4 -3
  12. package/src/cli/ask-seed.ts +5 -6
  13. package/src/cli/client-add.test.ts +2 -1
  14. package/src/cli/dev.test.ts +116 -0
  15. package/src/cli/dev.ts +107 -18
  16. package/src/cli/project-state.test.ts +50 -0
  17. package/src/cli/project-state.ts +123 -0
  18. package/src/cli/vault-cmd.test.ts +47 -18
  19. package/src/cli/vault-cmd.ts +2 -1
  20. package/src/compiler/extract.ts +12 -1
  21. package/src/console/server/console.test.ts +3 -1
  22. package/src/console/server/operator-db.test.ts +48 -17
  23. package/src/console/server/operator-db.ts +5 -1
  24. package/src/docker/derive.ts +24 -3
  25. package/src/docker/docker.test.ts +4 -2
  26. package/src/docker/index.ts +1 -0
  27. package/src/docker/recipes/index.ts +1 -0
  28. package/src/docker/recipes/llama-cpp.ts +20 -4
  29. package/src/drivers/ai-openai-compatible.ts +15 -3
  30. package/src/drivers/vault-builtin.test.ts +50 -42
  31. package/src/elements/ai/declare.ts +73 -3
  32. package/src/elements/ai/errors.test.ts +35 -0
  33. package/src/elements/ai/errors.ts +139 -0
  34. package/src/elements/ai/eval.ts +26 -1
  35. package/src/elements/ai/runtime.ts +140 -80
  36. package/src/elements/ai/tools.test.ts +1 -1
  37. package/src/elements/ai.test.ts +99 -2
  38. package/src/elements/ai.ts +11 -1
  39. package/src/elements/index.ts +2 -0
  40. package/src/elements/store/index-boot.test.ts +23 -6
  41. package/src/elements/store/resource.test.ts +38 -19
  42. package/src/elements/store/sql-session.test.ts +55 -58
  43. package/src/elements/vault/builtin-adapter.test.ts +115 -58
  44. package/src/elements/vault/builtin-adapter.ts +241 -47
  45. package/src/elements/vault/chaos-child.ts +424 -0
  46. package/src/elements/vault/chaos.test.ts +651 -0
  47. package/src/elements/vault/resilience.ts +6 -1
  48. package/src/elements/vault/security-checklist.test.ts +10 -8
  49. package/src/elements/vault/storage.ts +130 -27
  50. package/src/elements/vault/test-helpers.ts +368 -0
  51. package/src/elements/vault.ts +6 -0
  52. package/src/index.ts +2 -0
  53. package/src/kernel/app.ts +83 -10
  54. package/src/kernel/auto-registry.test.ts +52 -1
  55. package/src/kernel/element-registries.ts +19 -4
  56. package/src/kernel/errors.ts +3 -3
  57. package/src/kernel/fx.test.ts +25 -0
  58. package/src/kernel/fx.ts +4 -1
  59. package/src/manifest/types.ts +4 -0
  60. package/src/test/create-test-app.ts +16 -11
  61. package/src/test/reset-element-registries.ts +17 -9
@@ -0,0 +1,35 @@
1
+ /**
2
+ * AI error classification + timeout parsing.
3
+ */
4
+
5
+ import { describe, expect, test } from "bun:test";
6
+ import { aiHttpError, isRetryableAiError, outExpectsVia, resolveTimeoutMs } from "./errors.ts";
7
+
8
+ describe("resolveTimeoutMs", () => {
9
+ test("parses clock durations and ms numbers", () => {
10
+ expect(resolveTimeoutMs("30s")).toBe(30_000);
11
+ expect(resolveTimeoutMs(250)).toBe(250);
12
+ expect(resolveTimeoutMs(undefined)).toBeUndefined();
13
+ expect(resolveTimeoutMs("nope")).toBeUndefined();
14
+ });
15
+ });
16
+
17
+ describe("isRetryableAiError", () => {
18
+ test("classifies status and HTTP message", () => {
19
+ expect(isRetryableAiError(aiHttpError("rate", 429))).toBe(true);
20
+ expect(isRetryableAiError(aiHttpError("boom", 503))).toBe(true);
21
+ expect(isRetryableAiError(aiHttpError("auth", 401))).toBe(false);
22
+ expect(isRetryableAiError(new Error("openai-compatible HTTP 401"))).toBe(false);
23
+ const abort = new Error("aborted");
24
+ abort.name = "AbortError";
25
+ expect(isRetryableAiError(abort)).toBe(true);
26
+ });
27
+ });
28
+
29
+ describe("outExpectsVia", () => {
30
+ test("detects JSON-schema and Zod-like shape", () => {
31
+ expect(outExpectsVia({ properties: { via: { type: "string" } } })).toBe(true);
32
+ expect(outExpectsVia({ shape: { via: {} } })).toBe(true);
33
+ expect(outExpectsVia({ properties: { summary: { type: "string" } } })).toBe(false);
34
+ });
35
+ });
@@ -0,0 +1,139 @@
1
+ /**
2
+ * AI provider / transport error classification for recovery chains.
3
+ *
4
+ * Retryable failures may retry once on the same model, then advance `via`.
5
+ * Permanent failures stop the chain immediately.
6
+ */
7
+
8
+ import { parseDurationMs } from "../clock/duration.ts";
9
+ import type { AiTimeout } from "./declare.ts";
10
+
11
+ /** Optional structured fields drivers may attach to thrown errors. */
12
+ export interface AiErrorFields {
13
+ readonly status?: number;
14
+ readonly code?: string;
15
+ }
16
+
17
+ /**
18
+ * Resolve a prompt/ask timeout to milliseconds.
19
+ *
20
+ * @param timeout - Duration string (`"30s"`) or ms number
21
+ * @returns Milliseconds, or `undefined` when unset / invalid
22
+ */
23
+ export function resolveTimeoutMs(timeout: AiTimeout | undefined): number | undefined {
24
+ if (timeout === undefined) return undefined;
25
+ if (typeof timeout === "number") {
26
+ return Number.isFinite(timeout) && timeout > 0 ? timeout : undefined;
27
+ }
28
+ const ms = parseDurationMs(timeout);
29
+ return ms > 0 ? ms : undefined;
30
+ }
31
+
32
+ /**
33
+ * Merge an optional deadline into an ambient abort signal.
34
+ *
35
+ * @param timeoutMs - Deadline in ms
36
+ * @param ambient - Existing signal (e.g. request cancel)
37
+ */
38
+ export function mergeAskAbortSignal(
39
+ timeoutMs: number | undefined,
40
+ ambient?: AbortSignal,
41
+ ): AbortSignal | undefined {
42
+ if (timeoutMs === undefined) return ambient;
43
+ const deadline = AbortSignal.timeout(timeoutMs);
44
+ if (!ambient) return deadline;
45
+ if (typeof AbortSignal.any === "function") {
46
+ return AbortSignal.any([ambient, deadline]);
47
+ }
48
+ return deadline;
49
+ }
50
+
51
+ /**
52
+ * HTTP / abort / network failures that may succeed on retry or another model.
53
+ *
54
+ * @param err - Thrown value from a model attempt
55
+ */
56
+ export function isRetryableAiError(err: unknown): boolean {
57
+ if (err == null) return false;
58
+
59
+ const status = readStatus(err);
60
+ if (status !== undefined) {
61
+ if (status === 429) return true;
62
+ if (status >= 500 && status <= 599) return true;
63
+ if (status >= 400 && status <= 499) return false;
64
+ }
65
+
66
+ const name = err instanceof Error ? err.name : "";
67
+ const message = err instanceof Error ? err.message : String(err);
68
+ const lower = message.toLowerCase();
69
+
70
+ const httpMatch = /\bHTTP\s*(\d{3})\b/i.exec(message);
71
+ if (httpMatch?.[1]) {
72
+ const code = Number(httpMatch[1]);
73
+ if (code === 429 || (code >= 500 && code <= 599)) return true;
74
+ if (code >= 400 && code <= 499) return false;
75
+ }
76
+
77
+ if (name === "AbortError" || name === "TimeoutError") return true;
78
+ if (lower.includes("aborterror") || lower.includes("timeout")) return true;
79
+ if (
80
+ lower.includes("econnreset") ||
81
+ lower.includes("econnrefused") ||
82
+ lower.includes("enotfound") ||
83
+ lower.includes("socket hang up") ||
84
+ lower.includes("fetch failed") ||
85
+ lower.includes("network")
86
+ ) {
87
+ return true;
88
+ }
89
+
90
+ // Unclassified provider/transport errors — allow recovery to the next model.
91
+ return err instanceof Error;
92
+ }
93
+
94
+ /**
95
+ * Attach an HTTP status onto an Error for {@link isRetryableAiError}.
96
+ *
97
+ * @param message - Error message
98
+ * @param status - HTTP status
99
+ */
100
+ export function aiHttpError(message: string, status: number): Error {
101
+ const err = new Error(message) as Error & { status: number };
102
+ err.status = status;
103
+ return err;
104
+ }
105
+
106
+ function readStatus(err: unknown): number | undefined {
107
+ if (typeof err !== "object" || err === null) return undefined;
108
+ const status = (err as AiErrorFields).status;
109
+ return typeof status === "number" && Number.isFinite(status) ? status : undefined;
110
+ }
111
+
112
+ /**
113
+ * Whether a declared prompt `out` schema expects a `via` field
114
+ * (JSON-schema properties or Zod `.shape`).
115
+ *
116
+ * @param schema - Prompt `out` declaration
117
+ */
118
+ export function outExpectsVia(schema: unknown): boolean {
119
+ if (schema == null || typeof schema !== "object") return false;
120
+ if (
121
+ "properties" in schema &&
122
+ schema.properties &&
123
+ typeof schema.properties === "object" &&
124
+ !Array.isArray(schema.properties) &&
125
+ "via" in (schema.properties as Record<string, unknown>)
126
+ ) {
127
+ return true;
128
+ }
129
+ if (
130
+ "shape" in schema &&
131
+ schema.shape &&
132
+ typeof schema.shape === "object" &&
133
+ !Array.isArray(schema.shape) &&
134
+ "via" in (schema.shape as Record<string, unknown>)
135
+ ) {
136
+ return true;
137
+ }
138
+ return false;
139
+ }
@@ -47,13 +47,38 @@ export interface RunPromptEvalsOptions {
47
47
  readonly equals?: (actual: unknown, expect: unknown) => boolean;
48
48
  }
49
49
 
50
+ /**
51
+ * Deep equality for eval expectations — ignore a runtime-stamped `via`
52
+ * when the case did not declare one.
53
+ *
54
+ * @param actual - Ask output
55
+ * @param expect - Case expectation
56
+ */
57
+ function defaultEvalEquals(actual: unknown, expect: unknown): boolean {
58
+ if (JSON.stringify(actual) === JSON.stringify(expect)) return true;
59
+ if (
60
+ actual &&
61
+ typeof actual === "object" &&
62
+ !Array.isArray(actual) &&
63
+ expect &&
64
+ typeof expect === "object" &&
65
+ !Array.isArray(expect) &&
66
+ !("via" in expect) &&
67
+ "via" in actual
68
+ ) {
69
+ const { via: _via, ...rest } = actual as Record<string, unknown>;
70
+ return JSON.stringify(rest) === JSON.stringify(expect);
71
+ }
72
+ return false;
73
+ }
74
+
50
75
  /**
51
76
  * Run a prompt eval set. Fails CI when any case fails (`ok === false`).
52
77
  *
53
78
  * @param options - Cases + ask fn
54
79
  */
55
80
  export async function runPromptEvals(options: RunPromptEvalsOptions): Promise<EvalSuiteResult> {
56
- const equals = options.equals ?? ((a, b) => JSON.stringify(a) === JSON.stringify(b));
81
+ const equals = options.equals ?? defaultEvalEquals;
57
82
  const results: EvalCaseResult[] = [];
58
83
 
59
84
  for (let i = 0; i < options.cases.length; i++) {
@@ -12,7 +12,13 @@ import type { IndexStore } from "../../drivers/types.ts";
12
12
  import { maskRedactedDeep } from "../../kernel/redacted.ts";
13
13
  import type { GatePolicyContext } from "../gate/declare.ts";
14
14
  import type { GateRuntime } from "../gate/runtime.ts";
15
- import type { AiAgentDecl, AiEmbedDecl, AiModelDecl, AiPromptDecl } from "./declare.ts";
15
+ import type { AiAgentDecl, AiEmbedDecl, AiModelDecl, AiPromptDecl, AiTimeout } from "./declare.ts";
16
+ import {
17
+ isRetryableAiError,
18
+ mergeAskAbortSignal,
19
+ outExpectsVia,
20
+ resolveTimeoutMs,
21
+ } from "./errors.ts";
16
22
  import {
17
23
  AiSchemaValidationError,
18
24
  coerceModelObject,
@@ -20,6 +26,9 @@ import {
20
26
  type AiSchemaMismatch,
21
27
  } from "./schema.ts";
22
28
 
29
+ /** Brief pause before the same-model retry on a retryable failure. */
30
+ const AI_SAME_MODEL_RETRY_BACKOFF_MS = 250;
31
+
23
32
  /** Default bound for tool / agent loops. */
24
33
  export const AI_DEFAULT_MAX_STEPS = 6;
25
34
 
@@ -146,6 +155,8 @@ export interface CreateAiRuntimeOptions {
146
155
  /** Ask options. */
147
156
  export interface AiAskOptions {
148
157
  readonly via?: readonly string[];
158
+ /** Per-call deadline — overrides prompt `timeout` (`"30s"` or ms). */
159
+ readonly timeout?: AiTimeout;
149
160
  readonly allowPii?: boolean;
150
161
  /** Flow names offered as tools — each model call dispatches via `callTool`. */
151
162
  readonly tools?: readonly string[];
@@ -275,6 +286,8 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
275
286
  const model = models.get(name);
276
287
  const opened = await options.defaultDriver.open({
277
288
  model: model?.model ?? name,
289
+ ...(model?.baseUrl !== undefined ? { baseUrl: model.baseUrl } : {}),
290
+ ...(model?.apiKey !== undefined ? { apiKey: model.apiKey } : {}),
278
291
  });
279
292
  clients.set(name, opened);
280
293
  return opened;
@@ -389,6 +402,7 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
389
402
  readonly maxSteps: number;
390
403
  readonly agentLabel: string;
391
404
  readonly responseFormat?: unknown;
405
+ readonly signal?: AbortSignal;
392
406
  readonly callTool?: (name: string, input: unknown) => Promise<unknown>;
393
407
  readonly auth?: GatePolicyContext["auth"];
394
408
  readonly operator?: GatePolicyContext["operator"];
@@ -421,6 +435,7 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
421
435
  messages,
422
436
  tools: defs.length > 0 ? defs : undefined,
423
437
  responseFormat: opts.responseFormat,
438
+ ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
424
439
  });
425
440
  cost += result.usage?.cost ?? 0;
426
441
  lastText = result.text;
@@ -498,6 +513,7 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
498
513
  const version = decl.version;
499
514
  const started = now();
500
515
  const tools = opts?.tools ?? [];
516
+ const signal = mergeAskAbortSignal(resolveTimeoutMs(opts?.timeout ?? decl.timeout));
501
517
 
502
518
  // Replay from journal when input matches (nondeterministic contract)
503
519
  if (journalingForced && tools.length === 0) {
@@ -514,7 +530,8 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
514
530
  }
515
531
  }
516
532
 
517
- const via = opts?.via ?? (decl.model ? [decl.model] : [...models.keys()].slice(0, 1));
533
+ const via =
534
+ opts?.via ?? decl.via ?? (decl.model ? [decl.model] : [...models.keys()].slice(0, 1));
518
535
  const attempts: AiFallbackAttempt[] = [];
519
536
  let lastError: string | undefined;
520
537
  let lastSchema: AiSchemaMismatch | undefined;
@@ -522,72 +539,66 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
522
539
  const userContent = promptContentFromInput(input);
523
540
 
524
541
  for (const modelName of via) {
525
- const attemptStart = now();
526
- try {
527
- const client = await clientFor(modelName);
528
- let raw: unknown;
529
- let attemptCost = 0;
530
-
531
- if (tools.length > 0) {
532
- const loop = await toolLoop({
533
- client,
534
- modelName,
535
- messages: [{ role: "user", content: userContent }],
536
- tools,
537
- maxSteps: opts?.maxSteps ?? AI_DEFAULT_MAX_STEPS,
538
- agentLabel: prompt,
539
- responseFormat: decl.out,
540
- callTool: opts?.callTool,
541
- });
542
- raw = loop.lastToolResult !== undefined && !loop.text ? loop.lastToolResult : loop.raw;
543
- attemptCost = loop.cost;
544
- totalCost += attemptCost;
545
- if (loop.denials.length > 0 && loop.trail.every((t) => t.status === "denied")) {
546
- throw new Error(
547
- `ai: all tool calls denied for prompt "${prompt}": ${loop.denials[0]?.reason}`,
548
- );
549
- }
550
- } else {
551
- const result = await client.complete({
552
- model: wireModel(modelName, client),
553
- messages: [{ role: "user", content: userContent }],
554
- responseFormat: decl.out,
555
- });
556
- attemptCost = result.usage?.cost ?? 0;
557
- totalCost += attemptCost;
558
- raw = result.raw !== undefined ? result.raw : result.text;
559
- }
560
-
561
- const latencyMs = Math.max(0, now() - attemptStart);
562
-
542
+ let sameModelTries = 0;
543
+ let advance = true;
544
+ while (sameModelTries < 2 && advance) {
545
+ sameModelTries++;
546
+ const attemptStart = now();
563
547
  try {
564
- const output = decl.out
565
- ? validatePromptOut(prompt, version, decl.out, raw)
566
- : coerceModelObject(raw);
567
- attempts.push({
568
- model: modelName,
569
- ok: true,
570
- cost: attemptCost,
571
- latencyMs,
572
- at: now(),
573
- });
574
- if (journalingForced) {
575
- journal.push({
576
- prompt,
577
- ...(version !== undefined ? { version } : {}),
578
- input,
579
- output,
580
- attempts: [...attempts],
581
- outcome: "ok",
582
- cost: totalCost,
583
- latencyMs: Math.max(0, now() - started),
584
- at: now(),
548
+ const client = await clientFor(modelName);
549
+ let raw: unknown;
550
+ let attemptCost = 0;
551
+
552
+ if (tools.length > 0) {
553
+ const loop = await toolLoop({
554
+ client,
555
+ modelName,
556
+ messages: [{ role: "user", content: userContent }],
557
+ tools,
558
+ maxSteps: opts?.maxSteps ?? AI_DEFAULT_MAX_STEPS,
559
+ agentLabel: prompt,
560
+ responseFormat: decl.out,
561
+ callTool: opts?.callTool,
562
+ ...(signal !== undefined ? { signal } : {}),
563
+ });
564
+ raw =
565
+ loop.lastToolResult !== undefined && !loop.text ? loop.lastToolResult : loop.raw;
566
+ attemptCost = loop.cost;
567
+ totalCost += attemptCost;
568
+ if (loop.denials.length > 0 && loop.trail.every((t) => t.status === "denied")) {
569
+ throw new Error(
570
+ `ai: all tool calls denied for prompt "${prompt}": ${loop.denials[0]?.reason}`,
571
+ );
572
+ }
573
+ } else {
574
+ const result = await client.complete({
575
+ model: wireModel(modelName, client),
576
+ messages: [{ role: "user", content: userContent }],
577
+ responseFormat: decl.out,
578
+ ...(signal !== undefined ? { signal } : {}),
585
579
  });
580
+ attemptCost = result.usage?.cost ?? 0;
581
+ totalCost += attemptCost;
582
+ // Prefer assistant text — `raw` is often the transport envelope
583
+ // (OpenAI chat.completion object), which must not shadow the content.
584
+ raw =
585
+ typeof result.text === "string" && result.text.length > 0
586
+ ? result.text
587
+ : result.raw !== undefined
588
+ ? result.raw
589
+ : result.text;
586
590
  }
587
- return output;
588
- } catch (err) {
589
- if (err instanceof AiSchemaValidationError) {
590
- lastSchema = err.mismatch;
591
+
592
+ const latencyMs = Math.max(0, now() - attemptStart);
593
+
594
+ try {
595
+ const coerced = coerceModelObject(raw);
596
+ const prepared = outExpectsVia(decl.out) ? { ...coerced, via: modelName } : coerced;
597
+ const validated = decl.out
598
+ ? validatePromptOut(prompt, version, decl.out, prepared)
599
+ : prepared;
600
+ // Always report the winning logical model for recovery chains.
601
+ const output = { ...validated, via: modelName };
591
602
  attempts.push({
592
603
  model: modelName,
593
604
  ok: true,
@@ -600,30 +611,79 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
600
611
  prompt,
601
612
  ...(version !== undefined ? { version } : {}),
602
613
  input,
603
- output: coerceModelObject(raw),
614
+ output,
604
615
  attempts: [...attempts],
605
- outcome: "schema_invalid",
616
+ outcome: "ok",
606
617
  cost: totalCost,
607
618
  latencyMs: Math.max(0, now() - started),
608
- schemaMismatch: err.mismatch,
609
619
  at: now(),
610
620
  });
611
621
  }
622
+ return output;
623
+ } catch (err) {
624
+ if (err instanceof AiSchemaValidationError) {
625
+ lastSchema = err.mismatch;
626
+ attempts.push({
627
+ model: modelName,
628
+ ok: true,
629
+ cost: attemptCost,
630
+ latencyMs,
631
+ at: now(),
632
+ });
633
+ if (journalingForced) {
634
+ journal.push({
635
+ prompt,
636
+ ...(version !== undefined ? { version } : {}),
637
+ input,
638
+ output: coerceModelObject(raw),
639
+ attempts: [...attempts],
640
+ outcome: "schema_invalid",
641
+ cost: totalCost,
642
+ latencyMs: Math.max(0, now() - started),
643
+ schemaMismatch: err.mismatch,
644
+ at: now(),
645
+ });
646
+ }
647
+ throw err;
648
+ }
612
649
  throw err;
613
650
  }
614
- throw err;
651
+ } catch (err) {
652
+ if (err instanceof AiSchemaValidationError) throw err;
653
+ lastError = err instanceof Error ? err.message : String(err);
654
+ attempts.push({
655
+ model: modelName,
656
+ ok: false,
657
+ error: lastError,
658
+ cost: 0,
659
+ latencyMs: Math.max(0, now() - attemptStart),
660
+ at: now(),
661
+ });
662
+
663
+ if (!isRetryableAiError(err)) {
664
+ if (journalingForced) {
665
+ journal.push({
666
+ prompt,
667
+ ...(version !== undefined ? { version } : {}),
668
+ input,
669
+ output: { error: lastError },
670
+ attempts,
671
+ outcome: "provider_error",
672
+ cost: totalCost,
673
+ latencyMs: Math.max(0, now() - started),
674
+ at: now(),
675
+ });
676
+ }
677
+ throw err instanceof Error ? err : new Error(String(err));
678
+ }
679
+
680
+ if (sameModelTries < 2) {
681
+ await new Promise((r) => setTimeout(r, AI_SAME_MODEL_RETRY_BACKOFF_MS));
682
+ continue;
683
+ }
684
+ advance = true;
685
+ break;
615
686
  }
616
- } catch (err) {
617
- if (err instanceof AiSchemaValidationError) throw err;
618
- lastError = err instanceof Error ? err.message : String(err);
619
- attempts.push({
620
- model: modelName,
621
- ok: false,
622
- error: lastError,
623
- cost: 0,
624
- latencyMs: Math.max(0, now() - attemptStart),
625
- at: now(),
626
- });
627
687
  }
628
688
  }
629
689
 
@@ -58,7 +58,7 @@ describe("fx.ask tools via fx.call", () => {
58
58
  });
59
59
 
60
60
  const out = await fx.ask(prompt, { q: "status?" }, { tools: ["lookup.booking"], maxSteps: 4 });
61
- expect(out).toEqual({ answer: "found" });
61
+ expect(out).toEqual({ answer: "found", via: "smart" });
62
62
  expect(calls).toEqual([{ name: "lookup.booking", input: { id: "B9" } }]);
63
63
  expect(ledger.entries.some((e) => e.kind === "ask" && e.resource === "assistant")).toBe(true);
64
64
  expect(
@@ -29,10 +29,14 @@ describe("ai declaration", () => {
29
29
  version: 3,
30
30
  evals: "./evals/triage.jsonl",
31
31
  budget: { maxCostPerCall: 0.02 },
32
+ via: ["smart", "fast"],
33
+ timeout: "30s",
32
34
  });
33
35
  expect(triage.name).toBe("ticket-triage");
34
36
  expect(triage.version).toBe(3);
35
37
  expect(triage.model).toBe("smart");
38
+ expect(triage.via).toEqual(["smart", "fast"]);
39
+ expect(triage.timeout).toBe("30s");
36
40
 
37
41
  const agent = ai.agent("support", {
38
42
  tools: [{ name: "bookings.getBooking" }, "bookings.refundBooking"],
@@ -334,12 +338,105 @@ describe("model fallback chain", () => {
334
338
  },
335
339
  );
336
340
  expect(out.urgency).toBe("low");
341
+ expect(out.via).toBe("fast");
337
342
  const entry = runtime.journal[0]!;
338
- expect(entry.attempts).toHaveLength(2);
343
+ // Same-model retry (1) + second model success.
344
+ expect(entry.attempts.length).toBeGreaterThanOrEqual(2);
339
345
  expect(entry.attempts[0]).toMatchObject({ model: "smart", ok: false });
340
- expect(entry.attempts[1]).toMatchObject({ model: "fast", ok: true });
346
+ expect(entry.attempts.at(-1)).toMatchObject({ model: "fast", ok: true });
341
347
  expect(entry.outcome).toBe("ok");
342
348
  });
349
+
350
+ test("prompt.via is used when ask omits via", async () => {
351
+ const failing = {
352
+ driverId: "mock" as const,
353
+ model: "smart",
354
+ async complete() {
355
+ throw new Error("smart down");
356
+ },
357
+ };
358
+ const ok = await createMockAiDriver({
359
+ "*": { summary: "ok" },
360
+ }).open({ model: "local" });
361
+ const smart = ai.model("smart", { provider: "mock" });
362
+ const local = ai.model("local", { provider: "mock" });
363
+ const summarize = smart.prompt("summarize-note", {
364
+ via: ["smart", "local"],
365
+ timeout: "30s",
366
+ });
367
+ const runtime = createAiRuntime({
368
+ models: [smart, local],
369
+ prompts: [summarize],
370
+ clients: { smart: failing, local: ok },
371
+ });
372
+ const out = await runtime.ask("summarize-note", { body: "x" });
373
+ expect(out.summary).toBe("ok");
374
+ expect(out.via).toBe("local");
375
+ });
376
+
377
+ test("permanent 401 does not advance via", async () => {
378
+ const unauthorized = {
379
+ driverId: "mock" as const,
380
+ model: "smart",
381
+ async complete() {
382
+ const err = new Error("openai-compatible HTTP 401") as Error & { status: number };
383
+ err.status = 401;
384
+ throw err;
385
+ },
386
+ };
387
+ let localCalls = 0;
388
+ const local = {
389
+ driverId: "mock" as const,
390
+ model: "local",
391
+ async complete() {
392
+ localCalls++;
393
+ return { text: "{}", raw: { summary: "nope" }, model: "local", driverId: "mock" as const };
394
+ },
395
+ };
396
+ const smart = ai.model("smart", { provider: "mock" });
397
+ const localModel = ai.model("local", { provider: "mock" });
398
+ const prompt = smart.prompt("summarize-note", { via: ["smart", "local"] });
399
+ const runtime = createAiRuntime({
400
+ models: [smart, localModel],
401
+ prompts: [prompt],
402
+ clients: { smart: unauthorized, local },
403
+ });
404
+ await expect(runtime.ask("summarize-note", {})).rejects.toThrow(/401/);
405
+ expect(localCalls).toBe(0);
406
+ });
407
+
408
+ test("timeout aborts a hanging complete", async () => {
409
+ const hanging = {
410
+ driverId: "mock" as const,
411
+ model: "smart",
412
+ async complete(opts: { signal?: AbortSignal }) {
413
+ const signal = opts.signal;
414
+ if (!signal) throw new Error("expected abort signal");
415
+ await new Promise<void>((_resolve, reject) => {
416
+ const onAbort = () => {
417
+ const err = new Error("aborted");
418
+ err.name = "AbortError";
419
+ reject(err);
420
+ };
421
+ if (signal.aborted) {
422
+ onAbort();
423
+ return;
424
+ }
425
+ signal.addEventListener("abort", onAbort, { once: true });
426
+ });
427
+ return { text: "late", raw: { text: "late" }, model: "smart", driverId: "mock" as const };
428
+ },
429
+ };
430
+ const smart = ai.model("smart", { provider: "mock" });
431
+ const prompt = smart.prompt("hang", { timeout: 40 });
432
+ const runtime = createAiRuntime({
433
+ models: [smart],
434
+ prompts: [prompt],
435
+ clients: { smart: hanging },
436
+ forceJournal: false,
437
+ });
438
+ await expect(runtime.ask("hang", {})).rejects.toThrow();
439
+ }, 2_000);
343
440
  });
344
441
 
345
442
  describe("schema-validation is its own class", () => {
@@ -12,7 +12,7 @@
12
12
  * @module
13
13
  */
14
14
 
15
- export { ai } from "./ai/declare.ts";
15
+ export { ai, listAiDecls, resetAiDecls } from "./ai/declare.ts";
16
16
  export type {
17
17
  AiAgentDecl,
18
18
  AiAgentOptions,
@@ -23,8 +23,18 @@ export type {
23
23
  AiModelOptions,
24
24
  AiPromptDecl,
25
25
  AiPromptOptions,
26
+ AiTimeout,
26
27
  } from "./ai/declare.ts";
27
28
 
29
+ export {
30
+ aiHttpError,
31
+ isRetryableAiError,
32
+ mergeAskAbortSignal,
33
+ outExpectsVia,
34
+ resolveTimeoutMs,
35
+ } from "./ai/errors.ts";
36
+ export type { AiErrorFields } from "./ai/errors.ts";
37
+
28
38
  export {
29
39
  createAiRuntime,
30
40
  AiSchemaValidationError,
@@ -163,6 +163,8 @@ export type {
163
163
 
164
164
  export {
165
165
  ai,
166
+ listAiDecls,
167
+ resetAiDecls,
166
168
  createAiRuntime,
167
169
  assertAllowPiiForAsk,
168
170
  AiPiiBuildError,