ai-runtime-engine 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/CHANGELOG.md +78 -0
  2. package/LICENSE +15 -0
  3. package/README.md +489 -0
  4. package/dist/artifacts/artifacts.d.ts +28 -0
  5. package/dist/artifacts/artifacts.js +46 -0
  6. package/dist/benchmark/benchmark.d.ts +23 -0
  7. package/dist/benchmark/benchmark.js +40 -0
  8. package/dist/cli/cli.d.ts +6 -0
  9. package/dist/cli/cli.js +161 -0
  10. package/dist/cli/commands/cleanup.d.ts +47 -0
  11. package/dist/cli/commands/cleanup.js +96 -0
  12. package/dist/cli/commands/config.d.ts +8 -0
  13. package/dist/cli/commands/config.js +28 -0
  14. package/dist/cli/commands/doctor.d.ts +57 -0
  15. package/dist/cli/commands/doctor.js +86 -0
  16. package/dist/cli/commands/executions.d.ts +9 -0
  17. package/dist/cli/commands/executions.js +25 -0
  18. package/dist/cli/commands/info.d.ts +43 -0
  19. package/dist/cli/commands/info.js +53 -0
  20. package/dist/cli/commands/init.d.ts +5 -0
  21. package/dist/cli/commands/init.js +75 -0
  22. package/dist/cli/commands/inspect.d.ts +16 -0
  23. package/dist/cli/commands/inspect.js +60 -0
  24. package/dist/cli/commands/phase2.d.ts +22 -0
  25. package/dist/cli/commands/phase2.js +83 -0
  26. package/dist/cli/commands/route.d.ts +14 -0
  27. package/dist/cli/commands/route.js +49 -0
  28. package/dist/cli/commands/run.d.ts +11 -0
  29. package/dist/cli/commands/run.js +37 -0
  30. package/dist/cli/commands/setup.d.ts +34 -0
  31. package/dist/cli/commands/setup.js +104 -0
  32. package/dist/cli/commands/skills.d.ts +28 -0
  33. package/dist/cli/commands/skills.js +48 -0
  34. package/dist/cli/commands/test.d.ts +7 -0
  35. package/dist/cli/commands/test.js +29 -0
  36. package/dist/cli/context.d.ts +12 -0
  37. package/dist/cli/context.js +16 -0
  38. package/dist/cli/interactive/repl.d.ts +6 -0
  39. package/dist/cli/interactive/repl.js +45 -0
  40. package/dist/cli/interactive/session.d.ts +36 -0
  41. package/dist/cli/interactive/session.js +356 -0
  42. package/dist/cli/prompt.d.ts +6 -0
  43. package/dist/cli/prompt.js +18 -0
  44. package/dist/cli/render.d.ts +7 -0
  45. package/dist/cli/render.js +14 -0
  46. package/dist/comparison/analysis.d.ts +46 -0
  47. package/dist/comparison/analysis.js +177 -0
  48. package/dist/comparison/comparator.d.ts +46 -0
  49. package/dist/comparison/comparator.js +270 -0
  50. package/dist/comparison/comparison.d.ts +140 -0
  51. package/dist/comparison/comparison.js +9 -0
  52. package/dist/comparison/render.d.ts +7 -0
  53. package/dist/comparison/render.js +66 -0
  54. package/dist/config/defaults.d.ts +52 -0
  55. package/dist/config/defaults.js +56 -0
  56. package/dist/config/load.d.ts +17 -0
  57. package/dist/config/load.js +50 -0
  58. package/dist/config/providerDefaults.d.ts +17 -0
  59. package/dist/config/providerDefaults.js +61 -0
  60. package/dist/config/schema.d.ts +9 -0
  61. package/dist/config/schema.js +78 -0
  62. package/dist/context/budget.d.ts +13 -0
  63. package/dist/context/budget.js +17 -0
  64. package/dist/context/compiler.d.ts +61 -0
  65. package/dist/context/compiler.js +125 -0
  66. package/dist/context/tokens.d.ts +19 -0
  67. package/dist/context/tokens.js +38 -0
  68. package/dist/conversations/conversations.d.ts +38 -0
  69. package/dist/conversations/conversations.js +64 -0
  70. package/dist/core/capabilities/evidence.d.ts +40 -0
  71. package/dist/core/capabilities/evidence.js +102 -0
  72. package/dist/core/capabilities/overlay.d.ts +15 -0
  73. package/dist/core/capabilities/overlay.js +0 -0
  74. package/dist/core/capabilities/taxonomy.d.ts +19 -0
  75. package/dist/core/capabilities/taxonomy.js +25 -0
  76. package/dist/core/fallback/errors.d.ts +30 -0
  77. package/dist/core/fallback/errors.js +80 -0
  78. package/dist/core/fallback/fallback.d.ts +40 -0
  79. package/dist/core/fallback/fallback.js +82 -0
  80. package/dist/core/fallback/retryPolicy.d.ts +11 -0
  81. package/dist/core/fallback/retryPolicy.js +14 -0
  82. package/dist/core/health/health.d.ts +3 -0
  83. package/dist/core/health/health.js +5 -0
  84. package/dist/core/health/monitor.d.ts +23 -0
  85. package/dist/core/health/monitor.js +82 -0
  86. package/dist/core/policies/budget.d.ts +19 -0
  87. package/dist/core/policies/budget.js +37 -0
  88. package/dist/core/registry/builtinTasks.d.ts +8 -0
  89. package/dist/core/registry/builtinTasks.js +54 -0
  90. package/dist/core/registry/registry.d.ts +18 -0
  91. package/dist/core/registry/registry.js +33 -0
  92. package/dist/core/registry/taskRegistry.d.ts +15 -0
  93. package/dist/core/registry/taskRegistry.js +30 -0
  94. package/dist/core/router/confidence.d.ts +7 -0
  95. package/dist/core/router/confidence.js +20 -0
  96. package/dist/core/router/dimensions.d.ts +16 -0
  97. package/dist/core/router/dimensions.js +60 -0
  98. package/dist/core/router/executor.d.ts +16 -0
  99. package/dist/core/router/executor.js +25 -0
  100. package/dist/core/router/filter.d.ts +34 -0
  101. package/dist/core/router/filter.js +113 -0
  102. package/dist/core/router/normalize.d.ts +30 -0
  103. package/dist/core/router/normalize.js +119 -0
  104. package/dist/core/router/request.d.ts +4 -0
  105. package/dist/core/router/request.js +21 -0
  106. package/dist/core/router/router.d.ts +32 -0
  107. package/dist/core/router/router.js +195 -0
  108. package/dist/core/router/routingPrefs.d.ts +11 -0
  109. package/dist/core/router/routingPrefs.js +30 -0
  110. package/dist/core/router/scorer.d.ts +19 -0
  111. package/dist/core/router/scorer.js +50 -0
  112. package/dist/core/router/weights.d.ts +9 -0
  113. package/dist/core/router/weights.js +31 -0
  114. package/dist/core/validation/validator.d.ts +16 -0
  115. package/dist/core/validation/validator.js +33 -0
  116. package/dist/discovery/modelCatalog.d.ts +28 -0
  117. package/dist/discovery/modelCatalog.js +105 -0
  118. package/dist/discovery/openapi.d.ts +25 -0
  119. package/dist/discovery/openapi.js +76 -0
  120. package/dist/executions/checkpoint.d.ts +26 -0
  121. package/dist/executions/checkpoint.js +114 -0
  122. package/dist/executions/execution.d.ts +51 -0
  123. package/dist/executions/execution.js +8 -0
  124. package/dist/executions/store.d.ts +52 -0
  125. package/dist/executions/store.js +124 -0
  126. package/dist/generation/generateAdapter.d.ts +17 -0
  127. package/dist/generation/generateAdapter.js +30 -0
  128. package/dist/index.d.ts +147 -0
  129. package/dist/index.js +107 -0
  130. package/dist/learning/feedback.d.ts +9 -0
  131. package/dist/learning/feedback.js +18 -0
  132. package/dist/learning/learningStore.d.ts +68 -0
  133. package/dist/learning/learningStore.js +138 -0
  134. package/dist/learning/performanceStore.d.ts +27 -0
  135. package/dist/learning/performanceStore.js +0 -0
  136. package/dist/marketplace/presets.d.ts +24 -0
  137. package/dist/marketplace/presets.js +52 -0
  138. package/dist/mcp/mcp.d.ts +31 -0
  139. package/dist/mcp/mcp.js +54 -0
  140. package/dist/memory/bm25.d.ts +16 -0
  141. package/dist/memory/bm25.js +56 -0
  142. package/dist/memory/classifier.d.ts +14 -0
  143. package/dist/memory/classifier.js +17 -0
  144. package/dist/memory/memory.d.ts +80 -0
  145. package/dist/memory/memory.js +191 -0
  146. package/dist/orchestration/executor.d.ts +35 -0
  147. package/dist/orchestration/executor.js +65 -0
  148. package/dist/orchestration/orchestrator.d.ts +42 -0
  149. package/dist/orchestration/orchestrator.js +63 -0
  150. package/dist/orchestration/plan.d.ts +37 -0
  151. package/dist/orchestration/plan.js +70 -0
  152. package/dist/orchestration/planner.d.ts +29 -0
  153. package/dist/orchestration/planner.js +69 -0
  154. package/dist/plugin/ai.d.ts +82 -0
  155. package/dist/plugin/ai.js +167 -0
  156. package/dist/probing/probe.d.ts +25 -0
  157. package/dist/probing/probe.js +63 -0
  158. package/dist/providers/factory.d.ts +18 -0
  159. package/dist/providers/factory.js +54 -0
  160. package/dist/providers/httpClient.d.ts +34 -0
  161. package/dist/providers/httpClient.js +80 -0
  162. package/dist/providers/httpProvider.d.ts +49 -0
  163. package/dist/providers/httpProvider.js +135 -0
  164. package/dist/providers/mock/demo.d.ts +13 -0
  165. package/dist/providers/mock/demo.js +58 -0
  166. package/dist/providers/mock/mockProvider.d.ts +35 -0
  167. package/dist/providers/mock/mockProvider.js +121 -0
  168. package/dist/providers/mock/scenarios.d.ts +44 -0
  169. package/dist/providers/mock/scenarios.js +30 -0
  170. package/dist/providers/provider.d.ts +26 -0
  171. package/dist/providers/provider.js +11 -0
  172. package/dist/providers/wire/anthropicWire.d.ts +6 -0
  173. package/dist/providers/wire/anthropicWire.js +83 -0
  174. package/dist/providers/wire/openaiWire.d.ts +7 -0
  175. package/dist/providers/wire/openaiWire.js +81 -0
  176. package/dist/providers/wire/registry.d.ts +8 -0
  177. package/dist/providers/wire/registry.js +20 -0
  178. package/dist/providers/wire/types.d.ts +39 -0
  179. package/dist/providers/wire/types.js +24 -0
  180. package/dist/runtime/config.d.ts +31 -0
  181. package/dist/runtime/config.js +121 -0
  182. package/dist/runtime/context.d.ts +34 -0
  183. package/dist/runtime/context.js +11 -0
  184. package/dist/runtime/events.d.ts +99 -0
  185. package/dist/runtime/events.js +82 -0
  186. package/dist/runtime/host.d.ts +27 -0
  187. package/dist/runtime/host.js +7 -0
  188. package/dist/runtime/intent/classifier.d.ts +30 -0
  189. package/dist/runtime/intent/classifier.js +60 -0
  190. package/dist/runtime/intent/signals.d.ts +19 -0
  191. package/dist/runtime/intent/signals.js +46 -0
  192. package/dist/runtime/modes/availability.d.ts +11 -0
  193. package/dist/runtime/modes/availability.js +17 -0
  194. package/dist/runtime/modes/chat.d.ts +18 -0
  195. package/dist/runtime/modes/chat.js +67 -0
  196. package/dist/runtime/modes/modeResolver.d.ts +43 -0
  197. package/dist/runtime/modes/modeResolver.js +78 -0
  198. package/dist/runtime/policy.d.ts +72 -0
  199. package/dist/runtime/policy.js +59 -0
  200. package/dist/runtime/providerView.d.ts +62 -0
  201. package/dist/runtime/providerView.js +105 -0
  202. package/dist/runtime/routing.d.ts +26 -0
  203. package/dist/runtime/routing.js +65 -0
  204. package/dist/runtime/runtime.d.ts +191 -0
  205. package/dist/runtime/runtime.js +718 -0
  206. package/dist/runtime/types.d.ts +153 -0
  207. package/dist/runtime/types.js +9 -0
  208. package/dist/runtime/workspace/detectors.d.ts +15 -0
  209. package/dist/runtime/workspace/detectors.js +57 -0
  210. package/dist/runtime/workspace/workspace.d.ts +29 -0
  211. package/dist/runtime/workspace/workspace.js +116 -0
  212. package/dist/security/credentials.d.ts +26 -0
  213. package/dist/security/credentials.js +34 -0
  214. package/dist/security/redact.d.ts +16 -0
  215. package/dist/security/redact.js +57 -0
  216. package/dist/skills/builtins/fileAnalyzer.d.ts +7 -0
  217. package/dist/skills/builtins/fileAnalyzer.js +47 -0
  218. package/dist/skills/builtins/repositoryAnalyzer.d.ts +6 -0
  219. package/dist/skills/builtins/repositoryAnalyzer.js +47 -0
  220. package/dist/skills/discovery.d.ts +61 -0
  221. package/dist/skills/discovery.js +211 -0
  222. package/dist/skills/manifest.d.ts +30 -0
  223. package/dist/skills/manifest.js +75 -0
  224. package/dist/skills/registry.d.ts +15 -0
  225. package/dist/skills/registry.js +22 -0
  226. package/dist/skills/skill.d.ts +64 -0
  227. package/dist/skills/skill.js +8 -0
  228. package/dist/store/area.d.ts +54 -0
  229. package/dist/store/area.js +164 -0
  230. package/dist/store/paths.d.ts +15 -0
  231. package/dist/store/paths.js +48 -0
  232. package/dist/store/store.d.ts +59 -0
  233. package/dist/store/store.js +140 -0
  234. package/dist/telemetry/sinks/file.d.ts +12 -0
  235. package/dist/telemetry/sinks/file.js +28 -0
  236. package/dist/telemetry/telemetry.d.ts +36 -0
  237. package/dist/telemetry/telemetry.js +63 -0
  238. package/dist/tools/builtins/filesystem.d.ts +7 -0
  239. package/dist/tools/builtins/filesystem.js +53 -0
  240. package/dist/tools/builtins/git.d.ts +10 -0
  241. package/dist/tools/builtins/git.js +66 -0
  242. package/dist/tools/builtins/shell.d.ts +17 -0
  243. package/dist/tools/builtins/shell.js +91 -0
  244. package/dist/tools/jail.d.ts +12 -0
  245. package/dist/tools/jail.js +98 -0
  246. package/dist/tools/permissions.d.ts +25 -0
  247. package/dist/tools/permissions.js +24 -0
  248. package/dist/tools/registry.d.ts +10 -0
  249. package/dist/tools/registry.js +20 -0
  250. package/dist/tools/runner.d.ts +23 -0
  251. package/dist/tools/runner.js +64 -0
  252. package/dist/tools/tool.d.ts +53 -0
  253. package/dist/tools/tool.js +24 -0
  254. package/dist/tools/untrusted.d.ts +13 -0
  255. package/dist/tools/untrusted.js +30 -0
  256. package/dist/types.d.ts +460 -0
  257. package/dist/types.js +12 -0
  258. package/dist/util/clock.d.ts +6 -0
  259. package/dist/util/clock.js +4 -0
  260. package/dist/util/extractJson.d.ts +8 -0
  261. package/dist/util/extractJson.js +54 -0
  262. package/dist/verification/verify.d.ts +26 -0
  263. package/dist/verification/verify.js +67 -0
  264. package/docs/GUIDE.md +358 -0
  265. package/docs/README.md +21 -0
  266. package/docs/architecture.md +78 -0
  267. package/docs/router.md +376 -0
  268. package/docs/security.md +55 -0
  269. package/package.json +67 -0
package/docs/router.md ADDED
@@ -0,0 +1,376 @@
1
+ # Router engine reference
2
+
3
+ The **router** is the engine beneath the [Runtime](../README.md#how-to-use-it): the vendor-agnostic
4
+ `AI` class that turns a task into a routed, executed, validated result. Most users only touch it through
5
+ the Runtime — but it is fully usable on its own (`import { AI } from 'ai-runtime-engine'`), and this document is
6
+ its complete reference: configuration, the `run()` API, providers, tasks/capabilities, routing internals,
7
+ reliability, and advanced features.
8
+
9
+ ```ts
10
+ import { AI } from 'ai-runtime-engine';
11
+ const ai = await AI.load(); // ai-runtime.yaml + .env
12
+ const r = await ai.run({ task: 'jira-to-test', input: ticketText });
13
+ if (r.ok) console.log(r.response?.text);
14
+ else console.error(r.error?.category, r.error?.message);
15
+ ```
16
+
17
+ `run()` **does not throw** for routing/provider failures — it returns `ok: false` with an actionable
18
+ `error` and the full `routing` trace. It only throws for programmer errors (e.g. an unknown preset id).
19
+
20
+ ## The pipeline
21
+
22
+ ```
23
+ run(request)
24
+ 1. Normalize task name → required/preferred capabilities, strategy, sensitivity, context floor,
25
+ output contract. Unknown task? infer requirements from request hints; baseline confidence 0.5.
26
+ 2. Filter drop candidates that can't/mustn't serve, each with a reason: not registered · unhealthy ·
27
+ denied · not allow-listed · pin mismatch · missing capability · context too small · privacy ·
28
+ over a hard cost/latency limit.
29
+ 3. Score seven 0..1 sub-scores per survivor × strategy-adjusted weights → one score; sort
30
+ deterministically (score → quality → latency → id).
31
+ 4-7. Execute + validate + fall back (one loop, bounded by maxFallbacks, default 4) top candidate →
32
+ provider.execute() → normalize → validate against the output contract; a thrown error or a
33
+ failed validation drops to the next candidate; a non-retryable error poisons that provider.
34
+ → confidence = topScore × (0.5 + 0.5 × evidence) × task.confidence
35
+ → RunResult { ok, response?, error?, confidence, routing }
36
+ ```
37
+
38
+ Validation is applied **inside** the fallback loop as a per-attempt check, so an invalid response falls
39
+ through to the next provider just like a thrown error does.
40
+
41
+ ---
42
+
43
+ ## Configuration
44
+
45
+ A config carries **structure and env-var names only** — never secret values. The file is validated by a
46
+ **strict** schema at load: an unknown key, or an `apiKeyEnv` that looks like a real key (`sk-…`, `gsk_…`,
47
+ `Bearer …`, or a 40+ char token), is **rejected** with a `CONFIG` error — structurally preventing a
48
+ committed secret.
49
+
50
+ ### Config-file keys (validated)
51
+
52
+ These are the only keys accepted in `ai-runtime.yaml` / `.yml` / `.json` (also valid as the router block
53
+ inside `.ai-runtime/config.yaml`):
54
+
55
+ ```yaml
56
+ strategy: best # best | fastest | cheapest | highest-quality | local-only |
57
+ # cloud-only | provider-specific | privacy-first | fallback-only
58
+
59
+ weights: # scoring weights (defaults shown; need not sum to 100)
60
+ capabilityFit: 30
61
+ quality: 20
62
+ reliability: 15
63
+ historicalSuccess: 10
64
+ latency: 10
65
+ cost: 10
66
+ userPreference: 5
67
+
68
+ defaults:
69
+ timeoutMs: 90000 # per-request HTTP timeout
70
+ maxFallbacks: 4 # max providers tried per run
71
+ minEvidence: documented # min evidence a capability needs to count as supported
72
+ minConfidence: 0.7 # below this → result.routing.belowConfidenceThreshold = true
73
+
74
+ privacy:
75
+ allowCloud: true # default true — a single cloud provider still works
76
+ allowLocal: true
77
+ sensitiveDataAllowedOnCloud: false # HIGH-sensitivity input is kept off cloud providers
78
+ treatUnknownAsSensitive: false # strict fail-closed: treat unknown sensitivity as sensitive
79
+ defaultSensitivity: unknown # low | high | unknown
80
+
81
+ telemetry:
82
+ enabled: true
83
+ sink: memory # memory | file
84
+ storePrompts: false # schema only allows false
85
+ path: .ai-runtime/telemetry.jsonl # required when sink: file (keep it gitignored)
86
+
87
+ providers: # the only required key
88
+ - id: gemini
89
+ kind: gemini # gemini | groq | anthropic | ollama | openai-compatible | custom | mock
90
+ apiKeyEnv: GEMINI_API_KEY # NAME of the env var, never the value
91
+ # optional: enabled, baseUrl, models, defaultModel, capabilities,
92
+ # privacyClass, wireShape, headers, weightOverrides
93
+
94
+ - id: internal # an unknown OpenAI-compatible provider self-declaring capabilities
95
+ kind: openai-compatible # openai-compatible/custom REQUIRE a baseUrl
96
+ baseUrl: https://ai.internal.example/v1
97
+ apiKeyEnv: INTERNAL_AI_API_KEY
98
+ models: [internal-large]
99
+ capabilities:
100
+ intelligence: [reasoning, code_generation, code_analysis]
101
+ output: [text, code, json]
102
+ contextWindow: 128000
103
+
104
+ tasks: # extend/override the built-in task registry
105
+ - id: my-task
106
+ required:
107
+ - { group: intelligence, key: reasoning }
108
+ - { group: output, key: json }
109
+ output: { format: json }
110
+ defaultStrategy: best
111
+ ```
112
+
113
+ Load with `AI.load()`, `AI.load('/path/to/config.yaml')`, or a remote URL
114
+ `AI.load('https://.../ai-runtime.yaml')` (remote configs get the same validation).
115
+
116
+ ### Programmatic-only options
117
+
118
+ `learning`, `verification`, `budget`, and `policy` are read by the router but are **not** part of the
119
+ strict file schema — set them by passing a config **object** to the constructor (a YAML file containing
120
+ them is rejected). `verification` and `budget` can also be set per run in `run()`.
121
+
122
+ ```ts
123
+ const ai = new AI({
124
+ providers: [{ id: 'gemini', kind: 'gemini', apiKeyEnv: 'GEMINI_API_KEY' }],
125
+ learning: { enabled: true }, // default true
126
+ verification: { enabled: false }, // default false
127
+ budget: { maxCostUsd: 0.10, maxCalls: 8 }, // per-run guardrail
128
+ policy: { allowProviders: ['gemini', 'groq'], // org guardrails on EVERY run
129
+ requireLocal: false, maxCostUsd: 0.05, strategy: 'cheapest' },
130
+ });
131
+ ```
132
+
133
+ ---
134
+
135
+ ## The `run()` API
136
+
137
+ ```ts
138
+ const result = await ai.run({
139
+ task: 'analyze-playwright-trace', // task name or alias; unknown tasks still work
140
+ input: trace, // string, or { text?, parts?: InputPart[] } for multimodal
141
+
142
+ system: 'You are a senior QA engineer.',
143
+ params: { temperature: 0.2, maxTokens: 2048, topP: 1, stop: ['\n\n'] },
144
+ requirements: { intelligence: ['code_analysis'], minContextWindow: 128000 },
145
+ constraints: {
146
+ maxCostUsd: 0.02, maxLatencyMs: 3000, minimumConfidence: 0.85,
147
+ allowProviders: ['gemini', 'groq'], denyProviders: ['ollama'],
148
+ },
149
+ strategy: 'highest-quality',
150
+ provider: 'gemini', // pin a provider (skips the capability filter; privacy/health still apply)
151
+ model: 'gemini-2.0-flash', // pin a model
152
+ routing: { excludeProviders: ['x'], preferModels: ['gemini:gemini-2.0-flash'] }, // exclude/prefer
153
+ sensitivity: 'high', // 'low' | 'high' | 'unknown'
154
+ output: { format: 'json', schema: myZodSchema }, // validated after execution
155
+ tools: [{ name: 'search', parameters: {} }],
156
+ budget: { maxCostUsd: 0.05, maxCalls: 3 }, // per-run spend guardrail
157
+ verification: true, // run a second model to verify the answer
158
+ mcp: true, // attach tools from registered MCP sources
159
+ });
160
+ ```
161
+
162
+ **`RunResult`:**
163
+
164
+ ```ts
165
+ {
166
+ ok: boolean;
167
+ response?: { // normalized — no vendor-specific parsing
168
+ text?: string; json?: unknown; toolCalls?: { name; arguments }[];
169
+ finishReason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | 'error';
170
+ usage?: { inputTokens?; outputTokens?; totalTokens? };
171
+ providerId: string; model: string; latencyMs: number;
172
+ };
173
+ error?: { category: ErrorCategory; message: string; retryable: boolean; providerId?; model? };
174
+ confidence: number; // 0..1
175
+ routing: { // the full decision trace
176
+ taskId, strategy, weights, requiredCapabilities, consideredCount,
177
+ filteredOut: { providerId, model, reason }[], // why each provider was excluded
178
+ ranked: { providerId, model, score, breakdown }[],
179
+ selected?: { providerId, model },
180
+ attempts: { providerId, model, outcome, category?, latencyMs }[], // incl. fallbacks
181
+ validation?, verification?, belowConfidenceThreshold?,
182
+ };
183
+ }
184
+ ```
185
+
186
+ ## `AI` class reference
187
+
188
+ | Method | Purpose |
189
+ |---|---|
190
+ | `new AI(config?, options?)` | Construct from a config object. `options`: `{ fetchImpl, clock, env, sinks }` |
191
+ | `static AI.load(pathOrUrl?, options?)` | Build from a local config file or an http(s) URL |
192
+ | `ai.run(request)` | Route and execute a task → `RunResult` |
193
+ | `ai.registerProvider(provider, enabled?)` | Register a provider instance (e.g. `MockProvider`) — chainable |
194
+ | `ai.registerTask(def)` | Add/override a task definition — chainable |
195
+ | `ai.addPreset(id, overrides?)` | Add a provider from the marketplace catalog |
196
+ | `ai.registerFromOpenApi(specOrAnalysis, overrides?)` | Generate + register a provider from an OpenAPI spec |
197
+ | `ai.registerMcpSource(source)` | Register an MCP tool source — chainable |
198
+ | `ai.providers()` / `ai.tasksList()` | Key-free provider listing / all registered tasks |
199
+ | `ai.discover()` / `ai.checkHealth()` | Provider metadata + models + capabilities / run each health check |
200
+ | `ai.probe(providerId, model, which?)` | Verify capabilities with live probes |
201
+ | `ai.telemetryEvents()` / `ai.performanceSnapshot()` | Recent metadata-only telemetry / learned performance |
202
+
203
+ ---
204
+
205
+ ## Providers
206
+
207
+ Every provider implements one `AIProvider` contract. Real providers (`HttpProvider`) ride **one HTTP path**
208
+ (native `fetch`, zero SDKs); `MockProvider` is a deterministic no-network implementation of the same
209
+ contract for tests.
210
+
211
+ ### Built-in kinds
212
+
213
+ | kind | wire | key env (default) | notes |
214
+ |---|---|---|---|
215
+ | `gemini` | OpenAI-compat | `GEMINI_API_KEY` | via Gemini's OpenAI-compatible endpoint |
216
+ | `groq` | OpenAI-compat | `GROQ_API_KEY` | |
217
+ | `anthropic` | native `/v1/messages` | `ANTHROPIC_API_KEY` | dedicated wire (`x-api-key` + `anthropic-version`) |
218
+ | `ollama` | OpenAI-compat | — | local, no key; degrades gracefully if absent |
219
+ | `openai-compatible` | OpenAI-compat | (you set `apiKeyEnv`) | any gateway — **`baseUrl` required** |
220
+ | `custom` | OpenAI-compat | (you set `apiKeyEnv`) | as above — **`baseUrl` required** |
221
+ | `mock` | — | — | deterministic, no-network; registered programmatically |
222
+
223
+ ### Marketplace presets
224
+
225
+ `ai-runtime presets` lists ready-to-use presets. **Native** presets (`gemini`, `groq`, `anthropic`,
226
+ `ollama`) come from the built-in defaults; **community** presets (`openrouter`, `cerebras`, `mistral`,
227
+ `together`, `deepseek`) are OpenAI-compatible. Add one: `ai.addPreset('openrouter', { models: ['…'] })`.
228
+
229
+ ### Unknown / self-declared providers
230
+
231
+ A provider with no catalog entry is still routable if it **declares its capabilities** in config (the
232
+ `internal` example above), or if you **pin** it in `run()`. Pinning routes on your assertion and skips the
233
+ capability filter — but privacy and health filters still apply.
234
+
235
+ ### From an OpenAPI spec
236
+
237
+ ```bash
238
+ ai-runtime discover ./company-ai.openapi.yaml # analyze + suggest a config
239
+ ```
240
+ ```ts
241
+ ai.registerFromOpenApi('./company-ai.openapi.yaml', { id: 'company', apiKeyEnv: 'COMPANY_KEY', models: ['big'] });
242
+ ```
243
+
244
+ Only **config** is generated (and validated) — never code. If the spec yields no OpenAI-compatible mapping,
245
+ generation throws a `CONFIG` error rather than guessing.
246
+
247
+ ---
248
+
249
+ ## Tasks & capabilities
250
+
251
+ You route by **task name**, not a capability list. Built-in tasks (extend via `registerTask()`,
252
+ `new AI({ tasks })`, or the YAML `tasks:` key):
253
+
254
+ | Task (aliases) | required capabilities | context floor | default strategy |
255
+ |---|---|---|---|
256
+ | `generate-playwright-tests` (`url-to-test`) | code_generation, reasoning, output.code | 32k | highest-quality |
257
+ | `playwright-trace-analysis` (`trace-analysis`, `analyze-playwright-trace`) | code_analysis, reasoning | 128k | best |
258
+ | `test-healer` | code_analysis, code_generation | 64k | — |
259
+ | `framework-converter` | code_generation, code_analysis | 64k | — |
260
+ | `feature-analyzer` | reasoning, summarization, extraction | 32k | — |
261
+ | `jira-to-test` (`ticket-router`) | reasoning, extraction, classification | — | — |
262
+
263
+ **Unknown tasks never fail** — requirements are inferred from your `run()` hints (output format, image/file
264
+ parts, tools) and the result's baseline `confidence` is lowered to 0.5.
265
+
266
+ **Capability taxonomy** (normalized, provider-agnostic):
267
+ - input: `text, image, audio, video, file, pdf`
268
+ - output: `text, json, structured_output, code, image, audio`
269
+ - intelligence: `reasoning, classification, summarization, code_generation, code_analysis, planning, translation, extraction`
270
+ - agent: `tool_calling, function_calling, mcp, agentic_loop`
271
+ - context: numeric `contextWindow`
272
+
273
+ Every capability carries **evidence**, ranked `verified > documented > inferred > unknown > unsupported`.
274
+ `unknown`/`unsupported` never satisfy a requirement, and evidence rank is never lowered except by a
275
+ capability probe promoting a fact to `verified`.
276
+
277
+ ---
278
+
279
+ ## Routing internals
280
+
281
+ ### Strategies
282
+
283
+ | Strategy | Effect |
284
+ |---|---|
285
+ | `best` (default) | base weights, no special filtering |
286
+ | `fastest` | latency weight ×4 |
287
+ | `cheapest` | cost weight ×4 |
288
+ | `highest-quality` | quality weight ×2.5 |
289
+ | `local-only` | filters out cloud providers |
290
+ | `cloud-only` | filters out local providers |
291
+ | `provider-specific` | set automatically when you pin a provider |
292
+ | `privacy-first` | accepted; behaves like `best` — use `local-only`, `sensitivity: 'high'`, or a task/policy `requireLocal` |
293
+ | `fallback-only` | base weights (relies on the fallback order) |
294
+
295
+ ### Scoring & confidence
296
+
297
+ Each survivor gets a 0..1 sub-score per dimension (`capabilityFit · quality · reliability ·
298
+ historicalSuccess · latency · cost · userPreference`); the final score is the weighted sum divided by the
299
+ weight sum. Ties break by score → quality → latency → id.
300
+
301
+ `confidence = topScore × (0.5 + 0.5 × evidenceFactor) × task.confidence`, where `task.confidence` is 1 for a
302
+ known task and 0.5 for an inferred one. Verification adjusts it (agree `+0.05`, capped at 1; disagree caps
303
+ at `minConfidence × 0.9`). Below `minConfidence` (default 0.7), `routing.belowConfidenceThreshold` is set.
304
+
305
+ ### Reliability & health
306
+
307
+ - **Retryable** (fall through / retry): `RATE_LIMIT`, `QUOTA`, `TIMEOUT`, `NETWORK`, `PROVIDER`.
308
+ - **Run-stopping**: exactly `BUDGET`, `CONFIG`, `PRIVACY_VIOLATION`.
309
+ - **Other non-retryable** (`AUTHENTICATION`, `MODEL_NOT_FOUND`, `CAPABILITY`, `INVALID_REQUEST`,
310
+ `RESPONSE_VALIDATION`): poison *that* provider and let fallback continue.
311
+ - **Two terminal outcomes:** `NO_CAPABLE_PROVIDER` (nothing eligible — with per-provider reasons) vs
312
+ `ALL_PROVIDERS_FAILED` (candidates existed but every attempt failed — with the full trace).
313
+ - **Health monitor:** routable states are `AVAILABLE`, `CONFIGURED`, `DEGRADED`, `UNKNOWN`. A rate-limited
314
+ or quota-exhausted provider gets a timed cooldown (default 60s). Same-provider HTTP retries (default 2)
315
+ handle 429/5xx (honoring `Retry-After`, capped at 30s), timeouts, and a `response_format` graceful degrade.
316
+
317
+ ### Error categories
318
+
319
+ ```
320
+ AUTHENTICATION · AUTHORIZATION · RATE_LIMIT · QUOTA · MODEL_NOT_FOUND · CAPABILITY ·
321
+ INVALID_REQUEST · TIMEOUT · NETWORK · PROVIDER · RESPONSE_VALIDATION · BUDGET ·
322
+ PRIVACY_VIOLATION · NO_CAPABLE_PROVIDER · ALL_PROVIDERS_FAILED · CONFIG · UNKNOWN
323
+ ```
324
+
325
+ ---
326
+
327
+ ## Advanced features
328
+
329
+ - **Learning-based routing** — observed success rate/latency per `(task, provider, model)` feed the
330
+ `historicalSuccess`/`latency` dimensions after **3 samples**; never overrides a hard constraint. Inspect
331
+ with `ai.performanceSnapshot()`; toggle via `learning.enabled`.
332
+ - **Capability probing** — `ai.probe('gemini', 'gemini-2.0-flash')` verifies what a model can actually do
333
+ (opt-in, minimal tokens, no secrets); positive probes promote capabilities to `verified`.
334
+ - **Multi-model verification** — `run({ …, verification: true })` has a second model judge the answer; the
335
+ verdict adjusts confidence and appears in `routing.verification`. Budget-aware (inconclusive if unaffordable).
336
+ - **Budgets** — `run({ …, budget: { maxCostUsd, maxCalls } })`; a spent budget stops the run with a distinct
337
+ `BUDGET` error.
338
+ - **MCP tools** — an integration layer for tools/context (not a provider). Register a source; its tools are
339
+ namespaced `${sourceId}.${toolName}` and attached to a tool-calling model on `mcp: true` runs.
340
+ - **Team policies** — a constructor-only `policy` applies org guardrails (allow/deny providers,
341
+ `requireLocal`, `maxCostUsd`, `strategy`) to every run, merged with per-run constraints.
342
+ - **Remote config** — `AI.load('https://…/ai-runtime.yaml')` (same strict validation).
343
+ - **Enterprise credentials** — `setCredentialResolver((name) => secrets.get(name))` (env wins when present).
344
+ - **Observability** — `new AI(config, { sinks: [new CallbackSink(send)] })`; sinks: `MemorySink`, `FileSink`,
345
+ `CallbackSink`, `MultiSink`, `NullSink` (redacted metadata only).
346
+
347
+ ## Engine CLI commands
348
+
349
+ | Command | Description |
350
+ |---|---|
351
+ | `ai-runtime init` | Scaffold `ai-runtime.yaml`, `.env.example`, `.gitignore` (the low-level scaffold; `setup` is the guided one) |
352
+ | `ai-runtime providers` / `models` / `capabilities` | List providers + health / known models / the capability matrix (`--json`) |
353
+ | `ai-runtime route <task>` | Route a task (`-i` input, `-s` strategy, `-p`/`-m` pin, `--json`) |
354
+ | `ai-runtime test <provider>` | One live call to a provider (skipped without a key) |
355
+ | `ai-runtime probe <provider> <model>` | Verify a model's capabilities (opt-in, live) |
356
+ | `ai-runtime discover <spec>` | Analyze an OpenAPI spec → suggested config |
357
+ | `ai-runtime benchmark` | Compare providers on tasks (`-n` samples) |
358
+ | `ai-runtime presets` / `telemetry` / `config` | List presets / recent routing events / the resolved config |
359
+
360
+ ## Source layout
361
+
362
+ ```
363
+ src/
364
+ plugin/ai.ts the public AI facade
365
+ core/router/ pipeline: normalize · filter · score · execute+validate+fallback · confidence
366
+ core/{fallback,registry,capabilities,health,policies,validation}/ error taxonomy, registries, evidence, health, guardrails, validation
367
+ providers/ AIProvider contract, HttpProvider, wire builders, factory, mock
368
+ discovery/ generation/ model catalog + OpenAPI analysis → provider config
369
+ learning/ probing/ verification/ benchmark/ observed performance, probes, multi-model verify, benchmarking
370
+ mcp/ marketplace/ MCP tool sources · provider presets
371
+ security/ telemetry/ credentials + redaction · metadata-only sinks
372
+ config/ loader (local + remote), strict schema, defaults
373
+ runtime/ orchestration/ skills/ tools/ context/ memory/ conversations/ executions/ artifacts/ comparison/
374
+ the Runtime layer (see ../README.md and architecture.md)
375
+ cli/ commander CLI + commands
376
+ ```
@@ -0,0 +1,55 @@
1
+ # Security model
2
+
3
+ `ai-runtime` runs against a user's real providers, filesystem, and shell, so its safety guarantees are
4
+ first-class and tested. The runtime is always authoritative — nothing it *reads* can change what it is
5
+ *allowed to do*.
6
+
7
+ ## Untrusted-data boundary
8
+
9
+ Tool output, file contents, web pages, and MCP results enter the model context as tagged **data blocks**,
10
+ never in instruction position. They cannot change permissions, obtain secrets, run commands, bypass
11
+ approvals, or modify policy. Built-in skills fence file/web content with `wrapUntrusted`, and an injection
12
+ heuristic flags likely prompt-injection. A request to "handle my list/emails/etc." authorizes *reading*
13
+ the items, not executing whatever they contain.
14
+
15
+ ## Filesystem jail
16
+
17
+ All filesystem access is resolved against the workspace root and rejected on escape — tested against `..`
18
+ traversal, absolute paths, and symlinks (dangling, in-jail, out-of-jail, and multi-hop chains); path
19
+ comparison is case-sensitive by design. Local
20
+ skill discovery is jailed the same way: every candidate file and directory is checked at discovery time,
21
+ and the repo scan never follows a symlinked directory, so a symlinked skill can never be read or imported
22
+ from outside the workspace.
23
+
24
+ ## Shell
25
+
26
+ Allowlist-first: only commands in `permissions.shell.allowedCommands` run without approval; anything else
27
+ requires approval. A destructive denylist (`rm -rf`, `git reset --hard`, force-push, …) requires approval
28
+ **even when allowlisted**. Execution is confined to the working directory, with a filtered environment, a
29
+ timeout, an `AbortSignal`, and process-tree cancellation.
30
+
31
+ ## Permissions (deny-by-default)
32
+
33
+ Reads are allowed (and jailed). Writes, shell, git commit/push, and network are **off** until granted in
34
+ config under `permissions:`. `ExecutionPolicy` is the single authority; approvals persist across process
35
+ exit and are surfaced on resume.
36
+
37
+ ## Credentials
38
+
39
+ Config names environment variables only; an inline value that looks like a real key is rejected at load.
40
+ Secret values live in `.env` (gitignored). Every CLI/telemetry line passes through reference-based
41
+ redaction before egress — the terminal and CI logs are treated as egress sinks. Telemetry is metadata
42
+ only and can never store prompts.
43
+
44
+ ## Learning can never override a hard gate
45
+
46
+ Learned outcomes and user preferences feed only **soft** scoring. A dedicated test proves that no
47
+ preference — user-set or learned — can override an exclusion, a privacy rule, a pin, a required
48
+ capability, or the budget. An excluded provider stays visible in the routing trace but is never selected.
49
+
50
+ ## Review discipline
51
+
52
+ Every phase ran an adversarial review (parallel reviewers by concern → each finding independently
53
+ verified → only confirmed defects fixed) before merge. Confirmed findings across the project — including a
54
+ symlink path-jail escape that could execute code on load, resume/lease concurrency races, and a routing
55
+ escape-hatch that dropped an exclusion — were fixed and covered by reproduction tests.
package/package.json ADDED
@@ -0,0 +1,67 @@
1
+ {
2
+ "name": "ai-runtime-engine",
3
+ "version": "1.1.0",
4
+ "description": "AI Runtime — a provider-agnostic AI runtime and orchestration platform. Point it at whatever AI providers you have; it routes each task to the best available model. Ships the `ai-runtime` CLI and the `Runtime`/`AI` library API.",
5
+ "type": "module",
6
+ "license": "ISC",
7
+ "author": "pavan",
8
+ "keywords": [
9
+ "ai", "llm", "ai-runtime", "runtime", "orchestration", "router", "provider-agnostic", "agent",
10
+ "openai", "gemini", "groq", "anthropic", "ollama", "openrouter", "mcp", "cli", "typescript"
11
+ ],
12
+ "engines": {
13
+ "node": ">=22"
14
+ },
15
+ "main": "./dist/index.js",
16
+ "types": "./dist/index.d.ts",
17
+ "bin": {
18
+ "ai-runtime": "./dist/cli/cli.js"
19
+ },
20
+ "exports": {
21
+ ".": {
22
+ "types": "./dist/index.d.ts",
23
+ "import": "./dist/index.js"
24
+ },
25
+ "./package.json": "./package.json"
26
+ },
27
+ "files": [
28
+ "dist",
29
+ "README.md",
30
+ "CHANGELOG.md",
31
+ "LICENSE",
32
+ "docs/GUIDE.md",
33
+ "docs/router.md",
34
+ "docs/architecture.md",
35
+ "docs/security.md"
36
+ ],
37
+ "publishConfig": {
38
+ "access": "public"
39
+ },
40
+ "workspaces": [
41
+ "packages/*"
42
+ ],
43
+ "scripts": {
44
+ "build": "tsc -p tsconfig.build.json",
45
+ "typecheck": "tsc --noEmit",
46
+ "test": "node --import tsx --test \"tests/**/*.test.ts\"",
47
+ "test:unit": "node --import tsx --test \"tests/unit/**/*.test.ts\"",
48
+ "test:integration": "node --import tsx --test \"tests/integration/**/*.test.ts\"",
49
+ "test:contract": "node --import tsx --test \"tests/contract/**/*.test.ts\"",
50
+ "test:chaos": "node --import tsx --test \"tests/chaos/**/*.test.ts\"",
51
+ "cli": "node --import tsx src/cli/cli.ts",
52
+ "example:fallback": "node --import tsx examples/fallback.ts",
53
+ "example:minimal": "node --import tsx examples/minimal.ts",
54
+ "prepublishOnly": "npm run build"
55
+ },
56
+ "dependencies": {
57
+ "commander": "^12.1.0",
58
+ "dotenv": "^17.2.3",
59
+ "yaml": "^2.6.1",
60
+ "zod": "^3.24.1"
61
+ },
62
+ "devDependencies": {
63
+ "@types/node": "^22.10.0",
64
+ "tsx": "^4.19.2",
65
+ "typescript": "^5.6.3"
66
+ }
67
+ }