@retinue/agentkit 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/README.md +1 -1
  2. package/dist/adapters/audio/openai.d.ts +36 -0
  3. package/dist/adapters/audio/openai.js +136 -0
  4. package/dist/adapters/bullmq/lock.d.ts +26 -2
  5. package/dist/adapters/memory/connections.d.ts +10 -0
  6. package/dist/adapters/memory/connections.js +87 -0
  7. package/dist/adapters/memory/graph.d.ts +17 -0
  8. package/dist/adapters/memory/graph.js +267 -0
  9. package/dist/adapters/memory/index.d.ts +3 -0
  10. package/dist/adapters/memory/index.js +3 -0
  11. package/dist/adapters/memory/rate-limit.d.ts +14 -0
  12. package/dist/adapters/memory/rate-limit.js +28 -0
  13. package/dist/adapters/memory/runtime.js +0 -0
  14. package/dist/adapters/postgres/connections.d.ts +15 -0
  15. package/dist/adapters/postgres/connections.js +134 -0
  16. package/dist/adapters/postgres/graph.d.ts +24 -0
  17. package/dist/adapters/postgres/graph.js +325 -0
  18. package/dist/adapters/postgres/index.d.ts +2 -0
  19. package/dist/adapters/postgres/index.js +2 -0
  20. package/dist/adapters/postgres/migrations.js +198 -0
  21. package/dist/adapters/postgres/run-store.js +26 -2
  22. package/dist/adapters/postgres/schema.d.ts +61 -2
  23. package/dist/adapters/postgres/schema.js +95 -10
  24. package/dist/adapters/redis/index.d.ts +1 -0
  25. package/dist/adapters/redis/index.js +1 -0
  26. package/dist/adapters/redis/rate-limit.d.ts +46 -0
  27. package/dist/adapters/redis/rate-limit.js +55 -0
  28. package/dist/adapters/supabase/index.d.ts +15 -0
  29. package/dist/adapters/supabase/index.js +15 -0
  30. package/dist/adapters/supabase/rls.js +33 -0
  31. package/dist/agents/agent.d.ts +78 -1
  32. package/dist/agents/agent.js +118 -5
  33. package/dist/agents/define.js +24 -4
  34. package/dist/agents/engine.d.ts +20 -0
  35. package/dist/agents/engine.js +177 -6
  36. package/dist/audio/index.d.ts +151 -0
  37. package/dist/audio/index.js +183 -0
  38. package/dist/connections/cipher.d.ts +103 -0
  39. package/dist/connections/cipher.js +141 -0
  40. package/dist/connections/index.d.ts +156 -0
  41. package/dist/connections/index.js +72 -0
  42. package/dist/connections/oauth/client.d.ts +84 -0
  43. package/dist/connections/oauth/client.js +141 -0
  44. package/dist/connections/oauth/index.d.ts +166 -0
  45. package/dist/connections/oauth/index.js +226 -0
  46. package/dist/connections/oauth/service.d.ts +90 -0
  47. package/dist/connections/oauth/service.js +132 -0
  48. package/dist/connections/pause.d.ts +68 -0
  49. package/dist/connections/pause.js +97 -0
  50. package/dist/connections/resolver.d.ts +59 -0
  51. package/dist/connections/resolver.js +138 -0
  52. package/dist/connections/resume.d.ts +64 -0
  53. package/dist/connections/resume.js +99 -0
  54. package/dist/connections/vault-cipher.d.ts +95 -0
  55. package/dist/connections/vault-cipher.js +206 -0
  56. package/dist/core/content-parts.d.ts +18 -2
  57. package/dist/core/content-parts.js +1 -0
  58. package/dist/core/context.d.ts +25 -0
  59. package/dist/core/errors.d.ts +1 -1
  60. package/dist/core/errors.js +15 -0
  61. package/dist/core/events.d.ts +44 -2
  62. package/dist/core/events.js +31 -2
  63. package/dist/core/validation.js +19 -0
  64. package/dist/entries/connections.d.ts +10 -0
  65. package/dist/entries/connections.js +10 -0
  66. package/dist/entries/knowledge.d.ts +8 -0
  67. package/dist/entries/knowledge.js +8 -0
  68. package/dist/entries/mcp-server.d.ts +9 -0
  69. package/dist/entries/mcp-server.js +9 -0
  70. package/dist/entries/runtime.d.ts +10 -0
  71. package/dist/entries/runtime.js +8 -0
  72. package/dist/entries/testing.d.ts +28 -0
  73. package/dist/entries/testing.js +28 -0
  74. package/dist/graphql/resolvers.d.ts +3 -1
  75. package/dist/graphql/resolvers.js +14 -0
  76. package/dist/knowledge/communities.d.ts +166 -0
  77. package/dist/knowledge/communities.js +377 -0
  78. package/dist/knowledge/graph-global.d.ts +153 -0
  79. package/dist/knowledge/graph-global.js +246 -0
  80. package/dist/knowledge/graph-retrieval.d.ts +150 -0
  81. package/dist/knowledge/graph-retrieval.js +303 -0
  82. package/dist/knowledge/graph.d.ts +223 -0
  83. package/dist/knowledge/graph.js +369 -0
  84. package/dist/knowledge/index.d.ts +24 -0
  85. package/dist/knowledge/index.js +36 -1
  86. package/dist/knowledge/retrieval.d.ts +26 -1
  87. package/dist/knowledge/retrieval.js +77 -1
  88. package/dist/mcp-server/index.d.ts +168 -0
  89. package/dist/mcp-server/index.js +175 -0
  90. package/dist/models/extraction.d.ts +73 -0
  91. package/dist/models/extraction.js +118 -0
  92. package/dist/models/index.d.ts +40 -1
  93. package/dist/models/index.js +22 -1
  94. package/dist/models/pricing.d.ts +2 -0
  95. package/dist/models/pricing.js +17 -1
  96. package/dist/models/provider-factory.js +9 -7
  97. package/dist/models/streaming.d.ts +89 -0
  98. package/dist/models/streaming.js +179 -2
  99. package/dist/persistence/index.d.ts +230 -0
  100. package/dist/persistence/index.js +2 -0
  101. package/dist/runtime/index.d.ts +1 -1
  102. package/dist/runtime/index.js +12 -0
  103. package/dist/runtime/worker.d.ts +36 -1
  104. package/dist/runtime/worker.js +37 -4
  105. package/dist/security/checklist.js +46 -0
  106. package/dist/server/bin.d.ts +19 -0
  107. package/dist/server/bin.js +226 -0
  108. package/dist/server/boot.d.ts +20 -2
  109. package/dist/server/boot.js +29 -3
  110. package/dist/server/cli.d.ts +13 -0
  111. package/dist/server/cli.js +7 -4
  112. package/dist/server/doctor.d.ts +74 -0
  113. package/dist/server/doctor.js +280 -0
  114. package/dist/skills/context.d.ts +61 -0
  115. package/dist/skills/context.js +107 -0
  116. package/dist/skills/index.d.ts +1 -0
  117. package/dist/skills/index.js +1 -0
  118. package/dist/telemetry/spans.js +5 -0
  119. package/dist/testing/conformance/artifact-exports.d.ts +20 -0
  120. package/dist/testing/conformance/artifact-exports.js +172 -0
  121. package/dist/testing/conformance/artifacts.d.ts +11 -0
  122. package/dist/testing/conformance/artifacts.js +316 -0
  123. package/dist/testing/conformance/audio.d.ts +38 -0
  124. package/dist/testing/conformance/audio.js +145 -0
  125. package/dist/testing/conformance/capability.d.ts +23 -0
  126. package/dist/testing/conformance/capability.js +27 -0
  127. package/dist/testing/conformance/checkpoint-store.d.ts +15 -0
  128. package/dist/testing/conformance/checkpoint-store.js +72 -0
  129. package/dist/testing/conformance/connections.d.ts +14 -0
  130. package/dist/testing/conformance/connections.js +171 -0
  131. package/dist/testing/conformance/conversation-store.d.ts +10 -0
  132. package/dist/testing/conformance/conversation-store.js +60 -0
  133. package/dist/testing/conformance/evaluation.d.ts +10 -0
  134. package/dist/testing/conformance/evaluation.js +295 -0
  135. package/dist/testing/conformance/files.d.ts +19 -0
  136. package/dist/testing/conformance/files.js +454 -0
  137. package/dist/testing/conformance/flows.d.ts +16 -0
  138. package/dist/testing/conformance/flows.js +193 -0
  139. package/dist/testing/conformance/graph.d.ts +22 -0
  140. package/dist/testing/conformance/graph.js +500 -0
  141. package/dist/testing/conformance/hitl.d.ts +25 -0
  142. package/dist/testing/conformance/hitl.js +523 -0
  143. package/dist/testing/conformance/index.d.ts +163 -0
  144. package/dist/testing/conformance/index.js +321 -0
  145. package/dist/testing/conformance/invariants.d.ts +23 -0
  146. package/dist/testing/conformance/invariants.js +80 -0
  147. package/dist/testing/conformance/knowledge.d.ts +41 -0
  148. package/dist/testing/conformance/knowledge.js +592 -0
  149. package/dist/testing/conformance/parents.d.ts +50 -0
  150. package/dist/testing/conformance/parents.js +39 -0
  151. package/dist/testing/conformance/rate-limit.d.ts +25 -0
  152. package/dist/testing/conformance/rate-limit.js +71 -0
  153. package/dist/testing/conformance/records.d.ts +36 -0
  154. package/dist/testing/conformance/records.js +400 -0
  155. package/dist/testing/conformance/rollups.d.ts +30 -0
  156. package/dist/testing/conformance/rollups.js +425 -0
  157. package/dist/testing/conformance/run-coordinator.d.ts +27 -0
  158. package/dist/testing/conformance/run-coordinator.js +150 -0
  159. package/dist/testing/conformance/run-event-log.d.ts +9 -0
  160. package/dist/testing/conformance/run-event-log.js +138 -0
  161. package/dist/testing/conformance/run-store.d.ts +12 -0
  162. package/dist/testing/conformance/run-store.js +294 -0
  163. package/dist/testing/conformance/session-state.d.ts +38 -0
  164. package/dist/testing/conformance/session-state.js +198 -0
  165. package/dist/testing/conformance/usage-limits.d.ts +10 -0
  166. package/dist/testing/conformance/usage-limits.js +309 -0
  167. package/dist/testing/conformance.d.ts +8 -0
  168. package/dist/testing/conformance.js +8 -0
  169. package/dist/testing/memory-backend.d.ts +39 -0
  170. package/dist/testing/memory-backend.js +41 -0
  171. package/dist/testing/pglite.d.ts +55 -0
  172. package/dist/testing/pglite.js +78 -0
  173. package/dist/testing/stub-model.d.ts +58 -0
  174. package/dist/testing/stub-model.js +71 -0
  175. package/dist/testing/supabase-storage-double.d.ts +35 -0
  176. package/dist/testing/supabase-storage-double.js +101 -0
  177. package/dist/toolkit/http.d.ts +27 -0
  178. package/dist/toolkit/http.js +48 -0
  179. package/dist/toolkit/index.d.ts +27 -1
  180. package/dist/toolkit/index.js +24 -1
  181. package/dist/toolkit/mime.d.ts +97 -0
  182. package/dist/toolkit/mime.js +233 -0
  183. package/dist/toolkit/ssrf.d.ts +139 -0
  184. package/dist/toolkit/ssrf.js +349 -0
  185. package/dist/toolkit/vendor.d.ts +91 -0
  186. package/dist/toolkit/vendor.js +153 -0
  187. package/dist/tools/credentials.d.ts +265 -6
  188. package/dist/tools/credentials.js +318 -3
  189. package/dist/tools/define.d.ts +14 -0
  190. package/dist/tools/define.js +3 -0
  191. package/dist/tools/index.d.ts +21 -0
  192. package/dist/tools/index.js +7 -0
  193. package/dist/tools/library/audio.d.ts +63 -0
  194. package/dist/tools/library/audio.js +148 -0
  195. package/dist/tools/library/index.d.ts +8 -0
  196. package/dist/tools/library/index.js +7 -0
  197. package/dist/tools/registry.d.ts +33 -0
  198. package/dist/tools/registry.js +72 -2
  199. package/dist/usage/index.d.ts +21 -0
  200. package/dist/usage/index.js +1 -0
  201. package/dist/usage/rate-limit.d.ts +122 -0
  202. package/dist/usage/rate-limit.js +131 -0
  203. package/dist/usage/recorder.js +3 -0
  204. package/package.json +29 -2
@@ -18,6 +18,7 @@
18
18
  * on a refusal the model cannot resolve.
19
19
  */
20
20
  import { AgentPlatformError, isAgentPlatformError } from "../core/errors.js";
21
+ import { connectionNeedOf } from "../connections/pause.js";
21
22
  import { asId } from "../core/ids.js";
22
23
  import { applyInputGuardrails, applyOutputGuardrails } from "../guardrails/index.js";
23
24
  import { streamModelTurn, turnText } from "../models/index.js";
@@ -118,9 +119,48 @@ export const createDefaultEngine = (deps) => {
118
119
  const now = deps.now ?? Date.now;
119
120
  const streamTurn = deps.streamTurn ?? streamModelTurn;
120
121
  return {
121
- async *run({ run, context, signal }) {
122
- const manifest = await deps.loadManifest({ agentId: run.agentId, version: run.agentVersion, context });
122
+ async *run({ run, context: hostContext, signal }) {
123
+ const manifest = await deps.loadManifest({
124
+ agentId: run.agentId,
125
+ version: run.agentVersion,
126
+ context: hostContext,
127
+ });
128
+ /**
129
+ * The agent's tool policy, put on the context for everything downstream — task #244.
130
+ *
131
+ * The engine is the only layer that holds both halves: the manifest (per agent) and the registry (per
132
+ * deployment). Rather than threading the policy through `buildTools`, tool execution, `execute_tool` and
133
+ * every delegating tool, it travels on the context — which a model cannot write to, so `excluded` cannot
134
+ * be widened from inside a turn. `ExecutionContext.shadow` is the precedent and the same argument.
135
+ *
136
+ * Scoped **once, here**, and used everywhere below. A host-supplied context that already carried a policy
137
+ * is overridden by the manifest's: the manifest is what the run's `agentVersion` pins, so a stored
138
+ * definition — not the caller — decides what this agent may reach.
139
+ */
140
+ const context = { ...hostContext, agentToolPolicy: manifest.toolPolicy };
123
141
  const resolved = deps.resolveModel(manifest, context);
142
+ /**
143
+ * A structured agent needs a model that can do it — task #243 AC-3.
144
+ *
145
+ * Checked at resolution, before a token is spent, and here rather than only in the host's `resolveModel`
146
+ * because that callback is the host's: a host that has not been updated would resolve a text-only model
147
+ * and the agent would silently get prose, which is the defect being fixed rather than a new one.
148
+ *
149
+ * Skipped when the host returned no `definition`, following the same rule `modelModalities` already uses
150
+ * (#185): a caller that did not say what the model can do has not said it cannot do this, and refusing
151
+ * every structured agent from every host that has not been updated would be an outage dressed as a check.
152
+ */
153
+ if (manifest.responseFormat?.kind === "structured" && resolved.definition !== undefined) {
154
+ if (resolved.definition.capabilities?.structuredOutput !== true)
155
+ throw new AgentPlatformError({
156
+ code: "capability_unavailable",
157
+ message: `agent "${manifest.id}" asks for a structured response format and the resolved model ` +
158
+ `${resolved.modelId} does not declare the \`structuredOutput\` capability. Add ` +
159
+ "`requiredCapabilities: { structuredOutput: true }` to the agent's model policy so resolution " +
160
+ "picks a model that can, rather than discovering it mid-turn.",
161
+ retryable: false,
162
+ });
163
+ }
124
164
  const system = (await (deps.systemPrompt?.(manifest, context) ?? manifest.instructions)) || undefined;
125
165
  const history = await deps.loadHistory(context, run);
126
166
  const built = deps.buildTools ? await deps.buildTools(context, manifest) : [];
@@ -142,7 +182,22 @@ export const createDefaultEngine = (deps) => {
142
182
  budget: deps.catalogBudget,
143
183
  tokensOf: turnToolTokens,
144
184
  nameOf: (tool) => tool.name,
145
- protect: (tool) => META_TOOLS.includes(tool.name),
185
+ /**
186
+ * Protected: meta-tools, and whatever the agent declared preloaded — task #244.
187
+ *
188
+ * `toolPolicy.preloaded` and `toolPolicy.categories` say "these are loaded up front; everything
189
+ * else is discovered lazily". In this architecture *resident* is what "loaded up front" means, so
190
+ * the honest interpretation of both fields is that a budget may not drop them. With no budget
191
+ * configured every tool is resident anyway and the fields are a no-op — correct, and the reason
192
+ * they cannot be enforced anywhere else.
193
+ *
194
+ * Note this cannot make a tool appear: an excluded tool never reaches `built` (the registry
195
+ * removed it), and naming an excluded tool as preloaded does not resurrect it. Exclusion is a
196
+ * permission and residency is a budget; the permission wins.
197
+ */
198
+ protect: (tool) => META_TOOLS.includes(tool.name) ||
199
+ (manifest.toolPolicy?.preloaded ?? []).includes(tool.name) ||
200
+ (tool.category !== undefined && (manifest.toolPolicy?.categories ?? []).includes(tool.category)),
146
201
  });
147
202
  const declared = budgetOutcome?.resident ?? built;
148
203
  if (budgetOutcome !== undefined && (budgetOutcome.dropped.length > 0 || budgetOutcome.overBudget)) {
@@ -232,6 +287,58 @@ export const createDefaultEngine = (deps) => {
232
287
  * carried on and completed. The person's answer arrived for a run that was already over.
233
288
  */
234
289
  let pendingQuestion = null;
290
+ /**
291
+ * A connection a tool needed and the tenant does not have — task #264.
292
+ *
293
+ * Tracked exactly like `pendingApproval` and `pendingQuestion`, and for the same reason: the run has to
294
+ * stop on an *event* the worker understands rather than on an error the model would try to work around.
295
+ */
296
+ /**
297
+ * Turns a marked connection failure into a pause, or leaves it to fail — task #264.
298
+ *
299
+ * Three ways to end up failing rather than pausing, and each is deliberate:
300
+ *
301
+ * - the failure is not a connection gap at all
302
+ * - no `connectionConsent` is wired, so this deployment has no flow to send anybody to
303
+ * - the callback returns `null`, meaning *this provider* is token-only and there is no login URL
304
+ *
305
+ * All three fail, because the alternative is a run parked for ever on a consent screen nobody can reach.
306
+ */
307
+ const consentMarker = async (thrown, toolName) => {
308
+ const need = connectionNeedOf(thrown);
309
+ if (need === null || deps.connectionConsent === undefined)
310
+ return null;
311
+ const offer = await deps.connectionConsent({ context, need: { ...need, toolName } });
312
+ if (offer === null)
313
+ return null;
314
+ return {
315
+ event: {
316
+ type: "connection.requested",
317
+ provider: need.provider,
318
+ loginUrl: offer.loginUrl,
319
+ scopes: offer.scopes,
320
+ toolName,
321
+ expiresAt: offer.expiresAt,
322
+ },
323
+ // Returned to the model, not thrown — the same choice the approval path makes. The tool call stays a
324
+ // real part of the record with a real result, and the run pauses on the event rather than on an error
325
+ // the model would try to work around.
326
+ marker: {
327
+ status: "connection_required",
328
+ provider: need.provider,
329
+ message: `${toolName} needs a ${need.provider} connection. The run is paused; do not retry.`,
330
+ },
331
+ };
332
+ };
333
+ /**
334
+ * The formed event, not the parts.
335
+ *
336
+ * Storing the pieces and building the event at the check site fought control-flow analysis: the
337
+ * assignment happens inside a tool's `execute` closure, so the outer `let` narrows to `never` at the
338
+ * check and every property access errors. Holding the event itself needs no property access there, and
339
+ * reads more like `pendingApproval` — a value the emit site simply yields.
340
+ */
341
+ let pendingConnection = null;
235
342
  /**
236
343
  * Citation candidates a tool handed back this turn, waiting for the claims they ground — #165.
237
344
  *
@@ -328,10 +435,15 @@ export const createDefaultEngine = (deps) => {
328
435
  }
329
436
  catch (thrown) {
330
437
  const parked = questionMarker(thrown, t.name);
331
- if (parked === null)
438
+ if (parked !== null) {
439
+ pendingQuestion = parked.interactionId;
440
+ return parked.marker;
441
+ }
442
+ const consent = await consentMarker(thrown, t.name);
443
+ if (consent === null)
332
444
  throw thrown;
333
- pendingQuestion = parked.interactionId;
334
- return parked.marker;
445
+ pendingConnection = consent.event;
446
+ return consent.marker;
335
447
  }
336
448
  }
337
449
  const outcome = await approvals.runTool(context, run.id, { name: t.name, input });
@@ -365,6 +477,8 @@ export const createDefaultEngine = (deps) => {
365
477
  let attempt = 1;
366
478
  for (;;) {
367
479
  let emitted = 0;
480
+ // Whether this turn produced the structured answer a structured agent promises — #243.
481
+ let sawStructured = false;
368
482
  const textParts = new Map();
369
483
  const controller = new AbortController();
370
484
  try {
@@ -402,6 +516,15 @@ export const createDefaultEngine = (deps) => {
402
516
  ...(resolved.definition === undefined
403
517
  ? {}
404
518
  : { modelModalities: resolved.definition.inputModalities }),
519
+ // Mapped from the manifest here, so the model layer stays free of any dependency on `agents/`.
520
+ ...(manifest.responseFormat?.kind === "structured"
521
+ ? { structuredOutput: { schema: manifest.responseFormat.schema } }
522
+ : {}),
523
+ // From the resolved model's own declaration — #247. Absent definition means send nothing, the same
524
+ // rule `modelModalities` follows: a host that has not said what its model does has not said it caches.
525
+ ...(resolved.definition?.capabilities?.promptCaching === undefined
526
+ ? {}
527
+ : { promptCaching: resolved.definition.capabilities.promptCaching }),
405
528
  tools,
406
529
  maxSteps,
407
530
  abortSignal: controller.signal,
@@ -415,6 +538,8 @@ export const createDefaultEngine = (deps) => {
415
538
  }
416
539
  for (const event of mapChunk(chunk, messageId, resolved, textParts, ranByCall)) {
417
540
  emitted += 1;
541
+ if (event.type === "part.added" && event.part.type === "structured")
542
+ sawStructured = true;
418
543
  yield event;
419
544
  }
420
545
  // Raised by a tool call this turn. Stop here rather than letting the model keep going: the
@@ -432,6 +557,13 @@ export const createDefaultEngine = (deps) => {
432
557
  yield { type: "question.requested", interactionId: asId(pendingQuestion) };
433
558
  return;
434
559
  }
560
+ // The third stop — #264. The event carries the provider, the scopes and the URL, and **no secret**:
561
+ // it is rendered in a UI and clicked by a person, so it goes wherever a screenshot goes.
562
+ if (pendingConnection !== null) {
563
+ controller.abort();
564
+ yield pendingConnection;
565
+ return;
566
+ }
435
567
  }
436
568
  /**
437
569
  * Citations last, grounding the claims that were actually written — #165.
@@ -445,6 +577,23 @@ export const createDefaultEngine = (deps) => {
445
577
  * After the stream, so a citation cannot appear above text the reader is already looking at — the
446
578
  * append-only property `citationViewModel` depends on.
447
579
  */
580
+ /**
581
+ * A structured agent must have produced a structured answer — task #243 AC-2.
582
+ *
583
+ * `streamModelTurn` already validates and fails, so in the normal path this never fires. It fires for
584
+ * a host that supplied its own `streamTurn`, and that is the case worth guarding: the guarantee a
585
+ * consumer bought is "structured or an error", and if it depended solely on the shipped model layer
586
+ * then any host replacing that layer would silently get prose again — the original defect, reachable
587
+ * through a documented extension point.
588
+ */
589
+ if (manifest.responseFormat?.kind === "structured" && !sawStructured)
590
+ throw new AgentPlatformError({
591
+ code: "provider_error",
592
+ message: `agent "${manifest.id}" asks for a structured response format and the turn produced none. ` +
593
+ "The run fails rather than returning the turn's text, which would be prose presented as a " +
594
+ "validated object.",
595
+ retryable: true,
596
+ });
448
597
  for (const record of pendingVerdicts.splice(0))
449
598
  yield verdictEvent(record);
450
599
  if (deps.citations !== undefined && pendingCitations.length > 0) {
@@ -631,6 +780,9 @@ ranByCall = new Map()) {
631
780
  inputTokens: chunk.usage.inputTokens,
632
781
  outputTokens: chunk.usage.outputTokens,
633
782
  cachedInputTokens: chunk.usage.cachedInputTokens,
783
+ // Carried only when the provider reported it, so a turn with no breakdown is distinguishable from a
784
+ // turn that wrote nothing — the same rule `imageCount` follows two lines down.
785
+ ...(chunk.usage.cacheWriteTokens === undefined ? {} : { cacheWriteTokens: chunk.usage.cacheWriteTokens }),
634
786
  ...(chunk.usage.reasoningTokens !== undefined ? { reasoningTokens: chunk.usage.reasoningTokens } : {}),
635
787
  // Counted at the send site (`nonTextCounts`), carried through so the ledger records it (#185).
636
788
  ...(chunk.usage.imageCount !== undefined ? { imageCount: chunk.usage.imageCount } : {}),
@@ -641,6 +793,25 @@ ranByCall = new Map()) {
641
793
  };
642
794
  return;
643
795
  }
796
+ case "structured-output": {
797
+ /**
798
+ * The validated answer of a structured agent — task #243.
799
+ *
800
+ * Emitted once, complete. `streamModelTurn` has already validated it against the schema and fails the turn
801
+ * if it does not conform, so reaching here means the value satisfies what the caller asked for. Nothing
802
+ * partial is ever emitted: a half-built object does not satisfy a schema, so streaming one would publish
803
+ * values that violate the contract.
804
+ */
805
+ const part = {
806
+ id: `${messageId}:structured`,
807
+ type: "structured",
808
+ schemaVersion: 1,
809
+ createdAt: new Date(0).toISOString(),
810
+ value: chunk.value,
811
+ };
812
+ yield { type: "part.added", messageId, part };
813
+ return;
814
+ }
644
815
  case "error":
645
816
  throw chunk.error instanceof Error ? chunk.error : new Error(String(chunk.error));
646
817
  }
@@ -0,0 +1,151 @@
1
+ /**
2
+ * Hearing and speaking — REQ-062 (#257), task #258, Part 2.
3
+ *
4
+ * Two ports, because the two acts are genuinely separate and a deployment will often want one and not the
5
+ * other: transcription turns audio into text so a text-only model can read it, and speech turns text into
6
+ * audio so an interface can play it.
7
+ *
8
+ * ## Why ports rather than tools that call an API
9
+ *
10
+ * The same reason `tools-search` supplies providers instead of tools. Whisper, Deepgram, AssemblyAI and a
11
+ * self-hosted `whisper.cpp` are four *values*, not four sets of tools — and which one a deployment uses is a
12
+ * decision about cost, residency and latency that a model should never spend a tool call discovering.
13
+ *
14
+ * It also keeps AC-2's real case honest: *"so a deployment can use a model that does not accept audio
15
+ * natively"*. Most models do not. Transcribing first and sending text is how audio reaches them at all, and
16
+ * that path has to exist independently of any one provider.
17
+ *
18
+ * ## Bounds are enforced before the upload, not after
19
+ *
20
+ * AC-4 says so and it is the part most easily got wrong: the natural implementation reads the file, stores it,
21
+ * and then checks. A rejected 200MB recording must not be stored first — it costs the blob write, the
22
+ * retention obligation and the deletion, all for something that was never going to be accepted.
23
+ *
24
+ * So the check takes a *declared* size and media type and answers before any bytes move. `assertWithinBounds`
25
+ * is the function a caller runs at the boundary; nothing here reads a file to find out whether it may.
26
+ *
27
+ * ## Duration is bounded too, and it cannot be checked from the size
28
+ *
29
+ * A byte ceiling is not a duration ceiling: an hour of 8kbps speech is smaller than a minute of uncompressed
30
+ * WAV. Both matter for different reasons — bytes bound the storage and the upload, seconds bound the *cost*,
31
+ * because audio is billed per second. A deployment that bounded only bytes would have an unbounded bill.
32
+ *
33
+ * Duration is therefore checked where it becomes known: declared by the caller if it knows, and reported by
34
+ * the provider afterwards. `assertWithinBounds` refuses a declared duration over the ceiling before the call;
35
+ * `TranscriptionResult.durationSeconds` is what usage is charged on.
36
+ */
37
+ import { type PlatformError } from "../core/errors.js";
38
+ /**
39
+ * The media types accepted, and it is a list rather than `audio/*` on purpose.
40
+ *
41
+ * A provider rejects an unknown container with a message about the request, not about the format, so the
42
+ * useful refusal happens here. Every entry is one that the shipped adapters actually accept.
43
+ */
44
+ export declare const AUDIO_MEDIA_TYPES: readonly ["audio/mpeg", "audio/mp4", "audio/wav", "audio/webm", "audio/ogg", "audio/flac", "audio/x-m4a"];
45
+ export type AudioMediaType = (typeof AUDIO_MEDIA_TYPES)[number];
46
+ export declare const isAudioMediaType: (mediaType: string) => mediaType is AudioMediaType;
47
+ /**
48
+ * 25MB, which is where every hosted transcription API this could target draws its own line.
49
+ *
50
+ * Matching the provider's limit rather than inventing a smaller one: a deployment that wants less sets its
51
+ * own, and a ceiling *above* the provider's would just move the failure later and make it theirs to explain.
52
+ */
53
+ export declare const MAX_AUDIO_BYTES: number;
54
+ /**
55
+ * One hour.
56
+ *
57
+ * The number is about cost rather than capability. At a typical per-second rate an hour is already a
58
+ * noticeable charge for a single tool call, and a run that transcribes a six-hour recording by accident is the
59
+ * kind of bill nobody notices until the invoice.
60
+ */
61
+ export declare const MAX_AUDIO_SECONDS = 3600;
62
+ /**
63
+ * The transcript ceiling — AC-5.
64
+ *
65
+ * An hour of speech is roughly 9,000 words, which fits. This bounds the *pathological* case: a provider that
66
+ * returns a repeated hallucination on silence, which is a known Whisper failure mode and produces megabytes of
67
+ * one phrase. Truncation is reported, never silent — a shortened transcript that claimed to be complete would
68
+ * make the model summarise a fragment as if it were the whole recording.
69
+ */
70
+ export declare const MAX_TRANSCRIPT_CHARS = 100000;
71
+ /** What speech generation may be asked to say. Bounds the cost the same way the audio ceiling does. */
72
+ export declare const MAX_SPEECH_CHARS = 4000;
73
+ export type AudioBounds = {
74
+ readonly maxBytes?: number;
75
+ readonly maxSeconds?: number;
76
+ };
77
+ export declare class AudioRejected extends Error {
78
+ readonly code: "invalid_input";
79
+ }
80
+ /**
81
+ * Refuses before anything is stored or sent — AC-4.
82
+ *
83
+ * Takes what a caller knows *without reading the file*: the declared byte length and media type from the
84
+ * upload, and a duration if the client measured one. Deliberately not given the bytes: a function that
85
+ * received them would invite reading them, and the whole point is to answer before they move.
86
+ */
87
+ export declare const assertWithinBounds: (input: {
88
+ readonly byteSize: number;
89
+ readonly mediaType: string;
90
+ readonly durationSeconds?: number;
91
+ }, bounds?: AudioBounds) => void;
92
+ /** A transcript, bounded, with the truncation reported rather than hidden. */
93
+ export type TranscriptionResult = {
94
+ readonly text: string;
95
+ /**
96
+ * What usage is charged on — AC-8. Reported by the provider, because only it knows.
97
+ *
98
+ * A provider that does not report it leaves this absent rather than guessing, and the recorder then charges
99
+ * nothing for the audio. That under-bills, which is why the shipped adapter asks for a response format that
100
+ * includes it.
101
+ */
102
+ readonly durationSeconds?: number;
103
+ readonly truncated: boolean;
104
+ /** The provider's own language detection, when it offers one. Not a translation — see the REQ's scope. */
105
+ readonly language?: string;
106
+ };
107
+ export type TranscriptionRequest = {
108
+ readonly audio: Uint8Array;
109
+ readonly mediaType: string;
110
+ readonly filename?: string;
111
+ /** A hint, not an instruction. A provider free to detect is usually better at it than a caller guessing. */
112
+ readonly languageHint?: string;
113
+ readonly maxTranscriptChars?: number;
114
+ };
115
+ export interface TranscriptionProvider {
116
+ readonly id: string;
117
+ transcribe(request: TranscriptionRequest): Promise<TranscriptionResult>;
118
+ }
119
+ export type SpeechRequest = {
120
+ readonly text: string;
121
+ /** The provider's own voice identifier. Opaque here: naming a fixed set would date immediately. */
122
+ readonly voice?: string;
123
+ readonly format?: "mp3" | "wav" | "opus" | "flac";
124
+ };
125
+ export type SpeechResult = {
126
+ readonly audio: Uint8Array;
127
+ readonly mediaType: string;
128
+ /** Absent when the provider does not report it. Charged on when present. */
129
+ readonly durationSeconds?: number;
130
+ };
131
+ export interface SpeechProvider {
132
+ readonly id: string;
133
+ speak(request: SpeechRequest): Promise<SpeechResult>;
134
+ }
135
+ /** Bounds a transcript and says whether it had to. */
136
+ export declare const boundTranscript: (text: string, maxChars?: number) => {
137
+ text: string;
138
+ truncated: boolean;
139
+ };
140
+ /**
141
+ * A provider failure, in the platform's vocabulary — AC-5.
142
+ *
143
+ * Deliberately the same shape and the same distinctions as `describeFetchFailure`: 429 and 5xx retryable,
144
+ * unreachable `provider_unavailable`, everything else terminal. A second vocabulary for audio would mean the
145
+ * runtime's retry logic treated a transcription rate limit differently from a scrape rate limit for no reason
146
+ * anybody chose.
147
+ */
148
+ export declare const describeAudioFailure: (error: unknown, provider: string) => PlatformError;
149
+ /** Throws the classified failure, for a caller that would rather not branch. */
150
+ export declare const failAudio: (error: unknown, provider: string) => never;
151
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,183 @@
1
+ /**
2
+ * Hearing and speaking — REQ-062 (#257), task #258, Part 2.
3
+ *
4
+ * Two ports, because the two acts are genuinely separate and a deployment will often want one and not the
5
+ * other: transcription turns audio into text so a text-only model can read it, and speech turns text into
6
+ * audio so an interface can play it.
7
+ *
8
+ * ## Why ports rather than tools that call an API
9
+ *
10
+ * The same reason `tools-search` supplies providers instead of tools. Whisper, Deepgram, AssemblyAI and a
11
+ * self-hosted `whisper.cpp` are four *values*, not four sets of tools — and which one a deployment uses is a
12
+ * decision about cost, residency and latency that a model should never spend a tool call discovering.
13
+ *
14
+ * It also keeps AC-2's real case honest: *"so a deployment can use a model that does not accept audio
15
+ * natively"*. Most models do not. Transcribing first and sending text is how audio reaches them at all, and
16
+ * that path has to exist independently of any one provider.
17
+ *
18
+ * ## Bounds are enforced before the upload, not after
19
+ *
20
+ * AC-4 says so and it is the part most easily got wrong: the natural implementation reads the file, stores it,
21
+ * and then checks. A rejected 200MB recording must not be stored first — it costs the blob write, the
22
+ * retention obligation and the deletion, all for something that was never going to be accepted.
23
+ *
24
+ * So the check takes a *declared* size and media type and answers before any bytes move. `assertWithinBounds`
25
+ * is the function a caller runs at the boundary; nothing here reads a file to find out whether it may.
26
+ *
27
+ * ## Duration is bounded too, and it cannot be checked from the size
28
+ *
29
+ * A byte ceiling is not a duration ceiling: an hour of 8kbps speech is smaller than a minute of uncompressed
30
+ * WAV. Both matter for different reasons — bytes bound the storage and the upload, seconds bound the *cost*,
31
+ * because audio is billed per second. A deployment that bounded only bytes would have an unbounded bill.
32
+ *
33
+ * Duration is therefore checked where it becomes known: declared by the caller if it knows, and reported by
34
+ * the provider afterwards. `assertWithinBounds` refuses a declared duration over the ceiling before the call;
35
+ * `TranscriptionResult.durationSeconds` is what usage is charged on.
36
+ */
37
+ import { AgentPlatformError } from "../core/errors.js";
38
+ /**
39
+ * The media types accepted, and it is a list rather than `audio/*` on purpose.
40
+ *
41
+ * A provider rejects an unknown container with a message about the request, not about the format, so the
42
+ * useful refusal happens here. Every entry is one that the shipped adapters actually accept.
43
+ */
44
+ export const AUDIO_MEDIA_TYPES = [
45
+ "audio/mpeg",
46
+ "audio/mp4",
47
+ "audio/wav",
48
+ "audio/webm",
49
+ "audio/ogg",
50
+ "audio/flac",
51
+ "audio/x-m4a",
52
+ ];
53
+ export const isAudioMediaType = (mediaType) => AUDIO_MEDIA_TYPES.includes(mediaType.split(";")[0]?.trim().toLowerCase() ?? "");
54
+ /**
55
+ * 25MB, which is where every hosted transcription API this could target draws its own line.
56
+ *
57
+ * Matching the provider's limit rather than inventing a smaller one: a deployment that wants less sets its
58
+ * own, and a ceiling *above* the provider's would just move the failure later and make it theirs to explain.
59
+ */
60
+ export const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
61
+ /**
62
+ * One hour.
63
+ *
64
+ * The number is about cost rather than capability. At a typical per-second rate an hour is already a
65
+ * noticeable charge for a single tool call, and a run that transcribes a six-hour recording by accident is the
66
+ * kind of bill nobody notices until the invoice.
67
+ */
68
+ export const MAX_AUDIO_SECONDS = 3_600;
69
+ /**
70
+ * The transcript ceiling — AC-5.
71
+ *
72
+ * An hour of speech is roughly 9,000 words, which fits. This bounds the *pathological* case: a provider that
73
+ * returns a repeated hallucination on silence, which is a known Whisper failure mode and produces megabytes of
74
+ * one phrase. Truncation is reported, never silent — a shortened transcript that claimed to be complete would
75
+ * make the model summarise a fragment as if it were the whole recording.
76
+ */
77
+ export const MAX_TRANSCRIPT_CHARS = 100_000;
78
+ /** What speech generation may be asked to say. Bounds the cost the same way the audio ceiling does. */
79
+ export const MAX_SPEECH_CHARS = 4_000;
80
+ export class AudioRejected extends Error {
81
+ code = "invalid_input";
82
+ }
83
+ /**
84
+ * Refuses before anything is stored or sent — AC-4.
85
+ *
86
+ * Takes what a caller knows *without reading the file*: the declared byte length and media type from the
87
+ * upload, and a duration if the client measured one. Deliberately not given the bytes: a function that
88
+ * received them would invite reading them, and the whole point is to answer before they move.
89
+ */
90
+ export const assertWithinBounds = (input, bounds = {}) => {
91
+ const maxBytes = bounds.maxBytes ?? MAX_AUDIO_BYTES;
92
+ const maxSeconds = bounds.maxSeconds ?? MAX_AUDIO_SECONDS;
93
+ if (!isAudioMediaType(input.mediaType)) {
94
+ throw new AudioRejected(`${input.mediaType} is not an audio format this deployment accepts. Accepted: ` +
95
+ `${AUDIO_MEDIA_TYPES.join(", ")}.`);
96
+ }
97
+ if (!Number.isFinite(input.byteSize) || input.byteSize <= 0) {
98
+ throw new AudioRejected("An audio attachment needs a declared size, and it must be greater than zero.");
99
+ }
100
+ if (input.byteSize > maxBytes) {
101
+ throw new AudioRejected(`That recording is ${Math.round(input.byteSize / 1024 / 1024)}MB and the limit is ` +
102
+ `${Math.round(maxBytes / 1024 / 1024)}MB. Refused before upload — nothing was stored.`);
103
+ }
104
+ /**
105
+ * Duration only when the caller declared one.
106
+ *
107
+ * Absent is not an error: a browser knows the duration of a recording it just made and a server receiving a
108
+ * file often does not, and demanding it would mean either refusing legitimate uploads or reading the file to
109
+ * find out — which is the thing this function exists to avoid.
110
+ */
111
+ if (input.durationSeconds !== undefined && input.durationSeconds > maxSeconds) {
112
+ throw new AudioRejected(`That recording is ${Math.round(input.durationSeconds / 60)} minutes and the limit is ` +
113
+ `${Math.round(maxSeconds / 60)}. Refused before upload — nothing was stored.`);
114
+ }
115
+ };
116
+ /** Bounds a transcript and says whether it had to. */
117
+ export const boundTranscript = (text, maxChars = MAX_TRANSCRIPT_CHARS) => {
118
+ if (text.length <= maxChars)
119
+ return { text, truncated: false };
120
+ /**
121
+ * Cut at a word boundary when there is one nearby.
122
+ *
123
+ * A transcript sliced mid-word reads as a transcription error rather than as a truncation, and a model
124
+ * summarising it will occasionally treat the fragment as a real word. The 200-character window is small
125
+ * enough that the ceiling still means what it says.
126
+ */
127
+ const hard = text.slice(0, maxChars);
128
+ const lastSpace = hard.lastIndexOf(" ");
129
+ /**
130
+ * `lastSpace > 0` matters, and its absence was a bug the tests caught.
131
+ *
132
+ * `lastIndexOf` returns `-1` when there is no space at all — and `-1 > maxChars - 200` is *true* for any
133
+ * ceiling under 200, so the original condition sliced to `-1` and silently dropped the final character of
134
+ * every space-free transcript. A one-character loss that no assertion about truncation would notice.
135
+ */
136
+ const useBoundary = lastSpace > 0 && lastSpace > maxChars - 200;
137
+ return { text: useBoundary ? hard.slice(0, lastSpace) : hard, truncated: true };
138
+ };
139
+ /**
140
+ * A provider failure, in the platform's vocabulary — AC-5.
141
+ *
142
+ * Deliberately the same shape and the same distinctions as `describeFetchFailure`: 429 and 5xx retryable,
143
+ * unreachable `provider_unavailable`, everything else terminal. A second vocabulary for audio would mean the
144
+ * runtime's retry logic treated a transcription rate limit differently from a scrape rate limit for no reason
145
+ * anybody chose.
146
+ */
147
+ export const describeAudioFailure = (error, provider) => {
148
+ if (error instanceof AudioRejected) {
149
+ return { code: "invalid_input", message: error.message, retryable: false };
150
+ }
151
+ const status = error.status;
152
+ const message = error instanceof Error ? error.message : String(error);
153
+ if (status === 429) {
154
+ return { code: "rate_limited", message: `${provider} is rate limiting: ${message}`, retryable: true };
155
+ }
156
+ if (status === 413) {
157
+ // Not retryable, and worth its own arm: the file is too large and will be next time too.
158
+ return {
159
+ code: "invalid_input",
160
+ message: `${provider} refused the recording as too large: ${message}`,
161
+ retryable: false,
162
+ };
163
+ }
164
+ if (status === 401 || status === 403) {
165
+ return {
166
+ code: "unauthorized",
167
+ message: `${provider} refused the credential: ${message}. Retrying will not help.`,
168
+ retryable: false,
169
+ };
170
+ }
171
+ if (status !== undefined && status >= 500) {
172
+ return { code: "provider_unavailable", message: `${provider} returned ${status}: ${message}`, retryable: true };
173
+ }
174
+ if (/timed out|etimedout|abort|fetch failed|econnrefused|enotfound/i.test(message)) {
175
+ return { code: "provider_unavailable", message: `${provider} could not be reached: ${message}`, retryable: true };
176
+ }
177
+ return { code: "provider_error", message: `${provider} could not process the audio: ${message}`, retryable: false };
178
+ };
179
+ /** Throws the classified failure, for a caller that would rather not branch. */
180
+ export const failAudio = (error, provider) => {
181
+ throw new AgentPlatformError(describeAudioFailure(error, provider));
182
+ };
183
+ //# sourceMappingURL=index.js.map