@andreprado/agentkit 0.1.0-alpha.15 → 0.1.0-alpha.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,7 @@
1
1
  import type {
2
2
  ChannelAdapter,
3
+ ChannelMediaDownloadInput,
4
+ ChannelMediaDownloadResult,
3
5
  ChannelSendInput,
4
6
  ChannelSendResult,
5
7
  RawWebhookEvent,
@@ -105,6 +107,22 @@ export const zapsterWhatsappChannelAdapter: ChannelAdapter = {
105
107
  },
106
108
  };
107
109
 
110
+ if (messageType === "audio" && message.audio) {
111
+ return [
112
+ {
113
+ ...base,
114
+ kind: "message",
115
+ message: {
116
+ role: "user",
117
+ contentType: "audio",
118
+ content: "",
119
+ providerMessageId: message.id,
120
+ audio: message.audio,
121
+ },
122
+ },
123
+ ];
124
+ }
125
+
108
126
  if (messageType !== "text") {
109
127
  return [
110
128
  {
@@ -140,6 +158,9 @@ export const zapsterWhatsappChannelAdapter: ChannelAdapter = {
140
158
  },
141
159
  ];
142
160
  },
161
+ async downloadMedia(input) {
162
+ return downloadZapsterMedia(input);
163
+ },
143
164
  async sendMessage(input) {
144
165
  return sendZapsterMessage(input);
145
166
  },
@@ -169,6 +190,15 @@ type NormalizedZapsterWebhookMessage = {
169
190
  contactId?: string;
170
191
  type?: string;
171
192
  text?: string;
193
+ audio?: {
194
+ providerMediaId: string;
195
+ kind: "voice" | "audio";
196
+ mimeType?: string;
197
+ filename?: string;
198
+ durationSeconds?: number;
199
+ sizeBytes?: number;
200
+ downloadUrl?: string;
201
+ };
172
202
  };
173
203
 
174
204
  function normalizeZapsterOfficialMessage(body: Record<string, unknown>): NormalizedZapsterWebhookMessage | null {
@@ -185,17 +215,48 @@ function normalizeZapsterOfficialMessage(body: Record<string, unknown>): Normali
185
215
  contactId: readString(sender, "id"),
186
216
  type: readString(data, "type"),
187
217
  text: readString(content, "text"),
218
+ audio: normalizeZapsterAudio(content),
188
219
  };
189
220
  }
190
221
 
191
222
  function normalizeZapsterLegacyMessage(body: Record<string, unknown>): NormalizedZapsterWebhookMessage {
192
223
  const message = readRecord(body, "message");
224
+ const audio = readRecord(message, "audio");
193
225
 
194
226
  return {
195
227
  id: readString(message, "id"),
196
228
  contactId: readString(message, "from"),
197
229
  type: readString(message, "type"),
198
230
  text: readString(readRecord(message, "text"), "body"),
231
+ audio: normalizeZapsterAudio(audio),
232
+ };
233
+ }
234
+
235
+ function normalizeZapsterAudio(value: Record<string, unknown>): NormalizedZapsterWebhookMessage["audio"] {
236
+ const mediaId = readString(value, "id") ?? readString(value, "media_id") ?? readString(value, "mediaId");
237
+ const downloadUrl =
238
+ readString(value, "url") ??
239
+ readString(value, "download_url") ??
240
+ readString(value, "downloadUrl") ??
241
+ readString(value, "media_url") ??
242
+ readString(value, "mediaUrl");
243
+
244
+ if (!mediaId && !downloadUrl) {
245
+ return undefined;
246
+ }
247
+
248
+ return {
249
+ providerMediaId: mediaId ?? downloadUrl ?? "unknown",
250
+ kind: readString(value, "ptt") === "true" ? "voice" : "audio",
251
+ ...(readString(value, "mime_type") ?? readString(value, "mimeType")
252
+ ? { mimeType: readString(value, "mime_type") ?? readString(value, "mimeType") }
253
+ : {}),
254
+ filename: readString(value, "filename") ?? readString(value, "file_name") ?? defaultZapsterAudioFilename(value),
255
+ ...(readNumber(value, "duration") !== undefined ? { durationSeconds: readNumber(value, "duration") } : {}),
256
+ ...(readNumber(value, "file_size") ?? readNumber(value, "size")
257
+ ? { sizeBytes: readNumber(value, "file_size") ?? readNumber(value, "size") }
258
+ : {}),
259
+ ...(downloadUrl ? { downloadUrl } : {}),
199
260
  };
200
261
  }
201
262
 
@@ -249,6 +310,8 @@ export type ZapsterSendMessageRequest = {
249
310
  type ZapsterFetch = (url: string, init: RequestInit) => Promise<Response>;
250
311
 
251
312
  const ZAPSTER_SEND_MESSAGE_URL = "https://api.zapsterapi.com/v1/wa/messages";
313
+ const ZAPSTER_DEFAULT_MEDIA_HOSTS = ["api.zapsterapi.com"];
314
+ const ZAPSTER_MAX_DOWNLOAD_BYTES = 25_000_000;
252
315
 
253
316
  export function buildZapsterSendMessageRequest(input: ChannelSendInput): ZapsterSendMessageRequest {
254
317
  return buildZapsterSendMessageRequestInternal(input, "<redacted>");
@@ -381,6 +444,239 @@ export async function sendZapsterMessage(
381
444
  };
382
445
  }
383
446
 
447
+ export async function downloadZapsterMedia(
448
+ input: ChannelMediaDownloadInput,
449
+ fetcher: ZapsterFetch = fetch,
450
+ ): Promise<ChannelMediaDownloadResult> {
451
+ const apiKey = input.secrets.ZAPSTER_API_KEY;
452
+ const audio = input.message.audio;
453
+
454
+ if (!apiKey) {
455
+ return {
456
+ ok: false,
457
+ retryable: false,
458
+ code: "channel_secret_missing",
459
+ message: "Secret ZAPSTER_API_KEY is missing.",
460
+ };
461
+ }
462
+
463
+ if (!audio?.downloadUrl) {
464
+ return {
465
+ ok: false,
466
+ retryable: false,
467
+ code: "channel_audio_download_unavailable",
468
+ message: "Zapster audio payload did not include a media download URL.",
469
+ };
470
+ }
471
+
472
+ const mediaUrl = validateZapsterMediaDownloadUrl(audio.downloadUrl);
473
+
474
+ if (!mediaUrl.ok) {
475
+ return mediaUrl;
476
+ }
477
+
478
+ let response: Response;
479
+
480
+ try {
481
+ response = await fetcher(mediaUrl.url, {
482
+ method: "GET",
483
+ headers: {
484
+ Authorization: `Bearer ${apiKey}`,
485
+ },
486
+ });
487
+ } catch (error) {
488
+ return {
489
+ ok: false,
490
+ retryable: true,
491
+ code: "channel_provider_unavailable",
492
+ message: `Zapster media download failed before a response: ${redactString(
493
+ error instanceof Error ? error.message : String(error),
494
+ [apiKey, mediaUrl.url],
495
+ )}`,
496
+ };
497
+ }
498
+
499
+ if (!response.ok) {
500
+ return {
501
+ ok: false,
502
+ retryable: response.status === 429 || response.status >= 500,
503
+ code: response.status === 429 || response.status >= 500
504
+ ? "channel_provider_unavailable"
505
+ : "channel_media_download_failed",
506
+ message: `Zapster media download returned HTTP ${response.status}.`,
507
+ providerMetadata: {
508
+ status: response.status,
509
+ },
510
+ };
511
+ }
512
+
513
+ const maxDownloadBytes = input.maxBytes ?? ZAPSTER_MAX_DOWNLOAD_BYTES;
514
+ const bytes = await readResponseBytesWithLimit(response, maxDownloadBytes);
515
+
516
+ if (!bytes.ok) {
517
+ return bytes;
518
+ }
519
+
520
+ return {
521
+ ok: true,
522
+ audio: bytes.audio,
523
+ filename: audio.filename ?? `${audio.providerMediaId}.ogg`,
524
+ ...(audio.mimeType ? { mimeType: audio.mimeType } : {}),
525
+ ...(audio.durationSeconds !== undefined ? { durationSeconds: audio.durationSeconds } : {}),
526
+ sizeBytes: bytes.audio.byteLength,
527
+ providerMetadata: {
528
+ status: response.status,
529
+ },
530
+ };
531
+ }
532
+
533
+ function validateZapsterMediaDownloadUrl(rawUrl: string):
534
+ | {
535
+ ok: true;
536
+ url: string;
537
+ }
538
+ | {
539
+ ok: false;
540
+ retryable: false;
541
+ code: "channel_audio_download_unavailable";
542
+ message: string;
543
+ } {
544
+ let url: URL;
545
+
546
+ try {
547
+ url = new URL(rawUrl);
548
+ } catch {
549
+ return {
550
+ ok: false,
551
+ retryable: false,
552
+ code: "channel_audio_download_unavailable",
553
+ message: "Zapster audio download URL is not a valid URL.",
554
+ };
555
+ }
556
+
557
+ if (url.protocol !== "https:") {
558
+ return {
559
+ ok: false,
560
+ retryable: false,
561
+ code: "channel_audio_download_unavailable",
562
+ message: "Zapster audio download URL must use HTTPS.",
563
+ };
564
+ }
565
+
566
+ const hostname = url.hostname.toLowerCase();
567
+ const allowed = ZAPSTER_DEFAULT_MEDIA_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`));
568
+
569
+ if (!allowed) {
570
+ return {
571
+ ok: false,
572
+ retryable: false,
573
+ code: "channel_audio_download_unavailable",
574
+ message: "Zapster audio download URL host is not allowed.",
575
+ };
576
+ }
577
+
578
+ url.username = "";
579
+ url.password = "";
580
+
581
+ return {
582
+ ok: true,
583
+ url: url.href,
584
+ };
585
+ }
586
+
587
+ async function readResponseBytesWithLimit(
588
+ response: Response,
589
+ maxBytes: number,
590
+ ): Promise<
591
+ | {
592
+ ok: true;
593
+ audio: Uint8Array;
594
+ }
595
+ | {
596
+ ok: false;
597
+ retryable: false;
598
+ code: "channel_audio_too_large";
599
+ message: string;
600
+ providerMetadata?: Record<string, unknown>;
601
+ }
602
+ > {
603
+ const contentLength = response.headers.get("content-length");
604
+ const declaredBytes = contentLength ? Number(contentLength) : undefined;
605
+
606
+ if (declaredBytes !== undefined && Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
607
+ return {
608
+ ok: false,
609
+ retryable: false,
610
+ code: "channel_audio_too_large",
611
+ message: `Audio response declares ${declaredBytes} bytes, which exceeds the configured ${maxBytes} byte limit.`,
612
+ providerMetadata: {
613
+ sizeBytes: declaredBytes,
614
+ },
615
+ };
616
+ }
617
+
618
+ if (!response.body) {
619
+ const audio = new Uint8Array(await response.arrayBuffer());
620
+
621
+ if (audio.byteLength > maxBytes) {
622
+ return {
623
+ ok: false,
624
+ retryable: false,
625
+ code: "channel_audio_too_large",
626
+ message: `Audio response is ${audio.byteLength} bytes, which exceeds the configured ${maxBytes} byte limit.`,
627
+ providerMetadata: {
628
+ sizeBytes: audio.byteLength,
629
+ },
630
+ };
631
+ }
632
+
633
+ return { ok: true, audio };
634
+ }
635
+
636
+ const reader = response.body.getReader();
637
+ const chunks: Uint8Array[] = [];
638
+ let totalBytes = 0;
639
+
640
+ try {
641
+ while (true) {
642
+ const chunk = await reader.read();
643
+
644
+ if (chunk.done) {
645
+ break;
646
+ }
647
+
648
+ totalBytes += chunk.value.byteLength;
649
+
650
+ if (totalBytes > maxBytes) {
651
+ await reader.cancel().catch(() => undefined);
652
+ return {
653
+ ok: false,
654
+ retryable: false,
655
+ code: "channel_audio_too_large",
656
+ message: `Audio response exceeded the configured ${maxBytes} byte limit while downloading.`,
657
+ providerMetadata: {
658
+ sizeBytes: totalBytes,
659
+ },
660
+ };
661
+ }
662
+
663
+ chunks.push(chunk.value);
664
+ }
665
+ } finally {
666
+ reader.releaseLock();
667
+ }
668
+
669
+ const audio = new Uint8Array(totalBytes);
670
+ let offset = 0;
671
+
672
+ for (const chunk of chunks) {
673
+ audio.set(chunk, offset);
674
+ offset += chunk.byteLength;
675
+ }
676
+
677
+ return { ok: true, audio };
678
+ }
679
+
384
680
  function buildZapsterSendMessageRequestInternal(
385
681
  input: ChannelSendInput,
386
682
  instanceId: string,
@@ -494,6 +790,29 @@ function readString(value: Record<string, unknown>, key: string): string | undef
494
790
  return typeof field === "string" && field.length > 0 ? field : undefined;
495
791
  }
496
792
 
793
+ function readNumber(value: Record<string, unknown>, key: string): number | undefined {
794
+ const field = value[key];
795
+ return typeof field === "number" && Number.isFinite(field) ? field : undefined;
796
+ }
797
+
798
+ function defaultZapsterAudioFilename(value: Record<string, unknown>): string {
799
+ const mimeType = readString(value, "mime_type") ?? readString(value, "mimeType");
800
+
801
+ if (mimeType?.includes("ogg")) {
802
+ return "audio.ogg";
803
+ }
804
+
805
+ if (mimeType?.includes("mpeg") || mimeType?.includes("mp3")) {
806
+ return "audio.mp3";
807
+ }
808
+
809
+ if (mimeType?.includes("wav")) {
810
+ return "audio.wav";
811
+ }
812
+
813
+ return "audio.ogg";
814
+ }
815
+
497
816
  function isPositiveIntegerHeader(value: string | null): value is string {
498
817
  return typeof value === "string" && /^[1-9]\d*$/.test(value);
499
818
  }
@@ -1,13 +1,17 @@
1
1
  import type { AgentChannel, ChannelProvider, ChannelType } from "../index";
2
2
 
3
3
  export type ChannelMessageRole = "user" | "assistant";
4
- export type ChannelMessageContentType = "text";
4
+ export type ChannelMessageContentType = "text" | "audio";
5
5
  export type ChannelEventKind = "message" | "delivery_status" | "unsupported";
6
6
  export type ChannelDeliveryDirection = "inbound" | "outbound";
7
7
  export type ChannelDeliveryState =
8
8
  | "webhook_received"
9
9
  | "received"
10
10
  | "validated"
11
+ | "audio_received"
12
+ | "audio_downloaded"
13
+ | "transcribing"
14
+ | "transcribed"
11
15
  | "duplicate"
12
16
  | "buffered"
13
17
  | "queued"
@@ -45,11 +49,22 @@ export type ChannelExternalIdentity = {
45
49
  redactedDisplay?: string;
46
50
  };
47
51
 
52
+ export type ChannelAudioAttachment = {
53
+ providerMediaId: string;
54
+ kind: "voice" | "audio";
55
+ mimeType?: string;
56
+ filename?: string;
57
+ durationSeconds?: number;
58
+ sizeBytes?: number;
59
+ downloadUrl?: string;
60
+ };
61
+
48
62
  export type NormalizedChannelMessage = {
49
63
  role: ChannelMessageRole;
50
64
  contentType: ChannelMessageContentType;
51
65
  content: string;
52
66
  providerMessageId?: string;
67
+ audio?: ChannelAudioAttachment;
53
68
  };
54
69
 
55
70
  export type NormalizedChannelEvent = ChannelAdapterId & {
@@ -87,6 +102,36 @@ export type ChannelSendInput = ChannelAdapterId & {
87
102
  secrets: Record<string, string>;
88
103
  };
89
104
 
105
+ export type ChannelMediaDownloadInput = ChannelAdapterId & {
106
+ channelId: string;
107
+ message: NormalizedChannelMessage;
108
+ secrets: Record<string, string>;
109
+ maxBytes?: number;
110
+ };
111
+
112
+ export type ChannelMediaDownloadResult =
113
+ | {
114
+ ok: true;
115
+ audio: Uint8Array;
116
+ filename: string;
117
+ mimeType?: string;
118
+ durationSeconds?: number;
119
+ sizeBytes?: number;
120
+ providerMetadata?: Record<string, unknown>;
121
+ }
122
+ | {
123
+ ok: false;
124
+ retryable: boolean;
125
+ code:
126
+ | "channel_secret_missing"
127
+ | "channel_audio_download_unavailable"
128
+ | "channel_audio_too_large"
129
+ | "channel_provider_unavailable"
130
+ | "channel_media_download_failed";
131
+ message: string;
132
+ providerMetadata?: Record<string, unknown>;
133
+ };
134
+
90
135
  export type ChannelSendResult =
91
136
  | {
92
137
  ok: true;
@@ -120,6 +165,7 @@ export type ChannelStatusResult = {
120
165
  export type ChannelAdapter = ChannelAdapterId & {
121
166
  requiredSecrets: string[];
122
167
  normalizeInbound(input: RawWebhookEvent): Promise<NormalizedChannelEvent[]> | NormalizedChannelEvent[];
168
+ downloadMedia?(input: ChannelMediaDownloadInput): Promise<ChannelMediaDownloadResult>;
123
169
  sendMessage(input: ChannelSendInput): Promise<ChannelSendResult>;
124
170
  verifyWebhook?(input: WebhookVerificationInput): Promise<WebhookVerificationResult> | WebhookVerificationResult;
125
171
  getStatus?(input: ChannelStatusInput): Promise<ChannelStatusResult> | ChannelStatusResult;
@@ -16,6 +16,8 @@ import type {
16
16
  TelegramAllowedUpdate,
17
17
  AgentKnowledgeEmbeddingProvider,
18
18
  ChannelBufferMode,
19
+ ChannelAudioMode,
20
+ TranscriptionProviderName,
19
21
  } from "../index";
20
22
  import { AgentKitError } from "./errors";
21
23
  import { isValidTimeZone } from "./prompt-context";
@@ -28,8 +30,10 @@ const TOOL_VISIBILITIES = new Set<ToolVisibility>(["user", "internal"]);
28
30
  const CHANNEL_TYPES = new Set<ChannelType>(["website", "telegram", "whatsapp"]);
29
31
  const CHANNEL_PROVIDERS = new Set<ChannelProvider>(["agentkit", "telegram", "zapster", "meta"]);
30
32
  const CHANNEL_BUFFER_MODES = new Set<ChannelBufferMode>(["off", "debounce"]);
33
+ const CHANNEL_AUDIO_MODES = new Set<ChannelAudioMode>(["off", "reject", "transcribe"]);
31
34
  const TELEGRAM_ALLOWED_UPDATES = new Set<TelegramAllowedUpdate>(["message"]);
32
35
  const KNOWLEDGE_EMBEDDING_PROVIDERS = new Set<AgentKnowledgeEmbeddingProvider>(["none", "test", "openai"]);
36
+ const TRANSCRIPTION_PROVIDERS = new Set<TranscriptionProviderName>(["test", "openai", "groq"]);
33
37
  const CHANNEL_NAME_PATTERN = /^[a-z][a-z0-9-]*$/;
34
38
  const SECRET_NAME_PATTERN = /^[A-Z_][A-Z0-9_]*$/;
35
39
  const moduleDir = dirname(fileURLToPath(import.meta.url));
@@ -230,7 +234,8 @@ function validateAgentConfig(value: unknown): asserts value is AgentConfig {
230
234
  }
231
235
  }
232
236
 
233
- validateChannels(value.channels, value.runtime as AgentRuntime);
237
+ validateTranscriptionConfig(value.transcription);
238
+ validateChannels(value.channels, value.runtime as AgentRuntime, Boolean(value.transcription));
234
239
  validateKnowledgeConfig(value.knowledge);
235
240
 
236
241
  if (!isRecord(value.access) || !ACCESS_MODES.has(value.access.mode as AccessMode)) {
@@ -253,6 +258,46 @@ function validateAgentConfig(value: unknown): asserts value is AgentConfig {
253
258
  validateFilesConfig(value.storage.files);
254
259
  }
255
260
 
261
+ function validateTranscriptionConfig(value: unknown): void {
262
+ if (value === undefined) {
263
+ return;
264
+ }
265
+
266
+ if (!isRecord(value)) {
267
+ throw invalidConfig("transcription must be an object when provided.");
268
+ }
269
+
270
+ if (!TRANSCRIPTION_PROVIDERS.has(value.provider as TranscriptionProviderName)) {
271
+ throw invalidConfig(`transcription.provider must be one of: ${Array.from(TRANSCRIPTION_PROVIDERS).join(", ")}.`);
272
+ }
273
+
274
+ expectString(value.model, "transcription.model");
275
+
276
+ if (value.secret !== undefined) {
277
+ expectString(value.secret, "transcription.secret");
278
+ validateSecretNames([value.secret], "transcription.secret");
279
+ }
280
+
281
+ if (value.language !== undefined) {
282
+ expectString(value.language, "transcription.language");
283
+ }
284
+
285
+ if (value.prompt !== undefined) {
286
+ expectString(value.prompt, "transcription.prompt");
287
+ }
288
+
289
+ if (value.limits !== undefined) {
290
+ if (!isRecord(value.limits)) {
291
+ throw invalidConfig("transcription.limits must be an object when provided.");
292
+ }
293
+
294
+ validateOptionalPositiveInteger(value.limits.maxDurationSeconds, "transcription.limits.maxDurationSeconds");
295
+ validateOptionalPositiveInteger(value.limits.maxBytes, "transcription.limits.maxBytes");
296
+ }
297
+
298
+ validateOptionalPositiveInteger(value.rawAudioTtlSeconds, "transcription.rawAudioTtlSeconds");
299
+ }
300
+
256
301
  function validateKnowledgeConfig(value: unknown): void {
257
302
  if (value === undefined) {
258
303
  return;
@@ -381,7 +426,7 @@ function validateKnowledgeRetrieval(value: unknown): void {
381
426
  }
382
427
  }
383
428
 
384
- function validateChannels(value: unknown, runtime: AgentRuntime): void {
429
+ function validateChannels(value: unknown, runtime: AgentRuntime, hasTranscriptionConfig: boolean): void {
385
430
  if (value === undefined) {
386
431
  return;
387
432
  }
@@ -397,7 +442,7 @@ function validateChannels(value: unknown, runtime: AgentRuntime): void {
397
442
  const names = new Set<string>();
398
443
 
399
444
  for (const [index, channel] of value.entries()) {
400
- validateChannel(channel, index);
445
+ validateChannel(channel, index, hasTranscriptionConfig);
401
446
 
402
447
  if (names.has(channel.name)) {
403
448
  throw invalidConfig(`channels[${index}].name must be unique within the Agent Capsule.`);
@@ -407,7 +452,7 @@ function validateChannels(value: unknown, runtime: AgentRuntime): void {
407
452
  }
408
453
  }
409
454
 
410
- function validateChannel(value: unknown, index: number): asserts value is AgentChannel {
455
+ function validateChannel(value: unknown, index: number, hasTranscriptionConfig: boolean): asserts value is AgentChannel {
411
456
  if (!isRecord(value)) {
412
457
  throw invalidConfig(`channels[${index}] must be an object.`);
413
458
  }
@@ -432,6 +477,7 @@ function validateChannel(value: unknown, index: number): asserts value is AgentC
432
477
  validateSecretNames(value.secrets, `channels[${index}].secrets`);
433
478
  validateChannelLimits(value.limits, index);
434
479
  validateChannelBuffer(value.buffer, index);
480
+ validateChannelAudio(value.audio, index, hasTranscriptionConfig);
435
481
 
436
482
  if (value.type === "website") {
437
483
  validateWebsiteChannel(value, index);
@@ -446,6 +492,31 @@ function validateChannel(value: unknown, index: number): asserts value is AgentC
446
492
  validateWhatsappChannel(value, index);
447
493
  }
448
494
 
495
+ function validateChannelAudio(value: unknown, index: number, hasTranscriptionConfig: boolean): void {
496
+ if (value === undefined) {
497
+ return;
498
+ }
499
+
500
+ if (!isRecord(value)) {
501
+ throw invalidConfig(`channels[${index}].audio must be an object when provided.`);
502
+ }
503
+
504
+ if (!CHANNEL_AUDIO_MODES.has(value.mode as ChannelAudioMode)) {
505
+ throw invalidConfig(`channels[${index}].audio.mode must be "off", "reject", or "transcribe".`);
506
+ }
507
+
508
+ if (value.mode === "transcribe" && !hasTranscriptionConfig) {
509
+ throw invalidConfig(`channels[${index}].audio.mode "transcribe" requires top-level transcription config.`);
510
+ }
511
+
512
+ if (value.mode !== "off") {
513
+ expectString(value.fallbackMessage, `channels[${index}].audio.fallbackMessage`);
514
+ }
515
+
516
+ validateOptionalPositiveInteger(value.maxDurationSeconds, `channels[${index}].audio.maxDurationSeconds`);
517
+ validateOptionalPositiveInteger(value.maxBytes, `channels[${index}].audio.maxBytes`);
518
+ }
519
+
449
520
  function validateWebsiteChannel(value: Record<string, unknown>, index: number): void {
450
521
  if (value.provider !== "agentkit") {
451
522
  throw invalidConfig(`channels[${index}].provider must be "agentkit" when type is "website".`);
@@ -562,6 +633,16 @@ function validateSecretNames(secrets: string[], field: string): void {
562
633
  }
563
634
  }
564
635
 
636
+ function validateOptionalPositiveInteger(value: unknown, field: string): void {
637
+ if (value === undefined) {
638
+ return;
639
+ }
640
+
641
+ if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
642
+ throw invalidConfig(`${field} must be a positive integer when provided.`);
643
+ }
644
+ }
645
+
565
646
  function expectRequiredSecrets(value: unknown, requiredSecrets: string[], field: string): void {
566
647
  const declaredSecrets = new Set(Array.isArray(value) ? value : []);
567
648
  const missingSecrets = requiredSecrets.filter((secret) => !declaredSecrets.has(secret));
@@ -4,6 +4,7 @@ import type { AgentConfig, AgentTool } from "../../index";
4
4
  import type { LoadedAgentCapsule } from "../config";
5
5
  import { resolveKnowledgeConfig } from "../knowledge/config";
6
6
  import { KNOWLEDGE_SEARCH_TOOL_NAME } from "../knowledge/tool";
7
+ import { resolveTranscriptionConfig } from "../transcription";
7
8
 
8
9
  export type ManifestSchema = {
9
10
  source: string;
@@ -28,6 +29,19 @@ export type SharedAgentManifest = {
28
29
  timeoutMs: number | null;
29
30
  }>;
30
31
  channels: NonNullable<AgentConfig["channels"]>;
32
+ transcription: {
33
+ enabled: boolean;
34
+ provider: string | null;
35
+ model: string | null;
36
+ secret: string | null;
37
+ language: string | null;
38
+ prompt: string | null;
39
+ limits: {
40
+ maxDurationSeconds: number | null;
41
+ maxBytes: number | null;
42
+ };
43
+ rawAudioTtlSeconds: number | null;
44
+ };
31
45
  knowledge: {
32
46
  enabled: boolean;
33
47
  sources: Array<{
@@ -80,6 +94,7 @@ export function buildSharedAgentManifest(
80
94
  const database = buildDatabaseManifest(capsule, options.databaseSchema ?? null);
81
95
  const files = buildFilesManifest(capsule);
82
96
  const knowledge = resolveKnowledgeConfig(capsule.config);
97
+ const transcription = resolveTranscriptionConfig(capsule.config.transcription);
83
98
 
84
99
  requiredSecrets.add("AGENTKIT_PRIVATE_TOKEN");
85
100
 
@@ -100,6 +115,10 @@ export function buildSharedAgentManifest(
100
115
  requiredSecrets.add(knowledge.embedding.secret);
101
116
  }
102
117
 
118
+ if (transcription.secret) {
119
+ requiredSecrets.add(transcription.secret);
120
+ }
121
+
103
122
  for (const tool of capsule.config.tools ?? []) {
104
123
  for (const secret of tool.secrets ?? []) {
105
124
  requiredSecrets.add(secret);
@@ -130,6 +149,19 @@ export function buildSharedAgentManifest(
130
149
  timeoutMs: tool.timeoutMs ?? null,
131
150
  })),
132
151
  channels: capsule.config.channels ?? [],
152
+ transcription: {
153
+ enabled: transcription.enabled,
154
+ provider: transcription.enabled ? transcription.provider : null,
155
+ model: transcription.enabled ? transcription.model : null,
156
+ secret: transcription.secret,
157
+ language: transcription.language,
158
+ prompt: transcription.prompt,
159
+ limits: {
160
+ maxDurationSeconds: transcription.limits.maxDurationSeconds ?? null,
161
+ maxBytes: transcription.limits.maxBytes ?? null,
162
+ },
163
+ rawAudioTtlSeconds: transcription.rawAudioTtlSeconds,
164
+ },
133
165
  knowledge: {
134
166
  enabled: knowledge.enabled,
135
167
  sources: knowledge.sources.map((source) => ({