@andreprado/agentkit 0.1.0-alpha.15 → 0.1.0-alpha.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/guides/add-channel.md +63 -0
- package/docs/guides/channel-security.md +32 -0
- package/docs/guides/connect-telegram.md +58 -0
- package/docs/guides/connect-whatsapp-zapster.md +65 -0
- package/package.json +1 -1
- package/src/cli/commands/channels.ts +2 -0
- package/src/index.ts +86 -0
- package/src/runtime/channel-test-harness.ts +2 -0
- package/src/runtime/channels/telegram.ts +326 -10
- package/src/runtime/channels/whatsapp-zapster.ts +319 -0
- package/src/runtime/channels.ts +47 -1
- package/src/runtime/config.ts +85 -4
- package/src/runtime/core/manifest.ts +32 -0
- package/src/runtime/dev-server.ts +72 -4
- package/src/runtime/inspect.ts +34 -0
- package/src/runtime/targets/cloudflare/build.ts +2 -0
- package/src/runtime/transcription.ts +483 -0
- package/src/templates/skills/agentkit-channels/SKILL.md +34 -1
- package/src/templates/skills/agentkit-channels/references/channel-debugging.md +13 -0
- package/src/templates/skills/agentkit-channels/references/telegram.md +32 -0
- package/src/templates/skills/agentkit-channels/references/whatsapp-zapster.md +29 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
ChannelAdapter,
|
|
3
|
+
ChannelMediaDownloadInput,
|
|
4
|
+
ChannelMediaDownloadResult,
|
|
3
5
|
ChannelSendInput,
|
|
4
6
|
ChannelSendResult,
|
|
5
7
|
RawWebhookEvent,
|
|
@@ -105,6 +107,22 @@ export const zapsterWhatsappChannelAdapter: ChannelAdapter = {
|
|
|
105
107
|
},
|
|
106
108
|
};
|
|
107
109
|
|
|
110
|
+
if (messageType === "audio" && message.audio) {
|
|
111
|
+
return [
|
|
112
|
+
{
|
|
113
|
+
...base,
|
|
114
|
+
kind: "message",
|
|
115
|
+
message: {
|
|
116
|
+
role: "user",
|
|
117
|
+
contentType: "audio",
|
|
118
|
+
content: "",
|
|
119
|
+
providerMessageId: message.id,
|
|
120
|
+
audio: message.audio,
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
];
|
|
124
|
+
}
|
|
125
|
+
|
|
108
126
|
if (messageType !== "text") {
|
|
109
127
|
return [
|
|
110
128
|
{
|
|
@@ -140,6 +158,9 @@ export const zapsterWhatsappChannelAdapter: ChannelAdapter = {
|
|
|
140
158
|
},
|
|
141
159
|
];
|
|
142
160
|
},
|
|
161
|
+
async downloadMedia(input) {
|
|
162
|
+
return downloadZapsterMedia(input);
|
|
163
|
+
},
|
|
143
164
|
async sendMessage(input) {
|
|
144
165
|
return sendZapsterMessage(input);
|
|
145
166
|
},
|
|
@@ -169,6 +190,15 @@ type NormalizedZapsterWebhookMessage = {
|
|
|
169
190
|
contactId?: string;
|
|
170
191
|
type?: string;
|
|
171
192
|
text?: string;
|
|
193
|
+
audio?: {
|
|
194
|
+
providerMediaId: string;
|
|
195
|
+
kind: "voice" | "audio";
|
|
196
|
+
mimeType?: string;
|
|
197
|
+
filename?: string;
|
|
198
|
+
durationSeconds?: number;
|
|
199
|
+
sizeBytes?: number;
|
|
200
|
+
downloadUrl?: string;
|
|
201
|
+
};
|
|
172
202
|
};
|
|
173
203
|
|
|
174
204
|
function normalizeZapsterOfficialMessage(body: Record<string, unknown>): NormalizedZapsterWebhookMessage | null {
|
|
@@ -185,17 +215,48 @@ function normalizeZapsterOfficialMessage(body: Record<string, unknown>): Normali
|
|
|
185
215
|
contactId: readString(sender, "id"),
|
|
186
216
|
type: readString(data, "type"),
|
|
187
217
|
text: readString(content, "text"),
|
|
218
|
+
audio: normalizeZapsterAudio(content),
|
|
188
219
|
};
|
|
189
220
|
}
|
|
190
221
|
|
|
191
222
|
function normalizeZapsterLegacyMessage(body: Record<string, unknown>): NormalizedZapsterWebhookMessage {
|
|
192
223
|
const message = readRecord(body, "message");
|
|
224
|
+
const audio = readRecord(message, "audio");
|
|
193
225
|
|
|
194
226
|
return {
|
|
195
227
|
id: readString(message, "id"),
|
|
196
228
|
contactId: readString(message, "from"),
|
|
197
229
|
type: readString(message, "type"),
|
|
198
230
|
text: readString(readRecord(message, "text"), "body"),
|
|
231
|
+
audio: normalizeZapsterAudio(audio),
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function normalizeZapsterAudio(value: Record<string, unknown>): NormalizedZapsterWebhookMessage["audio"] {
|
|
236
|
+
const mediaId = readString(value, "id") ?? readString(value, "media_id") ?? readString(value, "mediaId");
|
|
237
|
+
const downloadUrl =
|
|
238
|
+
readString(value, "url") ??
|
|
239
|
+
readString(value, "download_url") ??
|
|
240
|
+
readString(value, "downloadUrl") ??
|
|
241
|
+
readString(value, "media_url") ??
|
|
242
|
+
readString(value, "mediaUrl");
|
|
243
|
+
|
|
244
|
+
if (!mediaId && !downloadUrl) {
|
|
245
|
+
return undefined;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return {
|
|
249
|
+
providerMediaId: mediaId ?? downloadUrl ?? "unknown",
|
|
250
|
+
kind: readString(value, "ptt") === "true" ? "voice" : "audio",
|
|
251
|
+
...(readString(value, "mime_type") ?? readString(value, "mimeType")
|
|
252
|
+
? { mimeType: readString(value, "mime_type") ?? readString(value, "mimeType") }
|
|
253
|
+
: {}),
|
|
254
|
+
filename: readString(value, "filename") ?? readString(value, "file_name") ?? defaultZapsterAudioFilename(value),
|
|
255
|
+
...(readNumber(value, "duration") !== undefined ? { durationSeconds: readNumber(value, "duration") } : {}),
|
|
256
|
+
...(readNumber(value, "file_size") ?? readNumber(value, "size")
|
|
257
|
+
? { sizeBytes: readNumber(value, "file_size") ?? readNumber(value, "size") }
|
|
258
|
+
: {}),
|
|
259
|
+
...(downloadUrl ? { downloadUrl } : {}),
|
|
199
260
|
};
|
|
200
261
|
}
|
|
201
262
|
|
|
@@ -249,6 +310,8 @@ export type ZapsterSendMessageRequest = {
|
|
|
249
310
|
type ZapsterFetch = (url: string, init: RequestInit) => Promise<Response>;
|
|
250
311
|
|
|
251
312
|
const ZAPSTER_SEND_MESSAGE_URL = "https://api.zapsterapi.com/v1/wa/messages";
|
|
313
|
+
const ZAPSTER_DEFAULT_MEDIA_HOSTS = ["api.zapsterapi.com"];
|
|
314
|
+
const ZAPSTER_MAX_DOWNLOAD_BYTES = 25_000_000;
|
|
252
315
|
|
|
253
316
|
export function buildZapsterSendMessageRequest(input: ChannelSendInput): ZapsterSendMessageRequest {
|
|
254
317
|
return buildZapsterSendMessageRequestInternal(input, "<redacted>");
|
|
@@ -381,6 +444,239 @@ export async function sendZapsterMessage(
|
|
|
381
444
|
};
|
|
382
445
|
}
|
|
383
446
|
|
|
447
|
+
export async function downloadZapsterMedia(
|
|
448
|
+
input: ChannelMediaDownloadInput,
|
|
449
|
+
fetcher: ZapsterFetch = fetch,
|
|
450
|
+
): Promise<ChannelMediaDownloadResult> {
|
|
451
|
+
const apiKey = input.secrets.ZAPSTER_API_KEY;
|
|
452
|
+
const audio = input.message.audio;
|
|
453
|
+
|
|
454
|
+
if (!apiKey) {
|
|
455
|
+
return {
|
|
456
|
+
ok: false,
|
|
457
|
+
retryable: false,
|
|
458
|
+
code: "channel_secret_missing",
|
|
459
|
+
message: "Secret ZAPSTER_API_KEY is missing.",
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
if (!audio?.downloadUrl) {
|
|
464
|
+
return {
|
|
465
|
+
ok: false,
|
|
466
|
+
retryable: false,
|
|
467
|
+
code: "channel_audio_download_unavailable",
|
|
468
|
+
message: "Zapster audio payload did not include a media download URL.",
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
const mediaUrl = validateZapsterMediaDownloadUrl(audio.downloadUrl);
|
|
473
|
+
|
|
474
|
+
if (!mediaUrl.ok) {
|
|
475
|
+
return mediaUrl;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
let response: Response;
|
|
479
|
+
|
|
480
|
+
try {
|
|
481
|
+
response = await fetcher(mediaUrl.url, {
|
|
482
|
+
method: "GET",
|
|
483
|
+
headers: {
|
|
484
|
+
Authorization: `Bearer ${apiKey}`,
|
|
485
|
+
},
|
|
486
|
+
});
|
|
487
|
+
} catch (error) {
|
|
488
|
+
return {
|
|
489
|
+
ok: false,
|
|
490
|
+
retryable: true,
|
|
491
|
+
code: "channel_provider_unavailable",
|
|
492
|
+
message: `Zapster media download failed before a response: ${redactString(
|
|
493
|
+
error instanceof Error ? error.message : String(error),
|
|
494
|
+
[apiKey, mediaUrl.url],
|
|
495
|
+
)}`,
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
if (!response.ok) {
|
|
500
|
+
return {
|
|
501
|
+
ok: false,
|
|
502
|
+
retryable: response.status === 429 || response.status >= 500,
|
|
503
|
+
code: response.status === 429 || response.status >= 500
|
|
504
|
+
? "channel_provider_unavailable"
|
|
505
|
+
: "channel_media_download_failed",
|
|
506
|
+
message: `Zapster media download returned HTTP ${response.status}.`,
|
|
507
|
+
providerMetadata: {
|
|
508
|
+
status: response.status,
|
|
509
|
+
},
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const maxDownloadBytes = input.maxBytes ?? ZAPSTER_MAX_DOWNLOAD_BYTES;
|
|
514
|
+
const bytes = await readResponseBytesWithLimit(response, maxDownloadBytes);
|
|
515
|
+
|
|
516
|
+
if (!bytes.ok) {
|
|
517
|
+
return bytes;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
return {
|
|
521
|
+
ok: true,
|
|
522
|
+
audio: bytes.audio,
|
|
523
|
+
filename: audio.filename ?? `${audio.providerMediaId}.ogg`,
|
|
524
|
+
...(audio.mimeType ? { mimeType: audio.mimeType } : {}),
|
|
525
|
+
...(audio.durationSeconds !== undefined ? { durationSeconds: audio.durationSeconds } : {}),
|
|
526
|
+
sizeBytes: bytes.audio.byteLength,
|
|
527
|
+
providerMetadata: {
|
|
528
|
+
status: response.status,
|
|
529
|
+
},
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function validateZapsterMediaDownloadUrl(rawUrl: string):
|
|
534
|
+
| {
|
|
535
|
+
ok: true;
|
|
536
|
+
url: string;
|
|
537
|
+
}
|
|
538
|
+
| {
|
|
539
|
+
ok: false;
|
|
540
|
+
retryable: false;
|
|
541
|
+
code: "channel_audio_download_unavailable";
|
|
542
|
+
message: string;
|
|
543
|
+
} {
|
|
544
|
+
let url: URL;
|
|
545
|
+
|
|
546
|
+
try {
|
|
547
|
+
url = new URL(rawUrl);
|
|
548
|
+
} catch {
|
|
549
|
+
return {
|
|
550
|
+
ok: false,
|
|
551
|
+
retryable: false,
|
|
552
|
+
code: "channel_audio_download_unavailable",
|
|
553
|
+
message: "Zapster audio download URL is not a valid URL.",
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
if (url.protocol !== "https:") {
|
|
558
|
+
return {
|
|
559
|
+
ok: false,
|
|
560
|
+
retryable: false,
|
|
561
|
+
code: "channel_audio_download_unavailable",
|
|
562
|
+
message: "Zapster audio download URL must use HTTPS.",
|
|
563
|
+
};
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
const hostname = url.hostname.toLowerCase();
|
|
567
|
+
const allowed = ZAPSTER_DEFAULT_MEDIA_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`));
|
|
568
|
+
|
|
569
|
+
if (!allowed) {
|
|
570
|
+
return {
|
|
571
|
+
ok: false,
|
|
572
|
+
retryable: false,
|
|
573
|
+
code: "channel_audio_download_unavailable",
|
|
574
|
+
message: "Zapster audio download URL host is not allowed.",
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
url.username = "";
|
|
579
|
+
url.password = "";
|
|
580
|
+
|
|
581
|
+
return {
|
|
582
|
+
ok: true,
|
|
583
|
+
url: url.href,
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
async function readResponseBytesWithLimit(
|
|
588
|
+
response: Response,
|
|
589
|
+
maxBytes: number,
|
|
590
|
+
): Promise<
|
|
591
|
+
| {
|
|
592
|
+
ok: true;
|
|
593
|
+
audio: Uint8Array;
|
|
594
|
+
}
|
|
595
|
+
| {
|
|
596
|
+
ok: false;
|
|
597
|
+
retryable: false;
|
|
598
|
+
code: "channel_audio_too_large";
|
|
599
|
+
message: string;
|
|
600
|
+
providerMetadata?: Record<string, unknown>;
|
|
601
|
+
}
|
|
602
|
+
> {
|
|
603
|
+
const contentLength = response.headers.get("content-length");
|
|
604
|
+
const declaredBytes = contentLength ? Number(contentLength) : undefined;
|
|
605
|
+
|
|
606
|
+
if (declaredBytes !== undefined && Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
|
|
607
|
+
return {
|
|
608
|
+
ok: false,
|
|
609
|
+
retryable: false,
|
|
610
|
+
code: "channel_audio_too_large",
|
|
611
|
+
message: `Audio response declares ${declaredBytes} bytes, which exceeds the configured ${maxBytes} byte limit.`,
|
|
612
|
+
providerMetadata: {
|
|
613
|
+
sizeBytes: declaredBytes,
|
|
614
|
+
},
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
if (!response.body) {
|
|
619
|
+
const audio = new Uint8Array(await response.arrayBuffer());
|
|
620
|
+
|
|
621
|
+
if (audio.byteLength > maxBytes) {
|
|
622
|
+
return {
|
|
623
|
+
ok: false,
|
|
624
|
+
retryable: false,
|
|
625
|
+
code: "channel_audio_too_large",
|
|
626
|
+
message: `Audio response is ${audio.byteLength} bytes, which exceeds the configured ${maxBytes} byte limit.`,
|
|
627
|
+
providerMetadata: {
|
|
628
|
+
sizeBytes: audio.byteLength,
|
|
629
|
+
},
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
return { ok: true, audio };
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
const reader = response.body.getReader();
|
|
637
|
+
const chunks: Uint8Array[] = [];
|
|
638
|
+
let totalBytes = 0;
|
|
639
|
+
|
|
640
|
+
try {
|
|
641
|
+
while (true) {
|
|
642
|
+
const chunk = await reader.read();
|
|
643
|
+
|
|
644
|
+
if (chunk.done) {
|
|
645
|
+
break;
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
totalBytes += chunk.value.byteLength;
|
|
649
|
+
|
|
650
|
+
if (totalBytes > maxBytes) {
|
|
651
|
+
await reader.cancel().catch(() => undefined);
|
|
652
|
+
return {
|
|
653
|
+
ok: false,
|
|
654
|
+
retryable: false,
|
|
655
|
+
code: "channel_audio_too_large",
|
|
656
|
+
message: `Audio response exceeded the configured ${maxBytes} byte limit while downloading.`,
|
|
657
|
+
providerMetadata: {
|
|
658
|
+
sizeBytes: totalBytes,
|
|
659
|
+
},
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
chunks.push(chunk.value);
|
|
664
|
+
}
|
|
665
|
+
} finally {
|
|
666
|
+
reader.releaseLock();
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
const audio = new Uint8Array(totalBytes);
|
|
670
|
+
let offset = 0;
|
|
671
|
+
|
|
672
|
+
for (const chunk of chunks) {
|
|
673
|
+
audio.set(chunk, offset);
|
|
674
|
+
offset += chunk.byteLength;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
return { ok: true, audio };
|
|
678
|
+
}
|
|
679
|
+
|
|
384
680
|
function buildZapsterSendMessageRequestInternal(
|
|
385
681
|
input: ChannelSendInput,
|
|
386
682
|
instanceId: string,
|
|
@@ -494,6 +790,29 @@ function readString(value: Record<string, unknown>, key: string): string | undef
|
|
|
494
790
|
return typeof field === "string" && field.length > 0 ? field : undefined;
|
|
495
791
|
}
|
|
496
792
|
|
|
793
|
+
function readNumber(value: Record<string, unknown>, key: string): number | undefined {
|
|
794
|
+
const field = value[key];
|
|
795
|
+
return typeof field === "number" && Number.isFinite(field) ? field : undefined;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
function defaultZapsterAudioFilename(value: Record<string, unknown>): string {
|
|
799
|
+
const mimeType = readString(value, "mime_type") ?? readString(value, "mimeType");
|
|
800
|
+
|
|
801
|
+
if (mimeType?.includes("ogg")) {
|
|
802
|
+
return "audio.ogg";
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
if (mimeType?.includes("mpeg") || mimeType?.includes("mp3")) {
|
|
806
|
+
return "audio.mp3";
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
if (mimeType?.includes("wav")) {
|
|
810
|
+
return "audio.wav";
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
return "audio.ogg";
|
|
814
|
+
}
|
|
815
|
+
|
|
497
816
|
function isPositiveIntegerHeader(value: string | null): value is string {
|
|
498
817
|
return typeof value === "string" && /^[1-9]\d*$/.test(value);
|
|
499
818
|
}
|
package/src/runtime/channels.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import type { AgentChannel, ChannelProvider, ChannelType } from "../index";
|
|
2
2
|
|
|
3
3
|
export type ChannelMessageRole = "user" | "assistant";
|
|
4
|
-
export type ChannelMessageContentType = "text";
|
|
4
|
+
export type ChannelMessageContentType = "text" | "audio";
|
|
5
5
|
export type ChannelEventKind = "message" | "delivery_status" | "unsupported";
|
|
6
6
|
export type ChannelDeliveryDirection = "inbound" | "outbound";
|
|
7
7
|
export type ChannelDeliveryState =
|
|
8
8
|
| "webhook_received"
|
|
9
9
|
| "received"
|
|
10
10
|
| "validated"
|
|
11
|
+
| "audio_received"
|
|
12
|
+
| "audio_downloaded"
|
|
13
|
+
| "transcribing"
|
|
14
|
+
| "transcribed"
|
|
11
15
|
| "duplicate"
|
|
12
16
|
| "buffered"
|
|
13
17
|
| "queued"
|
|
@@ -45,11 +49,22 @@ export type ChannelExternalIdentity = {
|
|
|
45
49
|
redactedDisplay?: string;
|
|
46
50
|
};
|
|
47
51
|
|
|
52
|
+
export type ChannelAudioAttachment = {
|
|
53
|
+
providerMediaId: string;
|
|
54
|
+
kind: "voice" | "audio";
|
|
55
|
+
mimeType?: string;
|
|
56
|
+
filename?: string;
|
|
57
|
+
durationSeconds?: number;
|
|
58
|
+
sizeBytes?: number;
|
|
59
|
+
downloadUrl?: string;
|
|
60
|
+
};
|
|
61
|
+
|
|
48
62
|
export type NormalizedChannelMessage = {
|
|
49
63
|
role: ChannelMessageRole;
|
|
50
64
|
contentType: ChannelMessageContentType;
|
|
51
65
|
content: string;
|
|
52
66
|
providerMessageId?: string;
|
|
67
|
+
audio?: ChannelAudioAttachment;
|
|
53
68
|
};
|
|
54
69
|
|
|
55
70
|
export type NormalizedChannelEvent = ChannelAdapterId & {
|
|
@@ -87,6 +102,36 @@ export type ChannelSendInput = ChannelAdapterId & {
|
|
|
87
102
|
secrets: Record<string, string>;
|
|
88
103
|
};
|
|
89
104
|
|
|
105
|
+
export type ChannelMediaDownloadInput = ChannelAdapterId & {
|
|
106
|
+
channelId: string;
|
|
107
|
+
message: NormalizedChannelMessage;
|
|
108
|
+
secrets: Record<string, string>;
|
|
109
|
+
maxBytes?: number;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
export type ChannelMediaDownloadResult =
|
|
113
|
+
| {
|
|
114
|
+
ok: true;
|
|
115
|
+
audio: Uint8Array;
|
|
116
|
+
filename: string;
|
|
117
|
+
mimeType?: string;
|
|
118
|
+
durationSeconds?: number;
|
|
119
|
+
sizeBytes?: number;
|
|
120
|
+
providerMetadata?: Record<string, unknown>;
|
|
121
|
+
}
|
|
122
|
+
| {
|
|
123
|
+
ok: false;
|
|
124
|
+
retryable: boolean;
|
|
125
|
+
code:
|
|
126
|
+
| "channel_secret_missing"
|
|
127
|
+
| "channel_audio_download_unavailable"
|
|
128
|
+
| "channel_audio_too_large"
|
|
129
|
+
| "channel_provider_unavailable"
|
|
130
|
+
| "channel_media_download_failed";
|
|
131
|
+
message: string;
|
|
132
|
+
providerMetadata?: Record<string, unknown>;
|
|
133
|
+
};
|
|
134
|
+
|
|
90
135
|
export type ChannelSendResult =
|
|
91
136
|
| {
|
|
92
137
|
ok: true;
|
|
@@ -120,6 +165,7 @@ export type ChannelStatusResult = {
|
|
|
120
165
|
export type ChannelAdapter = ChannelAdapterId & {
|
|
121
166
|
requiredSecrets: string[];
|
|
122
167
|
normalizeInbound(input: RawWebhookEvent): Promise<NormalizedChannelEvent[]> | NormalizedChannelEvent[];
|
|
168
|
+
downloadMedia?(input: ChannelMediaDownloadInput): Promise<ChannelMediaDownloadResult>;
|
|
123
169
|
sendMessage(input: ChannelSendInput): Promise<ChannelSendResult>;
|
|
124
170
|
verifyWebhook?(input: WebhookVerificationInput): Promise<WebhookVerificationResult> | WebhookVerificationResult;
|
|
125
171
|
getStatus?(input: ChannelStatusInput): Promise<ChannelStatusResult> | ChannelStatusResult;
|
package/src/runtime/config.ts
CHANGED
|
@@ -16,6 +16,8 @@ import type {
|
|
|
16
16
|
TelegramAllowedUpdate,
|
|
17
17
|
AgentKnowledgeEmbeddingProvider,
|
|
18
18
|
ChannelBufferMode,
|
|
19
|
+
ChannelAudioMode,
|
|
20
|
+
TranscriptionProviderName,
|
|
19
21
|
} from "../index";
|
|
20
22
|
import { AgentKitError } from "./errors";
|
|
21
23
|
import { isValidTimeZone } from "./prompt-context";
|
|
@@ -28,8 +30,10 @@ const TOOL_VISIBILITIES = new Set<ToolVisibility>(["user", "internal"]);
|
|
|
28
30
|
const CHANNEL_TYPES = new Set<ChannelType>(["website", "telegram", "whatsapp"]);
|
|
29
31
|
const CHANNEL_PROVIDERS = new Set<ChannelProvider>(["agentkit", "telegram", "zapster", "meta"]);
|
|
30
32
|
const CHANNEL_BUFFER_MODES = new Set<ChannelBufferMode>(["off", "debounce"]);
|
|
33
|
+
const CHANNEL_AUDIO_MODES = new Set<ChannelAudioMode>(["off", "reject", "transcribe"]);
|
|
31
34
|
const TELEGRAM_ALLOWED_UPDATES = new Set<TelegramAllowedUpdate>(["message"]);
|
|
32
35
|
const KNOWLEDGE_EMBEDDING_PROVIDERS = new Set<AgentKnowledgeEmbeddingProvider>(["none", "test", "openai"]);
|
|
36
|
+
const TRANSCRIPTION_PROVIDERS = new Set<TranscriptionProviderName>(["test", "openai", "groq"]);
|
|
33
37
|
const CHANNEL_NAME_PATTERN = /^[a-z][a-z0-9-]*$/;
|
|
34
38
|
const SECRET_NAME_PATTERN = /^[A-Z_][A-Z0-9_]*$/;
|
|
35
39
|
const moduleDir = dirname(fileURLToPath(import.meta.url));
|
|
@@ -230,7 +234,8 @@ function validateAgentConfig(value: unknown): asserts value is AgentConfig {
|
|
|
230
234
|
}
|
|
231
235
|
}
|
|
232
236
|
|
|
233
|
-
|
|
237
|
+
validateTranscriptionConfig(value.transcription);
|
|
238
|
+
validateChannels(value.channels, value.runtime as AgentRuntime, Boolean(value.transcription));
|
|
234
239
|
validateKnowledgeConfig(value.knowledge);
|
|
235
240
|
|
|
236
241
|
if (!isRecord(value.access) || !ACCESS_MODES.has(value.access.mode as AccessMode)) {
|
|
@@ -253,6 +258,46 @@ function validateAgentConfig(value: unknown): asserts value is AgentConfig {
|
|
|
253
258
|
validateFilesConfig(value.storage.files);
|
|
254
259
|
}
|
|
255
260
|
|
|
261
|
+
function validateTranscriptionConfig(value: unknown): void {
|
|
262
|
+
if (value === undefined) {
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
if (!isRecord(value)) {
|
|
267
|
+
throw invalidConfig("transcription must be an object when provided.");
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
if (!TRANSCRIPTION_PROVIDERS.has(value.provider as TranscriptionProviderName)) {
|
|
271
|
+
throw invalidConfig(`transcription.provider must be one of: ${Array.from(TRANSCRIPTION_PROVIDERS).join(", ")}.`);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
expectString(value.model, "transcription.model");
|
|
275
|
+
|
|
276
|
+
if (value.secret !== undefined) {
|
|
277
|
+
expectString(value.secret, "transcription.secret");
|
|
278
|
+
validateSecretNames([value.secret], "transcription.secret");
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
if (value.language !== undefined) {
|
|
282
|
+
expectString(value.language, "transcription.language");
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (value.prompt !== undefined) {
|
|
286
|
+
expectString(value.prompt, "transcription.prompt");
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
if (value.limits !== undefined) {
|
|
290
|
+
if (!isRecord(value.limits)) {
|
|
291
|
+
throw invalidConfig("transcription.limits must be an object when provided.");
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
validateOptionalPositiveInteger(value.limits.maxDurationSeconds, "transcription.limits.maxDurationSeconds");
|
|
295
|
+
validateOptionalPositiveInteger(value.limits.maxBytes, "transcription.limits.maxBytes");
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
validateOptionalPositiveInteger(value.rawAudioTtlSeconds, "transcription.rawAudioTtlSeconds");
|
|
299
|
+
}
|
|
300
|
+
|
|
256
301
|
function validateKnowledgeConfig(value: unknown): void {
|
|
257
302
|
if (value === undefined) {
|
|
258
303
|
return;
|
|
@@ -381,7 +426,7 @@ function validateKnowledgeRetrieval(value: unknown): void {
|
|
|
381
426
|
}
|
|
382
427
|
}
|
|
383
428
|
|
|
384
|
-
function validateChannels(value: unknown, runtime: AgentRuntime): void {
|
|
429
|
+
function validateChannels(value: unknown, runtime: AgentRuntime, hasTranscriptionConfig: boolean): void {
|
|
385
430
|
if (value === undefined) {
|
|
386
431
|
return;
|
|
387
432
|
}
|
|
@@ -397,7 +442,7 @@ function validateChannels(value: unknown, runtime: AgentRuntime): void {
|
|
|
397
442
|
const names = new Set<string>();
|
|
398
443
|
|
|
399
444
|
for (const [index, channel] of value.entries()) {
|
|
400
|
-
validateChannel(channel, index);
|
|
445
|
+
validateChannel(channel, index, hasTranscriptionConfig);
|
|
401
446
|
|
|
402
447
|
if (names.has(channel.name)) {
|
|
403
448
|
throw invalidConfig(`channels[${index}].name must be unique within the Agent Capsule.`);
|
|
@@ -407,7 +452,7 @@ function validateChannels(value: unknown, runtime: AgentRuntime): void {
|
|
|
407
452
|
}
|
|
408
453
|
}
|
|
409
454
|
|
|
410
|
-
function validateChannel(value: unknown, index: number): asserts value is AgentChannel {
|
|
455
|
+
function validateChannel(value: unknown, index: number, hasTranscriptionConfig: boolean): asserts value is AgentChannel {
|
|
411
456
|
if (!isRecord(value)) {
|
|
412
457
|
throw invalidConfig(`channels[${index}] must be an object.`);
|
|
413
458
|
}
|
|
@@ -432,6 +477,7 @@ function validateChannel(value: unknown, index: number): asserts value is AgentC
|
|
|
432
477
|
validateSecretNames(value.secrets, `channels[${index}].secrets`);
|
|
433
478
|
validateChannelLimits(value.limits, index);
|
|
434
479
|
validateChannelBuffer(value.buffer, index);
|
|
480
|
+
validateChannelAudio(value.audio, index, hasTranscriptionConfig);
|
|
435
481
|
|
|
436
482
|
if (value.type === "website") {
|
|
437
483
|
validateWebsiteChannel(value, index);
|
|
@@ -446,6 +492,31 @@ function validateChannel(value: unknown, index: number): asserts value is AgentC
|
|
|
446
492
|
validateWhatsappChannel(value, index);
|
|
447
493
|
}
|
|
448
494
|
|
|
495
|
+
function validateChannelAudio(value: unknown, index: number, hasTranscriptionConfig: boolean): void {
|
|
496
|
+
if (value === undefined) {
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
if (!isRecord(value)) {
|
|
501
|
+
throw invalidConfig(`channels[${index}].audio must be an object when provided.`);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
if (!CHANNEL_AUDIO_MODES.has(value.mode as ChannelAudioMode)) {
|
|
505
|
+
throw invalidConfig(`channels[${index}].audio.mode must be "off", "reject", or "transcribe".`);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
if (value.mode === "transcribe" && !hasTranscriptionConfig) {
|
|
509
|
+
throw invalidConfig(`channels[${index}].audio.mode "transcribe" requires top-level transcription config.`);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
if (value.mode !== "off") {
|
|
513
|
+
expectString(value.fallbackMessage, `channels[${index}].audio.fallbackMessage`);
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
validateOptionalPositiveInteger(value.maxDurationSeconds, `channels[${index}].audio.maxDurationSeconds`);
|
|
517
|
+
validateOptionalPositiveInteger(value.maxBytes, `channels[${index}].audio.maxBytes`);
|
|
518
|
+
}
|
|
519
|
+
|
|
449
520
|
function validateWebsiteChannel(value: Record<string, unknown>, index: number): void {
|
|
450
521
|
if (value.provider !== "agentkit") {
|
|
451
522
|
throw invalidConfig(`channels[${index}].provider must be "agentkit" when type is "website".`);
|
|
@@ -562,6 +633,16 @@ function validateSecretNames(secrets: string[], field: string): void {
|
|
|
562
633
|
}
|
|
563
634
|
}
|
|
564
635
|
|
|
636
|
+
function validateOptionalPositiveInteger(value: unknown, field: string): void {
|
|
637
|
+
if (value === undefined) {
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
642
|
+
throw invalidConfig(`${field} must be a positive integer when provided.`);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
565
646
|
function expectRequiredSecrets(value: unknown, requiredSecrets: string[], field: string): void {
|
|
566
647
|
const declaredSecrets = new Set(Array.isArray(value) ? value : []);
|
|
567
648
|
const missingSecrets = requiredSecrets.filter((secret) => !declaredSecrets.has(secret));
|
|
@@ -4,6 +4,7 @@ import type { AgentConfig, AgentTool } from "../../index";
|
|
|
4
4
|
import type { LoadedAgentCapsule } from "../config";
|
|
5
5
|
import { resolveKnowledgeConfig } from "../knowledge/config";
|
|
6
6
|
import { KNOWLEDGE_SEARCH_TOOL_NAME } from "../knowledge/tool";
|
|
7
|
+
import { resolveTranscriptionConfig } from "../transcription";
|
|
7
8
|
|
|
8
9
|
export type ManifestSchema = {
|
|
9
10
|
source: string;
|
|
@@ -28,6 +29,19 @@ export type SharedAgentManifest = {
|
|
|
28
29
|
timeoutMs: number | null;
|
|
29
30
|
}>;
|
|
30
31
|
channels: NonNullable<AgentConfig["channels"]>;
|
|
32
|
+
transcription: {
|
|
33
|
+
enabled: boolean;
|
|
34
|
+
provider: string | null;
|
|
35
|
+
model: string | null;
|
|
36
|
+
secret: string | null;
|
|
37
|
+
language: string | null;
|
|
38
|
+
prompt: string | null;
|
|
39
|
+
limits: {
|
|
40
|
+
maxDurationSeconds: number | null;
|
|
41
|
+
maxBytes: number | null;
|
|
42
|
+
};
|
|
43
|
+
rawAudioTtlSeconds: number | null;
|
|
44
|
+
};
|
|
31
45
|
knowledge: {
|
|
32
46
|
enabled: boolean;
|
|
33
47
|
sources: Array<{
|
|
@@ -80,6 +94,7 @@ export function buildSharedAgentManifest(
|
|
|
80
94
|
const database = buildDatabaseManifest(capsule, options.databaseSchema ?? null);
|
|
81
95
|
const files = buildFilesManifest(capsule);
|
|
82
96
|
const knowledge = resolveKnowledgeConfig(capsule.config);
|
|
97
|
+
const transcription = resolveTranscriptionConfig(capsule.config.transcription);
|
|
83
98
|
|
|
84
99
|
requiredSecrets.add("AGENTKIT_PRIVATE_TOKEN");
|
|
85
100
|
|
|
@@ -100,6 +115,10 @@ export function buildSharedAgentManifest(
|
|
|
100
115
|
requiredSecrets.add(knowledge.embedding.secret);
|
|
101
116
|
}
|
|
102
117
|
|
|
118
|
+
if (transcription.secret) {
|
|
119
|
+
requiredSecrets.add(transcription.secret);
|
|
120
|
+
}
|
|
121
|
+
|
|
103
122
|
for (const tool of capsule.config.tools ?? []) {
|
|
104
123
|
for (const secret of tool.secrets ?? []) {
|
|
105
124
|
requiredSecrets.add(secret);
|
|
@@ -130,6 +149,19 @@ export function buildSharedAgentManifest(
|
|
|
130
149
|
timeoutMs: tool.timeoutMs ?? null,
|
|
131
150
|
})),
|
|
132
151
|
channels: capsule.config.channels ?? [],
|
|
152
|
+
transcription: {
|
|
153
|
+
enabled: transcription.enabled,
|
|
154
|
+
provider: transcription.enabled ? transcription.provider : null,
|
|
155
|
+
model: transcription.enabled ? transcription.model : null,
|
|
156
|
+
secret: transcription.secret,
|
|
157
|
+
language: transcription.language,
|
|
158
|
+
prompt: transcription.prompt,
|
|
159
|
+
limits: {
|
|
160
|
+
maxDurationSeconds: transcription.limits.maxDurationSeconds ?? null,
|
|
161
|
+
maxBytes: transcription.limits.maxBytes ?? null,
|
|
162
|
+
},
|
|
163
|
+
rawAudioTtlSeconds: transcription.rawAudioTtlSeconds,
|
|
164
|
+
},
|
|
133
165
|
knowledge: {
|
|
134
166
|
enabled: knowledge.enabled,
|
|
135
167
|
sources: knowledge.sources.map((source) => ({
|