@andreprado/agentkit 0.1.0-alpha.14 → 0.1.0-alpha.16

package/src/runtime/channels/whatsapp-zapster.ts

@@ -1,5 +1,7 @@
 import type {
   ChannelAdapter,
+  ChannelMediaDownloadInput,
+  ChannelMediaDownloadResult,
   ChannelSendInput,
   ChannelSendResult,
   RawWebhookEvent,
@@ -105,6 +107,22 @@ export const zapsterWhatsappChannelAdapter: ChannelAdapter = {
       },
     };
 
+    if (messageType === "audio" && message.audio) {
+      return [
+        {
+          ...base,
+          kind: "message",
+          message: {
+            role: "user",
+            contentType: "audio",
+            content: "",
+            providerMessageId: message.id,
+            audio: message.audio,
+          },
+        },
+      ];
+    }
+
     if (messageType !== "text") {
       return [
         {
@@ -140,6 +158,9 @@ export const zapsterWhatsappChannelAdapter: ChannelAdapter = {
       },
     ];
   },
+  async downloadMedia(input) {
+    return downloadZapsterMedia(input);
+  },
   async sendMessage(input) {
     return sendZapsterMessage(input);
   },
@@ -169,6 +190,15 @@ type NormalizedZapsterWebhookMessage = {
   contactId?: string;
   type?: string;
   text?: string;
+  audio?: {
+    providerMediaId: string;
+    kind: "voice" | "audio";
+    mimeType?: string;
+    filename?: string;
+    durationSeconds?: number;
+    sizeBytes?: number;
+    downloadUrl?: string;
+  };
 };
 
 function normalizeZapsterOfficialMessage(body: Record<string, unknown>): NormalizedZapsterWebhookMessage | null {
@@ -185,17 +215,48 @@ function normalizeZapsterOfficialMessage(body: Record<string, unknown>): Normali
     contactId: readString(sender, "id"),
     type: readString(data, "type"),
     text: readString(content, "text"),
+    audio: normalizeZapsterAudio(content),
   };
 }
 
 function normalizeZapsterLegacyMessage(body: Record<string, unknown>): NormalizedZapsterWebhookMessage {
   const message = readRecord(body, "message");
+  const audio = readRecord(message, "audio");
 
   return {
     id: readString(message, "id"),
     contactId: readString(message, "from"),
     type: readString(message, "type"),
     text: readString(readRecord(message, "text"), "body"),
+    audio: normalizeZapsterAudio(audio),
+  };
+}
+
+function normalizeZapsterAudio(value: Record<string, unknown>): NormalizedZapsterWebhookMessage["audio"] {
+  const mediaId = readString(value, "id") ?? readString(value, "media_id") ?? readString(value, "mediaId");
+  const downloadUrl =
+    readString(value, "url") ??
+    readString(value, "download_url") ??
+    readString(value, "downloadUrl") ??
+    readString(value, "media_url") ??
+    readString(value, "mediaUrl");
+
+  if (!mediaId && !downloadUrl) {
+    return undefined;
+  }
+
+  return {
+    providerMediaId: mediaId ?? downloadUrl ?? "unknown",
+    kind: readString(value, "ptt") === "true" ? "voice" : "audio",
+    ...(readString(value, "mime_type") ?? readString(value, "mimeType")
+      ? { mimeType: readString(value, "mime_type") ?? readString(value, "mimeType") }
+      : {}),
+    filename: readString(value, "filename") ?? readString(value, "file_name") ?? defaultZapsterAudioFilename(value),
+    ...(readNumber(value, "duration") !== undefined ? { durationSeconds: readNumber(value, "duration") } : {}),
+    ...(readNumber(value, "file_size") ?? readNumber(value, "size")
+      ? { sizeBytes: readNumber(value, "file_size") ?? readNumber(value, "size") }
+      : {}),
+    ...(downloadUrl ? { downloadUrl } : {}),
   };
 }
 
@@ -249,6 +310,8 @@ export type ZapsterSendMessageRequest = {
 type ZapsterFetch = (url: string, init: RequestInit) => Promise<Response>;
 
 const ZAPSTER_SEND_MESSAGE_URL = "https://api.zapsterapi.com/v1/wa/messages";
+const ZAPSTER_DEFAULT_MEDIA_HOSTS = ["api.zapsterapi.com"];
+const ZAPSTER_MAX_DOWNLOAD_BYTES = 25_000_000;
 
 export function buildZapsterSendMessageRequest(input: ChannelSendInput): ZapsterSendMessageRequest {
   return buildZapsterSendMessageRequestInternal(input, "<redacted>");
@@ -381,6 +444,239 @@ export async function sendZapsterMessage(
   };
 }
 
+export async function downloadZapsterMedia(
+  input: ChannelMediaDownloadInput,
+  fetcher: ZapsterFetch = fetch,
+): Promise<ChannelMediaDownloadResult> {
+  const apiKey = input.secrets.ZAPSTER_API_KEY;
+  const audio = input.message.audio;
+
+  if (!apiKey) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "channel_secret_missing",
+      message: "Secret ZAPSTER_API_KEY is missing.",
+    };
+  }
+
+  if (!audio?.downloadUrl) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "channel_audio_download_unavailable",
+      message: "Zapster audio payload did not include a media download URL.",
+    };
+  }
+
+  const mediaUrl = validateZapsterMediaDownloadUrl(audio.downloadUrl);
+
+  if (!mediaUrl.ok) {
+    return mediaUrl;
+  }
+
+  let response: Response;
+
+  try {
+    response = await fetcher(mediaUrl.url, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    });
+  } catch (error) {
+    return {
+      ok: false,
+      retryable: true,
+      code: "channel_provider_unavailable",
+      message: `Zapster media download failed before a response: ${redactString(
+        error instanceof Error ? error.message : String(error),
+        [apiKey, mediaUrl.url],
+      )}`,
+    };
+  }
+
+  if (!response.ok) {
+    return {
+      ok: false,
+      retryable: response.status === 429 || response.status >= 500,
+      code: response.status === 429 || response.status >= 500
+        ? "channel_provider_unavailable"
+        : "channel_media_download_failed",
+      message: `Zapster media download returned HTTP ${response.status}.`,
+      providerMetadata: {
+        status: response.status,
+      },
+    };
+  }
+
+  const maxDownloadBytes = input.maxBytes ?? ZAPSTER_MAX_DOWNLOAD_BYTES;
+  const bytes = await readResponseBytesWithLimit(response, maxDownloadBytes);
+
+  if (!bytes.ok) {
+    return bytes;
+  }
+
+  return {
+    ok: true,
+    audio: bytes.audio,
+    filename: audio.filename ?? `${audio.providerMediaId}.ogg`,
+    ...(audio.mimeType ? { mimeType: audio.mimeType } : {}),
+    ...(audio.durationSeconds !== undefined ? { durationSeconds: audio.durationSeconds } : {}),
+    sizeBytes: bytes.audio.byteLength,
+    providerMetadata: {
+      status: response.status,
+    },
+  };
+}
+
+function validateZapsterMediaDownloadUrl(rawUrl: string):
+  | {
+      ok: true;
+      url: string;
+    }
+  | {
+      ok: false;
+      retryable: false;
+      code: "channel_audio_download_unavailable";
+      message: string;
+    } {
+  let url: URL;
+
+  try {
+    url = new URL(rawUrl);
+  } catch {
+    return {
+      ok: false,
+      retryable: false,
+      code: "channel_audio_download_unavailable",
+      message: "Zapster audio download URL is not a valid URL.",
+    };
+  }
+
+  if (url.protocol !== "https:") {
+    return {
+      ok: false,
+      retryable: false,
+      code: "channel_audio_download_unavailable",
+      message: "Zapster audio download URL must use HTTPS.",
+    };
+  }
+
+  const hostname = url.hostname.toLowerCase();
+  const allowed = ZAPSTER_DEFAULT_MEDIA_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`));
+
+  if (!allowed) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "channel_audio_download_unavailable",
+      message: "Zapster audio download URL host is not allowed.",
+    };
+  }
+
+  url.username = "";
+  url.password = "";
+
+  return {
+    ok: true,
+    url: url.href,
+  };
+}
+
+async function readResponseBytesWithLimit(
+  response: Response,
+  maxBytes: number,
+): Promise<
+  | {
+      ok: true;
+      audio: Uint8Array;
+    }
+  | {
+      ok: false;
+      retryable: false;
+      code: "channel_audio_too_large";
+      message: string;
+      providerMetadata?: Record<string, unknown>;
+    }
+> {
+  const contentLength = response.headers.get("content-length");
+  const declaredBytes = contentLength ? Number(contentLength) : undefined;
+
+  if (declaredBytes !== undefined && Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
+    return {
+      ok: false,
+      retryable: false,
+      code: "channel_audio_too_large",
+      message: `Audio response declares ${declaredBytes} bytes, which exceeds the configured ${maxBytes} byte limit.`,
+      providerMetadata: {
+        sizeBytes: declaredBytes,
+      },
+    };
+  }
+
+  if (!response.body) {
+    const audio = new Uint8Array(await response.arrayBuffer());
+
+    if (audio.byteLength > maxBytes) {
+      return {
+        ok: false,
+        retryable: false,
+        code: "channel_audio_too_large",
+        message: `Audio response is ${audio.byteLength} bytes, which exceeds the configured ${maxBytes} byte limit.`,
+        providerMetadata: {
+          sizeBytes: audio.byteLength,
+        },
+      };
+    }
+
+    return { ok: true, audio };
+  }
+
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let totalBytes = 0;
+
+  try {
+    while (true) {
+      const chunk = await reader.read();
+
+      if (chunk.done) {
+        break;
+      }
+
+      totalBytes += chunk.value.byteLength;
+
+      if (totalBytes > maxBytes) {
+        await reader.cancel().catch(() => undefined);
+        return {
+          ok: false,
+          retryable: false,
+          code: "channel_audio_too_large",
+          message: `Audio response exceeded the configured ${maxBytes} byte limit while downloading.`,
+          providerMetadata: {
+            sizeBytes: totalBytes,
+          },
+        };
+      }
+
+      chunks.push(chunk.value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const audio = new Uint8Array(totalBytes);
+  let offset = 0;
+
+  for (const chunk of chunks) {
+    audio.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+
+  return { ok: true, audio };
+}
+
 function buildZapsterSendMessageRequestInternal(
   input: ChannelSendInput,
   instanceId: string,
@@ -494,6 +790,29 @@ function readString(value: Record<string, unknown>, key: string): string | undef
   return typeof field === "string" && field.length > 0 ? field : undefined;
 }
 
+function readNumber(value: Record<string, unknown>, key: string): number | undefined {
+  const field = value[key];
+  return typeof field === "number" && Number.isFinite(field) ? field : undefined;
+}
+
+function defaultZapsterAudioFilename(value: Record<string, unknown>): string {
+  const mimeType = readString(value, "mime_type") ?? readString(value, "mimeType");
+
+  if (mimeType?.includes("ogg")) {
+    return "audio.ogg";
+  }
+
+  if (mimeType?.includes("mpeg") || mimeType?.includes("mp3")) {
+    return "audio.mp3";
+  }
+
+  if (mimeType?.includes("wav")) {
+    return "audio.wav";
+  }
+
+  return "audio.ogg";
+}
+
 function isPositiveIntegerHeader(value: string | null): value is string {
   return typeof value === "string" && /^[1-9]\d*$/.test(value);
 }

package/src/runtime/channels.ts

@@ -1,13 +1,17 @@
 import type { AgentChannel, ChannelProvider, ChannelType } from "../index";
 
 export type ChannelMessageRole = "user" | "assistant";
-export type ChannelMessageContentType = "text";
+export type ChannelMessageContentType = "text" | "audio";
 export type ChannelEventKind = "message" | "delivery_status" | "unsupported";
 export type ChannelDeliveryDirection = "inbound" | "outbound";
 export type ChannelDeliveryState =
   | "webhook_received"
   | "received"
   | "validated"
+  | "audio_received"
+  | "audio_downloaded"
+  | "transcribing"
+  | "transcribed"
   | "duplicate"
   | "buffered"
   | "queued"
@@ -45,11 +49,22 @@ export type ChannelExternalIdentity = {
   redactedDisplay?: string;
 };
 
+export type ChannelAudioAttachment = {
+  providerMediaId: string;
+  kind: "voice" | "audio";
+  mimeType?: string;
+  filename?: string;
+  durationSeconds?: number;
+  sizeBytes?: number;
+  downloadUrl?: string;
+};
+
 export type NormalizedChannelMessage = {
   role: ChannelMessageRole;
   contentType: ChannelMessageContentType;
   content: string;
   providerMessageId?: string;
+  audio?: ChannelAudioAttachment;
 };
 
 export type NormalizedChannelEvent = ChannelAdapterId & {
@@ -87,6 +102,36 @@ export type ChannelSendInput = ChannelAdapterId & {
   secrets: Record<string, string>;
 };
 
+export type ChannelMediaDownloadInput = ChannelAdapterId & {
+  channelId: string;
+  message: NormalizedChannelMessage;
+  secrets: Record<string, string>;
+  maxBytes?: number;
+};
+
+export type ChannelMediaDownloadResult =
+  | {
+      ok: true;
+      audio: Uint8Array;
+      filename: string;
+      mimeType?: string;
+      durationSeconds?: number;
+      sizeBytes?: number;
+      providerMetadata?: Record<string, unknown>;
+    }
+  | {
+      ok: false;
+      retryable: boolean;
+      code:
+        | "channel_secret_missing"
+        | "channel_audio_download_unavailable"
+        | "channel_audio_too_large"
+        | "channel_provider_unavailable"
+        | "channel_media_download_failed";
+      message: string;
+      providerMetadata?: Record<string, unknown>;
+    };
+
 export type ChannelSendResult =
   | {
       ok: true;
@@ -120,6 +165,7 @@ export type ChannelStatusResult = {
 export type ChannelAdapter = ChannelAdapterId & {
   requiredSecrets: string[];
   normalizeInbound(input: RawWebhookEvent): Promise<NormalizedChannelEvent[]> | NormalizedChannelEvent[];
+  downloadMedia?(input: ChannelMediaDownloadInput): Promise<ChannelMediaDownloadResult>;
   sendMessage(input: ChannelSendInput): Promise<ChannelSendResult>;
   verifyWebhook?(input: WebhookVerificationInput): Promise<WebhookVerificationResult> | WebhookVerificationResult;
   getStatus?(input: ChannelStatusInput): Promise<ChannelStatusResult> | ChannelStatusResult;

package/src/runtime/chat.ts

@@ -10,6 +10,7 @@ import { AgentKitError, isAgentKitError } from "./errors";
 import { syncConfiguredKnowledgeSources } from "./knowledge/ingest";
 import { appendKnowledgePromptPolicy } from "./knowledge/prompt-policy";
 import { createKnowledgeSearchTool } from "./knowledge/tool";
+import { appendRuntimePromptContext, buildRuntimeClockContext } from "./prompt-context";
 import { createToolRuntime } from "./tools";
 
 export type RunAgentMessageOptions = {
@@ -17,6 +18,7 @@ export type RunAgentMessageOptions = {
   conversationId?: string;
   signal?: AbortSignal;
   runtime?: Partial<ToolRuntimeContext>;
+  now?: Date;
 };
 
 export type AgentRunResult = ProviderRunResult & PersistedChatRun;
@@ -41,51 +43,64 @@ export async function runAgentMessage(
   const adapter = getProviderAdapter(capsule.config.provider);
   const env = await loadCapsuleEnv(capsule.root);
   await syncConfiguredKnowledgeSources(capsule);
-  const { store } = await openPreparedCapsuleStore(capsule);
   const conversationId = normalizeConversationId(options.conversationId);
-  const { conversation, previousMessages } = resolveConversation(store, {
-    agentName: capsule.config.name,
-    conversationId,
-    firstMessage: content,
-  });
-  const messages: AgentMessage[] = [...previousMessages, { role: "user", content }];
-  const userMessage = store.appendMessage({
-    conversationId: conversation.id,
-    role: "user",
-    content,
-  });
-  const run = store.createRun({
-    conversationId: conversation.id,
-    provider: capsule.config.provider.name,
-    model: capsule.config.provider.model,
-  });
-  const runtimeTools = withKnowledgeTools(capsule);
-  const toolRuntime = createToolRuntime({
-    tools: runtimeTools,
-    store,
-    runId: run.id,
-    conversationId: conversation.id,
-    env,
-    signal: options.signal,
-    runtime: {
-      environment: "local",
-      invocation: "chat",
-      target: "bun",
-      database: "sqlite",
-      ...options.runtime,
-    },
-  });
+  const { store } = await openPreparedCapsuleStore(capsule);
+  let runId: string | null = null;
 
   try {
+    const { conversation, previousMessages } = resolveConversation(store, {
+      agentName: capsule.config.name,
+      conversationId,
+      firstMessage: content,
+    });
+    const messages: AgentMessage[] = [...previousMessages, { role: "user", content }];
+    const userMessage = store.appendMessage({
+      conversationId: conversation.id,
+      role: "user",
+      content,
+    });
+    const run = store.createRun({
+      conversationId: conversation.id,
+      provider: capsule.config.provider.name,
+      model: capsule.config.provider.model,
+    });
+    runId = run.id;
+    const runtimeTools = withKnowledgeTools(capsule);
+    const clock = buildRuntimeClockContext({
+      timeZone: capsule.config.timeZone,
+      env,
+      now: options.now,
+    });
+    const toolRuntime = createToolRuntime({
+      tools: runtimeTools,
+      store,
+      runId: run.id,
+      conversationId: conversation.id,
+      env,
+      signal: options.signal,
+      clock,
+      runtime: {
+        environment: "local",
+        invocation: "chat",
+        target: "bun",
+        database: "sqlite",
+        ...options.runtime,
+      },
+    });
     const result = await adapter.run({
       agent: {
         name: capsule.config.name,
         runtime: capsule.config.runtime,
         provider: capsule.config.provider,
       },
-      instructions: withToolVisibilityInstructions(
-        appendKnowledgePromptPolicy(capsule.instructions, capsule.config),
-        runtimeTools,
+      instructions: appendRuntimePromptContext(
+        withToolVisibilityInstructions(
+          appendKnowledgePromptPolicy(capsule.instructions, capsule.config),
+          runtimeTools,
+        ),
+        {
+          clock,
+        },
       ),
       messages,
       tools: toolRuntime.definitions,
@@ -114,12 +129,14 @@ export async function runAgentMessage(
       assistantMessageId: assistantMessage.id,
     };
   } catch (error) {
-    store.completeRun({
-      runId: run.id,
-      status: "failed",
-      errorCode: isAgentKitError(error) ? error.code : "runtime_error",
-      errorMessage: error instanceof Error ? error.message : String(error),
-    });
+    if (runId) {
+      store.completeRun({
+        runId,
+        status: "failed",
+        errorCode: isAgentKitError(error) ? error.code : "runtime_error",
+        errorMessage: error instanceof Error ? error.message : String(error),
+      });
+    }
 
     throw error;
   } finally {