@full-self-developing/fsd 1.0.0

package/extensions/deepgram/audio.live.test.ts

@@ -0,0 +1,75 @@
+import {
+  runRealtimeSttLiveTest,
+  synthesizeElevenLabsLiveSpeech,
+} from "openclaw/plugin-sdk/provider-test-contracts";
+import { isLiveTestEnabled } from "openclaw/plugin-sdk/test-env";
+import { describe, expect, it } from "vitest";
+import { transcribeDeepgramAudio } from "./audio.js";
+import { buildDeepgramRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
+
+const DEEPGRAM_KEY = process.env.DEEPGRAM_API_KEY ?? "";
+const ELEVENLABS_KEY = process.env.ELEVENLABS_API_KEY ?? "";
+const DEEPGRAM_MODEL = process.env.DEEPGRAM_MODEL?.trim() || "nova-3";
+const DEEPGRAM_BASE_URL = process.env.DEEPGRAM_BASE_URL?.trim();
+const SAMPLE_URL =
+  process.env.DEEPGRAM_SAMPLE_URL?.trim() ||
+  "https://static.deepgram.com/examples/Bueller-Life-moves-pretty-fast.wav";
+const LIVE = isLiveTestEnabled(["DEEPGRAM_LIVE_TEST"]);
+
+const describeLive = LIVE && DEEPGRAM_KEY ? describe : describe.skip;
+
+async function fetchSampleBuffer(url: string, timeoutMs: number): Promise<Buffer> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutMs));
+  try {
+    const res = await fetch(url, { signal: controller.signal });
+    if (!res.ok) {
+      throw new Error(`Sample download failed (HTTP ${res.status})`);
+    }
+    const data = await res.arrayBuffer();
+    return Buffer.from(data);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+describeLive("deepgram live", () => {
+  it("transcribes sample audio", async () => {
+    const buffer = await fetchSampleBuffer(SAMPLE_URL, 15000);
+    const result = await transcribeDeepgramAudio({
+      buffer,
+      fileName: "sample.wav",
+      mime: "audio/wav",
+      apiKey: DEEPGRAM_KEY,
+      model: DEEPGRAM_MODEL,
+      baseUrl: DEEPGRAM_BASE_URL,
+      timeoutMs: 20000,
+    });
+    expect(result.text.trim().length).toBeGreaterThan(0);
+  }, 30000);
+
+  it("streams realtime STT through the registered transcription provider", async () => {
+    if (!ELEVENLABS_KEY) {
+      throw new Error("ELEVENLABS_API_KEY required to synthesize live realtime STT input");
+    }
+    const provider = buildDeepgramRealtimeTranscriptionProvider();
+    const phrase = "Testing OpenClaw Deepgram realtime transcription integration OK.";
+    const speech = await synthesizeElevenLabsLiveSpeech({
+      text: phrase,
+      apiKey: ELEVENLABS_KEY,
+      outputFormat: "ulaw_8000",
+      timeoutMs: 30_000,
+    });
+    expect(speech.byteLength).toBeGreaterThan(0);
+
+    await runRealtimeSttLiveTest({
+      provider,
+      providerConfig: {
+        apiKey: DEEPGRAM_KEY,
+        language: "en-US",
+        endpointingMs: 500,
+      },
+      audio: Buffer.concat([Buffer.alloc(4000, 0xff), speech, Buffer.alloc(8000, 0xff)]),
+    });
+  }, 90_000);
+});

package/extensions/deepgram/audio.test.ts

@@ -0,0 +1,146 @@
+import {
+  createAuthCaptureJsonFetch,
+  createRequestCaptureJsonFetch,
+  installPinnedHostnameTestHooks,
+} from "openclaw/plugin-sdk/test-env";
+import { describe, expect, it, vi } from "vitest";
+import { transcribeDeepgramAudio } from "./audio.js";
+
+installPinnedHostnameTestHooks();
+
+describe("transcribeDeepgramAudio", () => {
+  it("respects lowercase authorization header overrides", async () => {
+    const { fetchFn, getAuthHeader } = createAuthCaptureJsonFetch({
+      results: { channels: [{ alternatives: [{ transcript: "ok" }] }] },
+    });
+
+    const result = await transcribeDeepgramAudio({
+      buffer: Buffer.from("audio"),
+      fileName: "note.mp3",
+      apiKey: "test-key",
+      timeoutMs: 1000,
+      headers: { authorization: "Token override" },
+      fetchFn,
+    });
+
+    expect(getAuthHeader()).toBe("Token override");
+    expect(result.text).toBe("ok");
+  });
+
+  it("builds the expected request payload", async () => {
+    const { fetchFn, getRequest } = createRequestCaptureJsonFetch({
+      results: { channels: [{ alternatives: [{ transcript: "hello" }] }] },
+    });
+
+    const result = await transcribeDeepgramAudio({
+      buffer: Buffer.from("audio-bytes"),
+      fileName: "voice.wav",
+      apiKey: "test-key",
+      timeoutMs: 1234,
+      baseUrl: "https://api.example.com/v1/",
+      model: " ",
+      language: " en ",
+      mime: "audio/wav",
+      headers: { "X-Custom": "1" },
+      query: {
+        punctuate: false,
+        smart_format: true,
+      },
+      fetchFn,
+    });
+    const { url: seenUrl, init: seenInit } = getRequest();
+
+    expect(result.model).toBe("nova-3");
+    expect(result.text).toBe("hello");
+    expect(seenUrl).toBe(
+      "https://api.example.com/v1/listen?model=nova-3&language=en&punctuate=false&smart_format=true",
+    );
+    if (!seenInit) {
+      throw new Error("Expected Deepgram fetch request init");
+    }
+    expect(seenInit.method).toBe("POST");
+    expect(seenInit.signal).toBeInstanceOf(AbortSignal);
+
+    const headers = new Headers(seenInit.headers);
+    expect(headers.get("authorization")).toBe("Token test-key");
+    expect(headers.get("x-custom")).toBe("1");
+    expect(headers.get("content-type")).toBe("audio/wav");
+    expect(seenInit.body).toBeInstanceOf(Uint8Array);
+  });
+
+  it("throws when the provider response omits transcript", async () => {
+    const { fetchFn } = createRequestCaptureJsonFetch({
+      results: { channels: [{ alternatives: [{}] }] },
+    });
+
+    await expect(
+      transcribeDeepgramAudio({
+        buffer: Buffer.from("audio-bytes"),
+        fileName: "voice.wav",
+        apiKey: "test-key",
+        timeoutMs: 1234,
+        fetchFn,
+      }),
+    ).rejects.toThrow("Audio transcription response missing transcript");
+  });
+
+  it("wraps malformed successful transcription JSON with a stable provider error", async () => {
+    const fetchFn = vi.fn<typeof fetch>().mockResolvedValueOnce(new Response("{ nope"));
+
+    await expect(
+      transcribeDeepgramAudio({
+        buffer: Buffer.from("audio-bytes"),
+        fileName: "voice.wav",
+        apiKey: "test-key",
+        timeoutMs: 1234,
+        fetchFn,
+      }),
+    ).rejects.toThrow("Audio transcription failed: malformed JSON response");
+  });
+
+  it("rejects non-object successful transcription JSON with a stable provider error", async () => {
+    const fetchFn = vi.fn<typeof fetch>().mockResolvedValueOnce(new Response(JSON.stringify([])));
+
+    await expect(
+      transcribeDeepgramAudio({
+        buffer: Buffer.from("audio-bytes"),
+        fileName: "voice.wav",
+        apiKey: "test-key",
+        timeoutMs: 1234,
+        fetchFn,
+      }),
+    ).rejects.toThrow("Audio transcription failed: malformed JSON response");
+  });
+
+  it("rejects wrong nested transcript shapes with a stable provider error", async () => {
+    const { fetchFn } = createRequestCaptureJsonFetch({
+      results: { channels: { alternatives: [{ transcript: "hello" }] } },
+    });
+
+    await expect(
+      transcribeDeepgramAudio({
+        buffer: Buffer.from("audio-bytes"),
+        fileName: "voice.wav",
+        apiKey: "test-key",
+        timeoutMs: 1234,
+        fetchFn,
+      }),
+    ).rejects.toThrow("Audio transcription failed: malformed JSON response");
+  });
+
+  it("rejects non-string transcript values with a stable provider error", async () => {
+    const { fetchFn } = createRequestCaptureJsonFetch({
+      results: { channels: [{ alternatives: [{ transcript: 123 }] }] },
+    });
+
+    await expect(
+      transcribeDeepgramAudio({
+        buffer: Buffer.from("audio-bytes"),
+        fileName: "voice.wav",
+        apiKey: "test-key",
+        timeoutMs: 1234,
+        fetchFn,
+      }),
+    ).rejects.toThrow("Audio transcription failed: malformed JSON response");
+  });
+});

package/extensions/deepgram/audio.ts

@@ -0,0 +1,109 @@
+import type {
+  AudioTranscriptionRequest,
+  AudioTranscriptionResult,
+} from "openclaw/plugin-sdk/media-understanding";
+import {
+  assertOkOrThrowHttpError,
+  postTranscriptionRequest,
+  readProviderJsonObjectResponse,
+  resolveProviderHttpRequestConfig,
+  requireTranscriptionText,
+} from "openclaw/plugin-sdk/provider-http";
+
+export const DEFAULT_DEEPGRAM_AUDIO_BASE_URL = "https://api.deepgram.com/v1";
+export const DEFAULT_DEEPGRAM_AUDIO_MODEL = "nova-3";
+
+function resolveModel(model?: string): string {
+  const trimmed = model?.trim();
+  return trimmed || DEFAULT_DEEPGRAM_AUDIO_MODEL;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function readDeepgramTranscript(payload: Record<string, unknown>): string | undefined {
+  const results = asRecord(payload.results);
+  if (!results) {
+    return undefined;
+  }
+  if (!Array.isArray(results.channels)) {
+    throw new Error("Audio transcription failed: malformed JSON response");
+  }
+  const channel = asRecord(results.channels[0]);
+  if (!channel) {
+    return undefined;
+  }
+  if (!Array.isArray(channel.alternatives)) {
+    throw new Error("Audio transcription failed: malformed JSON response");
+  }
+  const alternative = asRecord(channel.alternatives[0]);
+  if (!alternative) {
+    return undefined;
+  }
+  if (alternative.transcript !== undefined && typeof alternative.transcript !== "string") {
+    throw new Error("Audio transcription failed: malformed JSON response");
+  }
+  return alternative.transcript;
+}
+
+export async function transcribeDeepgramAudio(
+  params: AudioTranscriptionRequest,
+): Promise<AudioTranscriptionResult> {
+  const fetchFn = params.fetchFn ?? fetch;
+  const model = resolveModel(params.model);
+  const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
+    resolveProviderHttpRequestConfig({
+      baseUrl: params.baseUrl,
+      defaultBaseUrl: DEFAULT_DEEPGRAM_AUDIO_BASE_URL,
+      headers: params.headers,
+      request: params.request,
+      defaultHeaders: {
+        authorization: `Token ${params.apiKey}`,
+        "content-type": params.mime ?? "application/octet-stream",
+      },
+      provider: "deepgram",
+      capability: "audio",
+      transport: "media-understanding",
+    });
+
+  const url = new URL(`${baseUrl}/listen`);
+  url.searchParams.set("model", model);
+  if (params.language?.trim()) {
+    url.searchParams.set("language", params.language.trim());
+  }
+  if (params.query) {
+    for (const [key, value] of Object.entries(params.query)) {
+      if (value === undefined) {
+        continue;
+      }
+      url.searchParams.set(key, String(value));
+    }
+  }
+
+  const body = new Uint8Array(params.buffer);
+  const { response: res, release } = await postTranscriptionRequest({
+    url: url.toString(),
+    headers,
+    body,
+    timeoutMs: params.timeoutMs,
+    fetchFn,
+    allowPrivateNetwork,
+    dispatcherPolicy,
+  });
+
+  try {
+    await assertOkOrThrowHttpError(res, "Audio transcription failed");
+
+    const payload = await readProviderJsonObjectResponse(res, "Audio transcription failed");
+    const transcript = requireTranscriptionText(
+      readDeepgramTranscript(payload),
+      "Audio transcription response missing transcript",
+    );
+    return { text: transcript, model };
+  } finally {
+    await release();
+  }
+}

package/extensions/deepgram/index.ts

@@ -0,0 +1,13 @@
+import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
+import { deepgramMediaUnderstandingProvider } from "./media-understanding-provider.js";
+import { buildDeepgramRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
+
+export default definePluginEntry({
+  id: "deepgram",
+  name: "Deepgram Media Understanding",
+  description: "Bundled Deepgram audio transcription provider",
+  register(api) {
+    api.registerMediaUnderstandingProvider(deepgramMediaUnderstandingProvider);
+    api.registerRealtimeTranscriptionProvider(buildDeepgramRealtimeTranscriptionProvider());
+  },
+});

package/extensions/deepgram/media-understanding-provider.ts

@@ -0,0 +1,10 @@
+import type { MediaUnderstandingProvider } from "openclaw/plugin-sdk/media-understanding";
+import { transcribeDeepgramAudio } from "./audio.js";
+
+export const deepgramMediaUnderstandingProvider: MediaUnderstandingProvider = {
+  id: "deepgram",
+  capabilities: ["audio"],
+  defaultModels: { audio: "nova-3" },
+  autoPriority: { audio: 30 },
+  transcribeAudio: transcribeDeepgramAudio,
+};

package/extensions/deepgram/openclaw.plugin.json

@@ -0,0 +1,30 @@
+{
+  "id": "deepgram",
+  "activation": {
+    "onStartup": false
+  },
+  "enabledByDefault": true,
+  "providerAuthEnvVars": {
+    "deepgram": ["DEEPGRAM_API_KEY"]
+  },
+  "contracts": {
+    "mediaUnderstandingProviders": ["deepgram"],
+    "realtimeTranscriptionProviders": ["deepgram"]
+  },
+  "mediaUnderstandingProviderMetadata": {
+    "deepgram": {
+      "capabilities": ["audio"],
+      "defaultModels": {
+        "audio": "nova-3"
+      },
+      "autoPriority": {
+        "audio": 30
+      }
+    }
+  },
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {}
+  }
+}

package/extensions/deepgram/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@openclaw/deepgram-provider",
+  "version": "2026.5.21",
+  "private": true,
+  "description": "OpenClaw Deepgram media-understanding provider",
+  "type": "module",
+  "devDependencies": {
+    "@openclaw/plugin-sdk": "workspace:*"
+  },
+  "openclaw": {
+    "extensions": [
+      "./index.ts"
+    ]
+  }
+}

package/extensions/deepgram/realtime-transcription-provider.test.ts

@@ -0,0 +1,69 @@
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  testing,
+  buildDeepgramRealtimeTranscriptionProvider,
+} from "./realtime-transcription-provider.js";
+
+describe("buildDeepgramRealtimeTranscriptionProvider", () => {
+  afterEach(() => {
+    vi.unstubAllEnvs();
+  });
+
+  it("normalizes nested provider config", () => {
+    const provider = buildDeepgramRealtimeTranscriptionProvider();
+    const resolved = provider.resolveConfig?.({
+      cfg: {} as OpenClawConfig,
+      rawConfig: {
+        providers: {
+          deepgram: {
+            apiKey: "dg-key",
+            model: "nova-3",
+            encoding: "g711_ulaw",
+            sample_rate: "8000",
+            interim_results: "true",
+            endpointing: "500",
+            language: "en-US",
+          },
+        },
+      },
+    });
+
+    expect(resolved).toEqual({
+      apiKey: "dg-key",
+      baseUrl: undefined,
+      model: "nova-3",
+      language: "en-US",
+      sampleRate: 8000,
+      encoding: "mulaw",
+      interimResults: true,
+      endpointingMs: 500,
+    });
+  });
+
+  it("builds a Deepgram listen websocket URL", () => {
+    const url = testing.toDeepgramRealtimeWsUrl({
+      apiKey: "dg-key",
+      baseUrl: "https://api.deepgram.com/v1",
+      model: "nova-3",
+      providerConfig: {},
+      sampleRate: 8000,
+      encoding: "mulaw",
+      interimResults: true,
+      endpointingMs: 800,
+    });
+
+    expect(url).toContain("wss://api.deepgram.com/v1/listen?");
+    expect(url).toContain("model=nova-3");
+    expect(url).toContain("encoding=mulaw");
+    expect(url).toContain("sample_rate=8000");
+  });
+
+  it("requires an API key when creating sessions", () => {
+    vi.stubEnv("DEEPGRAM_API_KEY", "");
+    const provider = buildDeepgramRealtimeTranscriptionProvider();
+    expect(() => provider.createSession({ providerConfig: {} })).toThrow(
+      "Deepgram API key missing",
+    );
+  });
+});