@framers/agentos-ext-streaming-stt-whisper 0.2.0

package/dist/index.js

@@ -0,0 +1,78 @@
+/**
+ * @file index.ts
+ * @description Pack factory for the Whisper Chunked Streaming STT extension pack.
+ *
+ * This module exports the main {@link createWhisperChunkedSTT} factory function
+ * and the {@link createExtensionPack} bridge function that conforms to the AgentOS
+ * manifest factory convention.
+ *
+ * ### Usage (direct)
+ * ```ts
+ * import { createWhisperChunkedSTT } from '@framers/agentos-ext-streaming-stt-whisper';
+ *
+ * const stt = createWhisperChunkedSTT(process.env.OPENAI_API_KEY!);
+ * const session = await stt.startSession({ language: 'en' });
+ * ```
+ *
+ * ### Usage (manifest-driven)
+ * ```json
+ * { "packs": [{ "module": "@framers/agentos-ext-streaming-stt-whisper" }] }
+ * ```
+ *
+ * @module streaming-stt-whisper
+ */
+import { WhisperChunkedSTT } from './WhisperChunkedSTT.js';
+/** Kind constant matching packages/agentos/src/extensions/types.ts. */
+const EXTENSION_KIND_STREAMING_STT = 'streaming-stt-provider';
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+/**
+ * Create a standalone {@link WhisperChunkedSTT} instance.
+ *
+ * Use this when composing the provider programmatically outside of the
+ * AgentOS extension system.
+ *
+ * @param apiKey - OpenAI (or compatible) API key.
+ * @param baseUrl - Optional API base URL override.
+ * @returns Configured {@link WhisperChunkedSTT}.
+ */
+export function createWhisperChunkedSTT(apiKey, baseUrl) {
+    return new WhisperChunkedSTT(apiKey, baseUrl);
+}
+/**
+ * AgentOS manifest factory function.
+ *
+ * Reads the `OPENAI_API_KEY` secret from the context and returns an
+ * {@link ExtensionPack} containing a single `streaming-stt-provider` descriptor
+ * backed by {@link WhisperChunkedSTT}.
+ *
+ * @param context - Pack context supplied by the extension manager.
+ * @returns A fully configured {@link ExtensionPack}.
+ */
+export function createExtensionPack(context) {
+    const apiKey = context.getSecret?.('OPENAI_API_KEY') ?? '';
+    const baseUrl = context.options?.['baseUrl'];
+    const stt = new WhisperChunkedSTT(apiKey, baseUrl);
+    return {
+        id: 'streaming-stt-whisper',
+        descriptors: [
+            {
+                id: 'whisper-chunked-stt',
+                kind: EXTENSION_KIND_STREAMING_STT,
+                payload: stt,
+                enableByDefault: true,
+                metadata: { providerId: 'whisper-chunked' },
+            },
+        ],
+    };
+}
+// ---------------------------------------------------------------------------
+// Re-exports
+// ---------------------------------------------------------------------------
+export * from './types.js';
+export { WhisperChunkedSTT } from './WhisperChunkedSTT.js';
+export { WhisperChunkSession } from './WhisperChunkSession.js';
+export { SlidingWindowBuffer } from './SlidingWindowBuffer.js';
+export { DEFAULT_CHUNK_SIZE_SAMPLES, DEFAULT_OVERLAP_SAMPLES, } from './SlidingWindowBuffer.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AA6B3D,uEAAuE;AACvE,MAAM,4BAA4B,GAAG,wBAAwB,CAAC;AAE9D,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAc,EAAE,OAAgB;IACtE,OAAO,IAAI,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAA6B;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,SAAS,CAAuB,CAAC;IACnE,MAAM,GAAG,GAAG,IAAI,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEnD,OAAO;QACL,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,qBAAqB;gBACzB,IAAI,EAAE,4BAA4B;gBAClC,OAAO,EAAE,GAAG;gBACZ,eAAe,EAAE,IAAI;gBACrB,QAAQ,EAAE,EAAE,UAAU,EAAE,iBAAiB,EAAE;aAC5C;SACF;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,GACxB,MAAM,0BAA0B,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,102 @@
+/**
+ * @file types.ts
+ * @description Whisper-specific configuration types for the chunked streaming STT extension pack.
+ *
+ * These types define the configuration for the sliding-window Whisper adapter that
+ * accumulates audio into 1-second chunks and sends them to the Whisper HTTP API.
+ *
+ * @module streaming-stt-whisper/types
+ */
+/**
+ * Configuration for the Whisper chunked streaming STT session.
+ *
+ * All fields except `apiKey` are optional — sensible defaults are applied.
+ */
+export interface WhisperChunkedConfig {
+    /**
+     * OpenAI API key (or compatible provider key).
+     * Read from `OPENAI_API_KEY` when constructed via {@link createExtensionPack}.
+     */
+    apiKey: string;
+    /**
+     * Base URL for the Whisper API endpoint.
+     * Override to use a self-hosted or compatible provider (e.g. Groq, local Faster-Whisper).
+     * @defaultValue 'https://api.openai.com'
+     */
+    baseUrl?: string;
+    /**
+     * Whisper model name.
+     * @defaultValue 'whisper-1'
+     * @see {@link https://platform.openai.com/docs/models/whisper}
+     */
+    model?: string;
+    /**
+     * BCP-47 language hint (e.g. `'en'`, `'fr'`, `'de'`).
+     * When omitted Whisper auto-detects the language.
+     */
+    language?: string;
+    /**
+     * Optional initial prompt to bias the first chunk's transcription.
+     * Subsequent chunks automatically receive the previous chunk's transcript as prompt.
+     * @see {@link https://platform.openai.com/docs/guides/speech-to-text/prompting}
+     */
+    prompt?: string;
+    /**
+     * Size of each audio chunk in samples (at 16 kHz).
+     * @defaultValue 16000 (1 second at 16 kHz)
+     */
+    chunkSizeSamples?: number;
+    /**
+     * Number of samples to carry forward from each chunk as overlap.
+     * Prevents words at chunk boundaries from being silently dropped.
+     * @defaultValue 3200 (200 ms at 16 kHz)
+     */
+    overlapSamples?: number;
+}
+/** Minimal AudioFrame shape — mirrors packages/agentos/src/voice-pipeline/types.ts */
+export interface AudioFrame {
+    samples: Float32Array;
+    sampleRate: number;
+    timestamp: number;
+    speakerHint?: string;
+}
+/** A single recognised word with timing metadata. */
+export interface TranscriptWord {
+    word: string;
+    start: number;
+    end: number;
+    confidence: number;
+    speaker?: string;
+}
+/** A transcription result emitted by the session. */
+export interface TranscriptEvent {
+    text: string;
+    confidence: number;
+    words: TranscriptWord[];
+    isFinal: boolean;
+    durationMs?: number;
+}
+/** A single segment from Whisper's verbose_json response. */
+export interface WhisperSegment {
+    id: number;
+    start: number;
+    end: number;
+    text: string;
+    avg_logprob?: number;
+    words?: WhisperWord[];
+}
+/** A word-level entry from Whisper's verbose_json response (requires word timestamps). */
+export interface WhisperWord {
+    word: string;
+    start: number;
+    end: number;
+}
+/** Top-level shape of the Whisper verbose_json transcription response. */
+export interface WhisperTranscriptionResponse {
+    task?: string;
+    language?: string;
+    duration?: number;
+    text: string;
+    segments?: WhisperSegment[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAMD,sFAAsF;AACtF,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qDAAqD;AACrD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAMD,6DAA6D;AAC7D,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;CACvB;AAED,0FAA0F;AAC1F,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,0EAA0E;AAC1E,MAAM,WAAW,4BAA4B;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,cAAc,EAAE,CAAC;CAC7B"}

package/dist/types.js

@@ -0,0 +1,11 @@
+/**
+ * @file types.ts
+ * @description Whisper-specific configuration types for the chunked streaming STT extension pack.
+ *
+ * These types define the configuration for the sliding-window Whisper adapter that
+ * accumulates audio into 1-second chunks and sends them to the Whisper HTTP API.
+ *
+ * @module streaming-stt-whisper/types
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}

package/manifest.json

@@ -0,0 +1,8 @@
+{
+  "name": "@framers/agentos-ext-streaming-stt-whisper",
+  "version": "0.1.0",
+  "description": "Chunked sliding-window streaming STT via OpenAI Whisper HTTP API",
+  "kind": "streaming-stt-provider",
+  "extensionId": "streaming-stt-whisper",
+  "entryPoint": "./dist/index.js"
+}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@framers/agentos-ext-streaming-stt-whisper",
+  "version": "0.2.0",
+  "description": "Chunked sliding-window streaming STT via OpenAI Whisper HTTP API for AgentOS voice pipeline",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "SKILL.md",
+    "manifest.json"
+  ],
+  "peerDependencies": {
+    "@framers/agentos": "^0.1.0"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "typescript": "^5.5.0",
+    "vitest": "^1.6.0",
+    "@framers/agentos": "0.1.94"
+  },
+  "license": "MIT",
+  "author": "Frame.dev",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/framersai/agentos-extensions.git",
+    "directory": "registry/curated/voice/streaming-stt-whisper"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "vitest run"
+  }
+}

package/src/SlidingWindowBuffer.ts

@@ -0,0 +1,176 @@
+/**
+ * @file SlidingWindowBuffer.ts
+ * @description Ring buffer that accumulates Float32 audio frames into fixed-size chunks
+ * with configurable overlap between consecutive chunks.
+ *
+ * When {@link pushSamples} fills the internal buffer to {@link chunkSizeSamples}, it
+ * emits a `'chunk_ready'` event carrying the complete `Float32Array` chunk, copies the
+ * last {@link overlapSamples} samples to the head of the buffer as overlap context for
+ * the next chunk, and resets the write cursor accordingly.
+ *
+ * The overlap strategy prevents words straddling chunk boundaries from being silently
+ * dropped by the Whisper model.
+ *
+ * @module streaming-stt-whisper/SlidingWindowBuffer
+ */
+
+import { EventEmitter } from 'node:events';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/**
+ * Default chunk size in samples.
+ * At 16 kHz this corresponds to exactly 1 second of mono audio.
+ */
+export const DEFAULT_CHUNK_SIZE_SAMPLES = 16_000;
+
+/**
+ * Default overlap in samples carried forward to the next chunk.
+ * At 16 kHz this corresponds to 200 ms of audio context.
+ */
+export const DEFAULT_OVERLAP_SAMPLES = 3_200;
+
+// ---------------------------------------------------------------------------
+// Events interface (TypeScript augmentation for typed emit/on)
+// ---------------------------------------------------------------------------
+
+/** Event map for {@link SlidingWindowBuffer}. */
+export interface SlidingWindowBufferEvents {
+  /** Emitted when a complete chunk of {@link chunkSizeSamples} is ready. */
+  chunk_ready: [chunk: Float32Array];
+}
+
+// ---------------------------------------------------------------------------
+// Main class
+// ---------------------------------------------------------------------------
+
+/**
+ * Ring-buffer that accumulates raw PCM samples and emits fixed-size audio
+ * chunks with configurable overlap.
+ *
+ * @example
+ * ```ts
+ * const buf = new SlidingWindowBuffer(16_000, 3_200);
+ * buf.on('chunk_ready', (chunk) => sendToWhisper(chunk));
+ *
+ * microphone.on('frame', (f) => buf.pushSamples(f.samples));
+ * await buf.flush(); // emit any remaining samples
+ * ```
+ */
+export class SlidingWindowBuffer extends EventEmitter {
+  // -------------------------------------------------------------------------
+  // Private state
+  // -------------------------------------------------------------------------
+
+  /**
+   * Internal sample store.  Sized to {@link chunkSizeSamples} so a single
+   * allocation is reused for the lifetime of the session.
+   */
+  private buffer: Float32Array;
+
+  /**
+   * Current write position within {@link buffer}.
+   * Always in the range `[0, chunkSizeSamples)`.
+   */
+  private writePos = 0;
+
+  // -------------------------------------------------------------------------
+  // Constructor
+  // -------------------------------------------------------------------------
+
+  /**
+   * @param chunkSizeSamples - Number of samples per emitted chunk.
+   *   Defaults to {@link DEFAULT_CHUNK_SIZE_SAMPLES} (1 s at 16 kHz).
+   * @param overlapSamples - Number of samples carried forward from each chunk
+   *   to the start of the next.  Must be less than `chunkSizeSamples`.
+   *   Defaults to {@link DEFAULT_OVERLAP_SAMPLES} (200 ms at 16 kHz).
+   */
+  constructor(
+    private readonly chunkSizeSamples: number = DEFAULT_CHUNK_SIZE_SAMPLES,
+    private readonly overlapSamples: number = DEFAULT_OVERLAP_SAMPLES,
+  ) {
+    super();
+
+    if (overlapSamples >= chunkSizeSamples) {
+      throw new RangeError(
+        `overlapSamples (${overlapSamples}) must be less than chunkSizeSamples (${chunkSizeSamples})`,
+      );
+    }
+
+    this.buffer = new Float32Array(chunkSizeSamples);
+  }
+
+  // -------------------------------------------------------------------------
+  // Public API
+  // -------------------------------------------------------------------------
+
+  /**
+   * Append audio samples to the internal buffer.
+   *
+   * If the incoming batch causes the buffer to reach or exceed
+   * {@link chunkSizeSamples}, one or more `'chunk_ready'` events are emitted
+   * before the remainder is retained for the next chunk.  Each chunk includes
+   * an overlap region copied from the tail of the previous chunk.
+   *
+   * @param samples - Float32 PCM samples to append.
+   */
+  pushSamples(samples: Float32Array): void {
+    let srcOffset = 0;
+
+    while (srcOffset < samples.length) {
+      // How many samples can we copy into the current chunk before it is full?
+      const spaceLeft = this.chunkSizeSamples - this.writePos;
+      const copyCount = Math.min(spaceLeft, samples.length - srcOffset);
+
+      this.buffer.set(samples.subarray(srcOffset, srcOffset + copyCount), this.writePos);
+      this.writePos += copyCount;
+      srcOffset += copyCount;
+
+      if (this.writePos >= this.chunkSizeSamples) {
+        // Chunk is full — emit a copy (not a reference to the internal buffer).
+        this.emit('chunk_ready', this.buffer.slice());
+
+        // Copy the last `overlapSamples` to the beginning of the buffer so that
+        // the next chunk begins with audio context from the previous boundary.
+        const overlapStart = this.chunkSizeSamples - this.overlapSamples;
+        this.buffer.copyWithin(0, overlapStart, this.chunkSizeSamples);
+        this.writePos = this.overlapSamples;
+      }
+    }
+  }
+
+  /**
+   * Emit any samples currently held in the buffer as a final partial chunk.
+   *
+   * If the buffer contains no samples (`writePos === 0`), this is a no-op.
+   * After flushing, the buffer is reset to an empty state.
+   */
+  flush(): void {
+    if (this.writePos === 0) return;
+
+    // Emit only the samples that were actually written (not the whole buffer).
+    this.emit('chunk_ready', this.buffer.slice(0, this.writePos));
+    this.reset();
+  }
+
+  /**
+   * Clear all buffered samples and reset the write cursor to zero.
+   *
+   * Does NOT emit a `'chunk_ready'` event — use {@link flush} for that.
+   */
+  reset(): void {
+    this.buffer = new Float32Array(this.chunkSizeSamples);
+    this.writePos = 0;
+  }
+
+  // -------------------------------------------------------------------------
+  // Accessors (useful for testing)
+  // -------------------------------------------------------------------------
+
+  /** Number of samples currently held in the buffer. */
+  get bufferedSamples(): number {
+    return this.writePos;
+  }
+}