@framers/agentos-ext-google-cloud-stt 0.2.1 → 0.3.0

package/SKILL.md

@@ -14,12 +14,14 @@ Provide credentials via the `GOOGLE_CLOUD_STT_CREDENTIALS` secret. Accepts eithe
 - An absolute path to a service-account JSON key file (contains `/` or `\`)
 - A raw JSON string with the service-account credentials
 
+Leave the secret unset to use Google's Application Default Credentials (`GOOGLE_APPLICATION_CREDENTIALS`, `gcloud auth application-default login`, or the metadata server on Google Cloud).
+
 ## Features
 
-- LINEAR16 PCM audio transcription
+- WAV and FLAC files (Google reads the encoding and sample rate from the header) and raw LINEAR16 PCM
 - Configurable language code (BCP-47)
-- Confidence scores and word-level alternatives
-- Maps to standard `SpeechTranscriptionResult` shape
+- Returns the AgentOS `SpeechTranscriptionResult`: `text` (each stretch's most likely transcript, in order), mean `confidence`, `isFinal`, and `segments` with timing when Google reports end times
+- Batch only (`supportsStreaming: false`)
 
 ## Configuration
 

package/dist/GoogleCloudSTTProvider.d.ts

@@ -10,18 +10,39 @@
  * @module google-cloud-stt
  */
 /**
- * A single recognised audio segment returned by the provider.
- *
- * Mirrors the generic `SpeechTranscriptionResult` shape used across the
- * AgentOS voice pipeline.
+ * One stretch of recognised speech: Google returns a result per consecutive
+ * portion of the audio.
+ */
+export interface SpeechTranscriptionSegment {
+    /** The most likely transcript of this stretch. */
+    text: string;
+    /** Start in seconds from the beginning of the audio. */
+    startTime: number;
+    /** End in seconds from the beginning of the audio. */
+    endTime: number;
+    /** Confidence score in [0, 1], when Google reports one. */
+    confidence?: number;
+}
+/**
+ * The transcription result of the AgentOS `SpeechToTextProvider` contract
+ * (`SpeechTranscriptionResult` in `@framers/agentos`), mirrored here so the
+ * pack carries no type import from agentos.
  */
 export interface SpeechTranscriptionResult {
-    /** The recognised text. */
-    transcript: string;
-    /** Confidence score in [0, 1]. */
-    confidence: number;
-    /** Whether this is a final (non-speculative) result. Always `true` for batch. */
+    /** The recognised text: every stretch's most likely transcript, in order. */
+    text: string;
+    /** BCP-47 language of the transcript. */
+    language?: string;
+    /** Mean confidence of the stretches, in [0, 1], when Google reports any. */
+    confidence?: number;
+    /** Always `true`: batch recognition returns final results only. */
     isFinal: boolean;
+    /** Always 0; cost is tracked above the provider, as for AgentOS's core batch providers. */
+    cost: number;
+    /** Per-stretch transcripts with their timing, when Google reports end times. */
+    segments?: SpeechTranscriptionSegment[];
+    /** The raw `RecognizeResponse`. */
+    providerResponse?: unknown;
 }
 /**
  * Per-call transcription options forwarded to the Google Cloud API.
@@ -31,13 +52,18 @@ export interface GoogleCloudSTTOptions {
     language?: string;
 }
 /**
- * Audio frame passed to {@link GoogleCloudSTTProvider.transcribe}.
+ * Audio passed to {@link GoogleCloudSTTProvider.transcribe}: the fields of the
+ * AgentOS `SpeechAudioInput` this provider reads.
  */
 export interface AudioData {
-    /** Raw PCM bytes (LINEAR16). */
+    /** The audio bytes: a WAV or FLAC file, or raw LINEAR16 PCM. */
     data: Buffer;
-    /** Sample rate in Hz. @defaultValue `16000` */
+    /** Sample rate in Hz. Raw PCM defaults to 16000; a WAV or FLAC header supplies its own. */
     sampleRate?: number;
+    /** MIME type, such as `'audio/wav'` or `'audio/flac'`. */
+    mimeType?: string;
+    /** Container format, such as `'wav'` or `'flac'`. */
+    format?: string;
 }
 /**
  * Google Cloud Speech-to-Text batch provider.
@@ -49,6 +75,10 @@ export interface AudioData {
 export declare class GoogleCloudSTTProvider {
     /** Stable provider identifier used by the AgentOS extension registry. */
     readonly id = "google-cloud-stt";
+    /** Human-readable provider name. */
+    readonly displayName = "Google Cloud Speech-to-Text";
+    /** Batch recognition only: this provider does not stream. */
+    readonly supportsStreaming = false;
     /** Lazily initialised Speech client. */
     private _client;
     /** Resolved client constructor options (set in constructor, used in {@link _getClient}). */
@@ -70,16 +100,22 @@ export declare class GoogleCloudSTTProvider {
      */
     private _getClient;
     /**
-     * Transcribe a batch audio buffer using Google Cloud Speech-to-Text.
+     * The provider's display name, as the AgentOS speech contract requires.
+     *
+     * @returns `'Google Cloud Speech-to-Text'`.
+     */
+    getProviderName(): string;
+    /**
+     * Transcribe an audio file or raw PCM buffer using Google Cloud Speech-to-Text.
      *
-     * The audio must be encoded as LINEAR16 (raw PCM, 16-bit little-endian).
-     * The method returns all recognised alternatives from the first result, each
-     * mapped to a {@link SpeechTranscriptionResult}.
+     * Google returns one result per consecutive stretch of the audio, each with
+     * its alternatives ordered by likelihood. The transcript is every stretch's
+     * first alternative, in order.
      *
-     * @param audio   - Audio frame containing the raw PCM bytes and sample rate.
+     * @param audio   - WAV or FLAC file bytes, or raw LINEAR16 PCM with its sample rate.
      * @param options - Optional per-call parameters (language code).
-     * @returns Array of transcription results ordered by confidence (descending).
+     * @returns The transcription in the AgentOS `SpeechTranscriptionResult` shape.
      */
-    transcribe(audio: AudioData, options?: GoogleCloudSTTOptions): Promise<SpeechTranscriptionResult[]>;
+    transcribe(audio: AudioData, options?: GoogleCloudSTTOptions): Promise<SpeechTranscriptionResult>;
 }
 //# sourceMappingURL=GoogleCloudSTTProvider.d.ts.map

package/dist/GoogleCloudSTTProvider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"GoogleCloudSTTProvider.d.ts","sourceRoot":"","sources":["../src/GoogleCloudSTTProvider.ts"],"names":[],"mappings":"AACA;;;;;;;;;;GAUG;AAOH;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;GAMG;AACH,qBAAa,sBAAsB;IACjC,yEAAyE;IACzE,QAAQ,CAAC,EAAE,sBAAsB;IAEjC,wCAAwC;IACxC,OAAO,CAAC,OAAO,CAA6B;IAE5C,4FAA4F;IAC5F,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0B;IAEzD;;;;;;;OAOG;gBACS,WAAW,EAAE,MAAM;IAmB/B;;;;;OAKG;YACW,UAAU;IAaxB;;;;;;;;;;OAUG;IACG,UAAU,CACd,KAAK,EAAE,SAAS,EAChB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,yBAAyB,EAAE,CAAC;CA2BxC"}
+{"version":3,"file":"GoogleCloudSTTProvider.d.ts","sourceRoot":"","sources":["../src/GoogleCloudSTTProvider.ts"],"names":[],"mappings":"AACA;;;;;;;;;;GAUG;AAOH;;;GAGG;AACH,MAAM,WAAW,0BAA0B;IACzC,kDAAkD;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,WAAW,yBAAyB;IACxC,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,OAAO,EAAE,OAAO,CAAC;IACjB,2FAA2F;IAC3F,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,QAAQ,CAAC,EAAE,0BAA0B,EAAE,CAAC;IACxC,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA0BD;;;;;;GAMG;AACH,qBAAa,sBAAsB;IACjC,yEAAyE;IACzE,QAAQ,CAAC,EAAE,sBAAsB;IAEjC,oCAAoC;IACpC,QAAQ,CAAC,WAAW,iCAAiC;IAErD,6DAA6D;IAC7D,QAAQ,CAAC,iBAAiB,SAAS;IAEnC,wCAAwC;IACxC,OAAO,CAAC,OAAO,CAA6B;IAE5C,4FAA4F;IAC5F,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0B;IAEzD;;;;;;;OAOG;gBACS,WAAW,EAAE,MAAM;IAmB/B;;;;;OAKG;YACW,UAAU;IAaxB;;;;OAIG;IACH,eAAe,IAAI,MAAM;IAIzB;;;;;;;;;;OAUG;IACG,UAAU,CACd,KAAK,EAAE,SAAS,EAChB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,yBAAyB,CAAC;CA0CtC"}

package/dist/GoogleCloudSTTProvider.js

@@ -10,6 +10,29 @@
  *
  * @module google-cloud-stt
  */
+/**
+ * The encoding fields of the recognition config for this audio.
+ *
+ * WAV and FLAC files carry a header that states the encoding and sample rate.
+ * Google reads both from it and rejects a request whose stated values disagree
+ * (google.cloud.speech.v1 `RecognitionConfig`), so for those formats the
+ * encoding is left out and the sample rate is sent only when the caller gives
+ * one. Anything else is sent as raw LINEAR16 PCM.
+ */
+function encodingFor(audio) {
+    const declared = `${audio.mimeType ?? ''} ${audio.format ?? ''}`.toLowerCase();
+    if (declared.includes('wav') || declared.includes('flac')) {
+        return audio.sampleRate ? { sampleRateHertz: audio.sampleRate } : {};
+    }
+    return { encoding: 'LINEAR16', sampleRateHertz: audio.sampleRate ?? 16000 };
+}
+/** Seconds in a protobuf `Duration` (`{ seconds, nanos }`, seconds possibly a string). */
+function durationSeconds(duration) {
+    if (!duration)
+        return undefined;
+    const seconds = Number(duration.seconds ?? 0) + Number(duration.nanos ?? 0) / 1e9;
+    return Number.isFinite(seconds) ? seconds : undefined;
+}
 /**
  * Google Cloud Speech-to-Text batch provider.
  *
@@ -20,6 +43,10 @@
 export class GoogleCloudSTTProvider {
     /** Stable provider identifier used by the AgentOS extension registry. */
     id = 'google-cloud-stt';
+    /** Human-readable provider name. */
+    displayName = 'Google Cloud Speech-to-Text';
+    /** Batch recognition only: this provider does not stream. */
+    supportsStreaming = false;
     /** Lazily initialised Speech client. */
     _client = null;
     /** Resolved client constructor options (set in constructor, used in {@link _getClient}). */
@@ -69,38 +96,62 @@ export class GoogleCloudSTTProvider {
     // Public API
     // ---------------------------------------------------------------------------
     /**
-     * Transcribe a batch audio buffer using Google Cloud Speech-to-Text.
+     * The provider's display name, as the AgentOS speech contract requires.
+     *
+     * @returns `'Google Cloud Speech-to-Text'`.
+     */
+    getProviderName() {
+        return this.displayName;
+    }
+    /**
+     * Transcribe an audio file or raw PCM buffer using Google Cloud Speech-to-Text.
      *
-     * The audio must be encoded as LINEAR16 (raw PCM, 16-bit little-endian).
-     * The method returns all recognised alternatives from the first result, each
-     * mapped to a {@link SpeechTranscriptionResult}.
+     * Google returns one result per consecutive stretch of the audio, each with
+     * its alternatives ordered by likelihood. The transcript is every stretch's
+     * first alternative, in order.
      *
-     * @param audio   - Audio frame containing the raw PCM bytes and sample rate.
+     * @param audio   - WAV or FLAC file bytes, or raw LINEAR16 PCM with its sample rate.
      * @param options - Optional per-call parameters (language code).
-     * @returns Array of transcription results ordered by confidence (descending).
+     * @returns The transcription in the AgentOS `SpeechTranscriptionResult` shape.
      */
     async transcribe(audio, options) {
         const client = await this._getClient();
+        const languageCode = options?.language ?? 'en-US';
         const response = await client.recognize({
             audio: { content: audio.data.toString('base64') },
-            config: {
-                encoding: 'LINEAR16',
-                sampleRateHertz: audio.sampleRate ?? 16000,
-                languageCode: options?.language ?? 'en-US',
-            },
+            config: { ...encodingFor(audio), languageCode },
         });
-        const results = [];
-        for (const result of response[0]?.results ?? []) {
+        const recognized = response[0];
+        const stretches = [];
+        for (const result of recognized?.results ?? []) {
             const alt = result?.alternatives?.[0];
-            if (alt) {
-                results.push({
-                    transcript: alt.transcript ?? '',
-                    confidence: alt.confidence ?? 0,
-                    isFinal: true,
-                });
-            }
+            if (!alt)
+                continue;
+            stretches.push({
+                text: (alt.transcript ?? '').trim(),
+                confidence: typeof alt.confidence === 'number' ? alt.confidence : undefined,
+                endTime: durationSeconds(result.resultEndTime),
+            });
         }
-        return results;
+        const confidences = stretches.map((s) => s.confidence).filter((c) => c !== undefined);
+        // Timing is reported only when Google gives every stretch an end time.
+        let start = 0;
+        const segments = stretches.length > 0 && stretches.every((s) => s.endTime !== undefined)
+            ? stretches.map((s) => {
+                const segment = { text: s.text, startTime: start, endTime: s.endTime, confidence: s.confidence };
+                start = s.endTime;
+                return segment;
+            })
+            : undefined;
+        return {
+            text: stretches.map((s) => s.text).filter((t) => t.length > 0).join(' '),
+            language: recognized?.results?.[0]?.languageCode || languageCode,
+            confidence: confidences.length > 0 ? confidences.reduce((sum, c) => sum + c, 0) / confidences.length : undefined,
+            isFinal: true,
+            cost: 0,
+            segments,
+            providerResponse: recognized,
+        };
     }
 }
 //# sourceMappingURL=GoogleCloudSTTProvider.js.map

package/dist/GoogleCloudSTTProvider.js.map

@@ -1 +1 @@
-{"version":3,"file":"GoogleCloudSTTProvider.js","sourceRoot":"","sources":["../src/GoogleCloudSTTProvider.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;;;;;;;;GAUG;AAwCH;;;;;;GAMG;AACH,MAAM,OAAO,sBAAsB;IACjC,yEAAyE;IAChE,EAAE,GAAG,kBAAkB,CAAC;IAEjC,wCAAwC;IAChC,OAAO,GAAwB,IAAI,CAAC;IAE5C,4FAA4F;IAC3E,cAAc,CAA0B;IAEzD;;;;;;;OAOG;IACH,YAAY,WAAmB;QAC7B,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YACxB,wEAAwE;YACxE,wEAAwE;YACxE,yEAAyE;YACzE,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QAC3B,CAAC;aAAM,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACnE,wBAAwB;YACxB,IAAI,CAAC,cAAc,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,8CAA8C;YAC9C,IAAI,CAAC,cAAc,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAA4B,EAAE,CAAC;QAC5F,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAE9E;;;;;OAKG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,wEAAwE;YACxE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,8EAA8E;IAC9E,aAAa;IACb,8EAA8E;IAE9E;;;;;;;;;;OAUG;IACH,KAAK,CAAC,UAAU,CACd,KAAgB,EAChB,OAA+B;QAE/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAEvC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC;YACtC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACjD,MAAM,EAAE;gBACN,QAAQ,EAAE,UAAU;gBACpB,eAAe,EAAE,KAAK,CAAC,UAAU,IAAI,KAAK;gBAC1C,YAAY,EAAE,OAAO,EAAE,QAAQ,IAAI,OAAO;aAC3C;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAgC,EAAE,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAChD,MAAM,GAAG,GAAG,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,IAAI,CAAC;oBACX,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,EAAE;oBAChC,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,CAAC;oBAC/B,OAAO,EAAE,IAAI;iBACd,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
+{"version":3,"file":"GoogleCloudSTTProvider.js","sourceRoot":"","sources":["../src/GoogleCloudSTTProvider.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;;;;;;;;GAUG;AAmEH;;;;;;;;GAQG;AACH,SAAS,WAAW,CAAC,KAAgB;IACnC,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/E,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,KAAK,CAAC,UAAU,IAAI,KAAK,EAAE,CAAC;AAC9E,CAAC;AAED,0FAA0F;AAC1F,SAAS,eAAe,CAAC,QAAmE;IAC1F,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IAClF,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AACxD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,sBAAsB;IACjC,yEAAyE;IAChE,EAAE,GAAG,kBAAkB,CAAC;IAEjC,oCAAoC;IAC3B,WAAW,GAAG,6BAA6B,CAAC;IAErD,6DAA6D;IACpD,iBAAiB,GAAG,KAAK,CAAC;IAEnC,wCAAwC;IAChC,OAAO,GAAwB,IAAI,CAAC;IAE5C,4FAA4F;IAC3E,cAAc,CAA0B;IAEzD;;;;;;;OAOG;IACH,YAAY,WAAmB;QAC7B,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YACxB,wEAAwE;YACxE,wEAAwE;YACxE,yEAAyE;YACzE,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QAC3B,CAAC;aAAM,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACnE,wBAAwB;YACxB,IAAI,CAAC,cAAc,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,8CAA8C;YAC9C,IAAI,CAAC,cAAc,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAA4B,EAAE,CAAC;QAC5F,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAE9E;;;;;OAKG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,wEAAwE;YACxE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,8EAA8E;IAC9E,aAAa;IACb,8EAA8E;IAE9E;;;;OAIG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,UAAU,CACd,KAAgB,EAChB,OAA+B;QAE/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,MAAM,YAAY,GAAG,OAAO,EAAE,QAAQ,IAAI,OAAO,CAAC;QAElD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC;YACtC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACjD,MAAM,EAAE,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE,YAAY,EAAE;SAChD,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAE/B,MAAM,SAAS,GAAmE,EAAE,CAAC;QACrF,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAC/C,MAAM,GAAG,GAAG,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,SAAS,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;gBACnC,UAAU,EAAE,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC3E,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,aAAa,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QACnG,uEAAuE;QACvE,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC;YACtF,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAClB,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAiB,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC3G,KAAK,GAAG,CAAC,CAAC,OAAiB,CAAC;gBAC5B,OAAO,OAAO,CAAC;YACjB,CAAC,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QAEd,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YACxE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,IAAI,YAAY;YAChE,UAAU,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAChH,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,CAAC;YACP,QAAQ;YACR,gBAAgB,EAAE,UAAU;SAC7B,CAAC;IACJ,CAAC;CACF"}

package/dist/index.d.ts

@@ -62,5 +62,5 @@ export declare function createGoogleCloudSTT(credentials: string): GoogleCloudST
  */
 export declare function createExtensionPack(context: ExtensionPackContext): ExtensionPack;
 export { GoogleCloudSTTProvider } from './GoogleCloudSTTProvider.js';
-export type { SpeechTranscriptionResult, GoogleCloudSTTOptions, AudioData, } from './GoogleCloudSTTProvider.js';
+export type { SpeechTranscriptionResult, SpeechTranscriptionSegment, GoogleCloudSTTOptions, AudioData, } from './GoogleCloudSTTProvider.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAMrE,2DAA2D;AAC3D,UAAU,mBAAmB;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qDAAqD;AACrD,UAAU,aAAa;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,4DAA4D;AAC5D,UAAU,oBAAoB;IAC5B,SAAS,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AASD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,sBAAsB,CAEhF;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,oBAAoB,GAAG,aAAa,CAgBhF;AAMD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,YAAY,EACV,yBAAyB,EACzB,qBAAqB,EACrB,SAAS,GACV,MAAM,6BAA6B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAMrE,2DAA2D;AAC3D,UAAU,mBAAmB;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qDAAqD;AACrD,UAAU,aAAa;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,4DAA4D;AAC5D,UAAU,oBAAoB;IAC5B,SAAS,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AASD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,sBAAsB,CAEhF;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,oBAAoB,GAAG,aAAa,CAgBhF;AAMD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,YAAY,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,qBAAqB,EACrB,SAAS,GACV,MAAM,6BAA6B,CAAC"}

package/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "@framers/agentos-ext-google-cloud-stt",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Batch speech-to-text via Google Cloud Speech-to-Text API",
   "kind": "stt-provider",
   "extensionId": "google-cloud-stt",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@framers/agentos-ext-google-cloud-stt",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Batch speech-to-text via Google Cloud Speech-to-Text API for AgentOS voice pipeline",
   "type": "module",
   "main": "./dist/index.js",
@@ -22,7 +22,7 @@
     "@google-cloud/speech": "^6.0.0"
   },
   "devDependencies": {
-    "@framers/agentos": "^0.10.40",
+    "@framers/agentos": "^0.12.5",
     "@google-cloud/speech": "^6.0.0",
     "typescript": "^5.5.0",
     "vitest": "^3.2.7"

package/src/GoogleCloudSTTProvider.ts

@@ -17,18 +17,40 @@
 type SpeechClient = any;
 
 /**
- * A single recognised audio segment returned by the provider.
- *
- * Mirrors the generic `SpeechTranscriptionResult` shape used across the
- * AgentOS voice pipeline.
+ * One stretch of recognised speech: Google returns a result per consecutive
+ * portion of the audio.
+ */
+export interface SpeechTranscriptionSegment {
+  /** The most likely transcript of this stretch. */
+  text: string;
+  /** Start in seconds from the beginning of the audio. */
+  startTime: number;
+  /** End in seconds from the beginning of the audio. */
+  endTime: number;
+  /** Confidence score in [0, 1], when Google reports one. */
+  confidence?: number;
+}
+
+/**
+ * The transcription result of the AgentOS `SpeechToTextProvider` contract
+ * (`SpeechTranscriptionResult` in `@framers/agentos`), mirrored here so the
+ * pack carries no type import from agentos.
  */
 export interface SpeechTranscriptionResult {
-  /** The recognised text. */
-  transcript: string;
-  /** Confidence score in [0, 1]. */
-  confidence: number;
-  /** Whether this is a final (non-speculative) result. Always `true` for batch. */
+  /** The recognised text: every stretch's most likely transcript, in order. */
+  text: string;
+  /** BCP-47 language of the transcript. */
+  language?: string;
+  /** Mean confidence of the stretches, in [0, 1], when Google reports any. */
+  confidence?: number;
+  /** Always `true`: batch recognition returns final results only. */
   isFinal: boolean;
+  /** Always 0; cost is tracked above the provider, as for AgentOS's core batch providers. */
+  cost: number;
+  /** Per-stretch transcripts with their timing, when Google reports end times. */
+  segments?: SpeechTranscriptionSegment[];
+  /** The raw `RecognizeResponse`. */
+  providerResponse?: unknown;
 }
 
 /**
@@ -40,13 +62,42 @@ export interface GoogleCloudSTTOptions {
 }
 
 /**
- * Audio frame passed to {@link GoogleCloudSTTProvider.transcribe}.
+ * Audio passed to {@link GoogleCloudSTTProvider.transcribe}: the fields of the
+ * AgentOS `SpeechAudioInput` this provider reads.
  */
 export interface AudioData {
-  /** Raw PCM bytes (LINEAR16). */
+  /** The audio bytes: a WAV or FLAC file, or raw LINEAR16 PCM. */
   data: Buffer;
-  /** Sample rate in Hz. @defaultValue `16000` */
+  /** Sample rate in Hz. Raw PCM defaults to 16000; a WAV or FLAC header supplies its own. */
   sampleRate?: number;
+  /** MIME type, such as `'audio/wav'` or `'audio/flac'`. */
+  mimeType?: string;
+  /** Container format, such as `'wav'` or `'flac'`. */
+  format?: string;
+}
+
+/**
+ * The encoding fields of the recognition config for this audio.
+ *
+ * WAV and FLAC files carry a header that states the encoding and sample rate.
+ * Google reads both from it and rejects a request whose stated values disagree
+ * (google.cloud.speech.v1 `RecognitionConfig`), so for those formats the
+ * encoding is left out and the sample rate is sent only when the caller gives
+ * one. Anything else is sent as raw LINEAR16 PCM.
+ */
+function encodingFor(audio: AudioData): { encoding?: string; sampleRateHertz?: number } {
+  const declared = `${audio.mimeType ?? ''} ${audio.format ?? ''}`.toLowerCase();
+  if (declared.includes('wav') || declared.includes('flac')) {
+    return audio.sampleRate ? { sampleRateHertz: audio.sampleRate } : {};
+  }
+  return { encoding: 'LINEAR16', sampleRateHertz: audio.sampleRate ?? 16000 };
+}
+
+/** Seconds in a protobuf `Duration` (`{ seconds, nanos }`, seconds possibly a string). */
+function durationSeconds(duration: { seconds?: unknown; nanos?: unknown } | null | undefined): number | undefined {
+  if (!duration) return undefined;
+  const seconds = Number(duration.seconds ?? 0) + Number(duration.nanos ?? 0) / 1e9;
+  return Number.isFinite(seconds) ? seconds : undefined;
 }
 
 /**
@@ -60,6 +111,12 @@ export class GoogleCloudSTTProvider {
   /** Stable provider identifier used by the AgentOS extension registry. */
   readonly id = 'google-cloud-stt';
 
+  /** Human-readable provider name. */
+  readonly displayName = 'Google Cloud Speech-to-Text';
+
+  /** Batch recognition only: this provider does not stream. */
+  readonly supportsStreaming = false;
+
   /** Lazily initialised Speech client. */
   private _client: SpeechClient | null = null;
 
@@ -113,44 +170,68 @@ export class GoogleCloudSTTProvider {
   // ---------------------------------------------------------------------------
 
   /**
-   * Transcribe a batch audio buffer using Google Cloud Speech-to-Text.
+   * The provider's display name, as the AgentOS speech contract requires.
    *
-   * The audio must be encoded as LINEAR16 (raw PCM, 16-bit little-endian).
-   * The method returns all recognised alternatives from the first result, each
-   * mapped to a {@link SpeechTranscriptionResult}.
+   * @returns `'Google Cloud Speech-to-Text'`.
+   */
+  getProviderName(): string {
+    return this.displayName;
+  }
+
+  /**
+   * Transcribe an audio file or raw PCM buffer using Google Cloud Speech-to-Text.
    *
-   * @param audio   - Audio frame containing the raw PCM bytes and sample rate.
+   * Google returns one result per consecutive stretch of the audio, each with
+   * its alternatives ordered by likelihood. The transcript is every stretch's
+   * first alternative, in order.
+   *
+   * @param audio   - WAV or FLAC file bytes, or raw LINEAR16 PCM with its sample rate.
    * @param options - Optional per-call parameters (language code).
-   * @returns Array of transcription results ordered by confidence (descending).
+   * @returns The transcription in the AgentOS `SpeechTranscriptionResult` shape.
    */
   async transcribe(
     audio: AudioData,
     options?: GoogleCloudSTTOptions,
-  ): Promise<SpeechTranscriptionResult[]> {
+  ): Promise<SpeechTranscriptionResult> {
     const client = await this._getClient();
+    const languageCode = options?.language ?? 'en-US';
 
     const response = await client.recognize({
       audio: { content: audio.data.toString('base64') },
-      config: {
-        encoding: 'LINEAR16',
-        sampleRateHertz: audio.sampleRate ?? 16000,
-        languageCode: options?.language ?? 'en-US',
-      },
+      config: { ...encodingFor(audio), languageCode },
     });
+    const recognized = response[0];
 
-    const results: SpeechTranscriptionResult[] = [];
-
-    for (const result of response[0]?.results ?? []) {
+    const stretches: Array<{ text: string; confidence?: number; endTime?: number }> = [];
+    for (const result of recognized?.results ?? []) {
       const alt = result?.alternatives?.[0];
-      if (alt) {
-        results.push({
-          transcript: alt.transcript ?? '',
-          confidence: alt.confidence ?? 0,
-          isFinal: true,
-        });
-      }
+      if (!alt) continue;
+      stretches.push({
+        text: (alt.transcript ?? '').trim(),
+        confidence: typeof alt.confidence === 'number' ? alt.confidence : undefined,
+        endTime: durationSeconds(result.resultEndTime),
+      });
     }
 
-    return results;
+    const confidences = stretches.map((s) => s.confidence).filter((c): c is number => c !== undefined);
+    // Timing is reported only when Google gives every stretch an end time.
+    let start = 0;
+    const segments = stretches.length > 0 && stretches.every((s) => s.endTime !== undefined)
+      ? stretches.map((s) => {
+          const segment = { text: s.text, startTime: start, endTime: s.endTime as number, confidence: s.confidence };
+          start = s.endTime as number;
+          return segment;
+        })
+      : undefined;
+
+    return {
+      text: stretches.map((s) => s.text).filter((t) => t.length > 0).join(' '),
+      language: recognized?.results?.[0]?.languageCode || languageCode,
+      confidence: confidences.length > 0 ? confidences.reduce((sum, c) => sum + c, 0) / confidences.length : undefined,
+      isFinal: true,
+      cost: 0,
+      segments,
+      providerResponse: recognized,
+    };
   }
 }

package/src/index.ts

@@ -105,6 +105,7 @@ export function createExtensionPack(context: ExtensionPackContext): ExtensionPac
 export { GoogleCloudSTTProvider } from './GoogleCloudSTTProvider.js';
 export type {
   SpeechTranscriptionResult,
+  SpeechTranscriptionSegment,
   GoogleCloudSTTOptions,
   AudioData,
 } from './GoogleCloudSTTProvider.js';