@absolutejs/voice 0.0.22-beta.636 → 0.0.22-beta.637

package/dist/index.js

@@ -49563,6 +49563,46 @@ var runSTTAdapterFixture = async (adapter, fixture, options = {}) => {
   };
 };
 
+// src/testing/confidenceCalibration.ts
+var clampConfidence2 = (value) => Math.max(0, Math.min(1, value));
+var calibrateVoiceConfidence = (samples, binCount = 10) => {
+  const safeBinCount = Math.max(1, Math.round(binCount));
+  const bins = Array.from({ length: safeBinCount }, (_, index) => {
+    const lowerBound = index / safeBinCount;
+    return {
+      accuracy: 0,
+      averageConfidence: 0,
+      count: 0,
+      lowerBound,
+      upperBound: (index + 1) / safeBinCount
+    };
+  });
+  let brierTotal = 0;
+  for (const sample of samples) {
+    const confidence = clampConfidence2(sample.confidence);
+    const binIndex = Math.min(safeBinCount - 1, Math.floor(confidence * safeBinCount));
+    const bin = bins[binIndex];
+    bin.count += 1;
+    bin.averageConfidence += confidence;
+    bin.accuracy += sample.correct ? 1 : 0;
+    brierTotal += (confidence - (sample.correct ? 1 : 0)) ** 2;
+  }
+  let expectedCalibrationError = 0;
+  for (const bin of bins) {
+    if (bin.count === 0)
+      continue;
+    bin.averageConfidence /= bin.count;
+    bin.accuracy /= bin.count;
+    expectedCalibrationError += bin.count / Math.max(1, samples.length) * Math.abs(bin.accuracy - bin.averageConfidence);
+  }
+  return {
+    bins,
+    brierScore: samples.length > 0 ? brierTotal / samples.length : 0,
+    expectedCalibrationError,
+    sampleCount: samples.length
+  };
+};
+
 // src/testing/criticalFields.ts
 var normalizeText4 = (value) => value.normalize("NFKC").toLowerCase().replace(/[^\p{L}\p{N}@.%+$'-]+/gu, " ").replace(/\s+/g, " ").trim();
 var normalizeDigits = (value) => normalizeSpokenNumbers(value).replace(/\D/g, "");
@@ -49833,6 +49873,15 @@ var toFixtureBenchmarkResult = (fixture, result, elapsedMs) => {
   const expectedTerms = scoreExpectedTerms(result.finalText, fixture.expectedTerms);
   const criticalFields = scoreVoiceCriticalFields(result.finalText, fixture.expectedCriticalFields);
   const speakerTurns = scoreSpeakerTurns(fixture, result);
+  const transcriptConfidence = average2(result.finalEvents.map((event) => {
+    if (typeof event.transcript.confidence === "number") {
+      return event.transcript.confidence;
+    }
+    return average2([
+      ...(event.transcript.words ?? []).map((word) => word.confidence),
+      ...(event.transcript.tokens ?? []).map((token) => token.confidence)
+    ]);
+  }));
   return {
     accuracy: result.accuracy,
     closeCount: result.closeEvents.length,
@@ -49856,6 +49905,7 @@ var toFixtureBenchmarkResult = (fixture, result, elapsedMs) => {
     timeToEndOfTurnMs,
     timeToFirstFinalMs,
     timeToFirstPartialMs,
+    transcriptConfidence: roundMetric4(transcriptConfidence),
     title: fixture.title
   };
 };
@@ -49969,6 +50019,12 @@ var summarizeSTTBenchmark = (adapterId, fixtures) => {
   const passCount = fixtures.filter((fixture) => fixture.passes).length;
   return {
     adapterId,
+    confidenceCalibration: calibrateVoiceConfidence(fixtures.flatMap((fixture) => typeof fixture.transcriptConfidence === "number" ? [
+      {
+        confidence: fixture.transcriptConfidence,
+        correct: fixture.passes
+      }
+    ] : [])),
     averageCharErrorRate: roundMetric4(average2(fixtures.map((fixture) => fixture.accuracy.charErrorRate))) ?? 0,
     averageElapsedMs: roundMetric4(average2(fixtures.map((fixture) => fixture.elapsedMs)), 2) ?? 0,
     averageEndOfTurnCount: roundMetric4(average2(fixtures.map((fixture) => fixture.endOfTurnCount)), 2) ?? 0,

package/dist/testing/benchmark.d.ts

@@ -1,6 +1,7 @@
 import type { STTAdapter, STTAdapterOpenOptions } from "../core/types";
 import { type VoiceSTTAdapterHarnessOptions, type VoiceSTTAdapterHarnessResult } from "./stt";
 import type { VoiceTestFixture } from "./fixtures";
+import { type VoiceConfidenceCalibrationReport } from "./confidenceCalibration";
 import { type VoiceCriticalFieldAccuracy } from "./criticalFields";
 export type VoiceExpectedTermAccuracy = {
     allMatched: boolean;
@@ -41,6 +42,7 @@ export type VoiceSTTBenchmarkFixtureResult = {
     timeToEndOfTurnMs?: number;
     timeToFirstFinalMs?: number;
     timeToFirstPartialMs?: number;
+    transcriptConfidence?: number;
     title: string;
 };
 export type VoiceSTTBenchmarkSummary = {
@@ -67,6 +69,7 @@ export type VoiceSTTBenchmarkSummary = {
     totalErrorCount: number;
     wordAccuracyRate: number;
     groupSummaries: VoiceSTTBenchmarkFixtureSummary[];
+    confidenceCalibration: VoiceConfidenceCalibrationReport;
 };
 export type VoiceSTTBenchmarkFixtureSummary = {
     group: VoiceSTTFixtureEnvironment;

package/dist/testing/index.js

@@ -367,6 +367,46 @@ var runSTTAdapterFixture = async (adapter, fixture, options = {}) => {
   };
 };
 
+// src/testing/confidenceCalibration.ts
+var clampConfidence = (value) => Math.max(0, Math.min(1, value));
+var calibrateVoiceConfidence = (samples, binCount = 10) => {
+  const safeBinCount = Math.max(1, Math.round(binCount));
+  const bins = Array.from({ length: safeBinCount }, (_, index) => {
+    const lowerBound = index / safeBinCount;
+    return {
+      accuracy: 0,
+      averageConfidence: 0,
+      count: 0,
+      lowerBound,
+      upperBound: (index + 1) / safeBinCount
+    };
+  });
+  let brierTotal = 0;
+  for (const sample of samples) {
+    const confidence = clampConfidence(sample.confidence);
+    const binIndex = Math.min(safeBinCount - 1, Math.floor(confidence * safeBinCount));
+    const bin = bins[binIndex];
+    bin.count += 1;
+    bin.averageConfidence += confidence;
+    bin.accuracy += sample.correct ? 1 : 0;
+    brierTotal += (confidence - (sample.correct ? 1 : 0)) ** 2;
+  }
+  let expectedCalibrationError = 0;
+  for (const bin of bins) {
+    if (bin.count === 0)
+      continue;
+    bin.averageConfidence /= bin.count;
+    bin.accuracy /= bin.count;
+    expectedCalibrationError += bin.count / Math.max(1, samples.length) * Math.abs(bin.accuracy - bin.averageConfidence);
+  }
+  return {
+    bins,
+    brierScore: samples.length > 0 ? brierTotal / samples.length : 0,
+    expectedCalibrationError,
+    sampleCount: samples.length
+  };
+};
+
 // src/core/numberNormalizer.ts
 var ONES = {
   eight: 8,
@@ -822,6 +862,15 @@ var toFixtureBenchmarkResult = (fixture, result, elapsedMs) => {
   const expectedTerms = scoreExpectedTerms(result.finalText, fixture.expectedTerms);
   const criticalFields = scoreVoiceCriticalFields(result.finalText, fixture.expectedCriticalFields);
   const speakerTurns = scoreSpeakerTurns(fixture, result);
+  const transcriptConfidence = average(result.finalEvents.map((event) => {
+    if (typeof event.transcript.confidence === "number") {
+      return event.transcript.confidence;
+    }
+    return average([
+      ...(event.transcript.words ?? []).map((word) => word.confidence),
+      ...(event.transcript.tokens ?? []).map((token) => token.confidence)
+    ]);
+  }));
   return {
     accuracy: result.accuracy,
     closeCount: result.closeEvents.length,
@@ -845,6 +894,7 @@ var toFixtureBenchmarkResult = (fixture, result, elapsedMs) => {
     timeToEndOfTurnMs,
     timeToFirstFinalMs,
     timeToFirstPartialMs,
+    transcriptConfidence: roundMetric(transcriptConfidence),
     title: fixture.title
   };
 };
@@ -958,6 +1008,12 @@ var summarizeSTTBenchmark = (adapterId, fixtures) => {
   const passCount = fixtures.filter((fixture) => fixture.passes).length;
   return {
     adapterId,
+    confidenceCalibration: calibrateVoiceConfidence(fixtures.flatMap((fixture) => typeof fixture.transcriptConfidence === "number" ? [
+      {
+        confidence: fixture.transcriptConfidence,
+        correct: fixture.passes
+      }
+    ] : [])),
     averageCharErrorRate: roundMetric(average(fixtures.map((fixture) => fixture.accuracy.charErrorRate))) ?? 0,
     averageElapsedMs: roundMetric(average(fixtures.map((fixture) => fixture.elapsedMs)), 2) ?? 0,
     averageEndOfTurnCount: roundMetric(average(fixtures.map((fixture) => fixture.endOfTurnCount)), 2) ?? 0,
@@ -1030,45 +1086,6 @@ var summarizeSTTBenchmarkSeries = (input) => {
     }
   };
 };
-// src/testing/confidenceCalibration.ts
-var clampConfidence = (value) => Math.max(0, Math.min(1, value));
-var calibrateVoiceConfidence = (samples, binCount = 10) => {
-  const safeBinCount = Math.max(1, Math.round(binCount));
-  const bins = Array.from({ length: safeBinCount }, (_, index) => {
-    const lowerBound = index / safeBinCount;
-    return {
-      accuracy: 0,
-      averageConfidence: 0,
-      count: 0,
-      lowerBound,
-      upperBound: (index + 1) / safeBinCount
-    };
-  });
-  let brierTotal = 0;
-  for (const sample of samples) {
-    const confidence = clampConfidence(sample.confidence);
-    const binIndex = Math.min(safeBinCount - 1, Math.floor(confidence * safeBinCount));
-    const bin = bins[binIndex];
-    bin.count += 1;
-    bin.averageConfidence += confidence;
-    bin.accuracy += sample.correct ? 1 : 0;
-    brierTotal += (confidence - (sample.correct ? 1 : 0)) ** 2;
-  }
-  let expectedCalibrationError = 0;
-  for (const bin of bins) {
-    if (bin.count === 0)
-      continue;
-    bin.averageConfidence /= bin.count;
-    bin.accuracy /= bin.count;
-    expectedCalibrationError += bin.count / Math.max(1, samples.length) * Math.abs(bin.accuracy - bin.averageConfidence);
-  }
-  return {
-    bins,
-    brierScore: samples.length > 0 ? brierTotal / samples.length : 0,
-    expectedCalibrationError,
-    sampleCount: samples.length
-  };
-};
 // src/core/correction.ts
 var escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 var buildAliasMatcher = (alias) => new RegExp(`(?<![\\p{L}\\p{N}'])${escapeRegExp(alias)}(?![\\p{L}\\p{N}'])`, "giu");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@absolutejs/voice",
-  "version": "0.0.22-beta.636",
+  "version": "0.0.22-beta.637",
   "description": "Voice primitives and Elysia plugin for AbsoluteJS",
   "repository": {
     "type": "git",