@speakableio/core 1.0.26 → 1.0.28

package/dist/index.native.mjs

@@ -2134,8 +2134,209 @@ var createSetRepo = () => {
   };
 };
 
+// src/utils/ai/detect-transcript-hallucionation.ts
+var HALLUCINATION_THRESHOLDS = {
+  // Short repeats
+  MIN_CONSECUTIVE_REPEATS: 3,
+  MIN_WORDS_FOR_RATIO_CHECK: 10,
+  MAX_UNIQUE_WORDS_FOR_RATIO: 3,
+  MIN_REPETITION_RATIO: 3,
+  // Phrase repeats
+  MIN_SENTENCE_LENGTH: 10,
+  MIN_CONSECUTIVE_SIMILAR_SENTENCES: 2,
+  MIN_SENTENCES_FOR_DUPLICATE_CHECK: 3,
+  // Cyclic patterns
+  MIN_CYCLE_LENGTH: 20,
+  MIN_CYCLE_REPEATS: 3,
+  // Entropy detection
+  MIN_LENGTH_FOR_ENTROPY_CHECK: 50,
+  MAX_ENTROPY_THRESHOLD: 2.5,
+  // bits per character
+  // Similarity
+  SENTENCE_SIMILARITY_THRESHOLD: 0.8,
+  SEGMENT_SIMILARITY_THRESHOLD: 0.85
+};
+function detectTranscriptHallucinationWithDetails(transcript) {
+  if (!transcript || transcript.trim().length === 0) {
+    return { isHallucination: false };
+  }
+  const text = transcript.trim();
+  if (text.length < 10) {
+    return { isHallucination: false };
+  }
+  const shortRepeats = detectShortRepeats(text);
+  if (shortRepeats) {
+    return {
+      isHallucination: true,
+      reason: "Detected repeated short words or phrases",
+      confidence: 0.9
+    };
+  }
+  const phraseRepeats = detectPhraseRepeats(text);
+  if (phraseRepeats) {
+    return {
+      isHallucination: true,
+      reason: "Detected repeated sentences or phrases",
+      confidence: 0.85
+    };
+  }
+  const cyclicRepeats = detectCyclicPattern(text);
+  if (cyclicRepeats) {
+    return {
+      isHallucination: true,
+      reason: "Detected cyclic repetition pattern",
+      confidence: 0.8
+    };
+  }
+  if (text.length >= HALLUCINATION_THRESHOLDS.MIN_LENGTH_FOR_ENTROPY_CHECK) {
+    const entropy = calculateEntropy(text);
+    if (entropy < HALLUCINATION_THRESHOLDS.MAX_ENTROPY_THRESHOLD) {
+      return {
+        isHallucination: true,
+        reason: "Detected low entropy (likely gibberish or excessive repetition)",
+        confidence: 0.75
+      };
+    }
+  }
+  return { isHallucination: false };
+}
+function detectShortRepeats(text) {
+  const words = text.toLowerCase().split(/[\s,;.!?]+/).filter((w) => w.length > 0);
+  if (words.length < 4) return false;
+  let repeatCount = 1;
+  for (let i = 1; i < words.length; i++) {
+    if (words[i] === words[i - 1]) {
+      repeatCount++;
+      if (repeatCount >= HALLUCINATION_THRESHOLDS.MIN_CONSECUTIVE_REPEATS) {
+        return true;
+      }
+    } else {
+      repeatCount = 1;
+    }
+  }
+  const uniqueWords = new Set(words);
+  const repetitionRatio = words.length / uniqueWords.size;
+  if (words.length >= HALLUCINATION_THRESHOLDS.MIN_WORDS_FOR_RATIO_CHECK && uniqueWords.size <= HALLUCINATION_THRESHOLDS.MAX_UNIQUE_WORDS_FOR_RATIO && repetitionRatio >= HALLUCINATION_THRESHOLDS.MIN_REPETITION_RATIO) {
+    return true;
+  }
+  return false;
+}
+function detectPhraseRepeats(text) {
+  const sentences = text.split(/[.!?]+/).map((s) => s.trim().toLowerCase()).filter((s) => s.length > HALLUCINATION_THRESHOLDS.MIN_SENTENCE_LENGTH);
+  if (sentences.length < 2) return false;
+  for (let i = 0; i < sentences.length - 1; i++) {
+    let consecutiveRepeats = 1;
+    for (let j = i + 1; j < sentences.length; j++) {
+      if (isSimilarSentence(sentences[i], sentences[j])) {
+        consecutiveRepeats++;
+      } else {
+        break;
+      }
+    }
+    if (consecutiveRepeats >= HALLUCINATION_THRESHOLDS.MIN_CONSECUTIVE_SIMILAR_SENTENCES) {
+      return true;
+    }
+  }
+  const uniqueSentences = new Set(sentences);
+  if (sentences.length >= HALLUCINATION_THRESHOLDS.MIN_SENTENCES_FOR_DUPLICATE_CHECK && uniqueSentences.size === 1) {
+    return true;
+  }
+  return false;
+}
+function isSimilarSentence(s1, s2, threshold = HALLUCINATION_THRESHOLDS.SENTENCE_SIMILARITY_THRESHOLD) {
+  if (s1 === s2) return true;
+  const normalized1 = s1.replace(/\s+/g, " ").trim();
+  const normalized2 = s2.replace(/\s+/g, " ").trim();
+  if (normalized1 === normalized2) return true;
+  const words1 = normalized1.split(/\s+/);
+  const words2 = normalized2.split(/\s+/);
+  if (Math.abs(words1.length - words2.length) > 2) return false;
+  const set1 = new Set(words1);
+  const set2 = new Set(words2);
+  const intersection = new Set([...set1].filter((w) => set2.has(w)));
+  const similarity = intersection.size * 2 / (set1.size + set2.size);
+  return similarity >= threshold;
+}
+function detectCyclicPattern(text) {
+  const normalized = text.toLowerCase().replace(/\s+/g, " ").trim();
+  const length = normalized.length;
+  const minCycleLength = HALLUCINATION_THRESHOLDS.MIN_CYCLE_LENGTH;
+  const maxCycleLength = Math.floor(length / 2);
+  if (maxCycleLength < minCycleLength) return false;
+  const step = 5;
+  for (let cycleLen = minCycleLength; cycleLen <= maxCycleLength; cycleLen += step) {
+    const pattern = normalized.substring(0, cycleLen);
+    let matchCount = 0;
+    let pos = 0;
+    while (pos < length) {
+      const segment = normalized.substring(pos, pos + cycleLen);
+      if (segment.length < cycleLen) {
+        const partialMatch = pattern.startsWith(segment);
+        if (partialMatch && matchCount > 0) {
+          matchCount++;
+        }
+        break;
+      }
+      if (segment === pattern || isSegmentSimilar(segment, pattern)) {
+        matchCount++;
+        pos += cycleLen;
+      } else {
+        break;
+      }
+    }
+    if (matchCount >= HALLUCINATION_THRESHOLDS.MIN_CYCLE_REPEATS) {
+      return true;
+    }
+  }
+  return false;
+}
+function isSegmentSimilar(s1, s2) {
+  if (s1 === s2) return true;
+  if (s1.length !== s2.length) return false;
+  let matches = 0;
+  const minLength = Math.min(s1.length, s2.length);
+  for (let i = 0; i < minLength; i++) {
+    if (s1[i] === s2[i]) {
+      matches++;
+    }
+  }
+  const similarity = matches / minLength;
+  return similarity >= HALLUCINATION_THRESHOLDS.SEGMENT_SIMILARITY_THRESHOLD;
+}
+function calculateEntropy(text) {
+  if (!text || text.length === 0) {
+    return 0;
+  }
+  const frequencies = /* @__PURE__ */ new Map();
+  for (const char of text.toLowerCase()) {
+    frequencies.set(char, (frequencies.get(char) || 0) + 1);
+  }
+  let entropy = 0;
+  const length = text.length;
+  for (const count of frequencies.values()) {
+    const probability = count / length;
+    entropy -= probability * Math.log2(probability);
+  }
+  return entropy;
+}
+function cleanHallucinatedTranscript(transcript) {
+  var _a, _b;
+  const result = detectTranscriptHallucinationWithDetails(transcript);
+  if (result.isHallucination) {
+    console.warn(
+      "Hallucinated transcript detected and removed:",
+      transcript.substring(0, 100),
+      `
+Reason: ${(_a = result.reason) != null ? _a : "Unknown"}`,
+      `Confidence: ${String((_b = result.confidence) != null ? _b : "Unknown")}`
+    );
+    return "";
+  }
+  return transcript;
+}
+
 // src/utils/ai/get-transcript.ts
-async function getTranscript(model, args) {
+async function getTranscript(model, args, cleanHallucinations = true) {
   var _a, _b, _c, _d, _e, _f;
   const getGeminiTranscript = (_b = (_a = api).httpsCallable) == null ? void 0 : _b.call(_a, "getGeminiTranscript");
   const getAssemblyAITranscript = (_d = (_c = api).httpsCallable) == null ? void 0 : _d.call(_c, "transcribeAssemblyAIAudio");
@@ -2146,7 +2347,7 @@ async function getTranscript(model, args) {
         audioUrl: args.audioUrl,
         language: args.language
       }));
-      return data;
+      return cleanHallucinations ? cleanHallucinatedTranscript(data) : data;
     } catch (error) {
       console.error("Error getting transcript from Whisper:", error);
       throw error;
@@ -2159,7 +2360,7 @@ async function getTranscript(model, args) {
         targetLanguage: args.language,
         prompt: args.prompt
       }));
-      return data.transcript;
+      return cleanHallucinations ? cleanHallucinatedTranscript(data.transcript) : data.transcript;
     } catch (error) {
       console.error("Error getting transcript from Gemini:", error);
       throw error;
@@ -2171,7 +2372,7 @@ async function getTranscript(model, args) {
         audioUrl: args.audioUrl,
         language: args.language
       }));
-      return response.data;
+      return cleanHallucinations ? cleanHallucinatedTranscript(response.data) : response.data;
     } catch (error) {
       console.error("Error getting transcript from AssemblyAI:", error);
       throw error;
@@ -2179,6 +2380,37 @@ async function getTranscript(model, args) {
   }
   return null;
 }
+async function getTranscriptCycle(args) {
+  const models = ["whisper", "gemini", "assemblyai"];
+  let transcript = "";
+  let lastError = null;
+  for (const model of models) {
+    try {
+      const transcriptResult = await getTranscript(model, args, false);
+      const rawTranscript = transcriptResult || "";
+      transcript = cleanHallucinatedTranscript(rawTranscript);
+      if (transcript !== "") {
+        console.log(`Successfully got transcript from ${model}`);
+        break;
+      }
+      console.warn(`${model} returned empty transcript, trying next model`);
+    } catch (e) {
+      console.error(`Error with ${model} transcript:`, e);
+      lastError = e;
+    }
+  }
+  if (transcript === "") {
+    console.error("All transcript models failed or returned empty", lastError);
+    return {
+      transcript: "",
+      success: false
+    };
+  }
+  return {
+    transcript,
+    success: true
+  };
+}
 
 // src/constants/all-langs.json
 var all_langs_default = {
@@ -2925,6 +3157,17 @@ function useSpeakableTranscript() {
     mutation
   };
 }
+function useSpeakableTranscriptCycle() {
+  const mutation = useMutation3({
+    mutationFn: async (args) => {
+      return getTranscriptCycle(args);
+    },
+    retry: false
+  });
+  return {
+    mutationTranscriptCycle: mutation
+  };
+}
 
 // src/hooks/useUpdateStudentVoc.ts
 var useUpdateStudentVocab = (page) => {
@@ -3478,6 +3721,7 @@ export {
   getSetFromCache,
   getTotalCompletedCards,
   getTranscript,
+  getTranscriptCycle,
   getWordHash,
   purify,
   refsCardsFiresotre,
@@ -3502,6 +3746,7 @@ export {
   useSet,
   useSpeakableApi,
   useSpeakableTranscript,
+  useSpeakableTranscriptCycle,
   useSubmitAssignmentScore,
   useSubmitPracticeScore,
   useUpdateCardScore,