@speakableio/core 1.0.26 → 1.0.28

package/dist/index.web.d.mts

@@ -93,6 +93,11 @@ interface PageActivity {
     completed?: boolean;
     media_area_id?: string | null;
     media_area_layout?: 'left' | 'right' | null;
+    media_mode?: 'single' | 'media_area' | 'none' | null;
+    media?: {
+        type: 'image' | 'video';
+        url: string;
+    } | null;
     score?: number;
     verificationStatus?: VerificationCardStatus;
     native_text?: string;
@@ -191,6 +196,11 @@ declare function useCards({ cardIds, enabled, asObject, }: {
         completed?: boolean;
         media_area_id?: string | null;
         media_area_layout?: "left" | "right" | null;
+        media_mode?: "single" | "media_area" | "none" | null;
+        media?: {
+            type: "image" | "video";
+            url: string;
+        } | null;
         score?: number;
         verificationStatus?: VerificationCardStatus;
         native_text?: string;
@@ -266,6 +276,11 @@ declare function useCreateCards(): {
         completed?: boolean;
         media_area_id?: string | null;
         media_area_layout?: "left" | "right" | null;
+        media_mode?: "single" | "media_area" | "none" | null;
+        media?: {
+            type: "image" | "video";
+            url: string;
+        } | null;
         score?: number;
         verificationStatus?: VerificationCardStatus;
         native_text?: string;
@@ -349,6 +364,11 @@ declare function useGetCard({ cardId, enabled }: {
     completed?: boolean;
     media_area_id?: string | null;
     media_area_layout?: "left" | "right" | null;
+    media_mode?: "single" | "media_area" | "none" | null;
+    media?: {
+        type: "image" | "video";
+        url: string;
+    } | null;
     score?: number;
     verificationStatus?: VerificationCardStatus;
     native_text?: string;
@@ -424,6 +444,11 @@ declare const createCardRepo: () => {
         completed?: boolean;
         media_area_id?: string | null;
         media_area_layout?: "left" | "right" | null;
+        media_mode?: "single" | "media_area" | "none" | null;
+        media?: {
+            type: "image" | "video";
+            url: string;
+        } | null;
         score?: number;
         verificationStatus?: VerificationCardStatus;
         native_text?: string;
@@ -494,6 +519,11 @@ declare const createCardRepo: () => {
         completed?: boolean;
         media_area_id?: string | null;
         media_area_layout?: "left" | "right" | null;
+        media_mode?: "single" | "media_area" | "none" | null;
+        media?: {
+            type: "image" | "video";
+            url: string;
+        } | null;
         score?: number;
         verificationStatus?: VerificationCardStatus;
         native_text?: string;
@@ -1318,6 +1348,11 @@ declare function createFsClientBase({ db, helpers, httpsCallable, logEvent, }: {
             completed?: boolean;
             media_area_id?: string | null;
             media_area_layout?: "left" | "right" | null;
+            media_mode?: "single" | "media_area" | "none" | null;
+            media?: {
+                type: "image" | "video";
+                url: string;
+            } | null;
             score?: number;
             verificationStatus?: VerificationCardStatus;
             native_text?: string;
@@ -1388,6 +1423,11 @@ declare function createFsClientBase({ db, helpers, httpsCallable, logEvent, }: {
             completed?: boolean;
             media_area_id?: string | null;
             media_area_layout?: "left" | "right" | null;
+            media_mode?: "single" | "media_area" | "none" | null;
+            media?: {
+                type: "image" | "video";
+                url: string;
+            } | null;
             score?: number;
             verificationStatus?: VerificationCardStatus;
             native_text?: string;
@@ -1525,7 +1565,15 @@ declare function getTranscript(model: 'gemini' | 'assemblyai' | 'whisper', args:
     language: string;
     audioUrl: string;
     prompt?: string;
-}): Promise<string | null>;
+}, cleanHallucinations?: boolean): Promise<string | null>;
+declare function getTranscriptCycle(args: {
+    audioUrl: string;
+    language: string;
+    prompt: string;
+}): Promise<{
+    transcript: string;
+    success: boolean;
+}>;
 
 declare const getRespondCardTool: ({ language, standard, }: {
     language: string;
@@ -1633,6 +1681,11 @@ declare function useActivity({ id, isAssignment, onAssignmentSubmitted, ltiData,
             completed?: boolean;
             media_area_id?: string | null;
             media_area_layout?: "left" | "right" | null;
+            media_mode?: "single" | "media_area" | "none" | null;
+            media?: {
+                type: "image" | "video";
+                url: string;
+            } | null;
             score?: number;
             verificationStatus?: VerificationCardStatus;
             native_text?: string;
@@ -2782,6 +2835,16 @@ declare function useSpeakableTranscript(): {
         prompt?: string;
     }, unknown>;
 };
+declare function useSpeakableTranscriptCycle(): {
+    mutationTranscriptCycle: _tanstack_react_query.UseMutationResult<{
+        transcript: string;
+        success: boolean;
+    }, Error, {
+        audioUrl: string;
+        language: string;
+        prompt: string;
+    }, unknown>;
+};
 
 declare const useUpdateStudentVocab: (page: PageActivityWithId | null) => {
     studentVocabMarkVoiceSuccess: undefined;
@@ -2838,7 +2901,7 @@ declare const useBaseOpenAI: ({ onTranscriptSuccess, onTranscriptError, onComple
         noFeedbackAvailable: boolean;
         success: boolean;
         reason: string;
-        accessType: "ai_enabled" | "teacher_preview" | "student_with_teacher_plan" | "none";
+        accessType: "none" | "ai_enabled" | "teacher_preview" | "student_with_teacher_plan";
     } | {
         noFeedbackAvailable: boolean;
         success: boolean;
@@ -3164,6 +3227,11 @@ declare const createFsClientWeb: ({ db, httpsCallable, logEvent }: FsClientParam
             completed?: boolean;
             media_area_id?: string | null;
             media_area_layout?: "left" | "right" | null;
+            media_mode?: "single" | "media_area" | "none" | null;
+            media?: {
+                type: "image" | "video";
+                url: string;
+            } | null;
             score?: number;
             verificationStatus?: VerificationCardStatus;
             native_text?: string;
@@ -3234,6 +3302,11 @@ declare const createFsClientWeb: ({ db, httpsCallable, logEvent }: FsClientParam
             completed?: boolean;
             media_area_id?: string | null;
             media_area_layout?: "left" | "right" | null;
+            media_mode?: "single" | "media_area" | "none" | null;
+            media?: {
+                type: "image" | "video";
+                url: string;
+            } | null;
             score?: number;
             verificationStatus?: VerificationCardStatus;
             native_text?: string;
@@ -3296,4 +3369,4 @@ declare const createFsClientWeb: ({ db, httpsCallable, logEvent }: FsClientParam
     };
 };
 
-export { ActivityPageType, type Assignment, type AssignmentAnalyticsType$1 as AssignmentAnalyticsType, type AssignmentWithId, BASE_MULTIPLE_CHOICE_FIELD_VALUES, BASE_REPEAT_FIELD_VALUES, BASE_RESPOND_FIELD_VALUES, type CreditContract, FeedbackTypesCard, FsCtx, type InstitutionSubscription, LENIENCY_OPTIONS, LeniencyCard, MULTIPLE_CHOICE_PAGE_ACTIVITY_TYPES, type Organization, type OrganizationAccess, type PageActivity, type PageActivityWithId, type PageScore, REPEAT_PAGE_ACTIVITY_TYPES, RESPOND_AUDIO_PAGE_ACTIVITY_TYPES, RESPOND_PAGE_ACTIVITY_TYPES, RESPOND_WRITE_PAGE_ACTIVITY_TYPES, type RefsCardsFiresotre, type RefsSetsFirestore, SPEAKABLE_NOTIFICATIONS, STUDENT_LEVELS_OPTIONS, type Score, type ScoreWithId, type Set, type SetWithId, type SpeakableNotificationType, SpeakableNotificationTypes, SpeakableProvider, VerificationCardStatus, assignmentQueryKeys, cardsQueryKeys, checkIsMCPage, checkIsMediaPage, checkIsRepeatPage, checkIsRespondAudioPage, checkIsRespondPage, checkIsRespondWrittenPage, checkIsShortAnswerPage, checkTypePageActivity, cleanString, createAssignmentRepo, createCardRepo, createFsClientWeb as createFsClient, createSetRepo, creditQueryKeys, debounce, getCardFromCache, getLabelPage, getPagePrompt, getPhraseLength, getRespondCardTool, getSetFromCache, getTotalCompletedCards, getTranscript, getWordHash, purify, refsCardsFiresotre, refsSetsFirestore, scoreQueryKeys, setsQueryKeys, updateCardInCache, updateSetInCache, useActivity, useActivityFeedbackAccess, useAssignment, useBaseOpenAI, useCards, useClearScore, useClearScoreV2, useCreateCard, useCreateCards, useCreateNotification, useGetCard, useOrganizationAccess, useScore, useSet, useSpeakableApi, useSpeakableTranscript, useSubmitAssignmentScore, useSubmitPracticeScore, useUpdateCardScore, useUpdateScore, useUpdateStudentVocab, useUserCredits };
+export { ActivityPageType, type Assignment, type AssignmentAnalyticsType$1 as AssignmentAnalyticsType, type AssignmentWithId, BASE_MULTIPLE_CHOICE_FIELD_VALUES, BASE_REPEAT_FIELD_VALUES, BASE_RESPOND_FIELD_VALUES, type CreditContract, FeedbackTypesCard, FsCtx, type InstitutionSubscription, LENIENCY_OPTIONS, LeniencyCard, MULTIPLE_CHOICE_PAGE_ACTIVITY_TYPES, type Organization, type OrganizationAccess, type PageActivity, type PageActivityWithId, type PageScore, REPEAT_PAGE_ACTIVITY_TYPES, RESPOND_AUDIO_PAGE_ACTIVITY_TYPES, RESPOND_PAGE_ACTIVITY_TYPES, RESPOND_WRITE_PAGE_ACTIVITY_TYPES, type RefsCardsFiresotre, type RefsSetsFirestore, SPEAKABLE_NOTIFICATIONS, STUDENT_LEVELS_OPTIONS, type Score, type ScoreWithId, type Set, type SetWithId, type SpeakableNotificationType, SpeakableNotificationTypes, SpeakableProvider, VerificationCardStatus, assignmentQueryKeys, cardsQueryKeys, checkIsMCPage, checkIsMediaPage, checkIsRepeatPage, checkIsRespondAudioPage, checkIsRespondPage, checkIsRespondWrittenPage, checkIsShortAnswerPage, checkTypePageActivity, cleanString, createAssignmentRepo, createCardRepo, createFsClientWeb as createFsClient, createSetRepo, creditQueryKeys, debounce, getCardFromCache, getLabelPage, getPagePrompt, getPhraseLength, getRespondCardTool, getSetFromCache, getTotalCompletedCards, getTranscript, getTranscriptCycle, getWordHash, purify, refsCardsFiresotre, refsSetsFirestore, scoreQueryKeys, setsQueryKeys, updateCardInCache, updateSetInCache, useActivity, useActivityFeedbackAccess, useAssignment, useBaseOpenAI, useCards, useClearScore, useClearScoreV2, useCreateCard, useCreateCards, useCreateNotification, useGetCard, useOrganizationAccess, useScore, useSet, useSpeakableApi, useSpeakableTranscript, useSpeakableTranscriptCycle, useSubmitAssignmentScore, useSubmitPracticeScore, useUpdateCardScore, useUpdateScore, useUpdateStudentVocab, useUserCredits };

package/dist/index.web.js

@@ -2134,8 +2134,209 @@ var createSetRepo = () => {
   };
 };
 
+// src/utils/ai/detect-transcript-hallucionation.ts
+var HALLUCINATION_THRESHOLDS = {
+  // Short repeats
+  MIN_CONSECUTIVE_REPEATS: 3,
+  MIN_WORDS_FOR_RATIO_CHECK: 10,
+  MAX_UNIQUE_WORDS_FOR_RATIO: 3,
+  MIN_REPETITION_RATIO: 3,
+  // Phrase repeats
+  MIN_SENTENCE_LENGTH: 10,
+  MIN_CONSECUTIVE_SIMILAR_SENTENCES: 2,
+  MIN_SENTENCES_FOR_DUPLICATE_CHECK: 3,
+  // Cyclic patterns
+  MIN_CYCLE_LENGTH: 20,
+  MIN_CYCLE_REPEATS: 3,
+  // Entropy detection
+  MIN_LENGTH_FOR_ENTROPY_CHECK: 50,
+  MAX_ENTROPY_THRESHOLD: 2.5,
+  // bits per character
+  // Similarity
+  SENTENCE_SIMILARITY_THRESHOLD: 0.8,
+  SEGMENT_SIMILARITY_THRESHOLD: 0.85
+};
+function detectTranscriptHallucinationWithDetails(transcript) {
+  if (!transcript || transcript.trim().length === 0) {
+    return { isHallucination: false };
+  }
+  const text = transcript.trim();
+  if (text.length < 10) {
+    return { isHallucination: false };
+  }
+  const shortRepeats = detectShortRepeats(text);
+  if (shortRepeats) {
+    return {
+      isHallucination: true,
+      reason: "Detected repeated short words or phrases",
+      confidence: 0.9
+    };
+  }
+  const phraseRepeats = detectPhraseRepeats(text);
+  if (phraseRepeats) {
+    return {
+      isHallucination: true,
+      reason: "Detected repeated sentences or phrases",
+      confidence: 0.85
+    };
+  }
+  const cyclicRepeats = detectCyclicPattern(text);
+  if (cyclicRepeats) {
+    return {
+      isHallucination: true,
+      reason: "Detected cyclic repetition pattern",
+      confidence: 0.8
+    };
+  }
+  if (text.length >= HALLUCINATION_THRESHOLDS.MIN_LENGTH_FOR_ENTROPY_CHECK) {
+    const entropy = calculateEntropy(text);
+    if (entropy < HALLUCINATION_THRESHOLDS.MAX_ENTROPY_THRESHOLD) {
+      return {
+        isHallucination: true,
+        reason: "Detected low entropy (likely gibberish or excessive repetition)",
+        confidence: 0.75
+      };
+    }
+  }
+  return { isHallucination: false };
+}
+function detectShortRepeats(text) {
+  const words = text.toLowerCase().split(/[\s,;.!?]+/).filter((w) => w.length > 0);
+  if (words.length < 4) return false;
+  let repeatCount = 1;
+  for (let i = 1; i < words.length; i++) {
+    if (words[i] === words[i - 1]) {
+      repeatCount++;
+      if (repeatCount >= HALLUCINATION_THRESHOLDS.MIN_CONSECUTIVE_REPEATS) {
+        return true;
+      }
+    } else {
+      repeatCount = 1;
+    }
+  }
+  const uniqueWords = new Set(words);
+  const repetitionRatio = words.length / uniqueWords.size;
+  if (words.length >= HALLUCINATION_THRESHOLDS.MIN_WORDS_FOR_RATIO_CHECK && uniqueWords.size <= HALLUCINATION_THRESHOLDS.MAX_UNIQUE_WORDS_FOR_RATIO && repetitionRatio >= HALLUCINATION_THRESHOLDS.MIN_REPETITION_RATIO) {
+    return true;
+  }
+  return false;
+}
+function detectPhraseRepeats(text) {
+  const sentences = text.split(/[.!?]+/).map((s) => s.trim().toLowerCase()).filter((s) => s.length > HALLUCINATION_THRESHOLDS.MIN_SENTENCE_LENGTH);
+  if (sentences.length < 2) return false;
+  for (let i = 0; i < sentences.length - 1; i++) {
+    let consecutiveRepeats = 1;
+    for (let j = i + 1; j < sentences.length; j++) {
+      if (isSimilarSentence(sentences[i], sentences[j])) {
+        consecutiveRepeats++;
+      } else {
+        break;
+      }
+    }
+    if (consecutiveRepeats >= HALLUCINATION_THRESHOLDS.MIN_CONSECUTIVE_SIMILAR_SENTENCES) {
+      return true;
+    }
+  }
+  const uniqueSentences = new Set(sentences);
+  if (sentences.length >= HALLUCINATION_THRESHOLDS.MIN_SENTENCES_FOR_DUPLICATE_CHECK && uniqueSentences.size === 1) {
+    return true;
+  }
+  return false;
+}
+function isSimilarSentence(s1, s2, threshold = HALLUCINATION_THRESHOLDS.SENTENCE_SIMILARITY_THRESHOLD) {
+  if (s1 === s2) return true;
+  const normalized1 = s1.replace(/\s+/g, " ").trim();
+  const normalized2 = s2.replace(/\s+/g, " ").trim();
+  if (normalized1 === normalized2) return true;
+  const words1 = normalized1.split(/\s+/);
+  const words2 = normalized2.split(/\s+/);
+  if (Math.abs(words1.length - words2.length) > 2) return false;
+  const set1 = new Set(words1);
+  const set2 = new Set(words2);
+  const intersection = new Set([...set1].filter((w) => set2.has(w)));
+  const similarity = intersection.size * 2 / (set1.size + set2.size);
+  return similarity >= threshold;
+}
+function detectCyclicPattern(text) {
+  const normalized = text.toLowerCase().replace(/\s+/g, " ").trim();
+  const length = normalized.length;
+  const minCycleLength = HALLUCINATION_THRESHOLDS.MIN_CYCLE_LENGTH;
+  const maxCycleLength = Math.floor(length / 2);
+  if (maxCycleLength < minCycleLength) return false;
+  const step = 5;
+  for (let cycleLen = minCycleLength; cycleLen <= maxCycleLength; cycleLen += step) {
+    const pattern = normalized.substring(0, cycleLen);
+    let matchCount = 0;
+    let pos = 0;
+    while (pos < length) {
+      const segment = normalized.substring(pos, pos + cycleLen);
+      if (segment.length < cycleLen) {
+        const partialMatch = pattern.startsWith(segment);
+        if (partialMatch && matchCount > 0) {
+          matchCount++;
+        }
+        break;
+      }
+      if (segment === pattern || isSegmentSimilar(segment, pattern)) {
+        matchCount++;
+        pos += cycleLen;
+      } else {
+        break;
+      }
+    }
+    if (matchCount >= HALLUCINATION_THRESHOLDS.MIN_CYCLE_REPEATS) {
+      return true;
+    }
+  }
+  return false;
+}
+function isSegmentSimilar(s1, s2) {
+  if (s1 === s2) return true;
+  if (s1.length !== s2.length) return false;
+  let matches = 0;
+  const minLength = Math.min(s1.length, s2.length);
+  for (let i = 0; i < minLength; i++) {
+    if (s1[i] === s2[i]) {
+      matches++;
+    }
+  }
+  const similarity = matches / minLength;
+  return similarity >= HALLUCINATION_THRESHOLDS.SEGMENT_SIMILARITY_THRESHOLD;
+}
+function calculateEntropy(text) {
+  if (!text || text.length === 0) {
+    return 0;
+  }
+  const frequencies = /* @__PURE__ */ new Map();
+  for (const char of text.toLowerCase()) {
+    frequencies.set(char, (frequencies.get(char) || 0) + 1);
+  }
+  let entropy = 0;
+  const length = text.length;
+  for (const count of frequencies.values()) {
+    const probability = count / length;
+    entropy -= probability * Math.log2(probability);
+  }
+  return entropy;
+}
+function cleanHallucinatedTranscript(transcript) {
+  var _a, _b;
+  const result = detectTranscriptHallucinationWithDetails(transcript);
+  if (result.isHallucination) {
+    console.warn(
+      "Hallucinated transcript detected and removed:",
+      transcript.substring(0, 100),
+      `
+Reason: ${(_a = result.reason) != null ? _a : "Unknown"}`,
+      `Confidence: ${String((_b = result.confidence) != null ? _b : "Unknown")}`
+    );
+    return "";
+  }
+  return transcript;
+}
+
 // src/utils/ai/get-transcript.ts
-async function getTranscript(model, args) {
+async function getTranscript(model, args, cleanHallucinations = true) {
   var _a, _b, _c, _d, _e, _f;
   const getGeminiTranscript = (_b = (_a = api).httpsCallable) == null ? void 0 : _b.call(_a, "getGeminiTranscript");
   const getAssemblyAITranscript = (_d = (_c = api).httpsCallable) == null ? void 0 : _d.call(_c, "transcribeAssemblyAIAudio");
@@ -2146,7 +2347,7 @@ async function getTranscript(model, args) {
         audioUrl: args.audioUrl,
         language: args.language
       }));
-      return data;
+      return cleanHallucinations ? cleanHallucinatedTranscript(data) : data;
     } catch (error) {
       console.error("Error getting transcript from Whisper:", error);
       throw error;
@@ -2159,7 +2360,7 @@ async function getTranscript(model, args) {
         targetLanguage: args.language,
         prompt: args.prompt
       }));
-      return data.transcript;
+      return cleanHallucinations ? cleanHallucinatedTranscript(data.transcript) : data.transcript;
     } catch (error) {
       console.error("Error getting transcript from Gemini:", error);
       throw error;
@@ -2171,7 +2372,7 @@ async function getTranscript(model, args) {
         audioUrl: args.audioUrl,
         language: args.language
       }));
-      return response.data;
+      return cleanHallucinations ? cleanHallucinatedTranscript(response.data) : response.data;
     } catch (error) {
       console.error("Error getting transcript from AssemblyAI:", error);
       throw error;
@@ -2179,6 +2380,37 @@ async function getTranscript(model, args) {
   }
   return null;
 }
+async function getTranscriptCycle(args) {
+  const models = ["whisper", "gemini", "assemblyai"];
+  let transcript = "";
+  let lastError = null;
+  for (const model of models) {
+    try {
+      const transcriptResult = await getTranscript(model, args, false);
+      const rawTranscript = transcriptResult || "";
+      transcript = cleanHallucinatedTranscript(rawTranscript);
+      if (transcript !== "") {
+        console.log(`Successfully got transcript from ${model}`);
+        break;
+      }
+      console.warn(`${model} returned empty transcript, trying next model`);
+    } catch (e) {
+      console.error(`Error with ${model} transcript:`, e);
+      lastError = e;
+    }
+  }
+  if (transcript === "") {
+    console.error("All transcript models failed or returned empty", lastError);
+    return {
+      transcript: "",
+      success: false
+    };
+  }
+  return {
+    transcript,
+    success: true
+  };
+}
 
 // src/constants/all-langs.json
 var all_langs_default = {
@@ -2925,6 +3157,17 @@ function useSpeakableTranscript() {
     mutation
   };
 }
+function useSpeakableTranscriptCycle() {
+  const mutation = useMutation3({
+    mutationFn: async (args) => {
+      return getTranscriptCycle(args);
+    },
+    retry: false
+  });
+  return {
+    mutationTranscriptCycle: mutation
+  };
+}
 
 // src/hooks/useUpdateStudentVoc.ts
 var useUpdateStudentVocab = (page) => {
@@ -3478,6 +3721,7 @@ export {
   getSetFromCache,
   getTotalCompletedCards,
   getTranscript,
+  getTranscriptCycle,
   getWordHash,
   purify,
   refsCardsFiresotre,
@@ -3502,6 +3746,7 @@ export {
   useSet,
   useSpeakableApi,
   useSpeakableTranscript,
+  useSpeakableTranscriptCycle,
   useSubmitAssignmentScore,
   useSubmitPracticeScore,
   useUpdateCardScore,