@agent-native/core 0.103.1 → 0.104.0

package/corpus/templates/clips/chrome-extension/src/background.ts

@@ -1,3 +1,7 @@
+import {
+  shouldReconcilePersistedRecording,
+  type OffscreenRecordingState,
+} from "./native-recording-state";
 import { captureExtensionError, initExtensionSentry } from "./sentry";
 
 initExtensionSentry("background");
@@ -174,6 +178,7 @@ type OffscreenStatusMessage = {
   storageSetupRequired?: boolean;
   savedToDisk?: boolean;
   savedFilename?: string;
+  recordingStep?: string;
 };
 
 type ExtensionErrorMessage = {
@@ -1154,6 +1159,7 @@ async function startRecordingFromTab(args: {
   settings: ExtensionSettings;
 }) {
   const { tab, settings } = args;
+  await reconcilePersistedNativeRecording();
   // The persisted value is authoritative (survives a worker suspension during
   // arming); the in-memory var is only a same-tick fast path on top of it.
   // Stale (past-TTL) or legacy-shaped guards are treated as absent — see
@@ -1311,7 +1317,6 @@ async function armRecording(args: {
       hasCamera: cameraInvolved,
       hasAudio:
         settings.includeMicrophone || settings.captureSurface !== "camera",
-      visibility: "public",
       mimeType: "video/webm",
       requestStreaming: true,
     });
@@ -1382,6 +1387,7 @@ async function armRecording(args: {
       hasCamera: cameraInvolved,
       startDelayMs,
       authToken,
+      transcriptUrl: actionUrl(settings, "save-browser-transcript"),
     });
   } catch (err) {
     console.error("[clips-bg] arm: BEGIN failed", err);
@@ -1548,6 +1554,10 @@ async function handleOverlayRestart() {
       hasCamera: cameraInvolved,
       startDelayMs: cameraInvolved ? 20000 : COUNTDOWN_SECONDS * 1000 + 1000,
       authToken: restartAuthToken,
+      transcriptUrl: actionUrl(
+        settingsFromRecording(recording),
+        "save-browser-transcript",
+      ),
     });
   } catch (err) {
     // Re-arming failed: tear the countdown overlay back down and report the
@@ -1721,7 +1731,48 @@ async function clearNativeRecording(): Promise<void> {
   await chrome.action.setBadgeText({ text: "" });
 }
 
+async function reconcilePersistedNativeRecording(): Promise<void> {
+  const recording = activeNativeRecording;
+  if (!recording) return;
+  if (recording.status === "error" || recording.status === "complete") return;
+
+  try {
+    await ensureOffscreenDocument();
+    const state = await sendOffscreenMessage<OffscreenRecordingState>({
+      type: "CLIPS_OFFSCREEN_STATUS",
+    });
+    if (
+      !shouldReconcilePersistedRecording(
+        recording.status,
+        recording.sessionId,
+        state,
+      )
+    ) {
+      return;
+    }
+  } catch {
+    // A transient wake-up failure must not discard a real recording.
+    return;
+  }
+
+  console.warn(
+    "[clips-bg] clearing persisted recording with no live offscreen session",
+    recording.sessionId,
+  );
+  await deleteSession(recording.sessionId).catch((error) => {
+    console.warn(
+      "[clips-bg] stale recording cleanup could not detach tab",
+      error,
+    );
+  });
+  resetOverlay();
+  await broadcastUnmount();
+  broadcastOverlayState();
+  await clearNativeRecording();
+}
+
 async function handlePopupStatus() {
+  await reconcilePersistedNativeRecording();
   return {
     ok: true,
     activeRecording: activeNativeRecording,
@@ -2355,6 +2406,27 @@ async function dispatchRuntimeMessage(message: unknown): Promise<unknown> {
             ? status.savedFilename
             : undefined;
       }
+      if (
+        status.status === "error" &&
+        status.recordingStep === "recorder-start"
+      ) {
+        await deleteSession(activeNativeRecording.sessionId).catch((error) => {
+          console.warn(
+            "[clips-bg] failed recorder-start cleanup could not detach tab",
+            error,
+          );
+        });
+        await postAction(
+          settingsFromRecording(activeNativeRecording),
+          "trash-recording",
+          { id: activeNativeRecording.recordingId },
+        ).catch(() => undefined);
+        resetOverlay();
+        await broadcastUnmount();
+        broadcastOverlayState();
+        await clearNativeRecording();
+        return { ok: true };
+      }
       await saveActiveNativeRecording();
       // The offscreen recorder finished its pre-roll and actually started —
       // advance from countdown to recording (the reliable phase flip).

package/corpus/templates/clips/chrome-extension/src/native-recording-state.ts

@@ -0,0 +1,32 @@
+export type NativeRecordingStateStatus =
+  | "recording"
+  | "paused"
+  | "stopping"
+  | "uploading"
+  | "complete"
+  | "error";
+
+export type OffscreenRecordingState = {
+  activeSessionId?: string;
+  preparedSessionId?: string;
+};
+
+export function hasLiveOffscreenSession(
+  sessionId: string,
+  state: OffscreenRecordingState,
+): boolean {
+  return (
+    state.activeSessionId === sessionId || state.preparedSessionId === sessionId
+  );
+}
+
+export function shouldReconcilePersistedRecording(
+  status: NativeRecordingStateStatus,
+  sessionId: string,
+  state: OffscreenRecordingState,
+): boolean {
+  // Preserve terminal errors so the popup can explain an upload failure and
+  // offer the existing discard/re-upload path.
+  if (status === "error" || status === "complete") return false;
+  return !hasLiveOffscreenSession(sessionId, state);
+}

package/corpus/templates/clips/chrome-extension/src/native-transcription.ts

@@ -0,0 +1,248 @@
+export interface NativeTranscriptResult {
+  text: string;
+  failureReason: string | null;
+}
+
+interface SpeechRecognitionResultLike {
+  isFinal: boolean;
+  0?: { transcript?: string };
+}
+
+interface SpeechRecognitionEventLike {
+  resultIndex: number;
+  results: ArrayLike<SpeechRecognitionResultLike>;
+}
+
+interface SpeechRecognitionErrorEventLike {
+  error?: string;
+}
+
+interface SpeechRecognitionLike {
+  continuous: boolean;
+  interimResults: boolean;
+  lang: string;
+  maxAlternatives: number;
+  onresult: ((event: SpeechRecognitionEventLike) => void) | null;
+  onerror: ((event: SpeechRecognitionErrorEventLike) => void) | null;
+  onend: (() => void) | null;
+  start: () => void;
+  stop: () => void;
+  abort: () => void;
+}
+
+type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;
+
+export interface NativeTranscriptionCapture {
+  supported: boolean;
+  start(): void;
+  stop(): Promise<NativeTranscriptResult>;
+  cancel(): void;
+  pause(): void;
+  resume(): void;
+}
+
+const STOP_SETTLE_MS = 1_500;
+const RESTART_DELAY_MS = 250;
+
+function getSpeechRecognitionCtor(): SpeechRecognitionConstructor | null {
+  if (typeof window === "undefined") return null;
+  const scope = window as unknown as {
+    SpeechRecognition?: SpeechRecognitionConstructor;
+    webkitSpeechRecognition?: SpeechRecognitionConstructor;
+  };
+  return scope.SpeechRecognition ?? scope.webkitSpeechRecognition ?? null;
+}
+
+function appendTranscript(current: string, next: string): string {
+  const cleanNext = next.trim();
+  if (!cleanNext) return current;
+  return current ? `${current} ${cleanNext}` : cleanNext;
+}
+
+export function createNativeTranscriptionCapture(options?: {
+  enabled?: boolean;
+  lang?: string;
+}): NativeTranscriptionCapture {
+  const Ctor = getSpeechRecognitionCtor();
+  const enabled = options?.enabled ?? true;
+  const language =
+    options?.lang ??
+    (typeof navigator !== "undefined" ? navigator.language : "en-US");
+  const unsupportedReason = !enabled
+    ? "Native speech transcription requires microphone capture."
+    : !Ctor
+      ? "Chrome Web Speech recognition is unavailable in this context."
+      : null;
+
+  let recognition: SpeechRecognitionLike | null = null;
+  let finalText = "";
+  let interimText = "";
+  let stopped = false;
+  let paused = false;
+  let disposed = false;
+  let failureReason = unsupportedReason;
+  let restartTimer: ReturnType<typeof setTimeout> | null = null;
+  let stopPromise: Promise<NativeTranscriptResult> | null = null;
+  let resolveStop: ((result: NativeTranscriptResult) => void) | null = null;
+  let stopTimer: ReturnType<typeof setTimeout> | null = null;
+
+  const result = (): NativeTranscriptResult => {
+    const text = appendTranscript(finalText, interimText).trim();
+    return {
+      text,
+      failureReason:
+        text.length > 0
+          ? null
+          : failureReason ||
+            "Chrome Web Speech recognition returned no transcript.",
+    };
+  };
+
+  const settleStop = (): void => {
+    if (!resolveStop) return;
+    if (stopTimer !== null) {
+      clearTimeout(stopTimer);
+      stopTimer = null;
+    }
+    const resolve = resolveStop;
+    resolveStop = null;
+    disposed = true;
+    recognition = null;
+    resolve(result());
+  };
+
+  const scheduleRestart = (): void => {
+    if (restartTimer !== null) return;
+    restartTimer = setTimeout(() => {
+      restartTimer = null;
+      if (disposed || stopped || paused || !recognition) return;
+      try {
+        recognition.start();
+      } catch (error) {
+        failureReason =
+          error instanceof Error
+            ? `Chrome Web Speech recognition could not restart: ${error.message}`
+            : "Chrome Web Speech recognition could not restart.";
+      }
+    }, RESTART_DELAY_MS);
+  };
+
+  const start = (): void => {
+    if (!Ctor || !enabled || disposed || stopped || recognition) return;
+
+    const nextRecognition = new Ctor();
+    recognition = nextRecognition;
+    nextRecognition.continuous = true;
+    nextRecognition.interimResults = true;
+    nextRecognition.lang = language;
+    nextRecognition.maxAlternatives = 1;
+    nextRecognition.onresult = (event) => {
+      let nextInterim = "";
+      for (
+        let index = event.resultIndex;
+        index < event.results.length;
+        index++
+      ) {
+        const item = event.results[index];
+        const text = item?.[0]?.transcript ?? "";
+        if (item?.isFinal) finalText = appendTranscript(finalText, text);
+        else nextInterim = appendTranscript(nextInterim, text);
+      }
+      interimText = nextInterim;
+    };
+    nextRecognition.onerror = (event) => {
+      const error = event.error;
+      if (!error || error === "no-speech" || error === "aborted") return;
+      failureReason = `Chrome Web Speech recognition error: ${error}.`;
+      if (error === "not-allowed" || error === "service-not-allowed") {
+        stopped = true;
+      }
+    };
+    nextRecognition.onend = () => {
+      if (disposed) return;
+      if (stopped) {
+        settleStop();
+        return;
+      }
+      if (!paused) scheduleRestart();
+    };
+
+    try {
+      nextRecognition.start();
+    } catch (error) {
+      failureReason =
+        error instanceof Error
+          ? `Chrome Web Speech recognition could not start: ${error.message}`
+          : "Chrome Web Speech recognition could not start.";
+      recognition = null;
+    }
+  };
+
+  const stop = (): Promise<NativeTranscriptResult> => {
+    if (stopPromise) return stopPromise;
+    stopped = true;
+    paused = false;
+    if (restartTimer !== null) {
+      clearTimeout(restartTimer);
+      restartTimer = null;
+    }
+    if (!recognition) {
+      disposed = true;
+      return Promise.resolve(result());
+    }
+
+    stopPromise = new Promise<NativeTranscriptResult>((resolve) => {
+      resolveStop = resolve;
+      stopTimer = setTimeout(settleStop, STOP_SETTLE_MS);
+      try {
+        recognition?.stop();
+      } catch {
+        settleStop();
+      }
+    });
+    return stopPromise;
+  };
+
+  const cancel = (): void => {
+    stopped = true;
+    disposed = true;
+    if (restartTimer !== null) clearTimeout(restartTimer);
+    if (stopTimer !== null) clearTimeout(stopTimer);
+    resolveStop = null;
+    try {
+      recognition?.abort();
+    } catch {
+      // Ignore teardown failures.
+    }
+    recognition = null;
+  };
+
+  const pause = (): void => {
+    if (!recognition || stopped || disposed) return;
+    paused = true;
+    try {
+      recognition.stop();
+    } catch {
+      // The onend handler will settle the stopped recognition.
+    }
+  };
+
+  const resume = (): void => {
+    if (!recognition || stopped || disposed) return;
+    paused = false;
+    try {
+      recognition.start();
+    } catch {
+      failureReason = "Chrome Web Speech recognition could not resume.";
+    }
+  };
+
+  return {
+    supported: Boolean(Ctor && enabled),
+    start,
+    stop,
+    cancel,
+    pause,
+    resume,
+  };
+}

package/corpus/templates/clips/chrome-extension/src/offscreen.ts

@@ -36,6 +36,10 @@ import {
 import { MAX_UPLOAD_BYTES } from "@shared/upload-limits";
 
 import { waitForReadyRecordingAfterFinalizeError } from "./finalize-recovery";
+import {
+  createNativeTranscriptionCapture,
+  type NativeTranscriptionCapture,
+} from "./native-transcription";
 import {
   createRecordingDurationState,
   pauseRecordingDuration,
@@ -141,6 +145,7 @@ type BeginMessage = {
   // Bearer token so chunk uploads authenticate the same way create-recording
   // does. The offscreen document has no Clips session cookie of its own.
   authToken?: string;
+  transcriptUrl?: string;
 };
 
 type SimpleMessage = {
@@ -172,6 +177,8 @@ type UploadResult = {
   waitingForStorage?: boolean;
   storageSetupRequired?: boolean;
   error?: string;
+  nativeTranscript?: string;
+  nativeTranscriptFailure?: string;
 };
 
 type PreparedStreams = {
@@ -192,6 +199,7 @@ type ActiveRecording = {
   uploadUrl: string;
   uploadMode: UploadMode;
   authToken: string | null;
+  transcriptUrl: string | null;
   mode: CaptureMode;
   startedAtMs: number;
   duration: RecordingDurationState;
@@ -231,6 +239,10 @@ type ActiveRecording = {
   dimensions: { width: number; height: number };
   hasAudio: boolean;
   hasCamera: boolean;
+  nativeTranscription: NativeTranscriptionCapture;
+  nativeTranscript: string;
+  nativeTranscriptFailure: string | null;
+  nativeTranscriptSaved: boolean;
   stopped: Promise<UploadResult>;
   resolveStopped: (result: UploadResult) => void;
   rejectStopped: (error: Error) => void;
@@ -1224,6 +1236,7 @@ async function begin(message: BeginMessage): Promise<{
     uploadUrl: message.uploadUrl,
     uploadMode: message.uploadMode ?? "buffered",
     authToken: message.authToken ?? null,
+    transcriptUrl: message.transcriptUrl ?? null,
     mode: ready.mode,
     startedAtMs: 0,
     duration: createRecordingDurationState(),
@@ -1259,6 +1272,12 @@ async function begin(message: BeginMessage): Promise<{
       typeof message.hasCamera === "boolean"
         ? message.hasCamera
         : ready.mode === "camera",
+    nativeTranscription: createNativeTranscriptionCapture({
+      enabled: Boolean(ready.micStream),
+    }),
+    nativeTranscript: "",
+    nativeTranscriptFailure: null,
+    nativeTranscriptSaved: false,
     stopped,
     resolveStopped,
     rejectStopped,
@@ -1358,6 +1377,7 @@ function startRecorderNow(recording: ActiveRecording): void {
     // Chime first (on "Go") so it mostly lands before the recording begins.
     playStartChime();
     recording.recorder.start(2000);
+    recording.nativeTranscription.start();
     recording.startedAtMs = Date.now();
     recording.duration = startRecordingDuration(recording.startedAtMs);
     console.log("[clips-offscreen] recorder.start ok");
@@ -1383,9 +1403,14 @@ function startRecorderNow(recording: ActiveRecording): void {
         hasCamera: recording.hasCamera,
       },
     });
+    recording.cancelled = true;
+    recording.nativeTranscription.cancel();
+    cleanup(recording);
+    if (activeRecording === recording) activeRecording = null;
     reportStatus(recording.sessionId, "error", {
       recordingId: recording.recordingId,
       error: err instanceof Error ? err.message : "Could not start recording.",
+      recordingStep: "recorder-start",
     });
   }
 }
@@ -1400,6 +1425,47 @@ function recordingDownloadFilename(recording: ActiveRecording): string {
   return `clip-${stamp}.${ext}`;
 }
 
+async function saveNativeTranscript(recording: ActiveRecording): Promise<void> {
+  if (recording.nativeTranscriptSaved || !recording.transcriptUrl) return;
+  const captured = await recording.nativeTranscription.stop();
+  recording.nativeTranscript = captured.text.trim();
+  recording.nativeTranscriptFailure = captured.failureReason;
+
+  const body = recording.nativeTranscript
+    ? {
+        recordingId: recording.recordingId,
+        fullText: recording.nativeTranscript,
+        source: "web-speech",
+      }
+    : {
+        recordingId: recording.recordingId,
+        fullText: "",
+        source: "web-speech",
+        failureReason:
+          recording.nativeTranscriptFailure ||
+          "Chrome Web Speech recognition returned no transcript.",
+      };
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+    "X-Agent-Native-Frontend": "1",
+    ...(recording.authToken
+      ? { Authorization: `Bearer ${recording.authToken}` }
+      : {}),
+  };
+  const response = await fetch(recording.transcriptUrl, {
+    method: "POST",
+    headers,
+    credentials: "include",
+    cache: "no-store",
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(5_000),
+  });
+  if (!response.ok) {
+    throw new Error(`Native transcript save failed (${response.status}).`);
+  }
+  recording.nativeTranscriptSaved = true;
+}
+
 // Last-resort recovery: if a finished recording can't be uploaded, save the
 // locally-buffered bytes to the user's Downloads so the recording is never
 // lost. The offscreen document can't call chrome.downloads, so it hands the
@@ -1461,6 +1527,18 @@ async function finalizeStop(recording: ActiveRecording): Promise<void> {
   // spent paused is excluded by pauseRecordingDuration().
   recording.duration = pauseRecordingDuration(recording.duration, Date.now());
   const durationMs = recording.duration.elapsedMs;
+  try {
+    await saveNativeTranscript(recording);
+  } catch (error) {
+    console.warn("[clips-offscreen] native transcript save failed", error);
+    captureExtensionError(error, {
+      tags: {
+        surface: "offscreen",
+        recordingStep: "save-native-transcript",
+      },
+      extra: { recordingId: recording.recordingId },
+    });
+  }
   reportStatus(recording.sessionId, "uploading", {
     recordingId: recording.recordingId,
   });
@@ -1514,6 +1592,15 @@ async function finalizeStop(recording: ActiveRecording): Promise<void> {
         hasAudio: recording.hasAudio,
         hasCamera: recording.hasCamera,
       });
+      result = {
+        ...result,
+        ...(recording.nativeTranscript
+          ? { nativeTranscript: recording.nativeTranscript }
+          : {}),
+        ...(recording.nativeTranscriptFailure
+          ? { nativeTranscriptFailure: recording.nativeTranscriptFailure }
+          : {}),
+      };
     } catch (err) {
       const error = err instanceof Error ? err : new Error(String(err));
       (error as { finalUpload?: boolean }).finalUpload = true;
@@ -1590,6 +1677,7 @@ function pause(message: SimpleMessage): { ok: boolean } {
   if (recording && recording.sessionId === message.sessionId) {
     if (recording.recorder.state === "recording") {
       recording.recorder.pause();
+      recording.nativeTranscription.pause();
       recording.duration = pauseRecordingDuration(
         recording.duration,
         Date.now(),
@@ -1607,6 +1695,7 @@ function resume(message: SimpleMessage): { ok: boolean } {
   if (recording && recording.sessionId === message.sessionId) {
     if (recording.recorder.state === "paused") {
       recording.recorder.resume();
+      recording.nativeTranscription.resume();
       recording.duration = resumeRecordingDuration(
         recording.duration,
         Date.now(),
@@ -1644,6 +1733,7 @@ function cancel(message: SimpleMessage): { ok: boolean } {
   const recording = activeRecording;
   if (recording && recording.sessionId === message.sessionId) {
     recording.cancelled = true;
+    recording.nativeTranscription.cancel();
     if (recording.startTimer !== null) {
       clearTimeout(recording.startTimer);
       recording.startTimer = null;
@@ -1673,6 +1763,18 @@ function startNow(message: SimpleMessage): { ok: boolean } {
   return { ok: true };
 }
 
+function getRecordingState(): {
+  ok: boolean;
+  activeSessionId?: string;
+  preparedSessionId?: string;
+} {
+  return {
+    ok: true,
+    ...(activeRecording ? { activeSessionId: activeRecording.sessionId } : {}),
+    ...(prepared ? { preparedSessionId: prepared.sessionId } : {}),
+  };
+}
+
 // Restart: discard the in-progress recording but keep the same source streams
 // (so the user does not have to re-pick a screen), then re-home them into a
 // prepared slot. A fresh recorder is built on the next BEGIN.
@@ -1687,6 +1789,7 @@ async function restart(
   // handler ignores the final flush, so the source tracks stay live.
   recording.restarting = true;
   recording.cancelled = true;
+  recording.nativeTranscription.cancel();
   if (recording.recorder.state !== "inactive") recording.recorder.stop();
   // Tear down the old compositor + mixing context; keep the capture tracks alive
   // so the next BEGIN can build a fresh compositor/recorder on the same streams.
@@ -1741,6 +1844,9 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
     case "CLIPS_OFFSCREEN_START_NOW":
       task = Promise.resolve(startNow(message as SimpleMessage));
       break;
+    case "CLIPS_OFFSCREEN_STATUS":
+      task = Promise.resolve(getRecordingState());
+      break;
     case "CLIPS_OFFSCREEN_COPY_TEXT":
       task = copyTextToClipboard(message as CopyTextMessage);
       break;