@open-everywhere/adapters 0.5.0 → 0.6.0

package/agent/model/transcription-routes.ts

@@ -0,0 +1,347 @@
+// THE TRANSCRIPTION ROLE'S TRANSPORTS (OPE-2470) — speech to text as a model-stack role with a primary and a
+// fallback route. The app's `agent/model/stacks.ts` names the routes per stack; this file sends one audio chunk
+// along them and returns the text with its timestamps.
+//
+// THREE TRANSPORTS. Each route names one:
+//   • `gateway`     — the deployment's own AI Gateway (`gatewayTranscriptionModel`, the same provider and the same
+//                     host plane as the turn model). The Gateway returns segments with start and end seconds; on
+//                     `spacexai/grok-stt` each segment is one word (probed 2026-10-11).
+//   • `deepinfra`   — DeepInfra's native inference endpoint for an open-weights speech model. It returns `segments`
+//                     and `words`, each with `start` and `end` in seconds (DeepInfra's API page, 2026-10-11). The
+//                     key is the `DEEPINFRA_API_KEY` entry of the DB secret store (`oe secret set`), and NEVER the
+//                     environment: see {@link storeOnlySecret}.
+//   • `self-hosted` — an OpenAI-compatible `/audio/transcriptions` server that the deployer runs (vLLM serving
+//                     Qwen3-ASR). Its base URL is config in the stack, not a secret. The call asks for
+//                     `verbose_json` with word and segment timestamps and keeps whatever the server returns.
+//
+// THE FALLBACK. {@link transcribeWithFallback} tries the primary route, and on ANY failure (an outage, a 4xx from a
+// provider allow-list, a timeout) tries the fallback route. Both fail ⇒ it throws one error that names both. It
+// never tries a third route: the stack names what may hear the meeting, and nothing else may.
+//
+// 🔴 THE DISCLOSURE IS DERIVED. The approval card shows {@link roleDisclosure}, built from the SAME route list the
+// call walks, so the card cannot name one provider while the audio goes to another.
+
+import { experimental_transcribe as transcribe } from "ai";
+import { getOperatorSecret } from "@open-everywhere/core/agent/adapter-api.js";
+import { GATEWAY_TARGET_HOST, gatewayTranscriptionModel, sharedHostPlaneFetch } from "./gateway.js";
+
+/** How a route reaches its model. */
+export type TranscriptionVia = "gateway" | "deepinfra" | "self-hosted";
+
+/** One route of the transcription role. */
+export interface TranscriptionRouteSpec {
+  readonly via: TranscriptionVia;
+  /** The model: a gateway id (`gateway`), a DeepInfra model path (`deepinfra`), or the served model name (`self-hosted`). */
+  readonly id: string;
+  /** The server's OpenAI-compatible base URL, e.g. `http://qwen3-asr:8000/v1`. Required for `self-hosted`, refused elsewhere. */
+  readonly baseUrl?: string;
+}
+
+/** The transcription role of one stack: a primary route and an optional fallback route. */
+export interface TranscriptionRoleSpec {
+  readonly primary: TranscriptionRouteSpec;
+  readonly fallback?: TranscriptionRouteSpec;
+}
+
+/** One timed span of the transcript, in seconds from the start of the chunk. */
+export interface TimedSpan {
+  readonly text: string;
+  readonly startSecond: number;
+  readonly endSecond: number;
+}
+
+/** One chunk's transcript. `words` and `segments` are empty when the route returns no timestamps. */
+export interface TimedTranscript {
+  readonly text: string;
+  readonly segments: readonly TimedSpan[];
+  readonly words: readonly TimedSpan[];
+  readonly durationSeconds?: number;
+  /** The provider's own cost for the call, in USD, when it reports one (the Gateway and DeepInfra do). */
+  readonly costUsd?: number;
+}
+
+/** What one route call receives. `prompt` is context text (the previous chunk's tail, names) for a model that takes one. */
+export interface TranscribeRequest {
+  readonly audio: Uint8Array;
+  readonly mediaType: string;
+  readonly prompt?: string;
+  readonly abortSignal?: AbortSignal;
+}
+
+/** The injectable seams, so every transport is testable with no network. */
+export interface TranscriptionDeps {
+  /** The `gateway` transport. Default: the AI SDK's `transcribe` on the deployment's gateway provider. */
+  readonly gateway?: (modelId: string, req: TranscribeRequest) => Promise<TimedTranscript>;
+  /** The HTTP client for `deepinfra` and `self-hosted`. Default: the host-plane fetch on the proxy tier, else `fetch`. */
+  readonly fetch?: typeof fetch;
+  /** The host-plane seam: declare hosts, get the proxy-tier fetch (undefined off the tier). Default: gateway.ts `sharedHostPlaneFetch`. */
+  readonly hostPlane?: (hosts: readonly string[]) => typeof fetch | undefined;
+  /** Resolve the DeepInfra key. Default: {@link storeOnlySecret}. */
+  readonly secret?: (name: string) => Promise<string>;
+  /** One attempt's time limit, ms. Default {@link ATTEMPT_TIMEOUT_MS}. */
+  readonly attemptTimeoutMs?: number;
+  /** Where a fall-through is reported. Default: `console.warn`. */
+  readonly warn?: (message: string) => void;
+}
+
+/** The secret-store name of the DeepInfra key. A fixed name in code, so no config line can point it at another secret. */
+export const DEEPINFRA_SECRET = "DEEPINFRA_API_KEY";
+const DEEPINFRA_HOST = "api.deepinfra.com";
+
+/**
+ * One attempt's limit. A chunk is at most about 75 seconds of audio, and the probed routes answer a 20-second chunk
+ * in about one second. 40 seconds per attempt keeps primary plus fallback inside 80 seconds, so a hung provider
+ * falls through instead of holding the chunk.
+ */
+const ATTEMPT_TIMEOUT_MS = 40_000;
+
+/** Refuse a malformed route at boot, not at the first chunk of a live meeting. */
+export function assertTranscriptionRole(role: TranscriptionRoleSpec): void {
+  for (const [slot, r] of [["primary", role.primary], ["fallback", role.fallback]] as const) {
+    if (r === undefined) continue;
+    if (r.id.trim() === "") throw new Error(`transcription ${slot} route: the model id is empty`);
+    if (r.via === "self-hosted") {
+      if (r.baseUrl === undefined) throw new Error(`transcription ${slot} route: a self-hosted route needs baseUrl`);
+      const u = new URL(r.baseUrl);
+      if (u.protocol !== "http:" && u.protocol !== "https:") throw new Error(`transcription ${slot} route: baseUrl must be http or https`);
+      if (u.username !== "" || u.password !== "") throw new Error(`transcription ${slot} route: baseUrl must not carry credentials`);
+    } else if (r.baseUrl !== undefined) {
+      throw new Error(`transcription ${slot} route: baseUrl is for a self-hosted route only`);
+    }
+  }
+}
+
+/** The host a route's audio reaches. */
+function routeHost(r: TranscriptionRouteSpec): string {
+  if (r.via === "deepinfra") return DEEPINFRA_HOST;
+  if (r.via === "self-hosted") return new URL(r.baseUrl ?? "").hostname;
+  return "the AI Gateway";
+}
+
+function routePhrase(r: TranscriptionRouteSpec): string {
+  if (r.via === "gateway") return `this deployment's own AI Gateway to \`${r.id}\``;
+  if (r.via === "deepinfra") return `DeepInfra (${DEEPINFRA_HOST}) to \`${r.id}\``;
+  return `this deployment's own speech server at ${routeHost(r)} (\`${r.id}\`)`;
+}
+
+/** The approval card's sentence: every place the audio can go, in the order it is tried. */
+export function roleDisclosure(role: TranscriptionRoleSpec): string {
+  const head = `Audio goes, in chunks of about a minute, through ${routePhrase(role.primary)}.`;
+  return role.fallback === undefined ? head : `${head} If that fails, it goes through ${routePhrase(role.fallback)} instead.`;
+}
+
+/**
+ * Resolve a key from the DB-backed secret store, and from nothing else.
+ *
+ * `getOperatorSecret` falls back to an env var of the same name (its migration on-ramp). The DeepInfra key is not
+ * infrastructure, so it has no env home: a value that resolved from the environment is refused, and the refusal
+ * names the fix. The same rule as the SIEM export token (core audit/siem-export.ts).
+ */
+export async function storeOnlySecret(
+  name: string,
+  resolve: (name: string) => Promise<{ readonly value: string; readonly source: string }> = getOperatorSecret,
+): Promise<string> {
+  const resolved = await resolve(name);
+  if (resolved.source !== "db") {
+    throw new Error(`secret "${name}" resolves only from the environment; the DeepInfra transcription route reads the secret store and nothing else. Store it with \`oe secret set ${name}\``);
+  }
+  return resolved.value;
+}
+
+// ── wire parsing ────────────────────────────────────────────────────────────────────────────────────
+
+function num(v: unknown): number | undefined {
+  if (typeof v === "number" && Number.isFinite(v)) return v;
+  if (typeof v === "string" && v.trim() !== "" && Number.isFinite(Number(v))) return Number(v);
+  return undefined;
+}
+
+/** Read `[{text|word, start, end}]` into spans. An entry without a finite start and end is dropped. */
+function parseSpans(raw: unknown): TimedSpan[] {
+  if (!Array.isArray(raw)) return [];
+  const out: TimedSpan[] = [];
+  for (const e of raw as Record<string, unknown>[]) {
+    if (e === null || typeof e !== "object") continue;
+    const text = typeof e.text === "string" ? e.text : typeof e.word === "string" ? e.word : undefined;
+    const start = num(e.start ?? e.startSecond);
+    const end = num(e.end ?? e.endSecond);
+    if (text === undefined || start === undefined || end === undefined || end < start) continue;
+    out.push({ text: text.trim(), startSecond: start, endSecond: end });
+  }
+  return out;
+}
+
+/** True when every span holds one word. Then the segments are word timestamps (the Gateway's grok-stt shape). */
+function spansAreWords(spans: readonly TimedSpan[]): boolean {
+  return spans.length > 0 && spans.every((s) => s.text !== "" && !/\s/.test(s.text));
+}
+
+/** Parse an OpenAI-compatible `verbose_json` body, or DeepInfra's native body (the same field names). */
+export function parseTranscriptionBody(body: unknown): TimedTranscript {
+  if (body === null || typeof body !== "object") throw new Error("transcription response is not a JSON object");
+  const b = body as Record<string, unknown>;
+  if (typeof b.text !== "string") throw new Error("transcription response has no text");
+  const segments = parseSpans(b.segments);
+  const words = parseSpans(b.words);
+  const status = b.inference_status as Record<string, unknown> | undefined;
+  const cost = num(status?.cost);
+  const duration = num(b.duration);
+  return {
+    text: b.text.trim(),
+    segments,
+    words: words.length > 0 ? words : spansAreWords(segments) ? segments : [],
+    ...(duration !== undefined ? { durationSeconds: duration } : {}),
+    ...(cost !== undefined ? { costUsd: cost } : {}),
+  };
+}
+
+/** `fetch` looked up at call time, so instrumentation that wraps it after boot still sees the call. */
+const lateFetch: typeof fetch = (input, init) => globalThis.fetch(input, init);
+
+// ── transports ──────────────────────────────────────────────────────────────────────────────────────
+
+const defaultGateway = async (modelId: string, req: TranscribeRequest): Promise<TimedTranscript> => {
+  // The SDK reads the container format from the bytes themselves, so the media type is not forwarded. The prompt is
+  // not forwarded either: the Gateway takes no common prompt field for transcription models.
+  const r = await transcribe({ model: gatewayTranscriptionModel(modelId), audio: req.audio, ...(req.abortSignal ? { abortSignal: req.abortSignal } : {}) });
+  const segments = r.segments.map((s) => ({ text: s.text.trim(), startSecond: s.startSecond, endSecond: s.endSecond }));
+  const cost = num((r.providerMetadata?.gateway as Record<string, unknown> | undefined)?.cost);
+  return {
+    text: r.text.trim(),
+    segments,
+    words: spansAreWords(segments) ? segments : [],
+    ...(r.durationInSeconds !== undefined ? { durationSeconds: r.durationInSeconds } : {}),
+    ...(cost !== undefined ? { costUsd: cost } : {}),
+  };
+};
+
+function fileName(mediaType: string): string {
+  const sub = mediaType.split(";")[0]!.split("/")[1] ?? "bin";
+  return `chunk.${sub.replace(/[^a-z0-9]/gi, "") || "bin"}`;
+}
+
+async function postForm(f: typeof fetch, url: string, form: FormData, headers: Record<string, string>, signal: AbortSignal | undefined): Promise<TimedTranscript> {
+  const res = await f(url, { method: "POST", body: form, headers, ...(signal ? { signal } : {}) });
+  if (!res.ok) {
+    // The status only: a provider's error body can echo request details.
+    throw new Error(`HTTP ${res.status} from ${new URL(url).hostname}`);
+  }
+  return parseTranscriptionBody(await res.json());
+}
+
+function deepinfraCall(f: typeof fetch, secret: (name: string) => Promise<string>) {
+  return async (r: TranscriptionRouteSpec, req: TranscribeRequest): Promise<TimedTranscript> => {
+    const key = await secret(DEEPINFRA_SECRET);
+    const form = new FormData();
+    form.append("audio", new Blob([req.audio as Uint8Array<ArrayBuffer>], { type: req.mediaType }), fileName(req.mediaType));
+    const path = r.id.split("/").map(encodeURIComponent).join("/");
+    return await postForm(f, `https://${DEEPINFRA_HOST}/v1/inference/${path}`, form, { authorization: `Bearer ${key}` }, req.abortSignal);
+  };
+}
+
+function selfHostedCall(f: typeof fetch) {
+  return async (r: TranscriptionRouteSpec, req: TranscribeRequest): Promise<TimedTranscript> => {
+    const form = new FormData();
+    form.append("file", new Blob([req.audio as Uint8Array<ArrayBuffer>], { type: req.mediaType }), fileName(req.mediaType));
+    form.append("model", r.id);
+    form.append("response_format", "verbose_json");
+    form.append("timestamp_granularities[]", "word");
+    form.append("timestamp_granularities[]", "segment");
+    if (req.prompt !== undefined && req.prompt !== "") form.append("prompt", req.prompt);
+    const base = (r.baseUrl ?? "").replace(/\/+$/, "");
+    return await postForm(f, `${base}/audio/transcriptions`, form, {}, req.abortSignal);
+  };
+}
+
+/**
+ * The hosts a role's routes reach on the host plane: the gateway host FIRST, then every non-gateway route host.
+ * They join the process-wide host set in gateway.ts (`sharedHostPlaneFetch`), so the one shared fetch pushes the
+ * union and neither the turn model nor a transcription route drops the other's host.
+ */
+export function roleHostPlaneHosts(role: TranscriptionRoleSpec): string[] {
+  const hosts = [GATEWAY_TARGET_HOST];
+  for (const r of [role.primary, role.fallback]) {
+    if (r === undefined || r.via === "gateway") continue;
+    const h = routeHost(r);
+    if (!hosts.includes(h)) hosts.push(h);
+  }
+  return hosts;
+}
+
+/** The transport for one route. Exported for the bake-off harness, which calls each route on its own. */
+export function routeTransport(r: TranscriptionRouteSpec, deps: TranscriptionDeps = {}): (req: TranscribeRequest) => Promise<TimedTranscript> {
+  if (r.via === "gateway") {
+    const g = deps.gateway ?? defaultGateway;
+    return (req) => g(r.id, req);
+  }
+  const f = deps.fetch ?? (deps.hostPlane ?? sharedHostPlaneFetch)(roleHostPlaneHosts({ primary: r })) ?? lateFetch;
+  if (r.via === "deepinfra") {
+    const call = deepinfraCall(f, deps.secret ?? storeOnlySecret);
+    return (req) => call(r, req);
+  }
+  const call = selfHostedCall(f);
+  return (req) => call(r, req);
+}
+
+/** Which route answered. */
+export interface RoleTranscript extends TimedTranscript {
+  readonly answeredBy: "primary" | "fallback";
+  readonly route: TranscriptionRouteSpec;
+}
+
+function describe(r: TranscriptionRouteSpec): string {
+  return `${r.via}:${r.id}`;
+}
+
+function errText(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+async function attempt(run: (req: TranscribeRequest) => Promise<TimedTranscript>, req: TranscribeRequest, timeoutMs: number): Promise<TimedTranscript> {
+  // A plain timer, not `AbortSignal.timeout`: that timer is unref'd, so a process with nothing else pending exits
+  // before a hung attempt is cut.
+  const limit = new AbortController();
+  const timer = setTimeout(() => limit.abort(new Error(`no answer within ${timeoutMs} ms`)), timeoutMs);
+  const signal = req.abortSignal ? AbortSignal.any([req.abortSignal, limit.signal]) : limit.signal;
+  // The race cuts a transport that ignores the signal, too.
+  const aborted = new Promise<never>((_, reject) => {
+    if (signal.aborted) reject(signal.reason);
+    signal.addEventListener("abort", () => reject(signal.reason), { once: true });
+  });
+  try {
+    return await Promise.race([run({ ...req, abortSignal: signal }), aborted]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Build the role's transcriber: primary, then fallback on any failure. The routes are validated here, at build time.
+ */
+export function transcribeWithFallback(role: TranscriptionRoleSpec, deps: TranscriptionDeps = {}): (req: TranscribeRequest) => Promise<RoleTranscript> {
+  assertTranscriptionRole(role);
+  const timeoutMs = deps.attemptTimeoutMs ?? ATTEMPT_TIMEOUT_MS;
+  const warn = deps.warn ?? ((m: string) => console.warn(m));
+  // ONE host-plane fetch for the whole role, so the two routes never push host lists that drop each other.
+  const shared: TranscriptionDeps = { ...deps, fetch: deps.fetch ?? (deps.hostPlane ?? sharedHostPlaneFetch)(roleHostPlaneHosts(role)) ?? lateFetch };
+  const primary = routeTransport(role.primary, shared);
+  const fallback = role.fallback === undefined ? undefined : routeTransport(role.fallback, shared);
+  return async (req) => {
+    let first: unknown;
+    try {
+      return { ...(await attempt(primary, req, timeoutMs)), answeredBy: "primary", route: role.primary };
+    } catch (err) {
+      if (req.abortSignal?.aborted) throw err;
+      first = err;
+    }
+    if (fallback === undefined || role.fallback === undefined) {
+      throw new Error(`transcription failed on ${describe(role.primary)}: ${errText(first)}`);
+    }
+    warn(`[open-everywhere:transcription] ${describe(role.primary)} failed (${errText(first)}); trying the fallback ${describe(role.fallback)}`);
+    try {
+      return { ...(await attempt(fallback, req, timeoutMs)), answeredBy: "fallback", route: role.fallback };
+    } catch (second) {
+      throw new Error(`transcription failed on both routes — ${describe(role.primary)}: ${errText(first)}; ${describe(role.fallback)}: ${errText(second)}`);
+    }
+  };
+}

package/agent/register-defaults.ts

@@ -26,6 +26,8 @@ import { registerAwsAccess } from "./aws/register.js";
 import { awsAccessEnabled } from "./aws/config.js";
 import { registerDirectoryAdapter } from "./directory/register.js";
 import { registerGatewayRouteProbe } from "./model/gateway-probe.js";
+import { registerLongLivedBoxes } from "./sandbox/long-lived-box.js";
+import { registerNativeTranscriptSources } from "./meetings/native-sources.js";
 import { registerNeon } from "./postgres/neon.js";
 import {
   registerDbCredentialBroker,
@@ -197,6 +199,15 @@ async function wireDefaultAdapters(options: DefaultAdapterOptions): Promise<Defa
     provider.register();
   }
 
+  // The long-lived box (OPE-2459): one box backend per sandbox backend, picked by the same rule as the
+  // turn-end stopper. A box is what a meeting bot (and later the person's browser) runs in. just-bash
+  // registers a backend that refuses with a reason, so a feature never runs a box somewhere unreviewed.
+  registerLongLivedBoxes();
+
+  // The native meeting-transcript readers (OPE-2471): Teams, Meet and Zoom. Each reads only behind the meeting
+  // policy's `native` block and a member's `meeting.recap` card, so registering all three is harmless.
+  registerNativeTranscriptSources();
+
   // The deployment-status surface's model probe (OPE-2230). It registers a function, and nothing
   // calls it until somebody reads `GET /eve/v1/oe/status`.
   registerGatewayRouteProbe();