@crievo/omni 0.0.0-stage → 0.1.1

package/studio/api/services/classify.ts

@@ -0,0 +1,196 @@
+/* Typed decisions (System One: noul, choice, score) over text and images, for the Omni voice agent.
+   Two models behind one call, routed by size: clef-flash straight on Workers AI for short text and
+   images (fastest, ~0.2 s for a sentence), Jev 1.14 through OpenRouter for long text (262K context,
+   steady ~0.6 s). Each is the other's fallback when a call fails or times out. Both speak the same
+   question format, so the caller never knows which one answered, except through `model`. */
+
+import { mkdirSync, readFileSync, existsSync, statSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { extname, join } from "node:path";
+import { readEnvKeys } from "./services";
+
+export type Question = {
+  type: "noul" | "choice" | "score";
+  instructions: string;
+  criteria?: Record<string, string | null> | string[];
+};
+export type Answer =
+  | { type: "noul"; noul: number }
+  | { type: "choice"; choice: string; probabilities: Record<string, number>; confidence?: number }
+  | { type: "score"; score: number; probabilities?: Record<string, number> };
+
+export type ClassifyInput = {
+  state: string;
+  /* absolute paths of PNG/JPEG/WebP files */
+  images?: string[];
+  questions: Record<string, Question>;
+  /* skip the router (tests, A/B) */
+  model?: Model;
+};
+export type ClassifyResult =
+  | {
+      ok: true;
+      answers: Record<string, Answer>;
+      model: Model;
+      ms: number;
+      input_tokens: number;
+      cost_usd: number;
+      /* the model that failed before this one answered */
+      fallback_from?: Model;
+    }
+  | { ok: false; error: string };
+
+export type Model = "clef-flash" | "jev-1.14";
+
+/* $ per input token (output is free on both) */
+const PRICE: Record<Model, number> = { "clef-flash": 0.038 / 1e6, "jev-1.14": 0.042 / 1e6 };
+/* clef-flash's window is 24,576 tokens and it silently truncates `state` past it: anything that may
+   not fit goes to Jev. ~3.5 characters per token in Portuguese, ~1,100 tokens per image. */
+const CLEF_MAX_TOKENS = 20_000;
+const CLEF_MAX_IMAGES = 4;
+const TIMEOUT_MS = 10_000;
+
+export const estimateTokens = (state: string, images = 0) => Math.ceil(state.length / 3.5) + images * 1_100;
+
+export function route(input: ClassifyInput): Model {
+  if (input.model) return input.model;
+  const images = input.images?.length ?? 0;
+  if (images > CLEF_MAX_IMAGES) return "jev-1.14";
+  return estimateTokens(input.state, images) > CLEF_MAX_TOKENS ? "jev-1.14" : "clef-flash";
+}
+
+export async function classify(input: ClassifyInput): Promise<ClassifyResult> {
+  if (!Object.keys(input.questions).length) return { ok: false, error: "nenhuma pergunta" };
+  for (const path of input.images ?? [])
+    if (!existsSync(path)) return { ok: false, error: `imagem não encontrada: ${path}` };
+  const first = route(input);
+  const second: Model = first === "clef-flash" ? "jev-1.14" : "clef-flash";
+  const a = await callModel(first, input);
+  if (a.ok) return a;
+  /* clef-flash can't take more than 4 images or a long text: then there is no fallback */
+  const fits =
+    second === "jev-1.14" ||
+    ((input.images?.length ?? 0) <= CLEF_MAX_IMAGES &&
+      estimateTokens(input.state, input.images?.length) <= CLEF_MAX_TOKENS);
+  if (!fits) return a;
+  const b = await callModel(second, input);
+  return b.ok ? { ...b, fallback_from: first } : { ok: false, error: `${first}: ${a.error} · ${second}: ${b.error}` };
+}
+
+async function callModel(model: Model, input: ClassifyInput): Promise<ClassifyResult> {
+  const env = readEnvKeys();
+  const t0 = performance.now();
+  try {
+    const r = model === "clef-flash" ? await callClef(env, input) : await callJev(env, input);
+    const ms = Math.round(performance.now() - t0);
+    return { ok: true, answers: r.answers, model, ms, input_tokens: r.tokens, cost_usd: r.tokens * PRICE[model] };
+  } catch (e) {
+    return { ok: false, error: String((e as Error)?.message ?? e) };
+  }
+}
+
+type Raw = { answers: Record<string, Answer>; tokens: number };
+
+async function post(url: string, key: string, body: unknown): Promise<any> {
+  const r = await fetch(url, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(TIMEOUT_MS),
+  });
+  const text = await r.text();
+  let json: any = null;
+  try {
+    json = JSON.parse(text);
+  } catch {
+    /* not JSON: the status says it */
+  }
+  if (!r.ok) {
+    const message = json?.errors?.[0]?.message ?? json?.error?.message ?? json?.error ?? text.slice(0, 200);
+    throw new Error(`${r.status} ${typeof message === "string" ? message : JSON.stringify(message)}`);
+  }
+  return json;
+}
+
+/* Workers AI REST: images go in their own `images` array as data URLs */
+async function callClef(env: Record<string, string>, input: ClassifyInput): Promise<Raw> {
+  const account = env.CLOUDFLARE_ACCOUNT_ID;
+  const token = env.CLOUDFLARE_AI_TOKEN;
+  if (!account || !token) throw new Error("falta CLOUDFLARE_ACCOUNT_ID ou CLOUDFLARE_AI_TOKEN no .env");
+  const body = {
+    model: "clef-flash",
+    state: input.state,
+    questions: input.questions,
+    ...(input.images?.length ? { images: input.images.map(dataUrl) } : {}),
+  };
+  const j = await post(
+    `https://api.cloudflare.com/client/v4/accounts/${account}/ai/run/@cf/cloudflare/clef-flash`,
+    token,
+    body,
+  );
+  const out = j?.result ?? j;
+  if (!out?.answers) throw new Error("resposta sem answers");
+  return { answers: out.answers, tokens: out.usage?.input_tokens ?? out.usage?.prompt_tokens ?? 0 };
+}
+
+/* OpenRouter Decisions API: images are parts of the `state` array; Jev wants PNG/JPEG with each side
+   between 256 and 1,568 px, so larger files are resized (sips) into a cache first */
+async function callJev(env: Record<string, string>, input: ClassifyInput): Promise<Raw> {
+  const key = env.OPENROUTER_API_KEY;
+  if (!key) throw new Error("falta OPENROUTER_API_KEY no .env");
+  const images = await Promise.all((input.images ?? []).map(jevImage));
+  const state = images.length
+    ? [input.state, ...images.map(p => ({ type: "image_url", image_url: { url: dataUrl(p) } }))]
+    : input.state;
+  const j = await post("https://openrouter.ai/api/alpha/decisions", key, {
+    model: "typesafe/jev-1.14",
+    state,
+    questions: input.questions,
+  });
+  if (!j?.answers) throw new Error("resposta sem answers");
+  return { answers: j.answers, tokens: j.usage?.input_tokens ?? 0 };
+}
+
+const MIME: Record<string, string> = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".webp": "image/webp",
+};
+
+function dataUrl(path: string) {
+  const mime = MIME[extname(path).toLowerCase()] ?? "image/png";
+  return `data:${mime};base64,${readFileSync(path).toString("base64")}`;
+}
+
+const JEV_CACHE = join(tmpdir(), "omnivideo-classify");
+const JEV_MAX_SIDE = 1568;
+
+async function jevImage(path: string): Promise<string> {
+  const ext = extname(path).toLowerCase();
+  const { width, height } = await imageSize(path);
+  if ((ext === ".png" || ext === ".jpg" || ext === ".jpeg") && Math.max(width, height) <= JEV_MAX_SIDE) return path;
+  mkdirSync(JEV_CACHE, { recursive: true });
+  const out = join(JEV_CACHE, `${Bun.hash(`${path}:${statSync(path).mtimeMs}`).toString(36)}.jpg`);
+  if (!existsSync(out)) {
+    const p = Bun.spawn(
+      ["sips", "-s", "format", "jpeg", "-s", "formatOptions", "85", "-Z", String(JEV_MAX_SIDE), path, "--out", out],
+      {
+        stdout: "ignore",
+        stderr: "pipe",
+      },
+    );
+    if ((await p.exited) !== 0) throw new Error(`sips: ${await new Response(p.stderr).text()}`);
+  }
+  return out;
+}
+
+async function imageSize(path: string): Promise<{ width: number; height: number }> {
+  const p = Bun.spawn(["sips", "-g", "pixelWidth", "-g", "pixelHeight", path], { stdout: "pipe", stderr: "ignore" });
+  const text = await new Response(p.stdout).text();
+  await p.exited;
+  return {
+    width: Number(/pixelWidth: (\d+)/.exec(text)?.[1] ?? 0),
+    height: Number(/pixelHeight: (\d+)/.exec(text)?.[1] ?? 0),
+  };
+}

package/studio/api/services/clone-phrases.ts

@@ -0,0 +1,75 @@
+/* Phrases the user reads to clone their voice. The context is the language they
+   speak in the videos: the clone learns what it hears, so the sample needs the
+   same kind of speech the channel will have (Portuguese only, English only, or Portuguese
+   with English terms and phrases mixed in, which is the app's default).
+
+   Each phrase takes 6 to 10 s at a natural pace and mixes different sounds: question,
+   number, list, pause, short sentence. The mixed-context ones switch language
+   mid-sentence, the way the channel actually speaks. */
+
+export type Phrase = { id: string; text: string };
+export type CloneContext = {
+  id: "pt" | "en" | "mixed";
+  name: string;
+  description: string;
+  language: "pt" | "en";
+  phrases: Phrase[];
+};
+
+const phraseList = (prefix: string, texts: string[]): Phrase[] =>
+  texts.map((text, i) => ({ id: `${prefix}${i + 1}`, text }));
+
+export const CLONE_CONTEXTS: CloneContext[] = [
+  {
+    id: "mixed",
+    name: "Português com inglês",
+    description:
+      "Fala em português e mistura termos e frases em inglês, como em vídeo de tecnologia e criador de conteúdo.",
+    language: "pt",
+    phrases: phraseList("mx", [
+      "Fala, pessoal! Hoje eu vou mostrar como fazer o deploy de um projeto novo, direto do GitHub, sem enrolação.",
+      "Primeiro, abre o terminal, roda o install e espera o build terminar. Se der erro, não entra em pânico.",
+      "O feedback do pessoal foi muito bom, mas o engagement do vídeo anterior caiu um pouco, então vamos ajustar o hook.",
+      "Eu gosto de começar pelo problema: qual é o pain point do usuário, e como a gente resolve isso de forma simples?",
+      "Welcome back to the channel! Hoje o assunto é produtividade, então pega um café e vem comigo.",
+      "Clica em settings, depois em upgrade, e escolhe o plano que faz mais sentido pro seu workflow.",
+      "A thumbnail precisa chamar atenção em dois segundos, e o título tem que prometer algo que o vídeo entrega.",
+      "No fim das contas, não existe mágica: é consistência, um bom roteiro e muita paciência. Simples assim.",
+      "That's it, é só isso. Se curtiu, deixa o like, se inscreve, e a gente se vê no próximo vídeo, valeu?",
+    ]),
+  },
+  {
+    id: "pt",
+    name: "Só português",
+    description: "Fala sempre em português do Brasil.",
+    language: "pt",
+    phrases: phraseList("pt", [
+      "Fala, pessoal! Hoje eu vou mostrar um jeito simples de organizar o seu dia e render muito mais.",
+      "Quando a gente começa um projeto novo, o mais importante é entender o problema antes de sair fazendo.",
+      "Você sabia que quase ninguém usa esse recurso? Pois é, ele muda completamente o resultado final.",
+      "Se der tudo certo, em cinco minutos você já tem a primeira versão funcionando, sem complicação nenhuma.",
+      "Vinte e três mil, quatrocentos e cinquenta e seis reais, mais ou menos, dependendo do mês e da cotação.",
+      "Calma, respira. Primeiro a gente resolve o mais urgente, depois cuida do resto, combinado?",
+      "Eu achei que ia ser difícil, mas na verdade foi bem mais tranquilo do que eu esperava.",
+      "Então é isso, gostou do vídeo? Deixa o seu like, se inscreve no canal e a gente se vê no próximo.",
+    ]),
+  },
+  {
+    id: "en",
+    name: "Só inglês",
+    description: "Fala sempre em inglês.",
+    language: "en",
+    phrases: phraseList("en", [
+      "Hey everyone, welcome back to the channel. Today we're going to build something really useful from scratch.",
+      "The first thing you want to do is understand the problem, then pick the simplest solution that works.",
+      "Did you know most people never touch this setting? It completely changes how the whole thing behaves.",
+      "In about five minutes you'll have a working version, and I promise it's easier than it sounds.",
+      "It costs around twenty-three thousand dollars, give or take, depending on the month and the exchange rate.",
+      "Take a breath. We'll fix the urgent stuff first, then we'll deal with everything else, okay?",
+      "I thought it was going to be hard, but honestly it turned out way easier than I expected.",
+      "So that's it. If you liked the video, leave a like, subscribe, and I'll see you in the next one.",
+    ]),
+  },
+];
+
+export const findCloneContext = (id: unknown) => CLONE_CONTEXTS.find(c => c.id === id);