ava-engine 0.0.0-stage → 0.2.0

package/lib/platforms.mjs

@@ -0,0 +1,201 @@
+// Complex platforms the onboarding agent can specialise in. Each entry gives the
+// talker/planner enough to act like a real product specialist for that product:
+// the sign-up → dashboard path, a clean short "journey" for the live progress
+// tracker, and a small knowledge base the agent can retrieve from (RAG).
+
+export const PLATFORMS = [
+  {
+    id: "todoist", name: "Todoist", url: "https://app.todoist.com",
+    category: "Productivity", accent: "#E44332",
+    tagline: "Dead-simple task manager — you're up and running in about a minute.",
+    firstWin: "your first task added and showing in your Today list",
+    path: [
+      "sign in or sign up (Google or email — it's quick)",
+      "you land on the Today view; click 'Add task' at the top (or just press the A key)",
+      "type a task, leave it due Today, and press Enter — it appears in your list, and that's the win",
+    ],
+    optional: "Projects, labels, and reminders are all there when you want them, but you don't need any of that to get started.",
+    journey: ["Sign in", "Add your first task", "See it in Today"],
+    etaMin: 2,
+    knowledge: [
+      { k: "add task new create shortcut a key", text: "To add a task, click 'Add task' at the top of the list or just press the A key. Type it and hit Enter to save." },
+      { k: "today view due date schedule tomorrow", text: "The Today view shows what's due today. When adding a task you can type a due date in plain words like 'tomorrow 5pm' and Todoist schedules it automatically." },
+      { k: "complete task done check circle", text: "Click the circle to the left of a task to complete it — it checks off and leaves your list." },
+      { k: "project inbox organize sidebar", text: "Projects in the left sidebar group related tasks. You don't need one to start — new tasks land in Inbox or Today by default." },
+    ],
+  },
+  {
+    id: "supabase", name: "Supabase", url: "https://supabase.com/dashboard",
+    category: "Developer backend", accent: "#3ECF8E",
+    tagline: "Postgres, auth, storage, edge functions — a lot of surface for a newcomer.",
+    firstWin: "your first project's dashboard, open and showing Healthy",
+    path: [
+      "sign in (GitHub or email)",
+      "create an organization (name it, choose the free plan)",
+      "create a new project — name it, set a strong database password (the Generate button works), pick the closest region",
+      "click Create new project and wait for it to provision",
+      "land on the project dashboard — once it shows Healthy, they're in and onboarding is DONE",
+    ],
+    optional: "If they want their API keys afterward, they live under Settings → API Keys (Project URL is on that same page). This is a bonus, not required to finish.",
+    journey: ["Sign in", "Create an organization", "Create a project", "Wait for provisioning", "Open your dashboard"],
+    etaMin: 6,
+    knowledge: [
+      { k: "password database generate strong project", text: "On the Create Project screen, the database password must be strong. The 'Generate a password' link fills a strong one automatically — click it if theirs is rejected as weak." },
+      { k: "region project location latency", text: "Pick the region closest to the user or their users — it can't be changed after the project is created, only by making a new project." },
+      { k: "api keys anon publishable url settings", text: "The Project URL and API keys live under Settings → API Keys. The 'publishable'/anon key is safe for browser code; the secret/service key must stay server-side." },
+      { k: "free plan pricing organization tier", text: "The Free plan is enough to start: 2 free projects, 500MB database, 1GB storage. You choose the plan when creating the organization." },
+      { k: "provision wait healthy time", text: "A new project takes about 2 minutes to provision. The dashboard shows 'Setting up' then 'Healthy' when it's ready to use." },
+    ],
+  },
+  {
+    // Fictional product used by demo mode (/app?demo=simple). Not in the lobby.
+    id: "basewell", name: "Basewell", url: "https://app.basewell.example", demoOnly: true,
+    category: "Developer backend", accent: "#F2541B",
+    tagline: "Postgres, auth and file storage in one place: quick to start, lots to learn.",
+    firstWin: "your first project's dashboard, open and showing Healthy",
+    path: [
+      "sign in (GitHub or email)",
+      "create a workspace (name it, keep the Free plan)",
+      "create a new project: name it, set a strong database password (the 'Generate a password' link works), pick the closest region",
+      "click Create new project and wait for it to provision",
+      "land on the project dashboard; once it shows Healthy, they're in and onboarding is DONE",
+    ],
+    optional: "Their Project URL and public API key are on the dashboard under 'Get your API keys'. A bonus, not required to finish.",
+    journey: ["Sign in", "Create a workspace", "Create a project", "Wait for provisioning", "Open your dashboard"],
+    etaMin: 5,
+    knowledge: [
+      { k: "password database generate strong project weak", text: "On Create a new project, the database password must be strong. The 'Generate a password' link (right above the field) fills a strong one automatically." },
+      { k: "region project location latency", text: "Pick the region closest to the user's customers. It can't be changed later without creating a new project." },
+      { k: "api keys public url connect", text: "The Project URL and the public API key are on the project dashboard under 'Get your API keys'. The public key is safe in browser code." },
+      { k: "free plan pricing workspace organization tier", text: "The Free plan is enough to start. The plan is chosen when the workspace is created and can be upgraded later." },
+      { k: "provision wait healthy time setting up", text: "A new project takes about a minute to provision. The dashboard shows 'Setting up your project', then 'Project is healthy' when it's ready." },
+    ],
+  },
+  {
+    id: "stripe", name: "Stripe", url: "https://dashboard.stripe.com",
+    category: "Payments", accent: "#635BFF",
+    tagline: "Powerful but sprawling — activation, test mode, API keys, first payment.",
+    firstWin: "your dashboard in test mode with your API keys located",
+    path: [
+      "sign in or create an account",
+      "confirm you're in Test mode (toggle, top right)",
+      "open the Developers → API keys page",
+      "note the publishable + secret test keys",
+      "explore the Payments tab where charges will appear",
+    ],
+    optional: "Activating live payments needs business details; that can wait until after they've tested.",
+    journey: ["Sign in", "Switch to Test mode", "Open API keys", "Copy your test keys", "See the Payments tab"],
+    etaMin: 5,
+    knowledge: [
+      { k: "test mode live toggle sandbox", text: "The Test mode toggle is top-right. In test mode no real money moves and you use test card 4242 4242 4242 4242. Live mode needs account activation with business details." },
+      { k: "api keys publishable secret developers", text: "API keys are under Developers → API keys. The publishable key (pk_) goes in client code; the secret key (sk_) must stay on your server." },
+      { k: "payment first charge test card", text: "Use card number 4242 4242 4242 4242 with any future expiry and any CVC to create a successful test payment." },
+    ],
+  },
+  {
+    id: "aws", name: "AWS Console", url: "https://console.aws.amazon.com",
+    category: "Cloud infrastructure", accent: "#FF9900",
+    tagline: "The classic maze — regions, services, IAM. Easy to get lost.",
+    firstWin: "the right region selected and the EC2 dashboard open",
+    path: [
+      "sign in to the console (root or IAM user)",
+      "check the region selector, top right — pick the closest region",
+      "search for the service you need in the top search bar (e.g. EC2)",
+      "open that service's dashboard",
+      "find the primary create/launch action",
+    ],
+    optional: "Setting up billing alerts and an IAM user instead of root is good hygiene, but not needed to look around.",
+    journey: ["Sign in", "Choose your region", "Find your service", "Open its dashboard", "Start your first resource"],
+    etaMin: 7,
+    knowledge: [
+      { k: "region selector location resources", text: "The region selector is top-right. Resources are region-specific — if something you created 'disappears', check you're in the same region you made it in." },
+      { k: "iam root user security", text: "Avoid using the root account for daily work. Create an IAM user with appropriate permissions and sign in as that instead." },
+      { k: "service search ec2 s3 find", text: "The top search bar jumps to any service by name — type 'EC2', 'S3', 'Lambda' etc. rather than hunting the menu." },
+    ],
+  },
+  {
+    id: "salesforce", name: "Salesforce", url: "https://login.salesforce.com",
+    category: "Enterprise CRM", accent: "#00A1E0",
+    tagline: "Dense enterprise UI — objects, tabs, setup. Notoriously steep.",
+    firstWin: "your Sales home with a first lead or contact created",
+    path: [
+      "log in to your org",
+      "make sure the Sales app is selected (App Launcher, top left)",
+      "open the Leads or Contacts tab",
+      "use the New button to create your first record",
+      "save and view it on the record page",
+    ],
+    optional: "Customising fields and page layouts lives in Setup (gear icon) — a later step.",
+    journey: ["Log in", "Open the Sales app", "Go to Leads", "Create a record", "Save and view it"],
+    etaMin: 6,
+    knowledge: [
+      { k: "app launcher switch sales waffle", text: "The App Launcher is the grid/waffle icon, top-left. Use it to switch between apps like Sales, Service, and Marketing." },
+      { k: "lead contact new record create", text: "Leads and Contacts each have a New button on their list view (top-right) to create a record. A Lead is an unqualified prospect; a Contact belongs to an Account." },
+      { k: "setup customize fields gear", text: "Setup (the gear icon, top-right) is where you customise objects, fields, and layouts — separate from day-to-day record work." },
+    ],
+  },
+  {
+    id: "datadog", name: "Datadog", url: "https://app.datadoghq.com",
+    category: "Observability", accent: "#632CA6",
+    tagline: "Deep product — agents, metrics, dashboards, monitors everywhere.",
+    firstWin: "a dashboard open with at least one metric graphed",
+    path: [
+      "log in",
+      "open the Dashboards section from the left nav",
+      "create a new dashboard",
+      "add a timeseries widget",
+      "pick a metric to graph and save",
+    ],
+    optional: "Installing the Agent to send real metrics is the bigger step; you can explore built-in integrations first.",
+    journey: ["Log in", "Open Dashboards", "Create a dashboard", "Add a widget", "Graph a metric"],
+    etaMin: 6,
+    knowledge: [
+      { k: "dashboard new create widget", text: "Dashboards are in the left nav. New Dashboard → then drag in a Timeseries widget to start graphing." },
+      { k: "metric graph timeseries query", text: "In a Timeseries widget, pick a metric in the query editor (e.g. system.cpu.user). Built-in integration metrics appear even before you install the Agent." },
+      { k: "agent install host metrics", text: "The Datadog Agent runs on your hosts to send metrics/logs. Install instructions are under Integrations → Agent." },
+    ],
+  },
+  {
+    id: "vercel", name: "Vercel", url: "https://vercel.com/dashboard",
+    category: "Deployment", accent: "#111827",
+    tagline: "Git-connected deploys, projects, domains, env vars.",
+    firstWin: "your first project deployed and its production URL live",
+    path: [
+      "sign in (usually with GitHub)",
+      "click Add New → Project",
+      "import a Git repository",
+      "confirm the framework preset and deploy",
+      "open the production URL when the build finishes",
+    ],
+    optional: "Custom domains and environment variables are set per-project after the first deploy.",
+    journey: ["Sign in", "Add new project", "Import a repo", "Deploy", "Open your live URL"],
+    etaMin: 5,
+    knowledge: [
+      { k: "import repository git github project", text: "Add New → Project lets you import a Git repo. Vercel auto-detects the framework (Next.js, Vite, etc.) — usually no config needed." },
+      { k: "environment variables env project settings", text: "Environment variables are set per project under Settings → Environment Variables, and apply to Production/Preview/Development." },
+      { k: "domain custom production url", text: "Every deploy gets a vercel.app URL. Add a custom domain under the project's Domains tab." },
+    ],
+  },
+  {
+    id: "google-workspace", name: "Google Workspace", url: "https://mail.google.com", demoOnly: true,
+    category: "Productivity", accent: "#4285F4",
+    tagline: "Gmail, Calendar, Docs — everyday multi-app work.",
+    firstWin: "the thing they came to do, done",
+    path: [],
+    journey: ["Read the email", "Reply", "Add it to Calendar", "Confirm the event"],
+    etaMin: 4,
+    knowledge: [
+      { k: "latest recent newest email top inbox order", text: "In Gmail the newest email is at the very top of the inbox list; each row shows sender, subject and time. 'Latest' means the top row." },
+      { k: "sender address email from who", text: "Open an email and the sender's name is at the top; their full address (name <address@domain>) shows next to it or when you click the sender name." },
+      { k: "reply respond email", text: "To reply, open the email and click Reply (bottom of the message, or the arrow top-right); the compose box opens with the recipient pre-filled." },
+      { k: "calendar event create meeting add schedule", text: "In Google Calendar, click Create (top-left) or click a time slot to add an event; set title, time and guests, then Save." },
+      { k: "guest invite attendee calendar", text: "Add people to an event in the 'Add guests' field; they get an email invite and it appears on their calendar." },
+    ],
+  },
+];
+
+export function platformById(id) { return PLATFORMS.find((p) => p.id === id) || null; }
+// Trimmed shape for the onboarding picker in the client (demo-only entries hidden).
+// Demo-only (fictional) products are included, flagged, so demo mode can select
+// them; the lobby's platform picker leaves them out.
+export function platformsView() { return PLATFORMS.map(({ id, name, url, category, tagline, accent, demoOnly }) => ({ id, name, url, category, tagline, accent, ...(demoOnly ? { demoOnly: true } : {}) })); }

package/lib/state.mjs

@@ -0,0 +1,150 @@
+// In-memory session store for the flexible agent.
+
+import { randomUUID } from "node:crypto";
+
+const sessions = new Map();
+const byKey = new Map(); // specialist key → session id (see mintSpecialistKey)
+
+function emptyScore() {
+  return { turns: 0, promptTokens: 0, completionTokens: 0, cost: 0, latencies: [], perception: { captured: 0, analysed: 0 } };
+}
+
+const MAX_SESSIONS = Number(process.env.MAX_SESSIONS) || 50;
+const MAX_AGE_MS = 6 * 3600 * 1000;
+const IDLE_MS = 30 * 60 * 1000;
+
+function drop(id) {
+  const s = sessions.get(id);
+  if (s?.specialistKey) revokeSpecialistKey(s);
+  sessions.delete(id);
+}
+
+/** Room for one more session? Expired and idle sessions are pruned first. */
+export function hasCapacity() {
+  const now = Date.now();
+  for (const [id, s] of sessions) { if (now - s.createdAt > MAX_AGE_MS || now - s.lastActive > IDLE_MS) drop(id); }
+  return sessions.size < MAX_SESSIONS;
+}
+
+export function newSession(opts = {}) {
+  if (!hasCapacity()) return null; // never evict someone's live call to make room
+  const id = randomUUID();
+  const mode = opts.mode === "onboarding" ? "onboarding" : "flexible";
+  // platform can attach in ANY mode — it supplies the RAG knowledge base; it only
+  // drives the persona + fixed goal when in onboarding mode.
+  const platform = opts.platform || null;
+  sessions.set(id, {
+    id,
+    createdAt: Date.now(),
+    lastActive: Date.now(),
+    specialistKey: null,         // capability for the human-specialist console; minted on handoff
+    ours: {
+      mode,                    // "flexible" | "onboarding"
+      platform,                // onboarding target {id,name,url,firstWin,path,...} or null
+      screenState: null,       // last structured screen state {app,page,elements,state,v}
+      screenVersion: 0,
+      lastHash: null,          // last perceptual hash we ran vision on
+      goal: mode === "onboarding" && platform ? `Get set up on ${platform.name} and into the dashboard` : "—", // the concrete goal the agent is driving toward
+      understanding: "—",      // planner's running read of what the user is trying to do
+      plan: [],                // planner's short grounded task list
+      guidance: "—",           // planner's tone/pace note
+      planApp: null,           // the app the current plan was made for (pivot detection)
+      frustration: 0,          // rolling frustration signal
+      turnsSincePlan: 0,
+      lastProactive: 0,        // timestamp of last proactive utterance
+      lastTalkerScreen: "",    // "app · page" the agent last addressed (to detect navigation since)
+      lastIntent: null,        // most recent intent read {intent, frustrated, request}
+      lastThink: null,         // most recent reasoning verdict {resolved, stuck, blocker, thought, nextTactic}
+      prevResolved: false,      // the previous turn's task was confirmed done → re-plan on the next message
+      done: false,              // the whole task/journey is finished on screen → stop guiding
+      doneAnnounced: false,     // already gave the one wrap-up celebration → stay quiet
+      // live workflow tracker — the journey the user is being walked through
+      workflow: (mode === "onboarding" && platform && platform.journey) ? {
+        title: `Getting set up on ${platform.name}`,
+        steps: platform.journey.map((t, i) => ({ title: t, status: i === 0 ? "active" : "todo" })),
+        current: 0,
+        etaMin: platform.etaMin || null,
+        note: "",
+      } : null,
+      handoff: { requested: false, state: "none" }, // none | connecting | waiting | joined
+      knowledgeUsed: [],       // snippets retrieved for the last reply (RAG)
+      bridge: { seq: 0, log: [], specialist: false }, // shared conversation the human specialist joins
+      lastSpokeAt: 0,          // when Ava last said anything (reply or proactive)
+      goalGuessed: false,      // goal came from the silent-browsing intention guesser
+      lastGuessAt: 0,          // when the intention guesser last ran
+      activity: [],            // inferred user actions over time [{at, action}] — the activity stream
+      history: [],             // compact [{role, text}]
+    },
+    score: { ours: emptyScore() },
+  });
+  return sessions.get(id);
+}
+
+export function getSession(id) {
+  const s = typeof id === "string" ? sessions.get(id) : undefined;
+  if (s) s.lastActive = Date.now();
+  return s;
+}
+
+// The specialist console gets its own key instead of the user's session id, so a
+// leaked handoff link can't act as the user, and cancelling the handoff revokes it.
+export function mintSpecialistKey(sess) {
+  if (sess.specialistKey) return sess.specialistKey;
+  sess.specialistKey = randomUUID();
+  byKey.set(sess.specialistKey, sess.id);
+  return sess.specialistKey;
+}
+const revokeHooks = [];
+/** Run fn(sess) whenever a specialist key is revoked (the server closes their live link). */
+export function onSpecialistRevoked(fn) { revokeHooks.push(fn); }
+export function revokeSpecialistKey(sess) {
+  if (!sess.specialistKey) return;
+  byKey.delete(sess.specialistKey);
+  sess.specialistKey = null;
+  for (const fn of revokeHooks) { try { fn(sess); } catch {} }
+}
+
+/** The customer hung up: forget the call now instead of waiting for it to idle out. */
+export function endSession(id) {
+  const s = sessions.get(id);
+  if (!s) return false;
+  revokeSpecialistKey(s);
+  drop(id);
+  return true;
+}
+export function getSessionBySpecialistKey(key) {
+  return typeof key === "string" ? getSession(byKey.get(key)) : undefined;
+}
+
+export function record(sess, arch, usages, hotLatencyMs) {
+  const sc = sess.score[arch];
+  sc.turns += 1;
+  for (const u of usages) {
+    sc.promptTokens += u.promptTokens || 0;
+    sc.completionTokens += u.completionTokens || 0;
+    sc.cost += u.cost || 0;
+  }
+  const lat = hotLatencyMs != null ? hotLatencyMs : usages.reduce((s, u) => s + (u.latencyMs || 0), 0);
+  sc.latencies.push(lat);
+  return lat;
+}
+
+// Add token/cost without counting a turn (used by the background observe loop).
+export function addTokens(sess, arch, usages) {
+  const sc = sess.score[arch];
+  for (const u of usages) { sc.promptTokens += u.promptTokens || 0; sc.completionTokens += u.completionTokens || 0; sc.cost += u.cost || 0; }
+}
+
+export function scoreView(sc) {
+  const lat = sc.latencies;
+  const avg = lat.length ? Math.round(lat.reduce((a, b) => a + b, 0) / lat.length) : 0;
+  return {
+    turns: sc.turns,
+    promptTokens: sc.promptTokens,
+    completionTokens: sc.completionTokens,
+    totalTokens: sc.promptTokens + sc.completionTokens,
+    cost: sc.cost,
+    avgLatencyMs: avg,
+    perception: sc.perception,
+  };
+}

package/lib/stt.mjs

@@ -0,0 +1,22 @@
+// Deepgram speech-to-text (nova-3): the cloud fallback that turns a recorded
+// utterance into text when the offline sherpa models aren't available.
+
+export const STT_ENABLED = !!process.env.DEEPGRAM_API_KEY;
+const MODEL = process.env.DEEPGRAM_MODEL || "nova-3";
+
+// buffer: raw audio bytes; contentType e.g. "audio/webm" or "audio/mpeg".
+export async function transcribe(buffer, contentType) {
+  const res = await fetch(`https://api.deepgram.com/v1/listen?model=${MODEL}&smart_format=true&punctuate=true`, {
+    method: "POST",
+    headers: { "Authorization": "Token " + process.env.DEEPGRAM_API_KEY, "Content-Type": contentType || "audio/webm" },
+    body: buffer,
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!res.ok) {
+    let msg = await res.text().catch(() => "");
+    try { const j = JSON.parse(msg); msg = j.err_msg || j.reason || msg; } catch {}
+    const e = new Error(`Deepgram ${res.status}: ${String(msg).slice(0, 200)}`); e.status = res.status; throw e;
+  }
+  const j = await res.json();
+  return j.results?.channels?.[0]?.alternatives?.[0]?.transcript || "";
+}

package/lib/tts.mjs

@@ -0,0 +1,178 @@
+// Text-to-speech providers. Voice is presentation only.
+// "kokoro" runs on the SERVER (free, no cloud/credits) so the client just plays
+// audio — works on the oldest device. ElevenLabs/Hume are cloud (paid). "browser"
+// is the on-device Web Speech fallback.
+
+import { loadKokoro, kokoroInstalled } from "./kokoro-model.mjs";
+
+const HAS_ELEVEN = !!process.env.ELEVENLABS_API_KEY;
+const HAS_HUME = !!process.env.HUME_API_KEY;
+const HAS_KOKORO = process.env.KOKORO !== "off"; // local, free — on by default
+
+const ELEVEN_VOICE = process.env.ELEVEN_VOICE_ID || "XrExE9yKIg1WjnnlVkGX";
+const ELEVEN_MODEL = process.env.ELEVEN_MODEL || "eleven_turbo_v2_5";
+const ELEVEN_LABEL = process.env.ELEVEN_LABEL || "Matilda";
+const HUME_VOICE = process.env.HUME_VOICE_ID || "a7ecc00a-6fc0-4546-8126-e12cfd8de3bf";
+const HUME_LABEL = process.env.HUME_LABEL || "Aria";
+const HUME_DESC = process.env.HUME_DESCRIPTION || "Warm, empathetic, natural and reassuring";
+const KOKORO_VOICE = process.env.KOKORO_VOICE || "af_heart";
+
+export const TTS_ENABLED = HAS_KOKORO || HAS_ELEVEN || HAS_HUME;
+const FETCH_TIMEOUT_MS = 20_000;
+const MAX_KOKORO_QUEUE = Number(process.env.KOKORO_MAX_QUEUE) || 6;
+
+/** Thrown when the local voice already has a full queue; the server answers 429. */
+export class TtsBusyError extends Error { constructor() { super("tts busy"); } }
+
+export function providers() {
+  return [
+    { id: "kokoro", label: "Kokoro · offline", enabled: HAS_KOKORO, note: "on-server, free" },
+    { id: "elevenlabs", label: ELEVEN_LABEL, enabled: HAS_ELEVEN },
+    { id: "hume", label: HUME_LABEL, enabled: HAS_HUME },
+    { id: "browser", label: "System", enabled: true },
+  ];
+}
+export function defaultProvider() {
+  // Explicit override wins, if that provider is actually available.
+  const want = process.env.TTS_DEFAULT;
+  if (want === "elevenlabs" && HAS_ELEVEN) return "elevenlabs";
+  if (want === "hume" && HAS_HUME) return "hume";
+  if (want === "kokoro" && HAS_KOKORO) return "kokoro";
+  if (want === "browser") return "browser";
+  if (HAS_KOKORO) return "kokoro"; // free + runs on any client
+  if (HAS_ELEVEN) return "elevenlabs";
+  if (HAS_HUME) return "hume";
+  return "browser";
+}
+
+/* ---- Kokoro (server-side, local, free) ---- */
+let kokoroTTS = null, kokoroInit = null;
+async function initKokoro() {
+  if (kokoroTTS) return kokoroTTS;
+  if (!kokoroInit) kokoroInit = (async () => {
+    if (!kokoroInstalled()) console.log("  kokoro: downloading the voice model (~90 MB, first run only)…");
+    kokoroTTS = await loadKokoro();
+    return kokoroTTS;
+  })();
+  return kokoroInit;
+}
+export function warmKokoro() { if (HAS_KOKORO) initKokoro().then(() => console.log("  kokoro voice ready")).catch((e) => console.log("  kokoro warm failed:", e.message)); }
+// Serialize generation: concurrent runs on the one ONNX session (prewarm +
+// preview + greeting can overlap) risk mid-stream failures, which the client
+// would misread as "kokoro broken" and fall back to the browser voice.
+// The queue is bounded so one client can't starve everyone else's voice.
+let kokoroLock = Promise.resolve(), kokoroQueued = 0;
+function withKokoroLock(fn) {
+  if (kokoroQueued >= MAX_KOKORO_QUEUE) return Promise.reject(new TtsBusyError());
+  kokoroQueued += 1;
+  const run = kokoroLock.then(fn, fn).finally(() => { kokoroQueued -= 1; });
+  kokoroLock = run.then(() => {}, () => {});
+  return run;
+}
+
+async function kokoro(text) {
+  const tts = await initKokoro();
+  const audio = await withKokoroLock(() => tts.generate(text, { voice: KOKORO_VOICE }));
+  return { audio: Buffer.from(await audio.toBlob().arrayBuffer()), model: "kokoro-82m", mime: "audio/wav" };
+}
+
+/* Break a reply into small CLAUSE-sized chunks for streaming synthesis. CPU
+   synthesis time scales with chunk length, and the client starts playing after
+   the FIRST chunk — so a whole-sentence first chunk (~1.6s) is the delay we're
+   killing. We split at sentence AND clause boundaries (commas, semicolons,
+   dashes) and keep the first chunk deliberately short so audio starts in
+   ~0.3-0.6s; later chunks are larger to keep prosody smooth and cut per-call
+   overhead. Splitting at natural pause points keeps it sounding natural. */
+function ttsChunks(text) {
+  const parts = (text.match(/[^.!?,;:—–\n]+[.!?,;:—–\n]*/g) || [text]).map((s) => s.trim()).filter(Boolean);
+  const chunks = [];
+  let cur = "";
+  for (const p of parts) {
+    const target = chunks.length === 0 ? 42 : 100; // first chunk short → fast first audio
+    if (cur && (cur.length + 1 + p.length) > target) { chunks.push(cur); cur = p; }
+    else cur = cur ? cur + " " + p : p;
+  }
+  if (cur) chunks.push(cur);
+  return chunks;
+}
+
+/* Streaming Kokoro: synthesize clause-sized chunks in order and yield each as
+   soon as it's ready — first audio after the (short) first chunk, not the whole
+   reply. onChunk(wavBuffer, chunkText) per chunk; the client schedules them
+   back-to-back for gapless playback. */
+export const KOKORO_STREAMS = HAS_KOKORO;
+export async function kokoroStream(text, onChunk) {
+  const clean = String(text || "").slice(0, 800).trim();
+  if (!clean) throw new Error("empty text");
+  const tts = await initKokoro();
+  const chunks = ttsChunks(clean);
+  return withKokoroLock(async () => {
+    let n = 0;
+    for (const chunk of chunks) {
+      const audio = await tts.generate(chunk, { voice: KOKORO_VOICE });
+      await onChunk(Buffer.from(await audio.toBlob().arrayBuffer()), chunk);
+      n += 1;
+    }
+    return n;
+  });
+}
+
+async function eleven(text) {
+  const res = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${ELEVEN_VOICE}?output_format=mp3_44100_128`, {
+    method: "POST",
+    headers: { "xi-api-key": process.env.ELEVENLABS_API_KEY, "Content-Type": "application/json", "Accept": "audio/mpeg" },
+    body: JSON.stringify({ text, model_id: ELEVEN_MODEL, voice_settings: { stability: 0.45, similarity_boost: 0.8, style: 0.3, use_speaker_boost: true } }),
+    signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+  });
+  if (!res.ok) throw new Error(`ElevenLabs ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`);
+  return { audio: Buffer.from(await res.arrayBuffer()), model: ELEVEN_MODEL, mime: "audio/mpeg" };
+}
+
+/* Streaming ElevenLabs: synthesize clause-sized chunks in order (same chunking
+   as Kokoro) so first audio lands in ~0.3-0.5s instead of waiting for the whole
+   clip. previous_text is passed so prosody stays continuous across chunks. Each
+   chunk is a complete MP3 the client decodes independently. onChunk(mp3, text). */
+export const ELEVEN_STREAMS = HAS_ELEVEN;
+export async function elevenStream(text, onChunk) {
+  const clean = String(text || "").slice(0, 800).trim();
+  if (!clean) throw new Error("empty text");
+  const chunks = ttsChunks(clean);
+  let prev = "", n = 0;
+  for (const chunk of chunks) {
+    const res = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${ELEVEN_VOICE}?output_format=mp3_44100_128`, {
+      method: "POST",
+      headers: { "xi-api-key": process.env.ELEVENLABS_API_KEY, "Content-Type": "application/json", "Accept": "audio/mpeg" },
+      body: JSON.stringify({ text: chunk, model_id: ELEVEN_MODEL, previous_text: prev || undefined, voice_settings: { stability: 0.45, similarity_boost: 0.8, style: 0.3, use_speaker_boost: true } }),
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+    });
+    if (!res.ok) { if (n === 0) throw new Error(`ElevenLabs ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`); break; }
+    await onChunk(Buffer.from(await res.arrayBuffer()), chunk);
+    prev = (prev + " " + chunk).trim().slice(-400);
+    n += 1;
+  }
+  return n;
+}
+async function hume(text) {
+  const res = await fetch("https://api.hume.ai/v0/tts/file", {
+    method: "POST",
+    headers: { "X-Hume-Api-Key": process.env.HUME_API_KEY, "Content-Type": "application/json" },
+    body: JSON.stringify({ utterances: [{ text, voice: { id: HUME_VOICE }, description: HUME_DESC }], format: { type: "mp3" }, num_generations: 1 }),
+    signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+  });
+  if (!res.ok) throw new Error(`Hume ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`);
+  return { audio: Buffer.from(await res.arrayBuffer()), model: "octave", mime: "audio/mpeg" };
+}
+
+// Returns { audio: Buffer, mime, chars, latencyMs, model, provider } or throws.
+export async function synth(provider, text) {
+  const clean = String(text || "").slice(0, 800).trim();
+  if (!clean) throw new Error("empty text");
+  if (provider === "elevenlabs" && !HAS_ELEVEN) throw new Error("elevenlabs not configured");
+  if (provider === "hume" && !HAS_HUME) throw new Error("hume not configured");
+  if (provider !== "hume" && provider !== "elevenlabs" && (provider !== "kokoro" || !HAS_KOKORO)) {
+    const e = new Error(`voice provider "${provider}" is not available`); e.status = 400; throw e; // KOKORO=off means off
+  }
+  const t0 = Date.now();
+  const out = provider === "hume" ? await hume(clean) : provider === "elevenlabs" ? await eleven(clean) : await kokoro(clean);
+  return { ...out, chars: clean.length, latencyMs: Date.now() - t0, provider };
+}

package/lib/workflow.mjs

@@ -0,0 +1,18 @@
+// The agent is a general, screen-aware, flexible assistant — no fixed script.
+// Its "job" is whatever the user is doing on screen or asks for. The value of
+// the architecture is that it adapts: perception reads any app, an intent read
+// figures out what they want, a planner keeps a running understanding, and the
+// talker helps — grounded, warm, and never looping.
+
+export const PERSONA =
+  "You are Ava, a warm, sharp, genuinely helpful assistant who can see the user's screen. " +
+  "You help with ANYTHING — whatever app they're in, whatever they're trying to do — grounded in what's actually on their screen. " +
+  "You speak in short, natural, human turns.";
+
+export const GUARDRAILS = [
+  "Ground everything in what's actually on the user's screen; never invent screen contents you can't see.",
+  "Meet the user where they are. Whatever app or task they're on, help with THAT — don't push your own agenda.",
+  "Never repeat the same instruction twice. If something didn't work, take a different angle.",
+  "If they're stuck or frustrated, acknowledge it honestly and offer real choices — try another way, or hand off to a human.",
+  "Be concise and concrete. One clear next thing at a time.",
+];