@urun-sh/sense 0.0.0-stage → 0.1.0

package/examples/cheese_demo.ts

@@ -0,0 +1,250 @@
+#!/usr/bin/env bun
+/**
+ * 🧀 Sense SDK cheese classifier — browser + mic (Bun server).
+ *
+ *   git clone https://github.com/urun-sh/sense-sdk.git
+ *   cd sense-sdk/typescript
+ *   bun install && bun run build
+ *   SENSE_API_KEY="<urun-key>" bun examples/cheese_demo.ts
+*/
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { homedir, networkInterfaces } from "node:os";
+import { join } from "node:path";
+import { Sense, type Session, type Wave } from "@urun-sh/sense";
+
+// -- Sense: define the questions, relay audio, read decisions -----------------
+
+const app = new Sense(); // apiKey / baseUrl default from SENSE_API_KEY / SENSE_BASE_URL
+
+app.noul("Is someone speaking right now?", { id: "speech" });
+app.noul("Is the speaker talking about cheese right now?", { id: "cheese" });
+app.choice(
+  "Which type of cheese are they talking about?",
+  ["cheddar", "brie", "gouda", "parmesan", "camembert", "mozzarella", "blue", "feta", "other"],
+  { id: "cheese_type" },
+);
+app.score("How does the speaker feel about that cheese?", ["hate", "dislike", "meh", "like", "love"], { id: "feeling" });
+
+/** A wave is a batch of decisions, not an audio response. */
+export type Verdict =
+  | { waveId: number; speaking: boolean; talking: false }
+  | { waveId: number; speaking: boolean; talking: true; kind: string; feeling: string; score: number; p: number };
+
+export function toVerdict(wave: Wave): Verdict {
+  const speaking = Boolean(wave.result("speech").value);
+  const detection = wave.result("cheese");
+  if (detection.p < 0.6) return { waveId: wave.waveId, speaking, talking: false };
+  const kind = wave.result("cheese_type");
+  const feeling = wave.result("feeling");
+  return {
+    waveId: wave.waveId,
+    speaking,
+    talking: true,
+    kind: kind.pOption(String(kind.value)) >= 0.6 ? String(kind.value) : "unknown type",
+    feeling: feeling.p >= 0.6 ? feeling.top : "meh",
+    score: Number(feeling.value),
+    p: detection.p,
+  };
+}
+
+// -- relay: browser WebSocket ⇄ SDK session -----------------------------------
+
+const CAP_MS = 300_000; // 5-minute session cap
+const MAX_PENDING_BYTES = 1_000_000; // ~20 s of mic audio; the dial takes seconds normally
+const MAX_RELAYS = 4; // the server spends its own API budget on LAN clients
+let activeRelays = 0;
+
+/** Relay state for one browser socket. Exported for tests. */
+export type Relay = {
+  session: Session | null;
+  send: (event: unknown) => void;
+  capTimer: ReturnType<typeof setTimeout> | null;
+  done: boolean;
+  closed: boolean;
+  pending: (Uint8Array | ArrayBuffer)[]; // mic frames buffered until the dial completes
+  streaming: boolean; // true once the run loop has started
+  wake: (() => void) | null;
+  end: () => void; // assigned by the ws handler; tears down the browser socket too
+};
+
+/** First close wins; returns false when the relay was already closed. */
+export function closeRelay(relay: Relay): boolean {
+  if (relay.closed) return false;
+  relay.closed = true;
+  if (relay.capTimer !== null) {
+    clearTimeout(relay.capTimer);
+    relay.capTimer = null;
+  }
+  relay.wake?.();
+  relay.session?.close();
+  return true;
+}
+
+export function handleAudio(relay: Relay, frame: Uint8Array | ArrayBuffer): void {
+  if (relay.closed) return;
+  if (relay.session && !relay.streaming) {
+    let total = frame.byteLength;
+    for (const f of relay.pending) total += f.byteLength;
+    if (total > MAX_PENDING_BYTES) {
+      // A stuck dial would grow this buffer unboundedly; give up instead.
+      relay.pending.length = 0;
+      relay.send({ type: "error", message: "upstream connection too slow — audio buffer overflow" });
+      relay.end();
+      return;
+    }
+    const first = relay.pending.length === 0;
+    relay.pending.push(frame);
+    if (first) relay.wake?.();
+    return;
+  }
+  if (relay.session) relay.session.audio.push(frame); // PCM16, mono, 24 kHz; one 40 ms block
+}
+
+export async function pump(relay: Relay): Promise<void> {
+  const session = relay.session;
+  if (!session) return;
+  try {
+    await session.connect();
+    if (relay.closed) {
+      // The client left during the dial; shut the session down instead of lingering.
+      session.close();
+      return;
+    }
+    // The turn opens on the first pushed audio frame, so wait for mic audio
+    // before starting the loop.
+    if (relay.pending.length === 0) await new Promise<void>((resolve) => (relay.wake = resolve));
+    if (relay.closed) {
+      session.close();
+      return;
+    }
+    relay.streaming = true;
+    for (const frame of relay.pending.splice(0)) session.audio.push(frame);
+    for await (const wave of session.run()) {
+      relay.send({ type: "wave", verdict: toVerdict(wave) });
+    }
+    if (relay.done) return; // cap path already reported stats and ended
+    relay.send({ type: "stats", stats: session.stats });
+    relay.send({ type: "ended", reason: "closed" });
+  } catch (err) {
+    relay.send({ type: "error", message: err instanceof Error ? err.message : String(err) });
+  }
+}
+
+function openRelay(send: (event: unknown) => void): Relay {
+  const relay: Relay = {
+    session: app.apiKey ? app.session() : null,
+    send,
+    capTimer: null,
+    done: false,
+    closed: false,
+    pending: [],
+    streaming: false,
+    wake: null,
+    end: () => {},
+  };
+  relay.send({ type: "hello", mode: relay.session ? "online" : "mic-only" });
+  if (relay.session) {
+    relay.capTimer = setTimeout(() => {
+      relay.done = true;
+      relay.send({ type: "stats", stats: relay.session!.stats });
+      relay.send({ type: "ended", reason: "cap" });
+      relay.end();
+    }, CAP_MS);
+    void pump(relay);
+  }
+  return relay;
+}
+
+// -- server: the demo page and the browser relay -------------------------------
+
+type WsData = { relay: Relay | null };
+
+function generateCert(): { cert: Buffer; key: Buffer } {
+  const dir = join(homedir(), ".cache", "sense-cheese-demo");
+  const certPath = join(dir, "cert.pem");
+  const keyPath = join(dir, "key.pem");
+  if (!existsSync(certPath) || !existsSync(keyPath)) {
+    mkdirSync(dir, { recursive: true });
+    const generated = spawnSync(
+      "openssl",
+      [
+        "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1",
+        "-nodes", "-keyout", keyPath, "-out", certPath, "-days", "365", "-subj", "/CN=cheese-demo",
+      ],
+      { stdio: "pipe" },
+    );
+    if (generated.error) throw new Error("openssl is required to generate the HTTPS certificate");
+    if (generated.status !== 0) {
+      throw new Error(`openssl certificate generation failed: ${(generated.stderr?.toString() ?? "").trim()}`);
+    }
+  }
+  return { cert: readFileSync(certPath), key: readFileSync(keyPath) };
+}
+
+function lanUrls(port: number): string[] {
+  const urls = [`https://localhost:${port}`];
+  for (const addrs of Object.values(networkInterfaces())) {
+    for (const addr of addrs ?? []) {
+      if (addr.family === "IPv4" && !addr.internal) urls.push(`https://${addr.address}:${port}`);
+    }
+  }
+  return urls;
+}
+
+if (import.meta.main) {
+  const port = Number(process.env.PORT) || 8080;
+  const { cert, key } = generateCert();
+  const page = readFileSync(join(import.meta.dir, "cheese_demo.html"));
+  console.log(`🧀 cheese demo (mode: ${app.apiKey ? "online" : "mic-only — set SENSE_API_KEY to classify"})`);
+  for (const url of lanUrls(port)) console.log(`  ${url}`);
+  console.log("  accept the self-signed certificate warning; HTTPS is required for phone mic access");
+  Bun.serve<WsData>({
+    port,
+    hostname: "0.0.0.0",
+    tls: { cert, key },
+    fetch: (req, server) => {
+      const { pathname } = new URL(req.url);
+      if (pathname === "/ws") {
+        // Browsers cannot set the bearer header Sense needs, so the server
+        // relays the browser's audio and events.
+        if (server.upgrade(req, { data: { relay: null } })) return undefined;
+        return new Response("upgrade failed", { status: 500 });
+      }
+      if (pathname === "/") return new Response(page, { headers: { "content-type": "text/html; charset=utf-8" } });
+      return new Response("not found", { status: 404 });
+    },
+    websocket: {
+      open(ws) {
+        if (activeRelays >= MAX_RELAYS) {
+          try {
+            ws.send(JSON.stringify({ type: "error", message: `relay busy: ${MAX_RELAYS} concurrent sessions max` }));
+          } catch {
+            /* the client is gone */
+          }
+          ws.close();
+          return;
+        }
+        activeRelays++;
+        const relay = openRelay((event) => {
+          try {
+            ws.send(JSON.stringify(event));
+          } catch {
+            /* the client is gone; the pump's remaining sends are best-effort */
+          }
+        });
+        relay.end = () => {
+          if (closeRelay(relay)) activeRelays--;
+          ws.close();
+        };
+        ws.data.relay = relay;
+      },
+      message(ws, message) {
+        if (ws.data.relay && typeof message !== "string") handleAudio(ws.data.relay, message);
+      },
+      close(ws) {
+        if (ws.data.relay && closeRelay(ws.data.relay)) activeRelays--;
+      },
+    },
+  });
+}

package/examples/voice_agent.ts

@@ -0,0 +1,166 @@
+/**
+ * Flagship full-duplex voice agent on @urun-sh/sense (TypeScript).
+ *
+ * Sense decides WHEN to respond, WHICH tool, and WHEN to interrupt; your code
+ * executes. Reads a wav (mic capture), streams it to the shared Sense
+ * endpoint, and reacts on edge-triggered predicates — including a barge-in
+ * that cancels the in-flight reply via ctx.cancel().
+ *
+ * Usage (run `bun install && bun run build` first — the demo imports the
+ * built package and the build stages the bundled sample wav):
+ *   SENSE_API_KEY=<key> bun examples/voice_agent.ts
+ */
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { Score, Sense, type Context, type Wave } from "@urun-sh/sense";
+
+// -- questions ----------------------------------------------------------------
+
+const app = new Sense({}); // apiKey/baseUrl default from SENSE_API_KEY / SENSE_BASE_URL
+const finished = app.noul("Did the current speaker just finish their turn?", { id: "finished" });
+const userSpeaking = app.noul("Is a human speaking right now?", { id: "user_speaking" });
+const intent = app.choice("What does the user want?", ["support", "sales", "other"], { id: "intent" });
+const actions = app.choice("What should the agent do?", ["respond", "ignore", "wait"], { id: "actions" });
+app.score("How urgent is the request?", ["low", "medium", "high"], { id: "urgency" });
+const tool = app.choice("Should a tool run?", ["none", "weather", "search", "timer"], { id: "tool" });
+
+// The most recent trigger context — barge-in cancels tasks through it.
+let lastEvent: Context | null = null;
+
+// -- reactions (edge-triggered: fire on first true, then each false->true) ----
+
+app.when(
+  finished
+    .and(intent.eq("support"))
+    .and(actions.top.eq("respond")),
+  (event) => {
+    lastEvent = event;
+    const wave = event.latest()!;
+    console.log(`[respond] wave ${wave.waveId} t=${wave.tMs}ms urgency=${(wave.result("urgency") as Score).value.toFixed(2)}`);
+    // A tracked cancellable run: duplicate active keys reject; the barge-in
+    // watcher cancels it.
+    event
+      .run("reply", (ctx) => reply(wave, ctx.signal), {
+        onCancel: () => console.log("[respond] reply cancelled"),
+      })
+      .catch((err: unknown) => {
+        if ((err as Error).message?.includes("cancelled")) return;
+        throw err;
+      });
+  },
+);
+
+app.when(finished.and(intent.p("support").gte(0.7)), (event) => {
+  const wave = event.latest()!;
+  console.log(`[sales-handoff] wave ${wave.waveId}: P(support)=${wave.result("intent").pOption("support").toFixed(2)}`);
+});
+
+app.when(tool.ne("none"), (event) => {
+  const wave = event.latest()!;
+  console.log(`[tool] wave ${wave.waveId}: invoking ${wave.result("tool").value}`);
+});
+
+// Turn boundary: the speaker finished AND went quiet — the moment a voice
+// agent would hand the floor to its reply path.
+app.when(finished.and(userSpeaking.not()), (event) => {
+  const wave = event.latest()!;
+  console.log(`[turn-boundary] wave ${wave.waveId} t=${wave.tMs}ms — speaker finished and quiet`);
+});
+
+// Probability predicate with hysteresis: true at P(finished) >= 0.8, resets
+// only below 0.6 — thresholds near the boundary flap and edge triggers re-fire.
+app.when(finished.p.between(0.8, 0.6), (event) => {
+  console.log(`[settled] wave ${event.latest()!.waveId}: speaker settled`);
+});
+
+app.onWave((event) => {
+  const w = event.latest()!;
+  if (w.result("actions").value === "ignore") return;
+  console.log(`[wave ${w.waveId}] t=${w.tMs}ms finished=${w.result("finished").value} intent=${w.result("intent").value}`);
+}, { coalesce: true });
+
+function delay(ms: number): Promise<void> {
+  const { promise, resolve } = Promise.withResolvers<void>();
+  setTimeout(resolve, ms);
+  return promise;
+}
+
+async function reply(wave: Wave, signal: AbortSignal | null): Promise<void> {
+  console.log(`[agent] replying to wave ${wave.waveId} (stub execution)`);
+  await delay(2000);
+  if (signal?.aborted) throw new Error(`reply for wave ${wave.waveId} cancelled`);
+  console.log(`[agent] reply for wave ${wave.waveId} complete`);
+}
+
+async function bargeInWatch(): Promise<void> {
+  for (;;) {
+    await delay(100);
+    if (!lastEvent) continue;
+    const spk = lastEvent.get("user_speaking");
+    if (spk && spk.value === true && spk.p >= 0.9) {
+      console.log("[barge-in] user resumed — cancelling the in-flight reply");
+      lastEvent.cancel("reply");
+      return;
+    }
+  }
+}
+
+// -- the session ---------------------------------------------------------------
+
+/** 24 kHz PCM16 mono audio frames, paced like a live mic (48 kHz decimated). */
+async function* audioFrames(path: string, chunkMs = 40): AsyncGenerator<{ audio: Uint8Array }> {
+  const buf = readFileSync(path);
+  // Minimal RIFF walk to the data chunk.
+  let off = 12;
+  let rate = 48000;
+  let data: Uint8Array | null = null;
+  while (off + 8 <= buf.length) {
+    const id = buf.subarray(off, off + 4).toString("latin1");
+    const size = buf.readUInt32LE(off + 4);
+    const body = buf.subarray(off + 8, off + 8 + size);
+    if (id === "fmt ") rate = body.readUInt32LE(4);
+    if (id === "data") data = body;
+    off += 8 + size + (size % 2);
+  }
+  if (!data) throw new Error("no data chunk");
+  const samples = new Int16Array(data.buffer, data.byteOffset, data.byteLength >> 1);
+  const decimated = rate === 48000 ? samples.filter((_, i) => i % 2 === 0) : samples;
+  const n = Math.floor((24000 * chunkMs) / 1000);
+  for (let i = 0; i + n <= decimated.length; i += n) {
+    yield { audio: new Uint8Array(decimated.buffer, decimated.byteOffset + i * 2, n * 2).slice() };
+    await delay(chunkMs);
+  }
+}
+
+async function main(): Promise<number> {
+  const path = process.argv[2] ?? fileURLToPath(new URL("./assets/mic-question-48k-mono.wav", import.meta.url));
+  const session = app.session();
+  // Text input alongside the audio (e.g. a transcript from your own ASR): the model
+  // hears the audio directly; text is extra context and exercises session.text.
+  session.text.push("Hi — my internet has been down since this morning and I need help.");
+  void bargeInWatch();
+  // Demo bound: a live agent runs forever; the example stops after 20s.
+  const done = Promise.withResolvers<void>();
+  const timer = setTimeout(() => done.resolve(), 30_000);
+  try {
+    const waves = session.run(audioFrames(path))[Symbol.asyncIterator]();
+    for (;;) {
+      const next = await Promise.race([waves.next(), done.promise.then(() => null)]);
+      if (next === null || next.done) break;
+      void next.value; // handlers react; the run loop assembles and yields waves
+    }
+  } finally {
+    clearTimeout(timer);
+    await session.close();
+  }
+  console.log(`stats: ${JSON.stringify(session.stats)}`);
+  return 0;
+}
+
+main().then(
+  (code) => process.exitCode = code,
+  (err: unknown) => {
+    console.error(err);
+    process.exitCode = 1;
+  },
+);

package/package.json

@@ -1,6 +1,57 @@
 {
   "name": "@urun-sh/sense",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "TypeScript SDK for the Sense realtime question stream",
+  "license": "Apache-2.0",
+  "author": "Urun Platform, Inc",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "examples",
+    "LICENSE"
+  ],
+  "bin": {
+    "sense-cheese-demo": "./examples/cheese_demo.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/urun-sh/sense-sdk.git",
+    "directory": "typescript"
+  },
+  "homepage": "https://github.com/urun-sh/sense-sdk/tree/main/typescript#readme",
+  "bugs": {
+    "url": "https://github.com/urun-sh/sense-sdk/issues"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && node scripts/fix-dts.mjs && node scripts/stage-assets.mjs",
+    "prepack": "npm run build",
+    "test": "bun test",
+    "conformance": "bun test ../conformance/typescript_runner.ts",
+    "lint": "oxlint",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@msgpack/msgpack": "^3.1.3",
+    "ws": "^8.22.1"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "@types/ws": "^8.18.2",
+    "jscpd": "^5.4.2",
+    "oxlint": "^1.91.0",
+    "typescript": "^5.9.0"
+  }
+}