@kaleidorg/mind 0.3.0 → 0.4.0

package/src/qvac/assistant.ts

@@ -0,0 +1,146 @@
+/**
+ * Hands-free voice-assistant loop — the transcribe → reason → speak cycle that
+ * QVAC's `transcribeStream()` makes possible, lifted into shared code so mobile
+ * and desktop run the same orchestration.
+ *
+ * The host owns the I/O: it opens the SDK session (`transcribeStream` with
+ * `DEFAULT_VOICE_STREAM_PARAMS`), feeds mic audio via `session.write()`, and
+ * supplies `respond` (LLM/funnel turn → reply text) + `speak` (synth + play).
+ * This loop does the parts that must be identical everywhere: filter Whisper's
+ * silence hallucinations, and gate the mic during playback so the assistant
+ * never transcribes its own voice (QVAC's reference uses a mic-gate, not
+ * barge-in — we mirror that).
+ */
+
+/** A transcript event from a `transcribeStream` conversation session. */
+export interface VoiceTranscriptEvent {
+  type: string;
+  /** Present on `text` events — a committed utterance. */
+  text?: string;
+}
+
+/** The host's transcription session (the SDK's conversation session fits). */
+export type VoiceAssistantSession = AsyncIterable<VoiceTranscriptEvent>;
+
+export type VoiceAssistantState = 'listening' | 'thinking' | 'speaking';
+
+export interface VoiceAssistantHandlers {
+  /** Produce an assistant reply for a user utterance (wraps the LLM/funnel). */
+  respond: (transcript: string) => Promise<string>;
+  /** Speak the reply: synth + playback. Resolves when playback finishes. */
+  speak: (text: string) => Promise<void>;
+  /**
+   * Gate mic capture so the assistant doesn't hear itself. The host should drop
+   * (not buffer) audio while gated. Called `true` before speaking, `false` after
+   * the post-playback cooldown.
+   */
+  setMicGated?: (gated: boolean) => void;
+  /** A user utterance passed the filter and is about to be handled. */
+  onUserText?: (text: string) => void;
+  /** The assistant's reply, before it is spoken. */
+  onReply?: (text: string) => void;
+  /** UI state transitions. */
+  onState?: (state: VoiceAssistantState) => void;
+}
+
+export interface VoiceAssistantOptions {
+  /** Minimum utterance length to handle (drops "you", ".", etc.). Default 3. */
+  minChars?: number;
+  /** Utterances to ignore (case-insensitive, trailing punctuation stripped). */
+  ignoredUtterances?: Iterable<string>;
+  /** Pause after playback so speaker reverb settles before listening. Default 300ms. */
+  postPlaybackCooldownMs?: number;
+  /** Injected for tests; defaults to setTimeout. */
+  sleep?: (ms: number) => Promise<void>;
+  /** Stop the loop early. */
+  signal?: AbortSignal;
+}
+
+/**
+ * Whisper frequently hallucinates these from silence — drop them so the
+ * assistant doesn't answer phantom turns. (QVAC docs cite "you", ".", "Thanks.")
+ */
+export const DEFAULT_IGNORED_UTTERANCES: readonly string[] = [
+  'you', 'thank you', 'thanks', 'bye', 'okay', '.',
+];
+
+/**
+ * Should this utterance be handled? False for too-short text or a known Whisper
+ * hallucination. Pure + exported so it's directly testable.
+ */
+export function shouldHandleUtterance(
+  text: string,
+  options: { minChars?: number; ignoredUtterances?: Iterable<string> } = {},
+): boolean {
+  const trimmed = text.trim();
+  if (trimmed.length < (options.minChars ?? 3)) return false;
+  const norm = trimmed.toLowerCase().replace(/[.!?,]+$/, '').trim();
+  if (!norm) return false;
+  const ignored = new Set(
+    [...(options.ignoredUtterances ?? DEFAULT_IGNORED_UTTERANCES)].map((s) => s.toLowerCase()),
+  );
+  return !ignored.has(norm);
+}
+
+/**
+ * Run the hands-free loop until the session ends or `signal` aborts. Only `text`
+ * events drive a turn; `vad`/`segment`/`endOfTurn` events are ignored here (the
+ * host can read them off the session separately for UI). Always leaves the mic
+ * un-gated on exit.
+ */
+export async function runVoiceAssistant(
+  session: VoiceAssistantSession,
+  handlers: VoiceAssistantHandlers,
+  options: VoiceAssistantOptions = {},
+): Promise<void> {
+  const sleep = options.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
+  const cooldown = options.postPlaybackCooldownMs ?? 300;
+  let speaking = false;
+
+  handlers.onState?.('listening');
+  try {
+    for await (const event of session) {
+      if (options.signal?.aborted) break;
+      if (event.type !== 'text' || typeof event.text !== 'string') continue;
+      // Defensive: ignore anything heard mid-playback (host also gates the mic).
+      if (speaking) continue;
+
+      const transcript = event.text.trim();
+      if (!shouldHandleUtterance(transcript, options)) continue;
+
+      handlers.onUserText?.(transcript);
+      handlers.onState?.('thinking');
+
+      let reply: string;
+      try {
+        reply = await handlers.respond(transcript);
+      } catch {
+        handlers.onState?.('listening');
+        continue;
+      }
+      if (options.signal?.aborted) break;
+      if (!reply || !reply.trim()) {
+        handlers.onState?.('listening');
+        continue;
+      }
+
+      handlers.onReply?.(reply);
+      speaking = true;
+      handlers.setMicGated?.(true);
+      handlers.onState?.('speaking');
+      try {
+        await handlers.speak(reply);
+      } catch {
+        /* keep the loop alive on a playback error */
+      } finally {
+        await sleep(cooldown);
+        speaking = false;
+        handlers.setMicGated?.(false);
+        handlers.onState?.('listening');
+      }
+    }
+  } finally {
+    // Never leave the mic gated if the loop exits mid-turn.
+    handlers.setMicGated?.(false);
+  }
+}

package/src/qvac/config.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect } from 'vitest';
+import {
+  LOCAL_LLM_CONFIG,
+  LOCAL_LLM_CONFIG_GPU,
+  DELEGATE_LLM_CONFIG,
+  TTS_SAMPLE_RATE,
+  normalizeWhisperLang,
+} from './config.js';
+
+describe('model configs', () => {
+  it('CPU baseline runs on cpu with tools enabled', () => {
+    expect(LOCAL_LLM_CONFIG.device).toBe('cpu');
+    expect(LOCAL_LLM_CONFIG.tools).toBe(true);
+  });
+
+  it('GPU config offloads layers and grows the context', () => {
+    expect(LOCAL_LLM_CONFIG_GPU.device).toBe('gpu');
+    expect(LOCAL_LLM_CONFIG_GPU.gpu_layers).toBe(99);
+    expect(LOCAL_LLM_CONFIG_GPU.ctx_size).toBeGreaterThan(LOCAL_LLM_CONFIG.ctx_size);
+  });
+
+  it('delegate config gives the desktop the largest context', () => {
+    expect(DELEGATE_LLM_CONFIG.ctx_size).toBe(16384);
+    expect(DELEGATE_LLM_CONFIG.device).toBe('gpu');
+  });
+
+  it('TTS sample rate matches SUPERTONIC-2 output', () => {
+    expect(TTS_SAMPLE_RATE).toBe(44100);
+  });
+});
+
+describe('normalizeWhisperLang', () => {
+  it('extracts a supported 2-letter code from a locale', () => {
+    expect(normalizeWhisperLang('it-IT')).toBe('it');
+    expect(normalizeWhisperLang('en_US')).toBe('en');
+  });
+
+  it('falls back to en for unsupported or missing locales', () => {
+    expect(normalizeWhisperLang('xx-YY')).toBe('en');
+    expect(normalizeWhisperLang('')).toBe('en');
+    expect(normalizeWhisperLang(null)).toBe('en');
+    expect(normalizeWhisperLang(undefined)).toBe('en');
+  });
+});

package/src/qvac/config.ts

@@ -0,0 +1,76 @@
+/**
+ * QVAC model-load configs and constants, shared across every host. These are
+ * plain data (no SDK import) so they stay portable and testable; callers merge
+ * in SDK-specific bits like `verbosity: VERBOSITY.ERROR` at load time.
+ */
+
+/**
+ * CPU baseline for the local llamacpp model. Used as the GPU fallback and as the
+ * base the GPU attempt overrides (device + gpu_layers).
+ */
+export const LOCAL_LLM_CONFIG = {
+  device: 'cpu',
+  gpu_layers: 0,
+  ctx_size: 2048,
+  tools: true,
+} as const;
+
+/**
+ * GPU (Metal on iPhone) offload — far faster than CPU when llamacpp can init the
+ * Metal context in the worklet. Fall back to {@link LOCAL_LLM_CONFIG} if the GPU
+ * load throws. ctx 4096 fits the agentic prompt (system + tools + skills + a
+ * little history); 2048 overflowed immediately ("prompt exceeds context").
+ */
+export const LOCAL_LLM_CONFIG_GPU = {
+  ...LOCAL_LLM_CONFIG,
+  device: 'gpu',
+  gpu_layers: 99, // offload all layers; llamacpp clamps to the model's count
+  ctx_size: 4096,
+} as const;
+
+/**
+ * Delegated to a desktop provider — it has the RAM to run a big context, so give
+ * the agentic prompt plenty of room (Qwen3-600M supports up to 32k). 2048
+ * overflowed with the system prompt + tool/skill definitions alone.
+ */
+export const DELEGATE_LLM_CONFIG = {
+  ...LOCAL_LLM_CONFIG_GPU,
+  ctx_size: 16384,
+} as const;
+
+/** SUPERTONIC-2 TTS output sample rate (Hz). Used to build the WAV for playback. */
+export const TTS_SAMPLE_RATE = 44100;
+
+/**
+ * Default params for a hands-free `transcribeStream()` voice session (Whisper).
+ * `emitVadEvents` turns the session into a conversation stream (text + vad +
+ * endOfTurn events); `endOfTurnSilenceMs` is how long a pause must last before
+ * an utterance is committed — conservative so it doesn't cut speakers off mid
+ * sentence or trigger on TTS reverb. Hosts merge in `modelId` + spread these.
+ */
+export const DEFAULT_VOICE_STREAM_PARAMS = {
+  emitVadEvents: true,
+  endOfTurnSilenceMs: 700,
+} as const;
+
+/**
+ * Whisper languages we request directly from the device locale. whisper.cpp
+ * supports more, but the QVAC handler rejects "auto"/detect_language for these
+ * tiny models, so we pass a concrete code (and fall back to 'en').
+ */
+export const WHISPER_LANGS: ReadonlySet<string> = new Set([
+  'en', 'it', 'es', 'fr', 'de', 'pt', 'nl', 'ru', 'pl', 'uk', 'tr', 'ar',
+  'zh', 'ja', 'ko', 'hi', 'id', 'sv', 'no', 'da', 'fi', 'cs', 'ro', 'el',
+  'he', 'th', 'vi', 'hu', 'ca',
+]);
+
+/**
+ * Best-effort 2-letter Whisper language code from an OS locale string
+ * (e.g. "it-IT" → "it"), restricted to codes Whisper handles well. Falls back to
+ * 'en'. Pure: the host reads the locale (NativeModules etc.) and passes it here.
+ */
+export function normalizeWhisperLang(locale: string | null | undefined): string {
+  if (!locale) return 'en';
+  const code = String(locale).split(/[-_]/)[0]?.toLowerCase() ?? 'en';
+  return WHISPER_LANGS.has(code) ? code : 'en';
+}

package/src/qvac/delegate.test.ts

@@ -0,0 +1,68 @@
+import { describe, it, expect } from 'vitest';
+import {
+  allowListFirewall,
+  denyListFirewall,
+  firewallFromKeyList,
+  buildDelegateConfig,
+} from './delegate.js';
+
+describe('allowListFirewall', () => {
+  it('builds an allow-list, trimming + de-duping keys', () => {
+    expect(allowListFirewall([' k1 ', 'k2', 'k1', ''])).toEqual({
+      mode: 'allow',
+      publicKeys: ['k1', 'k2'],
+    });
+  });
+
+  it('is empty for no keys (caller must decide: open vs refuse)', () => {
+    expect(allowListFirewall([])).toEqual({ mode: 'allow', publicKeys: [] });
+  });
+});
+
+describe('denyListFirewall', () => {
+  it('builds a deny-list', () => {
+    expect(denyListFirewall(['bad'])).toEqual({ mode: 'deny', publicKeys: ['bad'] });
+  });
+});
+
+describe('firewallFromKeyList', () => {
+  it('parses comma/space/newline-separated keys into an allow-list', () => {
+    expect(firewallFromKeyList('k1, k2\nk3 k4')).toEqual({
+      mode: 'allow',
+      publicKeys: ['k1', 'k2', 'k3', 'k4'],
+    });
+  });
+
+  it('returns undefined for empty/missing input (advertise openly)', () => {
+    expect(firewallFromKeyList('')).toBeUndefined();
+    expect(firewallFromKeyList('   ')).toBeUndefined();
+    expect(firewallFromKeyList(null)).toBeUndefined();
+    expect(firewallFromKeyList(undefined)).toBeUndefined();
+  });
+});
+
+describe('buildDelegateConfig', () => {
+  it('defaults fallbackToLocal to false and trims the key', () => {
+    expect(buildDelegateConfig('  pk  ')).toEqual({
+      providerPublicKey: 'pk',
+      fallbackToLocal: false,
+    });
+  });
+
+  it('passes through fallbackToLocal, timeout, forceNewConnection when set', () => {
+    expect(
+      buildDelegateConfig('pk', { fallbackToLocal: true, timeout: 60000, forceNewConnection: true }),
+    ).toEqual({
+      providerPublicKey: 'pk',
+      fallbackToLocal: true,
+      timeout: 60000,
+      forceNewConnection: true,
+    });
+  });
+
+  it('omits optional fields that are not set', () => {
+    const cfg = buildDelegateConfig('pk', { fallbackToLocal: false });
+    expect('timeout' in cfg).toBe(false);
+    expect('forceNewConnection' in cfg).toBe(false);
+  });
+});

package/src/qvac/delegate.ts

@@ -0,0 +1,71 @@
+/**
+ * Delegation helpers — the provider firewall (who may connect) and the
+ * consumer-side delegate config. Pure data builders (no `@qvac/sdk` import) so
+ * they stay shared + testable; the host passes the result to
+ * `startQVACProvider({ firewall })` / `loadModel({ delegate })`.
+ *
+ * Security note: a QVAC provider is reachable by anyone who learns its
+ * Hyperswarm public key. Advertising with no firewall means any such peer can
+ * run inference on your machine. Use {@link allowListFirewall} so a desktop
+ * provider serves ONLY its paired phone(s).
+ */
+
+/** Firewall for `startQVACProvider` — restrict who may delegate to this provider. */
+export interface ProviderFirewall {
+  mode: 'allow' | 'deny';
+  publicKeys: string[];
+}
+
+function normalizeKeys(keys: Iterable<string>): string[] {
+  return [...new Set([...keys].map((k) => k.trim()).filter(Boolean))];
+}
+
+/**
+ * Allow ONLY these consumer public keys to delegate (zero-trust). Pass the
+ * paired phone(s)' public keys so no one else can use the desktop brain even if
+ * they learn its public key.
+ */
+export function allowListFirewall(consumerPublicKeys: Iterable<string>): ProviderFirewall {
+  return { mode: 'allow', publicKeys: normalizeKeys(consumerPublicKeys) };
+}
+
+/** Deny these consumer public keys; everyone else may connect. */
+export function denyListFirewall(consumerPublicKeys: Iterable<string>): ProviderFirewall {
+  return { mode: 'deny', publicKeys: normalizeKeys(consumerPublicKeys) };
+}
+
+/**
+ * Parse a comma/space/newline-separated key list (e.g. from an env var or a
+ * pairing store) into an allow-list firewall, or `undefined` when none are
+ * configured — the caller then advertises openly and should warn.
+ */
+export function firewallFromKeyList(raw: string | null | undefined): ProviderFirewall | undefined {
+  if (!raw) return undefined;
+  const keys = raw.split(/[\s,]+/).map((k) => k.trim()).filter(Boolean);
+  return keys.length ? allowListFirewall(keys) : undefined;
+}
+
+/** Consumer-side config for `loadModel({ delegate })`. */
+export interface DelegateConfig {
+  providerPublicKey: string;
+  fallbackToLocal: boolean;
+  timeout?: number;
+  forceNewConnection?: boolean;
+}
+
+/**
+ * Build the `delegate` config for a delegated `loadModel`. `fallbackToLocal`
+ * defaults to false (the host owns recovery), matching rate's existing
+ * LLM/Whisper/TTS delegated loads.
+ */
+export function buildDelegateConfig(
+  providerPublicKey: string,
+  opts: { fallbackToLocal?: boolean; timeout?: number; forceNewConnection?: boolean } = {},
+): DelegateConfig {
+  return {
+    providerPublicKey: providerPublicKey.trim(),
+    fallbackToLocal: opts.fallbackToLocal ?? false,
+    ...(opts.timeout != null ? { timeout: opts.timeout } : {}),
+    ...(opts.forceNewConnection != null ? { forceNewConnection: opts.forceNewConnection } : {}),
+  };
+}

package/src/qvac/index.ts

@@ -0,0 +1,72 @@
+/**
+ * @kaleidorg/mind-qvac — the single home for all @qvac/sdk logic behind
+ * @kaleidorg/mind. Hosts (rate mobile, desktop provider, cli) supply @qvac/sdk
+ * as a peer dependency; this package owns the orchestration so the logic lives
+ * in one place instead of drifting copies per host.
+ *
+ * This first slice exports the platform-agnostic core (pure text helpers, model
+ * configs, completion parsing). The QVAC-calling provider/voice/host wrappers
+ * land next, on top of these.
+ */
+export {
+  cleanAssistantVisibleText,
+  sanitizeForSupertonic,
+} from './text.js';
+
+export {
+  LOCAL_LLM_CONFIG,
+  LOCAL_LLM_CONFIG_GPU,
+  DELEGATE_LLM_CONFIG,
+  TTS_SAMPLE_RATE,
+  DEFAULT_VOICE_STREAM_PARAMS,
+  WHISPER_LANGS,
+  normalizeWhisperLang,
+} from './config.js';
+
+export {
+  finalToTurn,
+  type QvacFinalLike,
+  type ParsedTurn,
+} from './parse.js';
+
+export {
+  consumeRun,
+  type CompletionEventLike,
+  type CompletionRunLike,
+  type StreamHandlers,
+  type ConsumedTurn,
+} from './stream.js';
+
+export {
+  createQvacProvider,
+  type QvacProviderOptions,
+  type QvacTurnInput,
+} from './provider.js';
+
+export {
+  createQvacVoice,
+  type QvacVoice,
+  type QvacVoiceOptions,
+  type VoiceSession,
+  type PcmAudio,
+} from './voice.js';
+
+export {
+  runVoiceAssistant,
+  shouldHandleUtterance,
+  DEFAULT_IGNORED_UTTERANCES,
+  type VoiceAssistantSession,
+  type VoiceAssistantHandlers,
+  type VoiceAssistantOptions,
+  type VoiceAssistantState,
+  type VoiceTranscriptEvent,
+} from './assistant.js';
+
+export {
+  allowListFirewall,
+  denyListFirewall,
+  firewallFromKeyList,
+  buildDelegateConfig,
+  type ProviderFirewall,
+  type DelegateConfig,
+} from './delegate.js';

package/src/qvac/parse.test.ts

@@ -0,0 +1,52 @@
+import { describe, it, expect } from 'vitest';
+import { finalToTurn } from './parse.js';
+
+describe('finalToTurn', () => {
+  it('uses contentText for visible text and strips reasoning', () => {
+    const out = finalToTurn({ contentText: '<think>x</think>Hello' });
+    expect(out.text).toBe('Hello');
+  });
+
+  it('falls back to the streamed text when contentText is empty', () => {
+    const out = finalToTurn({ contentText: '' }, 'streamed answer');
+    expect(out.text).toBe('streamed answer');
+  });
+
+  it('prefers raw.fullText for rawContent (history push-back)', () => {
+    const out = finalToTurn({ contentText: 'Hi', raw: { fullText: 'FRAMED<tool/>Hi' } });
+    expect(out.rawContent).toBe('FRAMED<tool/>Hi');
+    expect(out.text).toBe('Hi');
+  });
+
+  it('falls back to the raw text for rawContent when no framed form', () => {
+    const out = finalToTurn({ contentText: 'Hi' });
+    expect(out.rawContent).toBe('Hi');
+  });
+
+  it('maps tool calls and defaults missing arguments to {}', () => {
+    const out = finalToTurn({
+      contentText: '',
+      toolCalls: [{ id: 'a', name: 'get_balance' }, { name: 'send', arguments: { sats: 5000 } }],
+    });
+    expect(out.toolCalls).toEqual([
+      { id: 'a', name: 'get_balance', arguments: {} },
+      { id: undefined, name: 'send', arguments: { sats: 5000 } },
+    ]);
+  });
+
+  it('flags truncation when the SDK stops on length', () => {
+    const out = finalToTurn({ contentText: 'partial', stopReason: 'length' });
+    expect(out.truncated).toBe(true);
+    expect(out.stopReason).toBe('length');
+  });
+
+  it('does not flag truncation on a natural stop', () => {
+    const out = finalToTurn({ contentText: 'done' });
+    expect(out.truncated).toBe(false);
+  });
+
+  it('handles an empty final without throwing', () => {
+    const out = finalToTurn({});
+    expect(out).toEqual({ text: '', rawContent: '', toolCalls: [], truncated: false, stopReason: undefined });
+  });
+});

package/src/qvac/parse.ts

@@ -0,0 +1,57 @@
+/**
+ * Pure mapping from a QVAC completion `final` frame to the shape the shared
+ * @kaleidorg/mind Engine consumes. Kept SDK-free (structural input type) so it
+ * is testable without loading a model, and so the same mapping runs on mobile,
+ * desktop, and the eval harness.
+ */
+import { cleanAssistantVisibleText } from './text.js';
+
+/** Structural subset of a QVAC `completion().final` we depend on. */
+export interface QvacFinalLike {
+  /** Visible assistant text (excludes `<think>` reasoning). */
+  contentText?: string;
+  /** Raw assistant frame, incl. tool-call framing, for history push-back. */
+  raw?: { fullText?: string };
+  /** Tool calls the model requested this turn (empty ⇒ final answer). */
+  toolCalls?: Array<{ id?: string; name: string; arguments?: Record<string, unknown> }>;
+  /**
+   * Why generation stopped. QVAC 0.13 emits `"length"` when the token budget is
+   * exhausted, `"cancelled"` on abort, `undefined` on a natural stop. We surface
+   * it so the funnel can tell a truncated tool-call from a complete one.
+   */
+  stopReason?: 'length' | 'cancelled' | string;
+}
+
+export interface ParsedTurn {
+  /** Cleaned assistant content for display. */
+  text: string;
+  /** Raw assistant frame to push back into history for the next turn. */
+  rawContent: string;
+  /** Tool calls the model requested (arguments defaulted to `{}`). */
+  toolCalls: Array<{ id?: string; name: string; arguments: Record<string, unknown> }>;
+  /** True when generation was cut off by the token budget (incomplete output). */
+  truncated: boolean;
+  /** Raw stop reason from the SDK, when provided. */
+  stopReason?: string;
+}
+
+/**
+ * Map a completion `final` (plus the streamed fallback text) into a ParsedTurn.
+ * `rawContent` prefers the SDK's framed `raw.fullText` so the Engine can anchor
+ * the next turn; falls back to the visible text when a provider has no raw form.
+ */
+export function finalToTurn(final: QvacFinalLike, streamed = ''): ParsedTurn {
+  const rawText = final.contentText || streamed;
+  const text = cleanAssistantVisibleText(rawText);
+  return {
+    text,
+    rawContent: final.raw?.fullText ?? rawText,
+    toolCalls: (final.toolCalls ?? []).map((c) => ({
+      id: c.id,
+      name: c.name,
+      arguments: c.arguments ?? {},
+    })),
+    truncated: final.stopReason === 'length',
+    stopReason: final.stopReason,
+  };
+}