@mlx-bun/module-chat 0.0.0-stage → 1.0.0

package/src/panel/template.ts

@@ -0,0 +1,189 @@
+// The panel's markup: the recent-chats sidebar, the conversation and composer, the adapter routing table's overlay and
+// the mobile drawer's backdrop. The ids are the panel's public surface for its host (the memory entry and consent
+// card in the sidebar, the composer's controls) and for the assistant, which snapshots them. `data-spotlight`
+// marks the controls the assistant can point at; `data-ui-chrome="content"` keeps the rendered conversation out of
+// that snapshot, since an `a[href]` inside a message is untrusted content, not app chrome.
+export const MARKUP = `
+    <aside id="chat-sidebar">
+      <button id="chat-new" data-spotlight="new-chat" title="Start a new chat (or type /new)">＋&nbsp; New chat</button>
+      <input type="search" id="chat-sess-search" data-spotlight="session-search" placeholder="Search chats…" aria-label="Search chats" autocomplete="off">
+      <div id="chat-sessions" class="sesslist"></div>
+      <button id="chat-memory-entry" data-spotlight="memory-entry" type="button" style="display:none"
+              aria-haspopup="dialog" aria-controls="mem-overlay" aria-expanded="false">
+        <span class="mi" aria-hidden="true">◆</span>
+        <span class="mt">Memory</span>
+        <span class="mcount" id="chat-memory-count">—</span>
+      </button>
+    </aside>
+    <div id="chat-main">
+    <div id="chat-scroll">
+      <div id="chat-hello">
+        <h1>Local. Private. <span class="grad g-hero">Yours.</span></h1>
+        <p id="chat-hello-sub">A full pi agent on this machine — local model, local tools, zero cloud. Every token below is generated here.</p>
+        <div class="chips" id="chat-hello-chips">
+          <button class="chip" data-q="What is mlx-bun and what can it do?">What is mlx-bun?</button>
+          <button class="chip" data-q="How do I load a bigger or different model?">Load a bigger model</button>
+          <button class="chip" data-q="Is this really private? Where does my data go?">Is this private?</button>
+          <button class="chip" data-q="What can you help me with right now?">What can you do?</button>
+        </div>
+        <div id="chat-consent" role="group" aria-label="Memory setup">
+          <span class="cc-icon" aria-hidden="true">◆</span>
+          <span class="cc-text">Let mlx-bun remember what matters? Creates a local, git-tracked vault you can open anytime.</span>
+          <span class="cc-actions">
+            <button class="btn ghost sm" id="chat-consent-skip" type="button">Skip</button>
+            <button class="btn primary sm" id="chat-consent-yes" type="button">Set up memory</button>
+          </span>
+        </div>
+      </div>
+      <div id="chat-thread" role="log" aria-live="polite" aria-atomic="false" data-ui-chrome="content"></div>
+    </div>
+    <div id="chat-foot">
+      <div class="queuebar" id="chat-queue"></div>
+      <div class="composer">
+        <div id="chat-attach" class="attach-row"></div>
+        <div class="inputrow">
+          <button id="chat-attach-btn" class="iconbtn" title="Attach files or images" aria-label="Attach files">＋</button>
+          <input type="file" id="chat-file-input" multiple style="display:none">
+          <button id="chat-mic-btn" class="iconbtn" title="Hold to talk (Whisper)" aria-label="Hold to talk" style="display:none">🎙</button>
+          <div id="chat-box-wrap">
+            <textarea id="chat-box" data-spotlight="composer" rows="1" placeholder="Message the local agent…"></textarea>
+            <div id="chat-mention-pop" class="mentionpop" role="listbox" aria-label="Attach a file or recall a memory"></div>
+          </div>
+          <button id="chat-stop" title="Abort the current turn">Stop</button>
+          <button id="chat-send" data-spotlight="send">Send</button>
+        </div>
+        <div class="chat-hint">
+          <span id="chat-status-line">Enter to send · Shift+Enter for a new line · type while it streams to steer</span>
+          <span class="hint-right">
+            <span id="chat-sysprompt-wrap">
+              <span id="chat-sysprompt" data-spotlight="prompt-pill" class="pill" role="button" tabindex="0" aria-haspopup="true" aria-expanded="false"
+                    title="System prompt (shapes how the assistant replies in this chat)"><span class="dot"></span>Prompt</span>
+              <div id="chat-sysprompt-pop" class="samplepop sysprompt-pop" role="dialog" aria-label="System prompt">
+                <h4>System prompt</h4>
+                <textarea id="sysprompt-text" rows="6" maxlength="4000"
+                  placeholder="e.g. Answer in French. Keep replies under 3 sentences. Prefer Python over JS."
+                  aria-label="Custom system prompt"></textarea>
+                <div class="sysprompt-foot-row">
+                  <span class="note" id="sysprompt-count">0 / 4000</span>
+                  <button id="sysprompt-clear" type="button">Clear</button>
+                </div>
+                <span class="note">Layered on top of the built-in assistant prompt — it doesn't replace mlx-bun's own
+                  identity/tool guidance. Takes effect on your next message (not retroactively).</span>
+                <div class="sysprompt-presets">
+                  <h4>Presets</h4>
+                  <div class="sysprompt-preset-row">
+                    <select id="sysprompt-preset-select" class="pill" aria-label="Saved presets">
+                      <option value="">— presets —</option>
+                    </select>
+                    <button id="sysprompt-preset-save" type="button" title="Save the current prompt + sampling as a named preset">Save</button>
+                    <button id="sysprompt-preset-delete" type="button" title="Delete the selected preset">Delete</button>
+                  </div>
+                  <span class="note">A preset bundles this system prompt with the current Sampling settings — applying one sets both at once.</span>
+                </div>
+              </div>
+            </span>
+            <span id="chat-sampling-wrap">
+              <span id="chat-sampling" data-spotlight="sampling-pill" class="pill" role="button" tabindex="0" aria-haspopup="true" aria-expanded="false"
+                    title="Sampling controls (temperature · top_p · top_k)"><span class="dot"></span>Sampling</span>
+              <div id="chat-sampling-pop" class="samplepop" role="dialog" aria-label="Sampling controls">
+                <div class="samp-head-row">
+                  <h4>Sampling</h4>
+                  <span id="samp-oneshot-chip" class="samp-oneshot-chip" style="display:none">next msg only</span>
+                </div>
+                <div class="samp-scope-row">
+                  <label for="samp-scope">Apply</label>
+                  <select id="samp-scope" aria-label="Sampling scope">
+                    <option value="session">this session</option>
+                    <option value="next_turn">next message only</option>
+                  </select>
+                </div>
+                <div class="samprow">
+                  <label for="samp-temp">Temperature</label>
+                  <input id="samp-temp" type="range" min="0" max="2" step="0.05" aria-label="Temperature">
+                  <output id="samp-temp-val" class="samp-val auto">0.70</output>
+                </div>
+                <div class="samprow">
+                  <label for="samp-topp">top_p</label>
+                  <input id="samp-topp" type="range" min="0" max="1" step="0.01" aria-label="top_p">
+                  <output id="samp-topp-val" class="samp-val auto">0.95</output>
+                </div>
+                <div class="samprow">
+                  <label for="samp-topk">top_k</label>
+                  <input id="samp-topk" type="range" min="0" max="100" step="1" aria-label="top_k">
+                  <output id="samp-topk-val" class="samp-val auto">off</output>
+                </div>
+                <details class="samp-adv" id="samp-advanced">
+                  <summary>Advanced</summary>
+                  <div class="samprow">
+                    <label for="samp-minp">min_p</label>
+                    <input id="samp-minp" type="range" min="0" max="0.5" step="0.01" aria-label="min_p">
+                    <output id="samp-minp-val" class="samp-val auto">off</output>
+                  </div>
+                  <div class="samprow">
+                    <label for="samp-xtcp">XTC prob</label>
+                    <input id="samp-xtcp" type="range" min="0" max="1" step="0.01" aria-label="XTC probability">
+                    <output id="samp-xtcp-val" class="samp-val auto">off</output>
+                  </div>
+                  <div class="samprow">
+                    <label for="samp-xtct">XTC thresh</label>
+                    <input id="samp-xtct" type="range" min="0" max="1" step="0.01" aria-label="XTC threshold">
+                    <output id="samp-xtct-val" class="samp-val auto">off</output>
+                  </div>
+                  <div class="samprow">
+                    <label for="samp-reppen">repetition</label>
+                    <input id="samp-reppen" type="range" min="1" max="2" step="0.01" aria-label="repetition penalty">
+                    <output id="samp-reppen-val" class="samp-val auto">off</output>
+                  </div>
+                  <div class="samprow">
+                    <label for="samp-repctx">rep. window</label>
+                    <input id="samp-repctx" type="range" min="0" max="1024" step="16" aria-label="repetition context size">
+                    <output id="samp-repctx-val" class="samp-val auto">off</output>
+                  </div>
+                  <div class="samprow">
+                    <label for="samp-prespen">presence</label>
+                    <input id="samp-prespen" type="range" min="-2" max="2" step="0.05" aria-label="presence penalty">
+                    <output id="samp-prespen-val" class="samp-val auto">off</output>
+                  </div>
+                  <div class="samprow">
+                    <label for="samp-freqpen">frequency</label>
+                    <input id="samp-freqpen" type="range" min="-2" max="2" step="0.05" aria-label="frequency penalty">
+                    <output id="samp-freqpen-val" class="samp-val auto">off</output>
+                  </div>
+                  <div class="seed-row">
+                    <label for="samp-seed">seed</label>
+                    <input id="samp-seed" type="text" inputmode="numeric" pattern="[0-9]*" placeholder="random" aria-label="seed">
+                  </div>
+                </details>
+                <div class="samppop-foot">
+                  <span class="note" id="samp-foot-note">Sliders default to this model's recommended values — temperature also follows the thinking toggle. Reset returns everything to defaults.</span>
+                  <button id="samp-reset" type="button">Reset</button>
+                </div>
+              </div>
+            </span>
+            <select id="chat-adapter" data-spotlight="adapter-select" class="pill" title="LoRA adapter for this chat (none = base model)">
+              <option value="">no adapter</option>
+            </select>
+            <button id="adapters-manage" type="button" title="Manage adapters — routing table, RAM cost, stacking"
+                    aria-haspopup="dialog" aria-controls="adapters-overlay" aria-expanded="false">⚙</button>
+            <span id="chat-think" class="pill" role="button" tabindex="0" title="Toggle the model's reasoning"><span class="dot"></span>Thinking</span>
+          </span>
+        </div>
+        <div id="chat-perf">
+          <span class="pf-item" id="pf-tps" title="Decode speed for the current/last turn"></span>
+          <span class="pf-item" id="pf-ttft" title="Time to first token"></span>
+          <span class="pf-item" id="pf-fill" title="Context window in use — auto-compacts before it fills"></span>
+          </div>
+      </div>
+    </div>
+    </div>
+<div id="adapters-overlay">
+  <div id="adapters-panel" role="dialog" aria-modal="true" aria-labelledby="adapters-title">
+    <div class="mem-head">
+      <h2 id="adapters-title">Adapters</h2>
+      <button class="mem-close" id="adapters-close" aria-label="Close adapter routing table">✕</button>
+    </div>
+    <div class="mem-body" id="adapters-body"></div>
+  </div>
+</div>
+<div id="chat-drawer-backdrop"></div>
+`;

package/src/panel/ui-catalog.ts

@@ -0,0 +1,77 @@
+// Part of the chat panel (`@mlx-bun/module-chat`): the app-aware assistant's catalog of the host page's routes,
+// overlays and spotlight targets (ids of the host page's own controls).
+//
+// Shared route IDs keep browser navigation and server validation in agreement.
+// Spotlight selectors and overlay IDs belong to the browser.
+
+/** A hash-routed page (shell.ts's Route union, minus "routes" — the DAG
+ *  diagram tab isn't a useful assistant destination and is itself
+ *  feature-detected/hidden when unavailable). */
+import { APP_ROUTE_IDS as ROUTE_IDS } from "../protocol";
+export { ROUTE_IDS };
+export type RouteId = (typeof ROUTE_IDS)[number];
+
+/** A non-route overlay/panel the assistant can navigate to or spotlight
+ *  within. Opening one of these does not change location.hash. */
+export const VIEW_IDS = ["memory-panel", "hub-panel", "settings", "adapters-panel"] as const;
+export type ViewId = (typeof VIEW_IDS)[number];
+
+export function isRouteId(v: string): v is RouteId {
+  return (ROUTE_IDS as readonly string[]).includes(v);
+}
+export function isViewId(v: string): v is ViewId {
+  return (VIEW_IDS as readonly string[]).includes(v);
+}
+
+/** Human labels for routes, used in the "unknown route" tool-error message
+ *  and nowhere else — kept tiny on purpose. */
+export const ROUTE_LABELS: Record<RouteId, string> = {
+  chat: "Chat",
+  quantize: "Quantize",
+  finetune: "Fine-tune",
+  dataset: "Build Dataset",
+  status: "Status",
+};
+
+/** A curated spotlight target: a stable id -> the selector to highlight +
+ *  a human label for the popover title, plus which route/view it lives on
+ *  (spotlight_ui navigates there first if the browser isn't already on it).
+ *  This is the "catalog target" resolution path in resolve-spotlight.ts's
+ *  terms — the fallback when ref/label/selector don't resolve from the live
+ *  DOM or the last uiSnapshot. Anchors added to app.html via data-spotlight
+ *  in this same wave; ids are stable and boring (element id minus "chat-"
+ *  prefix noise where it reads fine either way). */
+export interface SpotlightTarget {
+  selector: string;
+  label: string;
+  route?: RouteId;
+  view?: ViewId;
+}
+
+export const SPOTLIGHT_TARGETS: Record<string, SpotlightTarget> = {
+  composer: { selector: "#chat-box", label: "Message box", route: "chat" },
+  send: { selector: "#chat-send", label: "Send", route: "chat" },
+  "new-chat": { selector: "#chat-new", label: "New chat", route: "chat" },
+  "session-search": { selector: "#chat-sess-search", label: "Search chats", route: "chat" },
+  "sampling-pill": { selector: "#chat-sampling", label: "Sampling", route: "chat" },
+  "prompt-pill": { selector: "#chat-sysprompt", label: "System prompt", route: "chat" },
+  "adapter-select": { selector: "#chat-adapter", label: "LoRA adapter", route: "chat" },
+  "memory-entry": { selector: "#chat-memory-entry", label: "Memory", route: "chat" },
+  "developer-toggle": { selector: "#nav-developer", label: "Developer toggle" },
+  "model-picker": { selector: "#nav-model", label: "Active model" },
+  "hub-browse": { selector: "#nav-model", label: "Browse models" },
+  "quantize-source": { selector: "#q-model", label: "Source model", route: "quantize" },
+  "finetune-base": { selector: "#f-model", label: "Base model", route: "finetune" },
+};
+
+export function getSpotlightTarget(id: string): SpotlightTarget | null {
+  return SPOTLIGHT_TARGETS[id] ?? null;
+}
+
+/** Resolve a free-form route/page string (either a bare route id or a
+ *  "#/route" hash) the way navigate_app's tool param is described to the
+ *  model. Returns null for anything not in ROUTE_IDS. */
+export function resolveRouteId(routeOrPage: string): RouteId | null {
+  const bare = routeOrPage.trim().replace(/^#\/?/, "");
+  return isRouteId(bare) ? bare : null;
+}

package/src/panel/voice.ts

@@ -0,0 +1,141 @@
+// Hold-to-talk voice input for the composer: microphone → 16 kHz float32
+// (an AudioContext opened at 16 kHz lets the browser resample the mic) →
+// 250 ms chunks streamed to /v1/audio/sessions/<id>/audio while the button
+// is held → finish on release → transcript inserted at the caret. Works
+// with ANY served chat model: transcription is the Whisper companion, not
+// the chat model's audio input. Shown only when `ready.transcription` is
+// true (composer.ts updateAttachHint).
+
+import { $, toast } from "./dom";
+
+const SAMPLE_RATE = 16_000;
+const CHUNK_SECONDS = 0.25;
+
+interface Take {
+  id: string;
+  ctx: AudioContext;
+  stream: MediaStream;
+  node: ScriptProcessorNode;
+  pending: Float32Array[];
+  pendingLen: number;
+  sending: Promise<void>;
+  failed: string | null;
+}
+
+let take: Take | null = null;
+let starting = false;
+
+async function postChunk(id: string, pcm: Float32Array): Promise<void> {
+  const r = await fetch(`/v1/audio/sessions/${id}/audio`, {
+    method: "POST", headers: { "content-type": "audio/pcm;rate=16000" },
+    body: new Uint8Array(pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength) as ArrayBuffer),
+  });
+  if (!r.ok) throw new Error(`audio chunk rejected (${r.status})`);
+}
+
+function flush(t: Take): void {
+  if (t.pendingLen === 0) return;
+  const buf = new Float32Array(t.pendingLen);
+  let o = 0;
+  for (const p of t.pending) { buf.set(p, o); o += p.length; }
+  t.pending = []; t.pendingLen = 0;
+  t.sending = t.sending.then(() => postChunk(t.id, buf)).catch((e) => { t.failed = (e as Error).message; });
+}
+
+async function begin(btn: HTMLButtonElement): Promise<void> {
+  if (take || starting) return;
+  starting = true;
+  btn.classList.add("busy");
+  try {
+    const r = await fetch("/v1/audio/sessions", {
+      method: "POST", headers: { "content-type": "application/json" },
+      body: JSON.stringify({ vad: true, vad_min_speech_ms: 120, temperature: 0 }),
+    });
+    if (!r.ok) throw new Error(r.status === 503 ? "no Whisper model on this server" : `session failed (${r.status})`);
+    const { id } = await r.json() as { id: string };
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } });
+    const ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
+    const src = ctx.createMediaStreamSource(stream);
+    // ScriptProcessorNode is deprecated but universally available and enough
+    // for 16 kHz mono capture; an AudioWorklet needs a separately served file.
+    const node = ctx.createScriptProcessor(4096, 1, 1);
+    const t: Take = { id, ctx, stream, node, pending: [], pendingLen: 0, sending: Promise.resolve(), failed: null };
+    node.onaudioprocess = (e) => {
+      if (take !== t) return;
+      const data = e.inputBuffer.getChannelData(0);
+      t.pending.push(new Float32Array(data));
+      t.pendingLen += data.length;
+      if (t.pendingLen >= SAMPLE_RATE * CHUNK_SECONDS) flush(t);
+    };
+    src.connect(node);
+    node.connect(ctx.destination); // required for onaudioprocess to fire in some browsers
+    take = t;
+    btn.classList.add("recording");
+  } catch (e) {
+    toast(`Voice input failed: ${(e as Error).message}`, "err");
+  } finally {
+    starting = false;
+    btn.classList.remove("busy");
+  }
+}
+
+async function end(btn: HTMLButtonElement, box: HTMLTextAreaElement): Promise<void> {
+  const t = take;
+  if (!t) return;
+  take = null;
+  btn.classList.remove("recording");
+  btn.classList.add("busy");
+  try {
+    t.node.disconnect();
+    for (const track of t.stream.getTracks()) track.stop();
+    void t.ctx.close();
+    flush(t);
+    await t.sending;
+    if (t.failed) throw new Error(t.failed);
+    const r = await fetch(`/v1/audio/sessions/${t.id}/finish`, { method: "POST" });
+    if (!r.ok) throw new Error(`transcription failed (${r.status})`);
+    const body = await r.json() as { text: string; mlx_bun?: { vad?: { speech: boolean } } };
+    const text = (body.text || "").trim();
+    if (!text) { toast(body.mlx_bun?.vad?.speech === false ? "No speech detected" : "Nothing transcribed", ""); return; }
+    insertAtCaret(box, text);
+  } catch (e) {
+    toast(`Voice input failed: ${(e as Error).message}`, "err");
+  } finally {
+    btn.classList.remove("busy");
+  }
+}
+
+function insertAtCaret(box: HTMLTextAreaElement, text: string): void {
+  const start = box.selectionStart ?? box.value.length;
+  const endPos = box.selectionEnd ?? start;
+  const before = box.value.slice(0, start);
+  const after = box.value.slice(endPos);
+  const lead = before.length && !/\s$/.test(before) ? " " : "";
+  const trail = after.length && !/^\s/.test(after) ? " " : "";
+  box.value = before + lead + text + trail + after;
+  const caret = (before + lead + text + trail).length;
+  box.setSelectionRange(caret, caret);
+  box.dispatchEvent(new Event("input", { bubbles: true }));
+  box.focus();
+}
+
+/** Wire hold-to-talk on `btn`: pointer hold, or Space/Enter held while focused. */
+export function initVoiceInput(btn: HTMLButtonElement, box: HTMLTextAreaElement): void {
+  if (!btn) return;
+  if (!navigator.mediaDevices?.getUserMedia) { btn.title = "Microphone capture needs a secure context (https or localhost)"; }
+  btn.addEventListener("pointerdown", (e) => { e.preventDefault(); btn.setPointerCapture(e.pointerId); void begin(btn); });
+  const release = () => { void end(btn, box); };
+  btn.addEventListener("pointerup", release);
+  btn.addEventListener("pointercancel", release);
+  btn.addEventListener("lostpointercapture", release);
+  let keyHeld = false;
+  btn.addEventListener("keydown", (e) => {
+    if ((e.key === " " || e.key === "Enter") && !keyHeld) { e.preventDefault(); keyHeld = true; void begin(btn); }
+  });
+  btn.addEventListener("keyup", (e) => {
+    if ((e.key === " " || e.key === "Enter") && keyHeld) { e.preventDefault(); keyHeld = false; release(); }
+  });
+  btn.addEventListener("blur", () => { if (keyHeld) { keyHeld = false; release(); } });
+  // expose for tests / the palette
+  ($ as unknown as { voice?: unknown }).voice = { begin: () => begin(btn), end: () => end(btn, box) };
+}