@volter/twin-xai 0.1.0

package/src/xai-server.ts

@@ -0,0 +1,258 @@
+// xAI twin HTTP server — serve the full xAI twin handler over HTTP so the real `@ai-sdk/xai`
+// provider (constructed with `baseURL: http://127.0.0.1:<port>/v1`) works unmodified. JSON
+// bodies pass straight through. Writable by default; pass readOnly to reject mutations (D3).
+//
+// Streaming: when the request body has `"stream": true`, the server constructs a REAL SSE
+// response by feeding the handler an sseSink that writes each chunk onto the HTTP stream in
+// the `data: <json>\n\n` wire format, ending with `data: [DONE]\n\n`. (The handler itself
+// stays socket-free — the sink is the only place a socket is touched, on the live HTTP path.)
+//
+// Scenario scripting (xai-scenario.ts): a JSON scenario file — via the scenarioPath option,
+// the TWIN_XAI_SCENARIO env var, or `world-xai serve --scenario <path>` — scripts the exact
+// assistant turns for POST /v1/chat/completions. One session per server (nthCall/once state).
+import { serveHttp } from '@volter/world-core';
+import { twinManifest, twinPublicBase, worldNow } from '@volter/world-core';
+import { createXaiScenarioEngine, loadXaiScenarioDocument, type XaiScenarioEngine } from './xai-scenario.ts';
+import { handleXaiTwinRequest } from './xai-twin.ts';
+import { decideXaiTwinDeviceAuthorization } from './xai-oauth.ts';
+import {
+  deviceConsentPageHtml,
+  deviceDonePageHtml,
+  deviceEntryPageHtml,
+  xaiDeviceFavicon,
+  xaiDeviceAuthView,
+  xaiDeviceScript,
+  xaiDeviceStylesheet,
+  XAI_DEVICE_FAVICON_PATH,
+  XAI_DEVICE_SCRIPT_PATH,
+  XAI_DEVICE_STYLE_PATH,
+} from './xai-device-auth-ui.ts';
+import type { SseEvent } from './xai-types.ts';
+
+function wantsStream(body: string): boolean {
+  if (!body) return false;
+  try {
+    const parsed = JSON.parse(body) as { stream?: unknown; deferred?: unknown };
+    // deferred takes precedence over stream: the vendor answers a deferred request with
+    // { request_id }, not an event stream.
+    return parsed?.stream === true && parsed?.deferred !== true;
+  } catch {
+    return false;
+  }
+}
+
+function encodeSse(event: SseEvent): string {
+  if (event.done) return 'data: [DONE]\n\n';
+  return `data: ${JSON.stringify(event.data)}\n\n`;
+}
+
+const STREAMABLE = new Set(['/v1/chat/completions']);
+
+function html(body: string, status = 200): Response {
+  return new Response(body, {
+    status,
+    headers: {
+      'cache-control': 'no-store',
+      'content-security-policy': "default-src 'self'; style-src 'self'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; base-uri 'none'",
+      'content-type': 'text/html; charset=utf-8',
+      'referrer-policy': 'no-referrer',
+      'x-content-type-options': 'nosniff',
+    },
+  });
+}
+
+/** Options every xAI-twin HTTP surface needs, independent of who owns the socket. */
+export interface XaiTwinFetchOptions {
+  root?: string;
+  readOnly?: boolean;
+  scenarioPath?: string;
+}
+
+/**
+ * The pack's whole HTTP surface as a plain `fetch` — Request in, Response out, no listener.
+ *
+ * This is the composable form (runtime contract R12b): a Worker / Durable Object entry has NO
+ * loopback ports, so it must mount a pack's handler IN-PROCESS. `createXaiTwinServer` is
+ * nothing but `Bun.serve` wrapped around this closure, so the standalone (R1) and hosted
+ * surfaces are the SAME code — there is no second HTTP adaptation to drift. openai is the
+ * reference for this shape; xai is its generative sibling.
+ *
+ * WHAT IT SERVES IS UNCHANGED (R9): xai is a GENERATIVE pack, so chat completions answer a
+ * labeled deterministic stub or a scripted scenario — never a model. The only wall-clock-shaped
+ * call on this path is `worldNow()`, the world's frozen instant, and GET /twin is built from
+ * constants plus the scenario engine's own counters, so replaying it on identical state is
+ * byte-identical.
+ *
+ * NOTHING ON THIS PATH TOUCHES A FILESYSTEM. The scenario document is read through the ACTIVE
+ * WORLD STORE (xai-scenario.ts), once, when the factory is called; the device-authorization
+ * stylesheet, script and favicon are all served from committed constants
+ * (xai-device-auth-ui.ts), so the OAuth protocol UI is workerd-servable too.
+ */
+export function createXaiTwinFetch(options: XaiTwinFetchOptions = {}): (request: Request) => Promise<Response> {
+  const readOnly = options.readOnly ?? false;
+  const scenarioPath = options.scenarioPath ?? process.env.TWIN_XAI_SCENARIO;
+  const scenarioEngine: XaiScenarioEngine | undefined = scenarioPath ? createXaiScenarioEngine(loadXaiScenarioDocument(scenarioPath)) : undefined;
+  return async function xaiTwinFetch(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const path = url.pathname + (url.search || '');
+    const cleanPath = url.pathname.replace(/\/+$/, '');
+
+    // THE READ DOORS (TWIN-PROGRAMMING-MODEL): discovery + inspection, read-only.
+    if (request.method === 'GET' && cleanPath === '/twin') {
+      return Response.json(twinManifest({
+        vendor: 'xai',
+        twinOf: 'xAI Grok API (chat completions)',
+        stateSentence: 'Seed nothing — usage accrues from ordinary API use with any key.',
+        behaviorSentence: 'Completions are scripted by MSW-shaped handlers in the world dir (handlers/xai.json): {on:{userTextIncludes|anyTextIncludes|modelEquals|hasTool|toolResultFor|lastMessageIsToolResult|nthCall}, respond:{text|toolCalls, finishReason?}, once?, phase?}. Unmatched requests answer a labeled stub naming this door.',
+        exampleHandler: { on: { userTextIncludes: 'post', hasTool: 'draftPost' }, respond: { toolCalls: { name: 'draftPost', arguments: { text: 'demo' } } }, once: true },
+        engine: scenarioEngine as never,
+      }));
+    }
+    if (request.method === 'GET' && cleanPath === '/twin/scenario') {
+      return Response.json(scenarioEngine ? scenarioEngine.status() : { vendor: 'xai', handlers: [], misses: 0, recentMisses: [] });
+    }
+
+    if (request.method === 'GET' && cleanPath === XAI_DEVICE_STYLE_PATH) {
+      return new Response(xaiDeviceStylesheet(), {
+        headers: { 'cache-control': 'no-store', 'content-type': 'text/css; charset=utf-8', 'x-content-type-options': 'nosniff' },
+      });
+    }
+
+    if (request.method === 'GET' && (cleanPath === XAI_DEVICE_FAVICON_PATH || cleanPath === '/favicon.ico')) {
+      return new Response(xaiDeviceFavicon(), {
+        headers: { 'cache-control': 'public, max-age=86400', 'content-type': 'image/svg+xml', 'x-content-type-options': 'nosniff' },
+      });
+    }
+
+    if (request.method === 'GET' && cleanPath === XAI_DEVICE_SCRIPT_PATH) {
+      return new Response(xaiDeviceScript(), {
+        headers: { 'cache-control': 'no-store', 'content-type': 'text/javascript; charset=utf-8', 'x-content-type-options': 'nosniff' },
+      });
+    }
+
+    // Internal Twin control routes exist for in-process capability setup only. They are never
+    // part of the network surface; browser authorization uses the real device forms below.
+    if (cleanPath.startsWith('/twin/')) {
+      return new Response(JSON.stringify({ code: 'not_found', error: 'The requested endpoint does not exist.' }), {
+        status: 404,
+        headers: { 'content-type': 'application/json' },
+      });
+    }
+
+    if (request.method === 'GET' && cleanPath === '') {
+      return Response.redirect(`${twinPublicBase(request)}/oauth2/device`, 303);
+    }
+
+    if (request.method === 'GET' && cleanPath === '/sign-out') {
+      // Account-shell sign-out is not an OAuth decision. The virtual account has no retained
+      // browser cookie, so return to the signed-out entry view and leave the device grant pending.
+      return Response.redirect(`${twinPublicBase(request)}/oauth2/device`, 303);
+    }
+
+    if (request.method === 'GET' && cleanPath === '/oauth2/device') {
+      const invalid = url.searchParams.get('error') === 'invalid_code';
+      const userCode = invalid ? '' : url.searchParams.get('user_code') ?? '';
+      const view = xaiDeviceAuthView(userCode, options.root, invalid ? 'Invalid or expired code. Please try again.' : undefined, worldNow());
+      if (view.status === 'approved' || view.status === 'connected') return html(deviceDonePageHtml(true, twinPublicBase(request)));
+      if (view.status === 'denied') return html(deviceDonePageHtml(false, twinPublicBase(request)));
+      return html(deviceEntryPageHtml(view, twinPublicBase(request)));
+    }
+
+    if (request.method === 'GET' && cleanPath === '/oauth2/device/consent') {
+      const view = xaiDeviceAuthView(url.searchParams.get('user_code') ?? '', options.root, undefined, worldNow());
+      if (view.status === 'pending') return html(deviceConsentPageHtml(view, twinPublicBase(request)));
+      if (view.status === 'approved' || view.status === 'connected') return html(deviceDonePageHtml(true, twinPublicBase(request)));
+      if (view.status === 'denied') return html(deviceDonePageHtml(false, twinPublicBase(request)));
+      const message = view.status === 'expired' ? 'This device code has expired. Start sign-in again from your terminal.' : 'Enter a valid device code.';
+      return html(deviceEntryPageHtml({ ...view, error: message }, twinPublicBase(request)), view.status === 'unknown' ? 404 : 400);
+    }
+
+    if (request.method === 'GET' && cleanPath === '/oauth2/device/done') {
+      const view = xaiDeviceAuthView(url.searchParams.get('user_code') ?? '', options.root, undefined, worldNow());
+      if (view.status === 'approved' || view.status === 'connected') return html(deviceDonePageHtml(true, twinPublicBase(request)));
+      if (view.status === 'denied') return html(deviceDonePageHtml(false, twinPublicBase(request)));
+      const message = view.status === 'expired' ? 'This device code has expired. Start sign-in again from your terminal.' : 'This device authorization has not completed.';
+      return html(deviceEntryPageHtml({ ...view, error: message }, twinPublicBase(request)), view.status === 'unknown' ? 404 : 409);
+    }
+
+    // Pass through the headers the handler models (auth 401/403, the rate-limit trigger),
+    // lower-cased. Their presence is what makes the live wire auth/rate-limit-aware
+    // (in-process trusted calls omit them and are not gated).
+    const passHeaders: Record<string, string> = {};
+    for (const k of ['authorization', 'x-grok-model-override', 'x-twin-force-rate-limit', 'x-xai-token-auth']) {
+      const v = request.headers.get(k);
+      if (v !== null) passHeaders[k] = v;
+    }
+
+    let body = '';
+    if (request.method !== 'GET') body = await request.text();
+
+    if (request.method === 'POST' && cleanPath === '/oauth2/device/continue') {
+      const form = new URLSearchParams(body);
+      const view = xaiDeviceAuthView(form.get('user_code') ?? '', options.root, undefined, worldNow());
+      if (view.status !== 'pending') {
+        return Response.redirect(`${twinPublicBase(request)}/oauth2/device?error=invalid_code`, 303);
+      }
+      return Response.redirect(`${twinPublicBase(request)}/oauth2/device/consent?user_code=${encodeURIComponent(view.userCode)}`, 303);
+    }
+
+    if (request.method === 'POST' && cleanPath === '/oauth2/device/decision') {
+      if (readOnly) return new Response(JSON.stringify({ code: 'method_not_allowed', error: 'Twin is read-only' }), { status: 405, headers: { 'content-type': 'application/json' } });
+      const form = new URLSearchParams(body);
+      const approved = form.get('decision') === 'approve';
+      const decision = await decideXaiTwinDeviceAuthorization({
+        approved,
+        occurredAt: worldNow(),
+        ...(options.root !== undefined ? { root: options.root } : {}),
+        userCode: form.get('user_code') ?? '',
+      });
+      if (decision.status >= 400) return new Response(JSON.stringify(decision.body), { status: decision.status, headers: { 'content-type': 'application/json' } });
+      return Response.redirect(`${twinPublicBase(request)}/oauth2/device/done?user_code=${encodeURIComponent(String(decision.body.user_code ?? ''))}`, 303);
+    }
+
+    // Streaming POST → a real text/event-stream response built from the sink. The handler is
+    // synchronous-fast, so events are collected FIRST: a pre-stream failure (validation 400,
+    // auth 401/403, rate-limit 429) then returns the real vendor-shaped JSON error with its
+    // real status — the vendor rejects a bad request BEFORE opening the event stream, it does
+    // not wrap the error in a 200 SSE frame (§9 hardening).
+    if (!readOnly && request.method.toUpperCase() === 'POST' && STREAMABLE.has(cleanPath) && wantsStream(body)) {
+      const events: SseEvent[] = [];
+      const { status, body: out, headers: errHeaders } = await handleXaiTwinRequest({
+        method: request.method, path, body, readOnly, occurredAt: worldNow(), headers: passHeaders, origin: twinPublicBase(request),
+        requireTwinOauth: process.env.TWIN_XAI_AUTH_SEAM !== 'unsealed',
+        ...(options.root !== undefined ? { root: options.root } : {}),
+        ...(scenarioEngine ? { scenarioEngine } : {}),
+        sseSink: (e) => events.push(e),
+      });
+      if (status >= 400) {
+        return new Response(JSON.stringify(out), { status, headers: { 'content-type': 'application/json', 'x-request-id': 'req_twin', ...(errHeaders ?? {}) } });
+      }
+      const stream = new ReadableStream<Uint8Array>({
+        start(controller) {
+          const enc = new TextEncoder();
+          for (const e of events) controller.enqueue(enc.encode(encodeSse(e)));
+          controller.close();
+        },
+      });
+      return new Response(stream, { headers: { 'content-type': 'text/event-stream; charset=utf-8', 'cache-control': 'no-cache', 'x-request-id': 'req_twin' } });
+    }
+
+    const { status, body: out, headers: outHeaders } = await handleXaiTwinRequest({
+      method: request.method, path, body, readOnly,
+      occurredAt: worldNow(), headers: passHeaders, origin: twinPublicBase(request),
+      requireTwinOauth: process.env.TWIN_XAI_AUTH_SEAM !== 'unsealed',
+      ...(options.root !== undefined ? { root: options.root } : {}),
+      ...(scenarioEngine ? { scenarioEngine } : {}),
+    });
+    return new Response(JSON.stringify(out), { status, headers: { 'content-type': 'application/json', 'x-request-id': 'req_twin', ...(outHeaders ?? {}) } });
+  };
+}
+
+export async function createXaiTwinServer(options: { root?: string; port?: number; readOnly?: boolean; scenarioPath?: string }): Promise<{ port: number; stop: () => void }> {
+  const server = await serveHttp({
+    port: options.port ?? 0,
+    idleTimeout: 60,
+    fetch: createXaiTwinFetch(options),
+  });
+  return { port: server.port ?? options.port ?? 0, stop: () => server.stop(true) };
+}

package/src/xai-stub.ts

@@ -0,0 +1,213 @@
+// THE GENERATIVE-STUB CORE (this twin's answer for generated text).
+//
+// The twin CANNOT run Grok — there are no weights here. So POST /v1/chat/completions,
+// /v1/completions, and /v1/messages return a DETERMINISTIC STUB completion that is CLEARLY a
+// twin stub, NEVER pretending to be real model output; Live Search returns deterministic
+// labeled citation URLs, never real web/X retrieval. What IS faithful is the ENTIRE PROTOCOL
+// ENVELOPE: response shapes, streaming SSE chunk sequences, tool_calls, finish_reason,
+// reasoning_content/reasoning-token accounting, and deterministic usage.
+//
+// The protocol is real; the generation is a deterministic labeled stub.
+
+import type { ChatMessageParam, ChatToolCall } from './xai-types.ts';
+
+/** Deterministic token estimate for a string: ~1 token per 4 chars (faithful order of
+ *  magnitude; deterministic so usage counts are assertable, like the vendor's tokenizer on a
+ *  fixed input). Never zero for non-empty text. */
+export function estimateTokens(text: string): number {
+  if (!text) return 0;
+  return Math.max(1, Math.ceil(text.length / 4));
+}
+
+/** Flatten a chat message's content (string OR content-part array) to its text for token
+ *  counting / echo. Non-text parts contribute their JSON length so the count is deterministic
+ *  and reflects payload size. */
+export function contentToText(content: ChatMessageParam['content']): string {
+  if (typeof content === 'string') return content;
+  if (content === null || content === undefined) return '';
+  if (!Array.isArray(content)) return '';
+  return content
+    .map((part) => {
+      if (part && typeof part === 'object' && (part as { type?: string }).type === 'text') {
+        return String((part as { text?: unknown }).text ?? '');
+      }
+      return JSON.stringify(part);
+    })
+    .join('\n');
+}
+
+/** Deterministic per-modality prompt token counts (xAI's usage.prompt_tokens_details splits
+ *  text vs image tokens; the twin counts image_url parts at a fixed deterministic weight). */
+export function countPromptTokenDetails(messages: ChatMessageParam[]): { text: number; image: number } {
+  let text = 0;
+  let image = 0;
+  for (const m of messages) {
+    if (Array.isArray(m.content)) {
+      for (const part of m.content) {
+        const p = part as { type?: string; text?: unknown };
+        if (p?.type === 'image_url') image += 256; // fixed deterministic per-image weight
+        else if (p?.type === 'text') text += estimateTokens(String(p.text ?? ''));
+        else text += estimateTokens(JSON.stringify(part));
+      }
+    } else {
+      text += estimateTokens(contentToText(m.content));
+    }
+    if (m.name) text += estimateTokens(m.name);
+    for (const tc of m.tool_calls ?? []) text += estimateTokens(JSON.stringify(tc));
+  }
+  return { text, image };
+}
+
+/** Deterministic total prompt-token count for the full set of messages. */
+export function countPromptTokens(messages: ChatMessageParam[]): number {
+  const d = countPromptTokenDetails(messages);
+  return d.text + d.image;
+}
+
+/** The last user turn's text — the thing the stub echoes (deterministic, clearly labeled). */
+export function lastUserText(messages: ChatMessageParam[]): string {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i]!.role === 'user') return contentToText(messages[i]!.content);
+  }
+  // No user turn (e.g. only system) → fall back to the last message's text.
+  return messages.length ? contentToText(messages[messages.length - 1]!.content) : '';
+}
+
+/**
+ * Build the deterministic stub ASSISTANT text. It is unmistakably a twin stub: it carries the
+ * `[twin-stub:<model>]` marker and echoes the prompt, so no caller can mistake it for real
+ * Grok output. Deterministic for a given prompt → assertable in tests.
+ */
+export function stubAssistantText(messages: ChatMessageParam[], model: string): string {
+  const prompt = lastUserText(messages).trim();
+  const echo = prompt.length > 200 ? `${prompt.slice(0, 200)}…` : prompt;
+  return `[twin-stub:${model}] This is a deterministic stub from the xAI twin (no model weights are run). Echoing your last message: ${echo || '(empty)'}`;
+}
+
+/** The labeled-stub reasoning trace (grok-3-mini exposes reasoning_content; the twin's is
+ *  clearly not a real chain-of-thought). */
+export function stubReasoningText(messages: ChatMessageParam[], model: string, effort: string): string {
+  const prompt = lastUserText(messages).trim().slice(0, 80);
+  return `[twin-stub:${model}] deterministic reasoning trace (effort=${effort}); no real chain-of-thought is produced. Considering: ${prompt || '(empty)'}`;
+}
+
+/** Extract a tool's function name from a Chat Completions tool ({ type:'function',
+ *  function:{ name } }) or a bare { name } entry. */
+function toolName(t: unknown): string {
+  const o = t as { function?: { name?: unknown }; name?: unknown } | undefined;
+  if (o?.function && typeof o.function.name === 'string') return o.function.name;
+  if (typeof o?.name === 'string') return o.name;
+  return 'unknown_function';
+}
+
+function placeholderForSchema(def: unknown): unknown {
+  const d = def as { type?: unknown; enum?: unknown[] } | undefined;
+  if (Array.isArray(d?.enum) && d!.enum!.length) return d!.enum![0];
+  switch (d?.type) {
+    case 'number':
+    case 'integer': return 0;
+    case 'boolean': return false;
+    case 'array': return [];
+    case 'object': return {};
+    default: return '';
+  }
+}
+
+/**
+ * Build a deterministic stub argument string for a tool. When the tool declares a JSON-schema
+ * `parameters` object, real Grok emits arguments that validate against the schema; the twin
+ * synthesizes a deterministic object containing every declared property with a
+ * type-appropriate placeholder so strict callers parse it cleanly.
+ */
+export function stubToolArguments(tool: unknown): string {
+  const o = tool as { function?: { parameters?: unknown }; parameters?: unknown; input_schema?: unknown } | undefined;
+  const schema = (o?.function?.parameters ?? o?.parameters ?? o?.input_schema) as { properties?: Record<string, unknown> } | undefined;
+  const props = schema && typeof schema === 'object' ? schema.properties : undefined;
+  if (!props || typeof props !== 'object') return '{}';
+  const out: Record<string, unknown> = {};
+  for (const [key, def] of Object.entries(props)) out[key] = placeholderForSchema(def);
+  return JSON.stringify(out);
+}
+
+/**
+ * When tools are provided, real Grok may respond with `tool_calls` and
+ * `finish_reason:'tool_calls'`. The stub deterministically "calls" the tool selected by
+ * `forcedName` (a named tool_choice) or the FIRST provided tool. Arguments are synthesized
+ * from the tool's JSON schema — clearly a stub, but a vendor-faithful tool_calls envelope.
+ */
+export function stubToolCall(tools: unknown, seq: number, forcedName?: string): ChatToolCall | null {
+  if (!Array.isArray(tools) || tools.length === 0) return null;
+  const chosen = forcedName ? (tools.find((t) => toolName(t) === forcedName) ?? tools[0]) : tools[0];
+  return { id: `call_twin_${seq}`, type: 'function', function: { name: toolName(chosen), arguments: stubToolArguments(chosen) } };
+}
+
+/**
+ * Build a deterministic JSON-object stub for `response_format` json_object / json_schema
+ * (xAI structured outputs). Always valid JSON; for json_schema every declared property is
+ * filled with a schema-typed placeholder so the caller's strict parse succeeds.
+ */
+export function stubJsonObject(messages: ChatMessageParam[], model: string, jsonSchema?: unknown): string {
+  const schema = jsonSchema as { schema?: { properties?: Record<string, unknown> }; properties?: Record<string, unknown> } | undefined;
+  const props = schema?.schema?.properties ?? schema?.properties;
+  if (props && typeof props === 'object') {
+    const out: Record<string, unknown> = {};
+    for (const [key, def] of Object.entries(props)) out[key] = placeholderForSchema(def);
+    return JSON.stringify(out);
+  }
+  return JSON.stringify({ _twin_stub: true, model, echo: lastUserText(messages).slice(0, 200) });
+}
+
+// ── Deterministic hashing / ids ─────────────────────────────────────────────────────────
+/** A small deterministic 32-bit hash (FNV-1a) of a string. */
+export function fnv1a(text: string): number {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < text.length; i++) {
+    h ^= text.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+/** xAI response ids are UUID-formatted strings. Build a deterministic UUID-shaped id from a
+ *  seed string (stable + assertable, like the twin's other deterministic outputs). */
+export function uuidFromSeed(seed: string): string {
+  const h = (s: string) => fnv1a(s).toString(16).padStart(8, '0');
+  const a = h(seed);
+  const b = h(`${seed}#1`);
+  const c = h(`${seed}#2`);
+  const d = h(`${seed}#3`);
+  return `${a}-${b.slice(0, 4)}-${b.slice(4)}-${c.slice(0, 4)}-${c.slice(4)}${d.slice(0, 8)}`;
+}
+
+// ── Deterministic pseudo-tokenizer (POST /v1/tokenize-text) ─────────────────────────────
+export type XaiToken = { token_id: number; string_token: string; token_bytes: number[] };
+
+/**
+ * Deterministically tokenize text into the faithful /v1/tokenize-text shape. NOT the real
+ * Grok BPE — a whitespace-preserving split whose `string_token`s re-join into the exact
+ * input, with deterministic token ids seeded from each piece. Shape + determinism are
+ * faithful; the ids carry no model meaning.
+ */
+export function pseudoTokenize(text: string): XaiToken[] {
+  const pieces = text.match(/\s+|\S+/g) ?? [];
+  return pieces.map((piece) => ({
+    token_id: fnv1a(piece) % 200000,
+    string_token: piece,
+    token_bytes: Array.from(new TextEncoder().encode(piece)),
+  }));
+}
+
+// ── Live Search: deterministic labeled citations ────────────────────────────────────────
+/** Deterministic stub citation URLs for a Live Search request: seeded from the query + the
+ *  requested source types, on a clearly-non-real host; the envelope (citations[] +
+ *  usage.num_sources_used) is faithful. */
+export function stubCitations(query: string, sourceTypes: string[]): string[] {
+  const types = sourceTypes.length ? sourceTypes : ['web'];
+  const out: string[] = [];
+  for (const t of types) {
+    const seed = fnv1a(`${t}|${query}`).toString(36);
+    out.push(`https://twin.invalid/xai-live-search-stub/${t}/${seed}-1`);
+    out.push(`https://twin.invalid/xai-live-search-stub/${t}/${seed}-2`);
+  }
+  return out;
+}