@tanstack-ai-browser/wllama 0.0.0-stage → 0.2.0

package/src/runtime.test.ts

@@ -0,0 +1,220 @@
+/**
+ * The client over wllama: availability from its storage, one load per
+ * model and settings with the download's progress, a new load for other
+ * settings, and unloading. wllama is faked with the API the client uses.
+ */
+import assert from "node:assert/strict"
+import { after, test } from "node:test"
+import { mkdtemp, rm } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join, resolve } from "node:path"
+import { pathToFileURL } from "node:url"
+import { createRequire } from "node:module"
+
+import { wllamaText } from "./adapter.js"
+import { modelUrl } from "./catalog.js"
+
+const MODEL = "LiquidAI/LFM2.5-350M-GGUF"
+
+test("on a server, which has no Web Worker or Origin Private File System, the model is unavailable", async () => {
+  assert.equal(typeof Worker, "undefined")
+  assert.equal(await wllamaText(MODEL).availability(), "unavailable")
+})
+
+const root = resolve(process.cwd(), "../..")
+const require = createRequire(join(root, "packages/wllama/package.json"))
+const { build } = (await import(pathToFileURL(require.resolve("esbuild")).href)) as typeof import("esbuild")
+const temporary = await mkdtemp(join(tmpdir(), "wllama-runtime-"))
+after(() => rm(temporary, { recursive: true, force: true }))
+
+type Control = {
+  stored: Array<{ url: string; size: number }>
+  /** Holds each download until it resolves. */
+  gate: Promise<void>
+  /** Called once a download has reported progress. */
+  progressed: null | (() => void)
+  /** Each load: its URL and options. */
+  loads: Array<{ url: string; params: Record<string, unknown> }>
+  /** wllama instances made. */
+  instances: number
+  /** wllama instances ended. */
+  exits: number
+  /** Rejects the next loads with it. */
+  loadError?: Error
+}
+
+type Built = typeof import("./client.js") & Pick<typeof import("./adapter.js"), "wllamaText">
+
+/** Builds client.ts against a fake wllama, in a scope with a Worker class and OPFS, and runs `body` with it. */
+async function withClient(name: string, body: (client: Built, control: Control, release: () => void) => Promise<void>) {
+  const descriptors = Object.fromEntries(["navigator", "Worker"].map((key) => [key, Object.getOwnPropertyDescriptor(globalThis, key)]))
+  let release!: () => void
+  const control: Control = {
+    stored: [],
+    gate: new Promise<void>((resolve) => (release = resolve)),
+    progressed: null,
+    loads: [],
+    instances: 0,
+    exits: 0
+  }
+  ;(globalThis as { __wllamaRuntime?: Control }).__wllamaRuntime = control
+  const define = (key: string, value: unknown) => Object.defineProperty(globalThis, key, { configurable: true, writable: true, value })
+  define("navigator", { storage: { getDirectory: async () => ({}) } })
+  define("Worker", class {})
+  try {
+    const outfile = join(temporary, `${name}.mjs`)
+    await build({
+      stdin: { contents: 'export * from "./client.ts"; export { wllamaText } from "./adapter.ts"', resolveDir: join(root, "packages/wllama/src"), loader: "ts" },
+      outfile,
+      bundle: true,
+      format: "esm",
+      platform: "node",
+      logLevel: "error",
+      plugins: [
+        {
+          name: "fake-wllama",
+          setup(bundle) {
+            bundle.onResolve({ filter: /@wllama\/wllama\/esm\/index\.js$/ }, () => ({ path: "wllama", namespace: "fake" }))
+            bundle.onLoad({ filter: /.*/, namespace: "fake" }, () => ({
+              contents: `
+                const control=globalThis.__wllamaRuntime;
+                export const LoggerWithoutDebug={};
+                export class ModelManager { async getModels(){ return control.stored } }
+                export class Wllama {
+                  constructor(){control.instances++}
+                  setCompat(){}
+                  async loadModelFromUrl(url,params){
+                    control.loads.push({url,params});
+                    if(control.loadError) throw control.loadError;
+                    const stored=control.stored.find((m)=>m.url===url);
+                    if(stored){ params.progressCallback({loaded:stored.size,total:stored.size}); return }
+                    params.progressCallback({loaded:250,total:1000});
+                    control.progressed?.();
+                    await Promise.race([control.gate, new Promise((r)=>params.signal.addEventListener("abort",r))]);
+                    params.signal.throwIfAborted();
+                    params.progressCallback({loaded:1000,total:1000});
+                    control.stored.push({url,size:1000});
+                  }
+                  async exit(){control.exits++}
+                }
+              `
+            }))
+          }
+        }
+      ]
+    })
+    await body((await import(pathToFileURL(outfile).href)) as Built, control, release)
+  } finally {
+    for (const [key, descriptor] of Object.entries(descriptors)) {
+      if (descriptor) Object.defineProperty(globalThis, key, descriptor)
+      else Reflect.deleteProperty(globalThis, key)
+    }
+  }
+}
+
+test("availability: downloadable, downloading during the download, available once loaded and from storage after unloading", { timeout: 5000 }, async () => {
+  await withClient("states", async (client, control, release) => {
+    assert.equal(await client.availability(MODEL), "downloadable")
+    const progress: Array<number> = []
+    const progressed = new Promise<void>((resolve) => (control.progressed = resolve))
+    const loading = client.loadModel(MODEL, {}, { onDownloadProgress: (value) => progress.push(value) })
+    await progressed
+    assert.equal(await client.availability(MODEL), "downloading")
+    release()
+    await loading
+    assert.deepEqual(progress, [0.25, 1], "the download's progress, then 1 once ready")
+    assert.equal(await client.availability(MODEL), "available")
+    await client.unloadModel(MODEL)
+    assert.equal(control.exits, 1)
+    assert.equal(await client.availability(MODEL), "available", "its file stays in storage")
+    control.stored = []
+    assert.equal(await client.availability(MODEL), "downloadable")
+  })
+})
+
+test("a model in storage loads without download progress: just 1", { timeout: 5000 }, async () => {
+  await withClient("stored", async (client, control) => {
+    control.stored.push({ url: modelUrl(MODEL), size: 1000 })
+    const progress: Array<number> = []
+    await client.loadModel(MODEL, {}, { onDownloadProgress: (value) => progress.push(value) })
+    assert.deepEqual(progress, [1])
+  })
+})
+
+test("the adapter loads with its own settings; the same settings share a load, other settings load again", { timeout: 5000 }, async () => {
+  await withClient("settings", async (client, control, release) => {
+    release()
+    await client.wllamaText(MODEL, { contextLength: 2048, gpu: false, threads: 4 }).create()
+    assert.equal(control.loads.at(-1)?.url, modelUrl(MODEL))
+    assert.equal(control.loads.at(-1)?.params.n_ctx, 2048)
+    assert.equal(control.loads.at(-1)?.params.n_gpu_layers, 0)
+    assert.equal(control.loads.at(-1)?.params.n_threads, 4)
+    await client.wllamaText(MODEL, { contextLength: 2048, gpu: false, threads: 4 }).create()
+    assert.equal(control.loads.length, 1, "the same settings: no new load")
+    await client.wllamaText(MODEL).create()
+    assert.equal(control.loads.length, 2, "other settings load it again")
+    assert.equal(control.loads.at(-1)?.params.n_ctx, 0, "the model's training context by default")
+    assert.equal(control.exits, 1, "the previous load was freed")
+  })
+})
+
+test("aborting a wait stops only that caller; unloading stops the download", { timeout: 5000 }, async () => {
+  await withClient("abort", async (client, control) => {
+    const controller = new AbortController()
+    const waiting = client.loadModel(MODEL, {}, { signal: controller.signal })
+    controller.abort()
+    await assert.rejects(waiting)
+    assert.equal(control.loads.length, 1, "the load goes on")
+    const again = client.loadModel(MODEL, {})
+    await client.unloadModel()
+    await assert.rejects(again)
+    assert.equal(await client.availability(MODEL), "downloadable")
+  })
+})
+
+test("a failed load rejects, frees wllama, and the next call loads again", { timeout: 5000 }, async () => {
+  await withClient("failure", async (client, control, release) => {
+    release()
+    control.loadError = new Error("llama.cpp couldn't load it")
+    await assert.rejects(client.wllamaText(MODEL).create(), /couldn't load/)
+    await new Promise((resolve) => setTimeout(resolve, 0))
+    assert.equal(control.exits, 1)
+    control.loadError = undefined
+    await client.wllamaText(MODEL).create()
+    assert.equal(control.loads.length, 2)
+  })
+})
+
+test("an unlisted model is named by its GGUF file: owner/repo/file.gguf on Hugging Face, or a URL", { timeout: 5000 }, async () => {
+  await withClient("unlisted", async (client, control, release) => {
+    release()
+    await client.wllamaText("bartowski/SmolLM2-135M-Instruct-GGUF/SmolLM2-135M-Instruct-Q4_K_M.gguf").create()
+    assert.equal(control.loads.at(-1)?.url, "https://huggingface.co/bartowski/SmolLM2-135M-Instruct-GGUF/resolve/main/SmolLM2-135M-Instruct-Q4_K_M.gguf")
+    await client.wllamaText("https://example.com/m.gguf").create()
+    assert.equal(control.loads.at(-1)?.url, "https://example.com/m.gguf")
+    await assert.rejects(client.wllamaText("some/repo").create(), /name its GGUF file/)
+  })
+})
+
+test("simultaneous cold starts with the same settings share one load, which unloading frees", { timeout: 5000 }, async () => {
+  await withClient("simultaneous", async (client, control) => {
+    control.stored.push({ url: modelUrl(MODEL), size: 1000 })
+    const [first, second] = await Promise.all([client.loadModel(MODEL, {}), client.loadModel(MODEL, {})])
+    assert.equal(control.loads.length, 1)
+    assert.equal(first, second)
+    await client.unloadModel()
+    assert.equal(control.instances, 1)
+    assert.equal(control.exits, control.instances, "no wllama is left running")
+  })
+})
+
+test("unloading during a load from storage stops it: the load rejects and no wllama is left running", { timeout: 5000 }, async () => {
+  await withClient("unload-stored", async (client, control) => {
+    control.stored.push({ url: modelUrl(MODEL), size: 1000 })
+    const loading = client.loadModel(MODEL, {})
+    await client.unloadModel()
+    await assert.rejects(loading)
+    assert.equal(control.exits, control.instances, "no wllama is left running")
+    assert.equal(await client.availability(MODEL), "available", "its file stays in storage")
+  })
+})

package/src/wllama-version.test.ts

@@ -0,0 +1,19 @@
+import assert from "node:assert/strict"
+import { test } from "node:test"
+import { readFileSync } from "node:fs"
+import { createRequire } from "node:module"
+import { join, resolve } from "node:path"
+
+const pkg = resolve(process.cwd(), "../../packages/wllama")
+const require = createRequire(join(pkg, "package.json"))
+
+test("the default WebAssembly URL is the installed @wllama/wllama's version", () => {
+  const installed = (require("@wllama/wllama/package.json") as { version: string }).version
+  const source = readFileSync(join(pkg, "src/client.ts"), "utf8")
+  const pinned = /export const WLLAMA_VERSION = "([^"]+)"/.exec(source)?.[1]
+  assert.equal(pinned, installed, "update WLLAMA_VERSION in client.ts with the dependency")
+  // The file's path in the package has moved between versions (src/wasm in 3.8, esm/wasm in 3.13).
+  const url = /export const WLLAMA_WASM_URL = `([^`]+)`/.exec(source)?.[1].replace("${WLLAMA_VERSION}", installed)
+  const own = (require("@wllama/wllama/esm/wasm-from-cdn.cjs") as { default: { default: string } }).default.default
+  assert.equal(url, own, "WLLAMA_WASM_URL is wllama's own WasmFromCDN URL")
+})

package/src/wrapper.test.ts

@@ -0,0 +1,173 @@
+/**
+ * What the adapter adds to TanStack's Chat Completions adapter: llama.cpp's
+ * request fields over the model card's sampling, the chat template's thinking
+ * switch, reading thinking, schemas as given, and the caller's signal
+ * reaching wllama. llama.cpp is stubbed.
+ */
+import assert from "node:assert/strict"
+import { after, beforeEach, test } from "node:test"
+import { mkdtemp, rm } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join, resolve } from "node:path"
+import { pathToFileURL } from "node:url"
+import { createRequire } from "node:module"
+
+const root = resolve(process.cwd(), "../..")
+const require = createRequire(join(root, "packages/wllama/package.json"))
+const { build } = (await import(pathToFileURL(require.resolve("esbuild")).href)) as typeof import("esbuild")
+const temporary = await mkdtemp(join(tmpdir(), "wllama-wrapper-"))
+after(() => rm(temporary, { recursive: true, force: true }))
+
+/** What the stubbed llama.cpp receives, and the chunks it streams back. */
+const control = {
+  requests: [] as Array<{ request: Record<string, unknown>; signal?: AbortSignal }>,
+  chunks: [] as Array<unknown>
+}
+;(globalThis as { __wllamaWrapper?: typeof control }).__wllamaWrapper = control
+
+const outfile = join(temporary, "wllama-adapter.mjs")
+await build({
+  entryPoints: [join(root, "packages/wllama/src/adapter.ts")],
+  outfile,
+  bundle: true,
+  format: "esm",
+  platform: "node",
+  conditions: ["@tanstack-ai-browser/source"],
+  logLevel: "error",
+  plugins: [
+    {
+      name: "stub-llama-cpp",
+      setup(bundle) {
+        bundle.onResolve({ filter: /^\.\/client\.js$/ }, (args) =>
+          args.importer.endsWith("/wllama/src/adapter.ts") ? { path: "client", namespace: "stub" } : undefined
+        )
+        bundle.onLoad({ filter: /.*/, namespace: "stub" }, () => ({
+          contents: `
+            const control=globalThis.__wllamaWrapper;
+            export const availability=async()=>"available";export const unloadModel=async()=>{};
+            export const loadModel=async()=>({
+              createChatCompletion:async({abortSignal,...request})=>{
+                control.requests.push({request,signal:abortSignal});
+                if(request.stream) return (async function*(){yield* control.chunks})();
+                return {choices:[{index:0,finish_reason:"stop",message:{role:"assistant",content:'{"city":"Oslo"}'}}]};
+              }
+            });
+          `
+        }))
+      }
+    }
+  ]
+})
+
+type Chunk = { type: string; [key: string]: unknown }
+type Adapter = {
+  model: string
+  chatStream(options: Record<string, unknown>): AsyncIterable<Chunk>
+  structuredOutput(options: Record<string, unknown>): Promise<{ data: unknown }>
+}
+const { wllamaText } = (await import(pathToFileURL(outfile).href)) as { wllamaText: (model: string) => Adapter }
+
+const MODEL = "unsloth/Qwen3.5-2B-GGUF"
+const logger = { request() { return undefined }, errors() { return undefined }, provider() { return undefined }, warn() { return undefined } }
+const options = (extra: Record<string, unknown> = {}) => ({
+  logger,
+  model: MODEL,
+  threadId: "wrapper",
+  messages: [{ role: "user", content: "hi" }],
+  ...extra
+})
+const delta = (d: object, finish: string | null = null) => ({ id: "c", object: "chat.completion.chunk", created: 0, model: MODEL, choices: [{ index: 0, delta: d, finish_reason: finish }] })
+
+async function run(extra: Record<string, unknown> = {}) {
+  const events: Array<Chunk> = []
+  for await (const event of wllamaText(MODEL).chatStream(options(extra))) events.push(event)
+  return { events, request: control.requests.at(-1)!.request }
+}
+
+beforeEach(() => {
+  control.requests = []
+  control.chunks = [delta({ role: "assistant", content: "Hi" }), delta({}, "stop")]
+})
+
+test("modelOptions go to llama.cpp as given, over the model card's sampling", async () => {
+  const given = { max_tokens: 64, temperature: 0.2, top_p: 0.9, min_p: 0.05, frequency_penalty: 0.1, presence_penalty: 0.5, seed: 7, stop: ["END"] }
+  const { request } = await run({ modelOptions: given })
+  for (const [key, value] of Object.entries(given)) assert.deepEqual(request[key], value, key)
+  // Left out: the model card's.
+  const card = (await run()).request
+  assert.equal(typeof card.top_k, "number")
+  assert.equal(typeof card.repeat_penalty, "number")
+})
+
+test("Qwen3.5's card sampling: its non-thinking settings by default, its thinking settings while thinking", async () => {
+  const plain = (await run()).request
+  assert.deepEqual(
+    [plain.temperature, plain.top_p, plain.top_k, plain.min_p, plain.presence_penalty, plain.repeat_penalty],
+    [1, 1, 20, 0, 2, 1]
+  )
+  const thinking = (await run({ reasoning: { level: "high", summary: false } })).request
+  assert.deepEqual([thinking.top_p, thinking.presence_penalty], [0.95, 1.5])
+})
+
+test("parallel_tool_calls goes to llama.cpp as given; the request asks for usage", async () => {
+  const tools = [{ name: "lookup", description: "Look up", inputSchema: { type: "object", properties: { q: { type: "string" } } } }]
+  const request = (await run({ tools, modelOptions: { parallel_tool_calls: false } })).request
+  assert.equal(request.parallel_tool_calls, false)
+  assert.deepEqual(request.stream_options, { include_usage: true })
+})
+
+test("reasoning: the chat template's enable_thinking, a budget when thinking, nothing when not asked", async () => {
+  const off = (await run({ reasoning: { level: "off", summary: false } })).request
+  assert.deepEqual(off.chat_template_kwargs, { enable_thinking: false })
+  assert.equal(off.reasoning_effort, undefined)
+  const high = (await run({ reasoning: { level: "high", summary: false, budgetTokens: 256 } })).request
+  assert.deepEqual(high.chat_template_kwargs, { enable_thinking: true })
+  assert.equal(high.reasoning_budget_tokens, 256)
+  assert.equal((await run()).request.chat_template_kwargs, undefined)
+})
+
+test("thinking streamed as reasoning_content comes back as REASONING events", async () => {
+  control.chunks = [delta({ role: "assistant", reasoning_content: "Let me think." }), delta({ content: "Hi" }), delta({}, "stop")]
+  const { events } = await run()
+  assert.equal(events.find((e) => e.type === "REASONING_MESSAGE_CONTENT")?.delta, "Let me think.")
+})
+
+test("structured output sends the schema as given; the caller's signal reaches wllama", async () => {
+  const schema = { type: "object", properties: { city: { type: "string" }, note: { type: "string" } }, required: ["city"] }
+  const controller = new AbortController()
+  const result = await wllamaText(MODEL).structuredOutput({
+    chatOptions: options({ request: { signal: controller.signal } }),
+    outputSchema: schema
+  })
+  assert.deepEqual(result.data, { city: "Oslo" })
+  const sent = control.requests.at(-1)!
+  assert.deepEqual((sent.request.response_format as { json_schema: { schema: unknown } }).json_schema.schema, schema)
+  assert.equal(sent.signal, controller.signal)
+})
+
+test("an audio clip and an image reach llama.cpp as their bytes, in their places", async () => {
+  // A WAV file's first bytes, and three of an image.
+  const wav = Uint8Array.from([..."RIFF", 0, 0, 0, 0, ..."WAVE", 7, 8, 9].map((v) => (typeof v === "string" ? v.charCodeAt(0) : v)))
+  const data = (bytes: Uint8Array) => Buffer.from(bytes).toString("base64")
+  const content = [
+    { type: "audio", source: { type: "data", value: data(wav), mimeType: "audio/wav" } },
+    { type: "text", content: "What is said here, and what is in the picture?" },
+    { type: "image", source: { type: "data", value: data(Uint8Array.from([1, 2, 3])), mimeType: "image/png" } }
+  ]
+  const { request } = await run({ messages: [{ role: "user", content }] })
+  const messages = request.messages as Array<{ content: Array<{ type: string; data?: ArrayBuffer }> }>
+  const parts = messages.at(-1)?.content ?? []
+  assert.deepEqual(parts.map((part) => part.type), ["audio", "text", "image"])
+  const [audio, , image] = parts
+  assert.deepEqual(new Uint8Array(audio?.data ?? new ArrayBuffer(0)), wav)
+  assert.deepEqual([...new Uint8Array(image?.data ?? new ArrayBuffer(0))], [1, 2, 3])
+})
+
+test("audio kept as an uploaded file is refused: wllama reads bytes", async () => {
+  const content = [{ type: "audio", source: { type: "file", value: "file-123" } }]
+  const events: Array<Chunk> = []
+  for await (const event of wllamaText(MODEL).chatStream(options({ messages: [{ role: "user", content }] }))) events.push(event)
+  const failure = events.find((event) => event.type === "RUN_ERROR")
+  assert.match(JSON.stringify(failure), /not as uploaded files/)
+  assert.equal(control.requests.length, 0)
+})