localm-web 0.3.0 → 0.5.0

package/dist/index.js

@@ -1,10 +1,3 @@
-const DOWNLOAD_PATTERN = /\b(fetch|download|loading from cache|cache hit|param)/i;
-const COMPILE_PATTERN = /\b(compil|shader|kernel|tensor|init|allocat|warm)/i;
-function classifyLoadPhase(text) {
-  if (DOWNLOAD_PATTERN.test(text)) return "downloading";
-  if (COMPILE_PATTERN.test(text)) return "compiling";
-  return "loading";
-}
 class LocalmWebError extends Error {
   /**
    * @param message - Human-readable description of the error.
@@ -30,6 +23,301 @@ class QuotaExceededError extends LocalmWebError {
 }
 class BackendNotAvailableError extends LocalmWebError {
 }
+class StructuredOutputError extends LocalmWebError {
+}
+const DOWNLOAD_PATTERN = /\b(fetch|download|loading from cache|cache hit|param)/i;
+const COMPILE_PATTERN = /\b(compil|shader|kernel|tensor|init|allocat|warm)/i;
+function classifyLoadPhase(text) {
+  if (DOWNLOAD_PATTERN.test(text)) return "downloading";
+  if (COMPILE_PATTERN.test(text)) return "compiling";
+  return "loading";
+}
+let transformersModulePromise$2 = null;
+async function loadTransformers$2() {
+  if (!transformersModulePromise$2) {
+    transformersModulePromise$2 = import("@huggingface/transformers");
+  }
+  return transformersModulePromise$2;
+}
+function buildSamplingKwargs(options) {
+  const kwargs = {};
+  if (options.maxTokens !== void 0) kwargs.max_new_tokens = options.maxTokens;
+  if (options.temperature !== void 0) kwargs.temperature = options.temperature;
+  if (options.topP !== void 0) kwargs.top_p = options.topP;
+  if (options.topK !== void 0) kwargs.top_k = options.topK;
+  if (options.temperature !== void 0 && options.temperature > 0) {
+    kwargs.do_sample = true;
+  }
+  return kwargs;
+}
+function toChatMessages$1(messages) {
+  return messages.map((m) => ({ role: m.role, content: m.content }));
+}
+function lastAssistantContent(output, promptText) {
+  const item = Array.isArray(output) ? output[0] : output;
+  if (!item) return "";
+  const generated = item.generated_text;
+  if (typeof generated === "string") {
+    return generated.startsWith(promptText) ? generated.slice(promptText.length) : generated;
+  }
+  if (Array.isArray(generated)) {
+    for (let i = generated.length - 1; i >= 0; i -= 1) {
+      const turn = generated[i];
+      if (turn && turn.role === "assistant") return turn.content;
+    }
+  }
+  return "";
+}
+function createAsyncQueue() {
+  const buffer = [];
+  let waiters = [];
+  let finished = false;
+  let pendingError = null;
+  const drain = () => {
+    while (buffer.length > 0 && waiters.length > 0) {
+      const resolver = waiters.shift();
+      const value = buffer.shift();
+      resolver?.({ value, done: false });
+    }
+    if ((finished || pendingError) && waiters.length > 0) {
+      const all = waiters;
+      waiters = [];
+      for (const w of all) {
+        if (pendingError) {
+          w({ value: void 0, done: true });
+        } else {
+          w({ value: void 0, done: true });
+        }
+      }
+    }
+  };
+  return {
+    push(item) {
+      buffer.push(item);
+      drain();
+    },
+    end(error) {
+      finished = true;
+      if (error) pendingError = error;
+      drain();
+    },
+    iterator: {
+      [Symbol.asyncIterator]() {
+        return {
+          next() {
+            if (buffer.length > 0) {
+              return Promise.resolve({ value: buffer.shift(), done: false });
+            }
+            if (pendingError) {
+              const err = pendingError;
+              pendingError = null;
+              return Promise.reject(err);
+            }
+            if (finished) {
+              return Promise.resolve({ value: void 0, done: true });
+            }
+            return new Promise((resolve) => waiters.push(resolve));
+          }
+        };
+      }
+    }
+  };
+}
+class TransformersTextEngine {
+  generator = null;
+  currentAbortController = null;
+  isLoaded() {
+    return this.generator !== null;
+  }
+  async load(modelId, onProgress) {
+    const transformers = await loadTransformers$2();
+    try {
+      const generator = await transformers.pipeline("text-generation", modelId, {
+        progress_callback: (report) => {
+          const progress = typeof report.progress === "number" ? report.progress / 100 : 0;
+          const text = report.status ?? "loading";
+          onProgress?.({
+            progress,
+            text,
+            loaded: 0,
+            total: 0,
+            phase: classifyLoadPhase(text)
+          });
+        }
+      });
+      this.generator = generator;
+      onProgress?.({
+        progress: 1,
+        text: "Model ready.",
+        loaded: 0,
+        total: 0,
+        phase: "ready"
+      });
+    } catch (err) {
+      throw new ModelLoadError(`Failed to load transformers model "${modelId}".`, err);
+    }
+  }
+  async generate(messages, options = {}) {
+    const generator = this.requireGenerator();
+    if (options.signal?.aborted) {
+      throw new GenerationAbortedError("Generation aborted before start.");
+    }
+    const chat = toChatMessages$1(messages);
+    try {
+      const output = await generator(chat, buildSamplingKwargs(options));
+      return lastAssistantContent(output, "");
+    } catch (err) {
+      if (err instanceof GenerationAbortedError) throw err;
+      throw new ModelLoadError("Transformers generation failed.", err);
+    }
+  }
+  async *stream(messages, options = {}) {
+    const generator = this.requireGenerator();
+    if (options.signal?.aborted) {
+      throw new GenerationAbortedError("Generation aborted before start.");
+    }
+    const transformers = await loadTransformers$2();
+    const queue = createAsyncQueue();
+    let index = 0;
+    const tokenizer = generator.tokenizer;
+    const streamer = new transformers.TextStreamer(tokenizer, {
+      skip_prompt: true,
+      skip_special_tokens: true,
+      callback_function: (text) => {
+        if (text) {
+          queue.push({ text, index, done: false });
+          index += 1;
+        }
+      }
+    });
+    const abortPromise = new Promise((_, reject) => {
+      if (options.signal) {
+        const onAbort = () => {
+          reject(new GenerationAbortedError("Generation aborted by signal."));
+        };
+        options.signal.addEventListener("abort", onAbort, { once: true });
+      }
+    });
+    const chat = toChatMessages$1(messages);
+    const generation = generator(chat, { ...buildSamplingKwargs(options), streamer }).then(() => {
+      queue.push({ text: "", index, done: true });
+      queue.end();
+    }).catch((err) => {
+      queue.end(err instanceof Error ? err : new Error(String(err)));
+    });
+    void Promise.race([generation, abortPromise]).catch((err) => {
+      if (err instanceof GenerationAbortedError) queue.end(err);
+    });
+    for await (const chunk of queue.iterator) {
+      yield chunk;
+    }
+  }
+  async complete(prompt, options = {}) {
+    const generator = this.requireGenerator();
+    if (options.signal?.aborted) {
+      throw new GenerationAbortedError("Generation aborted before start.");
+    }
+    try {
+      const output = await generator(prompt, buildSamplingKwargs(options));
+      return lastAssistantContent(output, prompt);
+    } catch (err) {
+      if (err instanceof GenerationAbortedError) throw err;
+      throw new ModelLoadError("Transformers completion failed.", err);
+    }
+  }
+  async *streamCompletion(prompt, options = {}) {
+    const generator = this.requireGenerator();
+    if (options.signal?.aborted) {
+      throw new GenerationAbortedError("Generation aborted before start.");
+    }
+    const transformers = await loadTransformers$2();
+    const queue = createAsyncQueue();
+    let index = 0;
+    const tokenizer = generator.tokenizer;
+    const streamer = new transformers.TextStreamer(tokenizer, {
+      skip_prompt: true,
+      skip_special_tokens: true,
+      callback_function: (text) => {
+        if (text) {
+          queue.push({ text, index, done: false });
+          index += 1;
+        }
+      }
+    });
+    generator(prompt, { ...buildSamplingKwargs(options), streamer }).then(() => {
+      queue.push({ text: "", index, done: true });
+      queue.end();
+    }).catch((err) => {
+      queue.end(err instanceof Error ? err : new Error(String(err)));
+    });
+    if (options.signal) {
+      options.signal.addEventListener(
+        "abort",
+        () => {
+          queue.end(new GenerationAbortedError("Generation aborted by signal."));
+        },
+        { once: true }
+      );
+    }
+    for await (const chunk of queue.iterator) {
+      yield chunk;
+    }
+  }
+  async unload() {
+    if (this.generator) {
+      const disposable = this.generator;
+      if (typeof disposable.dispose === "function") {
+        await disposable.dispose();
+      }
+      this.generator = null;
+    }
+    this.currentAbortController?.abort();
+    this.currentAbortController = null;
+  }
+  requireGenerator() {
+    if (!this.generator) {
+      throw new ModelNotLoadedError(
+        "TransformersTextEngine not loaded. Call load() before generation."
+      );
+    }
+    return this.generator;
+  }
+}
+function assertJsonSchema(schema) {
+  if (schema === null || typeof schema !== "object" || Array.isArray(schema)) {
+    throw new StructuredOutputError("jsonSchema must be a plain object describing a JSON Schema.");
+  }
+  const keys = Object.keys(schema);
+  const recognized = [
+    "type",
+    "$ref",
+    "oneOf",
+    "anyOf",
+    "allOf",
+    "enum",
+    "const",
+    "properties"
+  ];
+  if (!keys.some((key) => recognized.includes(key))) {
+    throw new StructuredOutputError(
+      "jsonSchema does not look like a JSON Schema (missing type/$ref/oneOf/anyOf/allOf/enum/const/properties)."
+    );
+  }
+}
+function serializeJsonSchema(schema) {
+  assertJsonSchema(schema);
+  return JSON.stringify(schema);
+}
+function parseStructuredOutput(text) {
+  try {
+    return JSON.parse(text);
+  } catch (err) {
+    throw new StructuredOutputError(
+      "Engine output is not valid JSON. The model may have ignored the constrained decoding directive.",
+      err
+    );
+  }
+}
 let webllmModulePromise = null;
 async function loadWebLLM() {
   if (!webllmModulePromise) {
@@ -47,6 +335,15 @@ function buildSamplingParams(options) {
   if (options.topP !== void 0) params.top_p = options.topP;
   return params;
 }
+function buildResponseFormat(options) {
+  if (options.jsonSchema !== void 0) {
+    return { type: "json_object", schema: serializeJsonSchema(options.jsonSchema) };
+  }
+  if (options.json) {
+    return { type: "json_object" };
+  }
+  return void 0;
+}
 function toChatMessages(messages) {
   return messages.map((m) => {
     switch (m.role) {
@@ -101,10 +398,12 @@ class WebLLMEngine {
     if (options.signal?.aborted) {
       throw new GenerationAbortedError("Generation aborted before start.");
     }
+    const responseFormat = buildResponseFormat(options);
     const completion = await engine.chat.completions.create({
       ...buildSamplingParams(options),
       messages: toChatMessages(messages),
-      stream: false
+      stream: false,
+      ...responseFormat ? { response_format: responseFormat } : {}
     });
     return completion.choices[0]?.message?.content ?? "";
   }
@@ -113,10 +412,12 @@ class WebLLMEngine {
     if (options.signal?.aborted) {
       throw new GenerationAbortedError("Generation aborted before start.");
     }
+    const responseFormat = buildResponseFormat(options);
     const completion = await engine.chat.completions.create({
       ...buildSamplingParams(options),
       messages: toChatMessages(messages),
-      stream: true
+      stream: true,
+      ...responseFormat ? { response_format: responseFormat } : {}
     });
     let index = 0;
     let finished = false;
@@ -150,10 +451,12 @@ class WebLLMEngine {
     if (options.signal?.aborted) {
       throw new GenerationAbortedError("Generation aborted before start.");
     }
+    const responseFormat = buildResponseFormat(options);
     const completion = await engine.completions.create({
       ...buildSamplingParams(options),
       prompt,
-      stream: false
+      stream: false,
+      ...responseFormat ? { response_format: responseFormat } : {}
     });
     return completion.choices[0]?.text ?? "";
   }
@@ -162,10 +465,12 @@ class WebLLMEngine {
     if (options.signal?.aborted) {
       throw new GenerationAbortedError("Generation aborted before start.");
     }
+    const responseFormat = buildResponseFormat(options);
     const completion = await engine.completions.create({
       ...buildSamplingParams(options),
       prompt,
-      stream: true
+      stream: true,
+      ...responseFormat ? { response_format: responseFormat } : {}
     });
     let index = 0;
     let finished = false;
@@ -491,6 +796,7 @@ const MODEL_PRESETS = Object.freeze({
     parameters: "3.8B",
     quantization: "q4f16_1",
     webllmId: "Phi-3.5-mini-instruct-q4f16_1-MLC",
+    transformersId: "onnx-community/Phi-3.5-mini-instruct-onnx-web",
     contextWindow: 4096,
     description: "Microsoft Phi-3.5 mini, INT4 quantized for browser inference."
   },
@@ -500,6 +806,7 @@ const MODEL_PRESETS = Object.freeze({
     parameters: "1B",
     quantization: "q4f16_1",
     webllmId: "Llama-3.2-1B-Instruct-q4f16_1-MLC",
+    transformersId: "onnx-community/Llama-3.2-1B-Instruct",
     contextWindow: 4096,
     description: "Meta Llama 3.2 1B Instruct, INT4 quantized."
   },
@@ -509,8 +816,19 @@ const MODEL_PRESETS = Object.freeze({
     parameters: "1.5B",
     quantization: "q4f16_1",
     webllmId: "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
+    transformersId: "onnx-community/Qwen2.5-1.5B-Instruct",
     contextWindow: 4096,
     description: "Alibaba Qwen 2.5 1.5B Instruct, INT4 quantized."
+  },
+  "smollm2-360m-int8": {
+    id: "smollm2-360m-int8",
+    family: "SmolLM2",
+    parameters: "360M",
+    quantization: "q8",
+    webllmId: "SmolLM2-360M-Instruct-q4f16_1-MLC",
+    transformersId: "HuggingFaceTB/SmolLM2-360M-Instruct",
+    contextWindow: 2048,
+    description: "HuggingFace SmolLM2 360M Instruct — smallest viable chat model, ideal for the fallback path on low-end devices."
   }
 });
 function resolveModelPreset(modelId) {
@@ -583,12 +901,33 @@ function listSupportedRerankerModels() {
 function createInferenceWorker() {
   return new Worker(new URL(
     /* @vite-ignore */
-    "/assets/inference.worker-CwvQtobb.js",
+    "/assets/inference.worker-DZbXKJZY.js",
     import.meta.url
   ), {
     type: "module"
   });
 }
+function defaultWebGPUDetector() {
+  return typeof navigator !== "undefined" && "gpu" in navigator;
+}
+function resolveBackend(choice, preset, webGPUAvailable) {
+  if (choice === "webllm") return "webllm";
+  if (choice === "transformers") {
+    if (!preset.transformersId) {
+      throw new BackendNotAvailableError(
+        `Model "${preset.id}" has no transformersId — cannot run on the transformers.js backend.`
+      );
+    }
+    return "transformers";
+  }
+  if (webGPUAvailable) return "webllm";
+  if (!preset.transformersId) {
+    throw new BackendNotAvailableError(
+      `WebGPU is unavailable and model "${preset.id}" has no transformersId for the fallback path.`
+    );
+  }
+  return "transformers";
+}
 class LMTask {
   constructor(engine, preset) {
     this.engine = engine;
@@ -604,13 +943,29 @@ class LMTask {
    */
   static async createEngine(modelId, options = {}) {
     const preset = resolveModelPreset(modelId);
-    const engine = options.engine ?? LMTask.defaultEngine(options);
+    if (options.engine) {
+      if (!options.engine.isLoaded()) {
+        await options.engine.load(preset.webllmId, options.onProgress);
+      }
+      return { engine: options.engine, preset };
+    }
+    const choice = options.backend ?? "auto";
+    const resolved = resolveBackend(
+      choice,
+      preset,
+      defaultWebGPUDetector()
+    );
+    const engine = LMTask.instantiateEngine(resolved, options);
+    const loadId = resolved === "transformers" ? preset.transformersId ?? "" : preset.webllmId;
     if (!engine.isLoaded()) {
-      await engine.load(preset.webllmId, options.onProgress);
+      await engine.load(loadId, options.onProgress);
     }
     return { engine, preset };
   }
-  static defaultEngine(options) {
+  static instantiateEngine(resolved, options) {
+    if (resolved === "transformers") {
+      return new TransformersTextEngine();
+    }
     const useWorker = options.inWorker ?? true;
     if (useWorker) {
       return new WorkerEngine(createInferenceWorker());
@@ -633,6 +988,20 @@ class ChatReply {
     this.tokensGenerated = tokensGenerated;
     this.finishReason = finishReason;
   }
+  /**
+   * Parse {@link ChatReply.text} as JSON.
+   *
+   * Intended for replies generated with `json: true` or `jsonSchema`.
+   * The result is cast to `T` without runtime validation; pair with Zod /
+   * Ajv on the call site if you need to verify the schema.
+   *
+   * @typeParam T - Expected parsed shape.
+   * @returns The parsed JSON value.
+   * @throws StructuredOutputError if the text is not valid JSON.
+   */
+  json() {
+    return parseStructuredOutput(this.text);
+  }
 }
 class CompletionResult {
   constructor(text, prompt, tokensGenerated, finishReason) {
@@ -641,6 +1010,19 @@ class CompletionResult {
     this.tokensGenerated = tokensGenerated;
     this.finishReason = finishReason;
   }
+  /**
+   * Parse {@link CompletionResult.text} as JSON.
+   *
+   * Intended for completions generated with `json: true` or `jsonSchema`.
+   * The result is cast to `T` without runtime validation.
+   *
+   * @typeParam T - Expected parsed shape.
+   * @returns The parsed JSON value.
+   * @throws StructuredOutputError if the text is not valid JSON.
+   */
+  json() {
+    return parseStructuredOutput(this.text);
+  }
 }
 class Chat extends LMTask {
   history = [];
@@ -1104,7 +1486,7 @@ async function* tap(stream, onChunk) {
     yield chunk;
   }
 }
-const VERSION = "0.3.0";
+const VERSION = "0.5.0";
 export {
   BackendNotAvailableError,
   Chat,
@@ -1123,18 +1505,25 @@ export {
   QuotaExceededError,
   RERANKER_PRESETS,
   Reranker,
+  StructuredOutputError,
+  TransformersTextEngine,
   UnknownModelError,
   VERSION,
   WebGPUUnavailableError,
+  WebLLMEngine,
   WorkerEngine,
+  assertJsonSchema,
   collectStream,
   createInferenceWorker,
   listSupportedEmbeddingModels,
   listSupportedModels,
   listSupportedRerankerModels,
+  parseStructuredOutput,
+  resolveBackend,
   resolveEmbeddingPreset,
   resolveModelPreset,
   resolveRerankerPreset,
+  serializeJsonSchema,
   tap
 };
 //# sourceMappingURL=index.js.map