@tryhamster/gerbil 1.0.0-rc.0

package/dist/browser/index.mjs

@@ -0,0 +1,755 @@
+import { a as resolveModel, t as BUILTIN_MODELS } from "../models-DKULvhOr.mjs";
+
+//#region src/browser/index.ts
+/**
+* Gerbil Browser Support
+*
+* Run LLMs directly in the browser with WebGPU acceleration.
+*
+* @example useChat (React)
+* ```tsx
+* import { useChat } from "@tryhamster/gerbil/browser";
+*
+* function Chat() {
+*   const { messages, input, setInput, handleSubmit, isLoading } = useChat();
+*
+*   if (isLoading) return <div>Loading model...</div>;
+*
+*   return (
+*     <form onSubmit={handleSubmit}>
+*       {messages.map(m => <div key={m.id}>{m.role}: {m.content}</div>)}
+*       <input value={input} onChange={e => setInput(e.target.value)} />
+*     </form>
+*   );
+* }
+* ```
+*
+* @example useCompletion (React)
+* ```tsx
+* import { useCompletion } from "@tryhamster/gerbil/browser";
+*
+* function App() {
+*   const { complete, completion, isLoading } = useCompletion();
+*   if (isLoading) return <div>Loading...</div>;
+*   return <button onClick={() => complete("Write a haiku")}>{completion}</button>;
+* }
+* ```
+*
+* @example Low-level API
+* ```ts
+* import { createGerbilWorker } from "@tryhamster/gerbil/browser";
+*
+* const gerbil = await createGerbilWorker({
+*   modelId: "qwen3-0.6b",
+*   onToken: (token) => console.log(token.text),
+* });
+* await gerbil.generate("Hello!");
+* gerbil.terminate();
+* ```
+*/
+/**
+* Create a Gerbil worker for streaming WebGPU inference
+*
+* Uses a Web Worker to keep the UI responsive during model loading
+* and text generation, with real-time token streaming.
+*/
+async function createGerbilWorker(options = {}) {
+	const { modelId = "qwen3-0.6b", onProgress, onToken, onComplete, onError } = options;
+	const source = resolveModel(modelId);
+	return new Promise((resolve, reject) => {
+		const blob = new Blob([`
+      import {
+        AutoTokenizer,
+        AutoModelForCausalLM,
+        TextStreamer,
+        InterruptableStoppingCriteria,
+      } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.0";
+
+      class ModelPipeline {
+        static tokenizer = null;
+        static model = null;
+        static modelId = "";
+
+        static async getInstance(modelId, options = {}, progressCallback) {
+          if (this.modelId !== modelId) {
+            this.tokenizer = null;
+            this.model = null;
+          }
+          this.modelId = modelId;
+
+          const dtype = options.dtype || "q4f16";
+          const device = options.device || "webgpu";
+
+          if (!this.tokenizer) {
+            this.tokenizer = await AutoTokenizer.from_pretrained(modelId, {
+              progress_callback: progressCallback,
+            });
+          }
+
+          if (!this.model) {
+            this.model = await AutoModelForCausalLM.from_pretrained(modelId, {
+              dtype,
+              device,
+              progress_callback: progressCallback,
+            });
+          }
+
+          return { tokenizer: this.tokenizer, model: this.model };
+        }
+      }
+
+      const stoppingCriteria = new InterruptableStoppingCriteria();
+      let pastKeyValuesCache = null;
+
+      async function load(data) {
+        const { modelId, options = {} } = data;
+        self.postMessage({ status: "loading", message: "Loading model..." });
+
+        // Track download state - if we see progress < 100, we're downloading
+        const downloadState = {
+          downloading: new Set(),  // Files currently downloading
+          completed: new Set(),    // Files completed
+          isDownloading: false,    // True if any file needed download
+        };
+
+        try {
+          const { tokenizer, model } = await ModelPipeline.getInstance(
+            modelId,
+            options,
+            (progress) => {
+              if (progress.status === "progress" && progress.file) {
+                const pct = Math.round(progress.progress || 0);
+                
+                // If we see progress < 100, this file is being downloaded (not from cache)
+                if (pct < 100) {
+                  downloadState.downloading.add(progress.file);
+                  downloadState.isDownloading = true;
+                } else if (pct === 100) {
+                  downloadState.downloading.delete(progress.file);
+                  downloadState.completed.add(progress.file);
+                }
+
+                // Only emit downloading status if actually downloading
+                if (downloadState.isDownloading) {
+                  self.postMessage({
+                    status: "downloading",
+                    file: progress.file,
+                    progress: pct,
+                    downloadCount: downloadState.downloading.size,
+                    totalFiles: downloadState.completed.size + downloadState.downloading.size,
+                  });
+                }
+              }
+            }
+          );
+
+          self.postMessage({ status: "loading", message: "Compiling shaders..." });
+          const warmupInputs = tokenizer("a");
+          await model.generate({ ...warmupInputs, max_new_tokens: 1 });
+
+          self.postMessage({ status: "ready" });
+        } catch (error) {
+          self.postMessage({ status: "error", error: error.message || String(error) });
+        }
+      }
+
+      async function generate(data) {
+        const { messages, options = {} } = data;
+        const { maxTokens = 256, temperature = 0.7, topP = 0.9, topK = 20, thinking = false } = options;
+
+        try {
+          const { tokenizer, model } = await ModelPipeline.getInstance(ModelPipeline.modelId, {});
+
+          const inputs = tokenizer.apply_chat_template(messages, {
+            add_generation_prompt: true,
+            return_dict: true,
+            enable_thinking: thinking,
+          });
+
+          let state = "answering";
+          const [START_THINKING_TOKEN_ID, END_THINKING_TOKEN_ID] = tokenizer.encode(
+            "<think></think>",
+            { add_special_tokens: false }
+          );
+
+          let startTime = null;
+          let numTokens = 0;
+
+          // Token callback for state tracking (receives raw token IDs)
+          const tokenCallback = (tokens) => {
+            startTime ??= performance.now();
+            numTokens++;
+            
+            const tokenId = Number(tokens[0]);
+            if (tokenId === START_THINKING_TOKEN_ID) {
+              state = "thinking";
+            } else if (tokenId === END_THINKING_TOKEN_ID) {
+              state = "answering";
+            }
+          };
+
+          // Text callback for streaming (receives decoded text)
+          const streamCallback = (text) => {
+            const tps = startTime ? (numTokens / (performance.now() - startTime)) * 1000 : 0;
+            self.postMessage({ status: "token", text, state, numTokens, tps });
+          };
+
+          const streamer = new TextStreamer(tokenizer, {
+            skip_prompt: true,
+            skip_special_tokens: true,
+            callback_function: streamCallback,
+            token_callback_function: tokenCallback,
+          });
+
+          self.postMessage({ status: "start" });
+
+          const { past_key_values, sequences } = await model.generate({
+            ...inputs,
+            past_key_values: pastKeyValuesCache,
+            do_sample: temperature > 0,
+            temperature: temperature > 0 ? temperature : undefined,
+            top_p: topP,
+            top_k: topK,
+            max_new_tokens: maxTokens,
+            streamer,
+            stopping_criteria: stoppingCriteria,
+            return_dict_in_generate: true,
+          });
+
+          pastKeyValuesCache = past_key_values;
+
+          const endTime = performance.now();
+          const totalTime = startTime ? endTime - startTime : 0;
+          const decoded = tokenizer.batch_decode(sequences, { skip_special_tokens: true });
+
+          self.postMessage({
+            status: "complete",
+            text: decoded[0] || "",
+            numTokens,
+            totalTime,
+            tps: totalTime > 0 ? (numTokens / totalTime) * 1000 : 0,
+          });
+        } catch (error) {
+          self.postMessage({ status: "error", error: error.message || String(error) });
+        }
+      }
+
+      self.addEventListener("message", async (e) => {
+        const { type, ...data } = e.data;
+        switch (type) {
+          case "load": await load(data); break;
+          case "generate": stoppingCriteria.reset(); await generate(data); break;
+          case "interrupt": stoppingCriteria.interrupt(); break;
+          case "reset": pastKeyValuesCache = null; stoppingCriteria.reset(); break;
+        }
+      });
+
+      self.postMessage({ status: "init" });
+    `], { type: "application/javascript" });
+		const workerUrl = URL.createObjectURL(blob);
+		const worker = new Worker(workerUrl, { type: "module" });
+		let isReady = false;
+		let currentResolve = null;
+		let currentReject = null;
+		let _generatedText = "";
+		worker.onmessage = (e) => {
+			const msg = e.data;
+			switch (msg.status) {
+				case "init":
+					worker.postMessage({
+						type: "load",
+						modelId: source.path
+					});
+					break;
+				case "loading":
+				case "downloading":
+					onProgress?.(msg);
+					break;
+				case "ready":
+					isReady = true;
+					onProgress?.(msg);
+					resolve(gerbilWorker);
+					break;
+				case "start":
+					_generatedText = "";
+					break;
+				case "token":
+					_generatedText += msg.text;
+					onToken?.(msg);
+					break;
+				case "complete":
+					onComplete?.(msg);
+					currentResolve?.(msg.text);
+					currentResolve = null;
+					currentReject = null;
+					break;
+				case "error":
+					onError?.(msg.error);
+					onProgress?.({
+						status: "error",
+						error: msg.error
+					});
+					if (currentReject) {
+						currentReject(new Error(msg.error));
+						currentResolve = null;
+						currentReject = null;
+					} else reject(new Error(msg.error));
+					break;
+			}
+		};
+		worker.onerror = (e) => {
+			const error = e.message || "Worker error";
+			onError?.(error);
+			reject(new Error(error));
+		};
+		const gerbilWorker = {
+			generate: (prompt, options$1 = {}) => {
+				return new Promise((res, rej) => {
+					currentResolve = res;
+					currentReject = rej;
+					const messages = [{
+						role: "system",
+						content: options$1.system || "You are a helpful assistant."
+					}, {
+						role: "user",
+						content: prompt
+					}];
+					worker.postMessage({
+						type: "generate",
+						messages,
+						options: {
+							maxTokens: options$1.maxTokens ?? 256,
+							temperature: options$1.temperature ?? .7,
+							topP: options$1.topP ?? .9,
+							topK: options$1.topK ?? 20,
+							thinking: options$1.thinking ?? false
+						}
+					});
+				});
+			},
+			interrupt: () => {
+				worker.postMessage({ type: "interrupt" });
+			},
+			reset: () => {
+				worker.postMessage({ type: "reset" });
+			},
+			terminate: () => {
+				worker.terminate();
+				URL.revokeObjectURL(workerUrl);
+			},
+			isReady: () => isReady
+		};
+	});
+}
+/**
+* React hook for chat with local LLM
+*
+* @example
+* ```tsx
+* import { useChat } from "@tryhamster/gerbil/browser";
+*
+* function Chat() {
+*   const { messages, input, setInput, handleSubmit, isLoading, isGenerating } = useChat();
+*
+*   if (isLoading) return <div>Loading model...</div>;
+*
+*   return (
+*     <div>
+*       {messages.map(m => (
+*         <div key={m.id}>{m.role}: {m.content}</div>
+*       ))}
+*       <form onSubmit={handleSubmit}>
+*         <input value={input} onChange={e => setInput(e.target.value)} />
+*         <button disabled={isGenerating}>Send</button>
+*       </form>
+*     </div>
+*   );
+* }
+* ```
+*/
+function useChat(options = {}) {
+	const React = globalThis.React;
+	if (!React) throw new Error("useChat requires React. Import React before using this hook.");
+	const { useState, useEffect, useRef, useCallback } = React;
+	const { model = "qwen3-0.6b", system = "You are a helpful assistant.", thinking: enableThinking = false, maxTokens = 512, temperature = .7, initialMessages = [], autoLoad = false, onReady, onError } = options;
+	const [messages, setMessages] = useState(initialMessages);
+	const [input, setInput] = useState("");
+	const [isLoading, setIsLoading] = useState(autoLoad);
+	const [loadingProgress, setLoadingProgress] = useState(null);
+	const [isGenerating, setIsGenerating] = useState(false);
+	const [thinking, setThinking] = useState("");
+	const [currentResponse, setCurrentResponse] = useState("");
+	const [tps, setTps] = useState(0);
+	const [error, setError] = useState(null);
+	const [isReady, setIsReady] = useState(false);
+	const [shouldLoad, setShouldLoad] = useState(autoLoad);
+	const workerRef = useRef(null);
+	const messageIdRef = useRef(0);
+	const mountedRef = useRef(true);
+	const load = useCallback(() => {
+		if (workerRef.current || isLoading) return;
+		setIsLoading(true);
+		setShouldLoad(true);
+	}, [isLoading]);
+	useEffect(() => {
+		if (!shouldLoad) return;
+		if (!isWebGPUSupported()) {
+			setError("WebGPU not supported. Use Chrome/Edge 113+.");
+			setIsLoading(false);
+			onError?.("WebGPU not supported");
+			return;
+		}
+		mountedRef.current = true;
+		createGerbilWorker({
+			modelId: model,
+			onProgress: (p) => {
+				if (!mountedRef.current) return;
+				setLoadingProgress(p);
+				if (p.status === "ready") {
+					setIsLoading(false);
+					setIsReady(true);
+					onReady?.();
+				}
+			},
+			onToken: (token) => {
+				if (!mountedRef.current) return;
+				setTps(token.tps);
+				if (token.state === "thinking") setThinking((t) => t + token.text);
+				else setCurrentResponse((r) => r + token.text);
+			},
+			onComplete: () => {
+				if (!mountedRef.current) return;
+				setIsGenerating(false);
+			},
+			onError: (err) => {
+				if (!mountedRef.current) return;
+				setError(err);
+				setIsGenerating(false);
+				onError?.(err);
+			}
+		}).then((worker) => {
+			if (mountedRef.current) workerRef.current = worker;
+			else worker.terminate();
+		}).catch((err) => {
+			if (mountedRef.current) {
+				setError(err.message);
+				setIsLoading(false);
+				onError?.(err.message);
+			}
+		});
+		return () => {
+			mountedRef.current = false;
+			workerRef.current?.terminate();
+		};
+	}, [model, shouldLoad]);
+	useEffect(() => {
+		if (!isGenerating && currentResponse) {
+			setMessages((msgs) => {
+				if (msgs.at(-1)?.role === "assistant") return msgs.map((m, i) => i === msgs.length - 1 ? {
+					...m,
+					content: currentResponse,
+					thinking: thinking || void 0
+				} : m);
+				return msgs;
+			});
+			setCurrentResponse("");
+			setThinking("");
+		}
+		return () => {};
+	}, [
+		isGenerating,
+		currentResponse,
+		thinking
+	]);
+	const pendingMessageRef = useRef(null);
+	const handleSubmit = useCallback((e) => {
+		e?.preventDefault?.();
+		if (!input.trim() || isGenerating) return;
+		const userMessage = {
+			id: `msg-${++messageIdRef.current}`,
+			role: "user",
+			content: input.trim()
+		};
+		const assistantMessage = {
+			id: `msg-${++messageIdRef.current}`,
+			role: "assistant",
+			content: ""
+		};
+		setMessages((msgs) => [
+			...msgs,
+			userMessage,
+			assistantMessage
+		]);
+		setInput("");
+		setCurrentResponse("");
+		setThinking("");
+		if (!workerRef.current) {
+			pendingMessageRef.current = userMessage.content;
+			load();
+			return;
+		}
+		setIsGenerating(true);
+		workerRef.current.generate(userMessage.content, {
+			system,
+			thinking: enableThinking,
+			maxTokens,
+			temperature
+		});
+	}, [
+		input,
+		isGenerating,
+		system,
+		enableThinking,
+		maxTokens,
+		temperature,
+		load
+	]);
+	useEffect(() => {
+		if (isReady && pendingMessageRef.current && workerRef.current) {
+			const pendingContent = pendingMessageRef.current;
+			pendingMessageRef.current = null;
+			setIsGenerating(true);
+			workerRef.current.generate(pendingContent, {
+				system,
+				thinking: enableThinking,
+				maxTokens,
+				temperature
+			});
+		}
+		return () => {};
+	}, [
+		isReady,
+		system,
+		enableThinking,
+		maxTokens,
+		temperature
+	]);
+	const stop = useCallback(() => {
+		workerRef.current?.interrupt();
+		setIsGenerating(false);
+	}, []);
+	const clear = useCallback(() => {
+		workerRef.current?.reset();
+		setMessages([]);
+		setCurrentResponse("");
+		setThinking("");
+	}, []);
+	return {
+		messages: messages.map((m, i) => {
+			if (i === messages.length - 1 && m.role === "assistant" && isGenerating) return {
+				...m,
+				content: currentResponse,
+				thinking: thinking || void 0
+			};
+			return m;
+		}),
+		input,
+		setInput,
+		handleSubmit,
+		isLoading,
+		loadingProgress,
+		isGenerating,
+		thinking,
+		stop,
+		clear,
+		tps,
+		isReady,
+		error,
+		load
+	};
+}
+/**
+* React hook for text completion with local LLM
+*
+* @example
+* ```tsx
+* import { useCompletion } from "@tryhamster/gerbil/browser";
+*
+* function App() {
+*   const { complete, completion, isLoading, isGenerating } = useCompletion();
+*
+*   if (isLoading) return <div>Loading...</div>;
+*
+*   return (
+*     <div>
+*       <button onClick={() => complete("Write a haiku")}>Generate</button>
+*       <p>{completion}</p>
+*     </div>
+*   );
+* }
+* ```
+*/
+function useCompletion(options = {}) {
+	const React = globalThis.React;
+	if (!React) throw new Error("useCompletion requires React. Import React before using this hook.");
+	const { useState, useEffect, useRef, useCallback } = React;
+	const { model = "qwen3-0.6b", system = "You are a helpful assistant.", thinking: enableThinking = false, maxTokens = 512, temperature = .7, autoLoad = false, onReady, onError } = options;
+	const [completion, setCompletion] = useState("");
+	const [thinking, setThinking] = useState("");
+	const [isLoading, setIsLoading] = useState(autoLoad);
+	const [loadingProgress, setLoadingProgress] = useState(null);
+	const [isGenerating, setIsGenerating] = useState(false);
+	const [tps, setTps] = useState(0);
+	const [error, setError] = useState(null);
+	const [isReady, setIsReady] = useState(false);
+	const [shouldLoad, setShouldLoad] = useState(autoLoad);
+	const workerRef = useRef(null);
+	const resolveRef = useRef(null);
+	const rejectRef = useRef(null);
+	const pendingPromptRef = useRef(null);
+	const mountedRef = useRef(true);
+	const load = useCallback(() => {
+		if (workerRef.current || isLoading) return;
+		setIsLoading(true);
+		setShouldLoad(true);
+	}, [isLoading]);
+	useEffect(() => {
+		if (!shouldLoad) return;
+		if (!isWebGPUSupported()) {
+			setError("WebGPU not supported. Use Chrome/Edge 113+.");
+			setIsLoading(false);
+			onError?.("WebGPU not supported");
+			return;
+		}
+		mountedRef.current = true;
+		createGerbilWorker({
+			modelId: model,
+			onProgress: (p) => {
+				if (!mountedRef.current) return;
+				setLoadingProgress(p);
+				if (p.status === "ready") {
+					setIsLoading(false);
+					setIsReady(true);
+					onReady?.();
+				}
+			},
+			onToken: (token) => {
+				if (!mountedRef.current) return;
+				setTps(token.tps);
+				if (token.state === "thinking") setThinking((t) => t + token.text);
+				else setCompletion((c) => c + token.text);
+			},
+			onComplete: (result) => {
+				if (!mountedRef.current) return;
+				setIsGenerating(false);
+				resolveRef.current?.(result.text);
+				resolveRef.current = null;
+			},
+			onError: (err) => {
+				if (!mountedRef.current) return;
+				setError(err);
+				setIsGenerating(false);
+				onError?.(err);
+			}
+		}).then((worker) => {
+			if (mountedRef.current) workerRef.current = worker;
+			else worker.terminate();
+		}).catch((err) => {
+			if (mountedRef.current) {
+				setError(err.message);
+				setIsLoading(false);
+				onError?.(err.message);
+			}
+		});
+		return () => {
+			mountedRef.current = false;
+			workerRef.current?.terminate();
+		};
+	}, [model, shouldLoad]);
+	const complete = useCallback((prompt) => {
+		return new Promise((resolve, reject) => {
+			setCompletion("");
+			setThinking("");
+			resolveRef.current = resolve;
+			rejectRef.current = reject;
+			if (!workerRef.current) {
+				pendingPromptRef.current = prompt;
+				load();
+				return;
+			}
+			setIsGenerating(true);
+			workerRef.current.generate(prompt, {
+				system,
+				thinking: enableThinking,
+				maxTokens,
+				temperature
+			});
+		});
+	}, [
+		system,
+		enableThinking,
+		maxTokens,
+		temperature,
+		load
+	]);
+	useEffect(() => {
+		if (isReady && pendingPromptRef.current && workerRef.current) {
+			const pendingPrompt = pendingPromptRef.current;
+			pendingPromptRef.current = null;
+			setIsGenerating(true);
+			workerRef.current.generate(pendingPrompt, {
+				system,
+				thinking: enableThinking,
+				maxTokens,
+				temperature
+			});
+		}
+		return () => {};
+	}, [
+		isReady,
+		system,
+		enableThinking,
+		maxTokens,
+		temperature
+	]);
+	return {
+		completion,
+		thinking,
+		complete,
+		isLoading,
+		loadingProgress,
+		isGenerating,
+		stop: useCallback(() => {
+			workerRef.current?.interrupt();
+			setIsGenerating(false);
+		}, []),
+		tps,
+		isReady,
+		error,
+		load
+	};
+}
+/**
+* Check if WebGPU is supported
+*/
+function isWebGPUSupported() {
+	if (typeof navigator === "undefined") return false;
+	return "gpu" in navigator;
+}
+/**
+* Get WebGPU adapter info
+*/
+async function getWebGPUInfo() {
+	if (!isWebGPUSupported()) return { supported: false };
+	try {
+		const adapter = await navigator.gpu.requestAdapter();
+		if (!adapter) return { supported: false };
+		const info = await adapter.requestAdapterInfo();
+		return {
+			supported: true,
+			adapter: info.vendor,
+			device: info.device
+		};
+	} catch {
+		return { supported: false };
+	}
+}
+var browser_default = {
+	isWebGPUSupported,
+	getWebGPUInfo,
+	createGerbilWorker
+};
+
+//#endregion
+export { BUILTIN_MODELS, createGerbilWorker, browser_default as default, getWebGPUInfo, isWebGPUSupported, useChat, useCompletion };
+//# sourceMappingURL=index.mjs.map