@tryhamster/gerbil 1.0.0-rc.0 → 1.0.0-rc.2

package/dist/gerbil-BfnsFWRE.mjs

@@ -1,644 +0,0 @@
-import { a as resolveModel, n as createExternalModelConfig, r as getModelConfig, t as BUILTIN_MODELS } from "./models-DKULvhOr.mjs";
-import { n as zodToJsonSchema, t as extractJson } from "./utils-7vXqtq2Q.mjs";
-import { AutoModelForCausalLM, AutoTokenizer, env, pipeline } from "@huggingface/transformers";
-
-//#region src/core/gerbil.ts
-/**
-* Gerbil - Local GPU-accelerated LLM inference
-*/
-const pipeline$1 = pipeline;
-function suppressNoisyWarnings(fn) {
-	const originalWarn = console.warn;
-	console.warn = (...args) => {
-		const msg = args[0]?.toString?.() || "";
-		if (msg.includes("content-length") || msg.includes("Unable to determine")) return;
-		originalWarn.apply(console, args);
-	};
-	return fn().finally(() => {
-		console.warn = originalWarn;
-	});
-}
-const isBrowser = typeof window !== "undefined";
-env.allowLocalModels = !isBrowser;
-env.useBrowserCache = isBrowser;
-let webgpuInitialized = false;
-let webgpuAvailable = false;
-/**
-* Initialize WebGPU for Node.js environments
-* Called automatically before model loading
-*/
-async function initNodeWebGPU() {
-	if (webgpuInitialized) return webgpuAvailable;
-	webgpuInitialized = true;
-	if (typeof window !== "undefined") {
-		webgpuAvailable = "gpu" in navigator;
-		return webgpuAvailable;
-	}
-	try {
-		const { create, globals } = await new Function("specifier", "return import(specifier)")("webgpu");
-		Object.assign(globalThis, globals);
-		if (!globalThis.navigator) globalThis.navigator = {};
-		globalThis.navigator.gpu = create([]);
-		webgpuAvailable = true;
-	} catch {
-		webgpuAvailable = false;
-	}
-	return webgpuAvailable;
-}
-var Gerbil = class {
-	generator = null;
-	tokenizer = null;
-	model = null;
-	embedder = null;
-	currentModel = null;
-	modelConfig = null;
-	config;
-	stats;
-	useDirect = false;
-	chromeBackend = null;
-	_deviceMode = "cpu";
-	constructor(config = {}) {
-		this.config = config;
-		this.stats = {
-			prompts: 0,
-			tokensIn: 0,
-			tokensOut: 0,
-			avgSpeed: 0,
-			totalTime: 0,
-			cacheHits: 0,
-			cacheMisses: 0
-		};
-	}
-	static listModels() {
-		return Object.values(BUILTIN_MODELS);
-	}
-	static getModel(modelId) {
-		return BUILTIN_MODELS[modelId];
-	}
-	/**
-	* Load a model
-	*
-	* @example
-	* ```ts
-	* // Built-in model
-	* await g.loadModel("qwen3-0.6b");
-	*
-	* // HuggingFace model
-	* await g.loadModel("hf:microsoft/Phi-3-mini");
-	*
-	* // Local model
-	* await g.loadModel("file:./models/my-model");
-	* ```
-	*/
-	async loadModel(modelId = "qwen3-0.6b", options = {}) {
-		await initNodeWebGPU();
-		const source = resolveModel(modelId);
-		const { onProgress, device = "auto", dtype: userDtype } = options;
-		let config = getModelConfig(modelId);
-		if (!config) config = createExternalModelConfig(modelId, source.path);
-		onProgress?.({ status: `Loading ${modelId}...` });
-		const isBrowser$1 = typeof window !== "undefined";
-		const fallbackDevice = isBrowser$1 ? "wasm" : "cpu";
-		let tfDevice = fallbackDevice;
-		if (device === "webgpu" || device === "gpu" || device === "auto") tfDevice = "webgpu";
-		const dtype = userDtype ?? (tfDevice === "webgpu" ? "q4f16" : "q4");
-		let isLoading = true;
-		let lastFile = "";
-		let lastPct = -1;
-		const progressCallback = (progress) => {
-			if (!isLoading) return;
-			if (progress.status === "progress" && progress.file) {
-				const pct = Math.round(progress.progress || 0);
-				if (progress.file !== lastFile || pct >= lastPct + 5) {
-					lastFile = progress.file;
-					lastPct = pct;
-					onProgress?.({
-						status: `Downloading ${progress.file}`,
-						progress: pct,
-						file: progress.file
-					});
-				}
-			}
-		};
-		try {
-			if (isBrowser$1 && tfDevice === "webgpu") {
-				onProgress?.({ status: "Loading tokenizer..." });
-				this.tokenizer = await suppressNoisyWarnings(() => AutoTokenizer.from_pretrained(source.path, { progress_callback: progressCallback }));
-				onProgress?.({ status: "Loading model..." });
-				this.model = await suppressNoisyWarnings(() => AutoModelForCausalLM.from_pretrained(source.path, {
-					dtype,
-					device: tfDevice,
-					progress_callback: progressCallback
-				}));
-				this.useDirect = true;
-				this._deviceMode = "webgpu";
-				isLoading = false;
-				this.currentModel = modelId;
-				this.modelConfig = config;
-				onProgress?.({ status: "Ready (WebGPU)!" });
-			} else if (!isBrowser$1 && tfDevice === "webgpu") {
-				onProgress?.({ status: "Starting Chrome WebGPU backend..." });
-				const { ChromeGPUBackend } = await import("./chrome-backend-CtwPENIW.mjs");
-				this.chromeBackend = await ChromeGPUBackend.create({
-					modelId: source.path,
-					onProgress
-				});
-				this.useDirect = false;
-				this._deviceMode = "webgpu";
-				isLoading = false;
-				this.currentModel = modelId;
-				this.modelConfig = config;
-			} else {
-				const pipelineOptions = {
-					dtype,
-					device: tfDevice,
-					progress_callback: progressCallback
-				};
-				this.generator = await suppressNoisyWarnings(() => pipeline$1("text-generation", source.path, pipelineOptions));
-				this.useDirect = false;
-				this._deviceMode = tfDevice;
-				isLoading = false;
-				this.currentModel = modelId;
-				this.modelConfig = config;
-				onProgress?.({ status: `Ready (${tfDevice.toUpperCase()})!` });
-			}
-		} catch (err) {
-			if (tfDevice !== fallbackDevice) {
-				onProgress?.({ status: `Using ${fallbackDevice.toUpperCase()}...` });
-				if (this.chromeBackend) {
-					await this.chromeBackend.dispose();
-					this.chromeBackend = null;
-				}
-				this.generator = await suppressNoisyWarnings(() => pipeline$1("text-generation", source.path, {
-					dtype: "q4",
-					device: fallbackDevice,
-					progress_callback: progressCallback
-				}));
-				this.useDirect = false;
-				this._deviceMode = fallbackDevice;
-				isLoading = false;
-				this.currentModel = modelId;
-				this.modelConfig = config;
-				onProgress?.({ status: `Ready (${fallbackDevice.toUpperCase()})!` });
-			} else throw err;
-		}
-	}
-	/**
-	* Check if a model is loaded
-	*/
-	isLoaded() {
-		return this.generator !== null || this.useDirect && this.model !== null || this.chromeBackend !== null;
-	}
-	/**
-	* Get current model info
-	*/
-	getModelInfo() {
-		return this.modelConfig;
-	}
-	/**
-	* Get current device mode (webgpu, cpu, or wasm)
-	*/
-	getDeviceMode() {
-		return this._deviceMode;
-	}
-	/**
-	* Get dtype used for current model
-	*/
-	getDtype() {
-		return this._deviceMode === "webgpu" ? "q4f16" : "q4";
-	}
-	/**
-	* Get Chrome backend status (if using WebGPU via Chrome)
-	*/
-	getChromeStatus() {
-		if (!this.chromeBackend) return null;
-		return this.chromeBackend.getStatus();
-	}
-	/**
-	* Get Chrome memory usage (if using WebGPU via Chrome)
-	* Returns JS heap memory in bytes
-	*/
-	async getChromeMemory() {
-		if (!this.chromeBackend) return null;
-		return this.chromeBackend.getMemoryUsage();
-	}
-	/**
-	* Get memory usage in GB (if using WebGPU via Chrome)
-	*/
-	async getMemoryUsage() {
-		if (!this.chromeBackend) return null;
-		return this.chromeBackend.getMemoryStats();
-	}
-	/**
-	* Clear KV cache to free memory
-	* This will reset the conversation context but free up memory
-	*/
-	async clearCache() {
-		if (this.chromeBackend) await this.chromeBackend.reset();
-	}
-	/**
-	* Check memory usage and cleanup if needed
-	* @param thresholdGB Memory threshold in GB (default: 8)
-	* @returns true if cleanup was performed
-	*/
-	async checkMemoryAndCleanup(thresholdGB = 8) {
-		if (!this.chromeBackend) return false;
-		return this.chromeBackend.checkMemoryAndCleanup(thresholdGB);
-	}
-	/**
-	* Generate text
-	*/
-	async generate(prompt, options = {}) {
-		if (!this.isLoaded()) await this.loadModel(this.config.model || "qwen3-0.6b");
-		const { maxTokens = 256, temperature = .7, topP = .9, topK = 50, thinking = false, system } = options;
-		const startTime = performance.now();
-		try {
-			let rawText = "";
-			if (this.chromeBackend) try {
-				rawText = await this.chromeBackend.generate(prompt, {
-					maxTokens,
-					temperature,
-					topP,
-					topK,
-					thinking,
-					system,
-					onToken: options.onToken ? (t) => options.onToken(t.text) : void 0
-				});
-			} catch (chromeErr) {
-				if (chromeErr?.message === "CHROME_BACKEND_DEAD" || !this.chromeBackend?.isAlive()) {
-					await this.chromeBackend?.dispose().catch(() => {});
-					this.chromeBackend = null;
-					this._deviceMode = "cpu";
-					this.generator = await pipeline$1("text-generation", this.currentModel || "qwen3-0.6b", {
-						dtype: "q4",
-						device: "cpu"
-					});
-					return this.generate(prompt, options);
-				}
-				throw chromeErr;
-			}
-			else if (this.useDirect && this.model && this.tokenizer) {
-				const messages = this.buildMessages(prompt, {
-					...options,
-					thinking
-				});
-				const inputs = this.tokenizer.apply_chat_template(messages, {
-					add_generation_prompt: true,
-					return_dict: true,
-					enable_thinking: thinking
-				});
-				const output = await this.model.generate({
-					...inputs,
-					max_new_tokens: maxTokens,
-					temperature: temperature > 0 ? temperature : void 0,
-					top_p: topP,
-					top_k: topK,
-					do_sample: temperature > 0
-				});
-				const inputLength = inputs.input_ids.dims?.[1] || inputs.input_ids.data?.length || 0;
-				const outputTokens = output.slice(null, [inputLength, null]);
-				rawText = this.tokenizer.batch_decode(outputTokens, { skip_special_tokens: true })[0] || "";
-				if (rawText.toLowerCase().includes("assistant")) {
-					const match = rawText.match(/assistant[:\s]*([\s\S]*)/i);
-					if (match) rawText = match[1].trim();
-				}
-			} else if (this.generator) {
-				const formattedPrompt = this.formatPrompt(prompt, {
-					...options,
-					thinking
-				});
-				const output = await this.generator(formattedPrompt, {
-					max_new_tokens: maxTokens,
-					temperature,
-					top_p: topP,
-					top_k: topK,
-					do_sample: temperature > 0,
-					return_full_text: false
-				});
-				if (Array.isArray(output) && output[0]) {
-					const result = output[0];
-					if (Array.isArray(result.generated_text)) rawText = result.generated_text.at(-1)?.content || "";
-					else rawText = result.generated_text || "";
-				}
-			} else throw new Error("No model loaded");
-			const totalTime = performance.now() - startTime;
-			rawText = this.cleanOutput(rawText);
-			const { thinking: thinkingText, response } = this.parseThinking(rawText);
-			const finalThinking = thinking ? thinkingText : void 0;
-			const tokensGenerated = Math.ceil(response.length / 4);
-			this.stats.prompts++;
-			this.stats.tokensOut += tokensGenerated;
-			this.stats.totalTime += totalTime;
-			this.stats.avgSpeed = this.stats.tokensOut / this.stats.totalTime * 1e3;
-			return {
-				text: response,
-				thinking: finalThinking,
-				tokensGenerated,
-				tokensPerSecond: tokensGenerated / totalTime * 1e3,
-				totalTime,
-				finishReason: "stop",
-				provider: "local",
-				cached: false
-			};
-		} catch (_error) {
-			return {
-				text: "",
-				tokensGenerated: 0,
-				tokensPerSecond: 0,
-				totalTime: performance.now() - startTime,
-				finishReason: "error",
-				provider: "local",
-				cached: false
-			};
-		}
-	}
-	/**
-	* Stream text generation (simulated token-by-token)
-	*
-	* Note: Yields the raw output including <think> tags if thinking mode is enabled.
-	* The final result has parsed thinking separated out.
-	*/
-	async *stream(prompt, options = {}) {
-		if (!this.isLoaded()) await this.loadModel(this.config.model || "qwen3-0.6b");
-		const startTime = performance.now();
-		if (this.chromeBackend) {
-			let fullText = "";
-			const tokenQueue = [];
-			let resolveNext = null;
-			let done = false;
-			const generatePromise = this.chromeBackend.generate(prompt, {
-				...options,
-				onToken: (token) => {
-					fullText += token.text;
-					if (resolveNext) {
-						resolveNext(token.text);
-						resolveNext = null;
-					} else tokenQueue.push(token.text);
-				}
-			}).then(() => {
-				done = true;
-				if (resolveNext) resolveNext(null);
-			}).catch((err) => {
-				done = true;
-				if (resolveNext) resolveNext(null);
-				throw err;
-			});
-			while (!done || tokenQueue.length > 0) if (tokenQueue.length > 0) {
-				const token = tokenQueue.shift();
-				yield token;
-				options.onToken?.(token);
-			} else if (!done) {
-				const token = await new Promise((resolve) => {
-					resolveNext = resolve;
-				});
-				if (token) {
-					yield token;
-					options.onToken?.(token);
-				}
-			}
-			await generatePromise;
-			const { thinking: thinkingText, response } = this.parseThinking(fullText);
-			const tokensGenerated = Math.ceil(response.length / 4);
-			const totalTime = performance.now() - startTime;
-			return {
-				text: response,
-				thinking: options.thinking ? thinkingText : void 0,
-				tokensGenerated,
-				totalTime,
-				tokensPerSecond: tokensGenerated / totalTime * 1e3,
-				finishReason: "stop"
-			};
-		}
-		const result = await this.generateRaw(prompt, options);
-		const words = result.rawText.split(/(\s+)/);
-		for (const word of words) if (word) {
-			yield word;
-			options.onToken?.(word);
-		}
-		return result.result;
-	}
-	/**
-	* Internal: Generate with raw text access for streaming
-	*/
-	async generateRaw(prompt, options = {}) {
-		const { maxTokens = 256, temperature = .7, topP = .9, topK = 50, thinking = false } = options;
-		const startTime = performance.now();
-		const formattedPrompt = this.formatPrompt(prompt, {
-			...options,
-			thinking
-		});
-		try {
-			const output = await this.generator(formattedPrompt, {
-				max_new_tokens: maxTokens,
-				temperature,
-				top_p: topP,
-				top_k: topK,
-				do_sample: temperature > 0,
-				return_full_text: false
-			});
-			const totalTime = performance.now() - startTime;
-			let rawText = "";
-			if (Array.isArray(output) && output[0]) {
-				const result = output[0];
-				if (Array.isArray(result.generated_text)) rawText = result.generated_text.at(-1)?.content || "";
-				else rawText = result.generated_text || "";
-			}
-			rawText = this.cleanOutput(rawText);
-			const { thinking: thinkingText, response } = this.parseThinking(rawText);
-			const finalThinking = thinking ? thinkingText : void 0;
-			const tokensGenerated = Math.ceil(response.length / 4);
-			this.stats.prompts++;
-			this.stats.tokensOut += tokensGenerated;
-			this.stats.totalTime += totalTime;
-			this.stats.avgSpeed = this.stats.tokensOut / this.stats.totalTime * 1e3;
-			return {
-				rawText,
-				result: {
-					text: response,
-					thinking: finalThinking,
-					tokensGenerated,
-					tokensPerSecond: tokensGenerated / totalTime * 1e3,
-					totalTime,
-					finishReason: "stop",
-					provider: "local",
-					cached: false
-				}
-			};
-		} catch (_error) {
-			return {
-				rawText: "",
-				result: {
-					text: "",
-					tokensGenerated: 0,
-					tokensPerSecond: 0,
-					totalTime: performance.now() - startTime,
-					finishReason: "error",
-					provider: "local",
-					cached: false
-				}
-			};
-		}
-	}
-	/**
-	* Generate structured JSON output
-	*/
-	async json(prompt, options) {
-		const { schema, retries = 3, temperature = .3 } = options;
-		const systemPrompt = `You are a JSON generator. You MUST respond with valid JSON only.
-No explanations, no markdown, no code blocks. Just pure JSON.
-The JSON must conform to this schema: ${JSON.stringify(zodToJsonSchema(schema))}`;
-		for (let attempt = 0; attempt < retries; attempt++) {
-			const result = await this.generate(prompt, {
-				system: options.system || systemPrompt,
-				temperature,
-				maxTokens: 1e3
-			});
-			try {
-				const jsonStr = extractJson(result.text);
-				const parsed = JSON.parse(jsonStr);
-				return schema.parse(parsed);
-			} catch (error) {
-				if (attempt === retries - 1) throw new Error(`Failed to generate valid JSON after ${retries} attempts: ${error}`);
-			}
-		}
-		throw new Error("Failed to generate valid JSON");
-	}
-	/**
-	* Generate embeddings
-	*/
-	async embed(text, options = {}) {
-		if (!this.embedder) this.embedder = await pipeline$1("feature-extraction", options.model || "Xenova/all-MiniLM-L6-v2");
-		const startTime = performance.now();
-		const output = await this.embedder(text, {
-			pooling: "mean",
-			normalize: options.normalize !== false
-		});
-		return {
-			vector: Array.from(output.data),
-			text,
-			totalTime: performance.now() - startTime
-		};
-	}
-	/**
-	* Generate embeddings for multiple texts
-	*/
-	async embedBatch(texts, options = {}) {
-		const results = [];
-		for (const text of texts) results.push(await this.embed(text, options));
-		return results;
-	}
-	/**
-	* Get session stats
-	*/
-	getStats() {
-		return { ...this.stats };
-	}
-	/**
-	* Get system info
-	*/
-	getInfo() {
-		return {
-			version: "1.0.0",
-			model: this.modelConfig,
-			device: {
-				backend: "transformers.js",
-				gpu: null,
-				vram: null,
-				status: this.isLoaded() ? "ready" : "loading"
-			},
-			context: {
-				max: this.modelConfig?.contextLength || 0,
-				used: 0,
-				available: this.modelConfig?.contextLength || 0
-			},
-			cache: {
-				location: "~/.gerbil/models",
-				size: "0 MB",
-				modelCount: 0
-			}
-		};
-	}
-	/**
-	* Reset stats
-	*/
-	resetStats() {
-		this.stats = {
-			prompts: 0,
-			tokensIn: 0,
-			tokensOut: 0,
-			avgSpeed: 0,
-			totalTime: 0,
-			cacheHits: 0,
-			cacheMisses: 0
-		};
-	}
-	/**
-	* Dispose of resources
-	*/
-	async dispose() {
-		if (this.chromeBackend) {
-			try {
-				await this.chromeBackend.dispose();
-			} catch {}
-			this.chromeBackend = null;
-		}
-		if (this.generator) {
-			if (typeof this.generator.dispose === "function") try {
-				await this.generator.dispose();
-			} catch {}
-			this.generator = null;
-		}
-		if (this.embedder) {
-			if (typeof this.embedder.dispose === "function") try {
-				await this.embedder.dispose();
-			} catch {}
-			this.embedder = null;
-		}
-		this.currentModel = null;
-		this.modelConfig = null;
-	}
-	formatPrompt(prompt, options) {
-		const system = options.system || "You are a helpful assistant.";
-		const isQwen = this.currentModel?.includes("qwen");
-		if (options.thinking && this.modelConfig?.supportsThinking) return `<|im_start|>system\n${`${system}\n\nThink step-by-step before answering. Wrap your reasoning in <think></think> tags, then provide your answer.`}<|im_end|>\n<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
-		if (isQwen) return `<|im_start|>system\n${system}<|im_end|>\n<|im_start|>user\n${prompt} /no_think<|im_end|>\n<|im_start|>assistant\n`;
-		return `<|im_start|>system\n${system}<|im_end|>\n<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
-	}
-	buildMessages(prompt, options) {
-		const system = options.system || "You are a helpful assistant.";
-		const messages = [];
-		messages.push({
-			role: "system",
-			content: system
-		});
-		messages.push({
-			role: "user",
-			content: prompt
-		});
-		return messages;
-	}
-	parseThinking(text) {
-		const match = text.match(/<think>([\s\S]*?)<\/think>/);
-		if (match) return {
-			thinking: match[1].trim(),
-			response: text.replace(/<think>[\s\S]*?<\/think>/, "").trim()
-		};
-		const unclosedMatch = text.match(/<think>([\s\S]*)$/);
-		if (unclosedMatch) {
-			const thinking = unclosedMatch[1].trim();
-			const response = text.replace(/<think>[\s\S]*$/, "").trim();
-			return {
-				thinking: thinking || void 0,
-				response
-			};
-		}
-		return { response: text.replace(/<\/?think>/g, "").trim() };
-	}
-	cleanOutput(text) {
-		return text.replace(/<\|im_end\|>/g, "").replace(/<\|im_start\|>/g, "").replace(/<\|endoftext\|>/g, "").replace(/<\/s>/g, "").replace(/^\/no_think\s*/i, "").replace(/^assistant\s*/i, "").replace(/^\s*\/no_think\s*/gim, "").replace(/^\s*assistant\s*/gim, "").replace(/^(system|user|assistant):\s*/gim, "").trim();
-	}
-};
-
-//#endregion
-export { Gerbil as t };
-//# sourceMappingURL=gerbil-BfnsFWRE.mjs.map