@tryhamster/gerbil 1.0.0-rc.0 → 1.0.0-rc.10

package/docs/vision.md

@@ -0,0 +1,396 @@
+# Vision Models in Gerbil
+
+Gerbil supports **Vision Language Models (VLMs)** like Ministral 3B that can understand and describe images. This guide covers how to use vision capabilities across all Gerbil interfaces.
+
+## Quick Start
+
+```typescript
+import { Gerbil } from "@tryhamster/gerbil";
+
+const g = new Gerbil();
+await g.loadModel("ministral-3b"); // Vision-capable model
+
+const result = await g.generate("What's in this image?", {
+  images: [{ source: "https://example.com/photo.jpg" }]
+});
+
+console.log(result.text);
+```
+
+## Supported Models
+
+| Model ID | Vision | Reasoning | Context | Size |
+|----------|--------|-----------|---------|------|
+| `ministral-3b` | ✅ | ✅ | 256K | ~2.5GB |
+
+More vision models coming soon as they become available in ONNX format.
+
+## Image Input Types
+
+Gerbil accepts images in several formats:
+
+```typescript
+// URL (recommended for web images)
+images: [{ source: "https://example.com/image.jpg" }]
+
+// Data URI (base64 encoded)
+images: [{ source: "data:image/png;base64,iVBORw0KGgo..." }]
+
+// Local file path (Node.js only, auto-converted to data URI)
+images: [{ source: "/path/to/image.png" }]
+
+// With alt text (optional, provides context)
+images: [{ source: "...", alt: "A photo of a sunset" }]
+```
+
+## Multiple Images
+
+You can pass multiple images for comparison or multi-image understanding:
+
+```typescript
+const result = await g.generate("What's the difference between these two images?", {
+  images: [
+    { source: "https://example.com/before.jpg" },
+    { source: "https://example.com/after.jpg" }
+  ]
+});
+```
+
+## Model Capability Detection
+
+Check if the loaded model supports vision:
+
+```typescript
+await g.loadModel("ministral-3b");
+
+if (g.supportsVision()) {
+  // Use vision features
+} else {
+  // Text-only mode
+}
+```
+
+## Graceful Fallback
+
+If you pass images to a non-vision model, Gerbil will:
+1. Log a warning to console
+2. Ignore the images
+3. Process the text prompt normally
+
+This allows you to write code that works with any model:
+
+```typescript
+// This works with any model - images are used if supported
+const result = await g.generate("Describe this", {
+  images: [{ source: imageUrl }]
+});
+```
+
+---
+
+## AI SDK Integration
+
+Use vision models with Vercel AI SDK v5+:
+
+```typescript
+import { generateText } from "ai";
+import { gerbil } from "@tryhamster/gerbil/ai";
+
+const { text } = await generateText({
+  model: gerbil("ministral-3b"),
+  messages: [
+    {
+      role: "user",
+      content: [
+        { type: "image", image: new URL("https://example.com/photo.jpg") },
+        { type: "text", text: "Describe this image in detail" },
+      ],
+    },
+  ],
+});
+```
+
+### Image Part Formats
+
+The AI SDK integration accepts images in these formats:
+
+```typescript
+// URL object
+{ type: "image", image: new URL("https://...") }
+
+// URL string
+{ type: "image", image: "https://..." }
+
+// Base64 string
+{ type: "image", image: "data:image/png;base64,..." }
+
+// Uint8Array with mime type
+{ type: "image", image: imageBytes, mimeType: "image/png" }
+```
+
+---
+
+## Express & Next.js Integration
+
+### Express
+
+```typescript
+import express from "express";
+import { gerbil } from "@tryhamster/gerbil/express";
+
+const app = express();
+app.use("/ai", gerbil({ model: "ministral-3b" })());
+
+// POST /ai/generate
+// Body: { prompt: "Describe this", images: [{ source: "https://..." }] }
+```
+
+### Next.js App Router
+
+```typescript
+// app/api/chat/route.ts
+import { gerbil } from "@tryhamster/gerbil/next";
+
+export const POST = gerbil.handler({ model: "ministral-3b" });
+
+// Fetch from client:
+// fetch("/api/chat", {
+//   method: "POST",
+//   body: JSON.stringify({
+//     prompt: "What's in this image?",
+//     images: [{ source: dataUri }]
+//   })
+// })
+```
+
+---
+
+## React Hooks (Browser)
+
+### useChat with Images
+
+```tsx
+import { useChat } from "@tryhamster/gerbil/browser";
+
+function VisionChat() {
+  const { 
+    messages, 
+    input, 
+    setInput, 
+    handleSubmit,
+    attachImage,
+    attachedImages,
+    clearImages,
+    sendWithImages,
+  } = useChat({ model: "ministral-3b" });
+
+  const handleFileSelect = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    if (file) {
+      const reader = new FileReader();
+      reader.onload = () => attachImage(reader.result as string);
+      reader.readAsDataURL(file);
+    }
+  };
+
+  return (
+    <div>
+      {/* Messages */}
+      {messages.map(m => (
+        <div key={m.id}>
+          {m.images?.map((img, i) => (
+            <img key={i} src={img} alt="" className="max-w-xs" />
+          ))}
+          <p>{m.content}</p>
+        </div>
+      ))}
+
+      {/* Image attachment */}
+      <input type="file" accept="image/*" onChange={handleFileSelect} />
+      
+      {attachedImages.length > 0 && (
+        <div>
+          📎 {attachedImages.length} image(s) attached
+          <button onClick={clearImages}>Clear</button>
+        </div>
+      )}
+
+      {/* Input */}
+      <form onSubmit={handleSubmit}>
+        <input 
+          value={input} 
+          onChange={e => setInput(e.target.value)}
+          placeholder="Describe the image..."
+        />
+        <button type="submit">Send</button>
+      </form>
+    </div>
+  );
+}
+```
+
+### Direct Image Send
+
+```tsx
+// Send a message with specific images
+sendWithImages("Compare these two photos", [image1DataUri, image2DataUri]);
+```
+
+---
+
+## Built-in Vision Skills
+
+Gerbil includes pre-built skills for common vision tasks:
+
+### Describe Image
+
+```typescript
+import { describeImage } from "@tryhamster/gerbil/skills";
+
+const description = await describeImage({
+  image: "https://example.com/photo.jpg",
+  focus: "details", // "general" | "details" | "text" | "objects" | "scene"
+  format: "bullets", // "paragraph" | "bullets" | "structured"
+});
+```
+
+### Analyze Screenshot
+
+```typescript
+import { analyzeScreenshot } from "@tryhamster/gerbil/skills";
+
+const analysis = await analyzeScreenshot({
+  image: screenshotDataUri,
+  type: "accessibility", // "ui-review" | "accessibility" | "suggestions" | "qa"
+});
+```
+
+### Extract from Image
+
+```typescript
+import { extractFromImage } from "@tryhamster/gerbil/skills";
+
+const extracted = await extractFromImage({
+  image: documentPhoto,
+  extract: "text", // "text" | "data" | "code" | "table" | "diagram"
+  outputFormat: "markdown", // "raw" | "json" | "markdown"
+});
+```
+
+### Compare Images
+
+```typescript
+import { compareImages } from "@tryhamster/gerbil/skills";
+
+const comparison = await compareImages({
+  image1: beforeScreenshot,
+  image2: afterScreenshot,
+  focus: "differences", // "differences" | "similarities" | "detailed"
+});
+```
+
+### Caption Image
+
+```typescript
+import { captionImage } from "@tryhamster/gerbil/skills";
+
+const caption = await captionImage({
+  image: photo,
+  style: "descriptive", // "concise" | "descriptive" | "creative" | "funny"
+});
+```
+
+---
+
+## Performance Tips
+
+### WebGPU Acceleration
+
+Vision models benefit significantly from GPU acceleration:
+
+```typescript
+// Node.js: Uses Chrome backend for WebGPU
+await g.loadModel("ministral-3b"); // Auto-detects WebGPU
+
+// Browser: Native WebGPU
+await g.loadModel("ministral-3b", { device: "webgpu" });
+```
+
+### Image Size
+
+- Larger images take longer to process
+- Consider resizing before sending to the model
+- 512x512 to 1024x1024 is generally optimal
+
+### Caching
+
+The model caches in the browser's IndexedDB (via Chrome backend in Node.js), so subsequent loads are fast.
+
+---
+
+## Troubleshooting
+
+### "Model doesn't support vision"
+
+Make sure you're using a vision-capable model like `ministral-3b`.
+
+### Slow image processing
+
+- Ensure WebGPU is being used (check `g.getDeviceMode()`)
+- Resize large images before sending
+- In Node.js, the Chrome backend provides GPU acceleration
+
+### Image not loading
+
+- Check the URL is accessible
+- For local files, ensure the path is absolute
+- Base64 data URIs must include the mime type prefix
+
+---
+
+## API Reference
+
+### ImageInput
+
+```typescript
+interface ImageInput {
+  /** Image source: URL, base64 data URI, or local file path */
+  source: string;
+  /** Optional alt text for context */
+  alt?: string;
+}
+```
+
+### GenerateOptions (with images)
+
+```typescript
+interface GenerateOptions {
+  // ... standard options ...
+  
+  /** Images to include (only used if model supports vision) */
+  images?: ImageInput[];
+}
+```
+
+### supportsVision()
+
+```typescript
+g.supportsVision(): boolean
+```
+
+Returns `true` if the loaded model supports vision input.
+
+### ModelConfig
+
+```typescript
+interface ModelConfig {
+  // ... standard properties ...
+  
+  /** Whether model supports vision/image input */
+  supportsVision?: boolean;
+  
+  /** Size of vision encoder (if applicable) */
+  visionEncoderSize?: string;
+}
+```
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryhamster/gerbil",
-  "version": "1.0.0-rc.0",
+  "version": "1.0.0-rc.10",
   "description": "Local LLM inference for Node.js. GPU-accelerated. Zero config. Works standalone or with Vercel AI SDK.",
   "type": "module",
   "main": "dist/index.mjs",
@@ -62,32 +62,18 @@
       "types": "./dist/integrations/mcp-client.d.mts"
     },
     "./browser": {
-      "import": "./dist/browser/index.mjs",
-      "types": "./dist/browser/index.d.mts"
+      "import": "./dist/browser/index.js",
+      "types": "./dist/browser/index.d.ts"
     }
   },
-  "scripts": {
-    "build": "tsdown",
-    "dev": "tsx src/cli/index.ts",
-    "typecheck": "tsc --noEmit",
-    "check": "ultracite check",
-    "fix": "ultracite fix",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "prepublishOnly": "pnpm build",
-    "changeset": "changeset",
-    "release": "changeset publish",
-    "prepare": "lefthook install"
-  },
   "dependencies": {
     "@huggingface/hub": "^2.7.1",
-    "@huggingface/transformers": "^3.8.0",
     "chalk": "^5.3.0",
     "cli-progress": "^3.12.0",
     "commander": "^12.1.0",
-    "onnxruntime-web": "^1.21.0-dev.20250114-228dd16893",
     "ora": "^8.0.1",
     "puppeteer-core": "^24.31.0",
+    "react": "^19.0.0",
     "webgpu": "^0.3.8",
     "zod": "^3.23.0"
   },
@@ -101,8 +87,7 @@
     "hono": ">=4.0.0",
     "langchain": ">=0.1.0",
     "llamaindex": ">=0.1.0",
-    "next": ">=14.0.0",
-    "react": ">=18.0.0"
+    "next": ">=14.0.0"
   },
   "peerDependenciesMeta": {
     "@ai-sdk/provider": {
@@ -142,6 +127,9 @@
   "devDependencies": {
     "@ai-sdk/provider": "^2.0.0",
     "@biomejs/biome": "^2.3.8",
+    "@huggingface/transformers": "^3.8.0",
+    "kokoro-js": "^1.2.1",
+    "onnxruntime-web": "^1.21.0-dev.20250114-228dd16893",
     "@changesets/changelog-github": "^0.5.1",
     "@changesets/cli": "^2.28.1",
     "@types/cli-progress": "^3.11.6",
@@ -158,7 +146,6 @@
     "ink-spinner": "^5.0.0",
     "ink-text-input": "^6.0.0",
     "lefthook": "^2.0.5",
-    "react": "^19.0.0",
     "tsdown": "^0.17.0-beta.3",
     "tsx": "^4.15.0",
     "typescript": "^5.4.5",
@@ -203,5 +190,17 @@
   ],
   "publishConfig": {
     "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsx src/cli/index.ts",
+    "typecheck": "tsc --noEmit",
+    "check": "ultracite check",
+    "fix": "ultracite fix",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "changeset": "changeset",
+    "version": "changeset version",
+    "release": "pnpm publish && changeset tag"
   }
-}
+}

package/dist/auto-update-BbNHbSU1.mjs

@@ -1,3 +0,0 @@
-import { i as installUpdate, n as checkForUpdate, r as compareVersions, t as CURRENT_VERSION } from "./cli.mjs";
-
-export { CURRENT_VERSION, checkForUpdate, installUpdate };