@tryhamster/gerbil 1.0.0-rc.23 → 1.0.0-rc.25

package/dist/{types-evP8RShr.d.mts → types-6uG8lC7u.d.mts}

@@ -144,6 +144,10 @@ type GerbilConfig = {
   cache?: CacheConfig;
   /** Fallback configuration */
   fallback?: FallbackConfig;
+  /** Telemetry hooks for observability (Sentry, logging, etc.) */
+  telemetry?: TelemetryConfig;
+  /** Concurrency control for request queuing */
+  concurrency?: ConcurrencyConfig;
 };
 type CacheConfig = {
   /** Enable caching (default: true) */
@@ -372,6 +376,65 @@ type StreamingTranscriptionSession = {
   /** Reset session (clear buffer and transcript) */
   reset: () => void;
 };
+/**
+ * Telemetry hooks for production observability.
+ * Pass your own Sentry instance or custom logging functions.
+ */
+type TelemetryConfig = {
+  /**
+   * Called after successful generation with full result and timing.
+   * Use for logging, metrics, or analytics.
+   */
+  onGenerate?: (event: GenerateEvent) => void;
+  /**
+   * Called when any error occurs during Gerbil operations.
+   * Perfect for Sentry.captureException() or similar.
+   */
+  onError?: (error: Error, context: ErrorContext) => void;
+  /**
+   * Called after model loading completes (success or failure).
+   */
+  onModelLoad?: (event: ModelLoadEvent) => void;
+  /**
+   * Called when a request is queued (if concurrency limit reached).
+   */
+  onQueueWait?: (waitTimeMs: number) => void;
+};
+type GenerateEvent = {
+  /** Model used for generation */
+  modelId: string;
+  /** Generation result */
+  result: GenerateResult;
+  /** Whether response came from cache */
+  cached: boolean;
+  /** Time spent waiting in queue (if any) */
+  queueTimeMs?: number;
+};
+/**
+ * Context passed to telemetry onError callback.
+ * Flexible record to allow any relevant context data.
+ */
+type ErrorContext = Record<string, unknown>;
+type ModelLoadEvent = {
+  /** Model that was loaded */
+  modelId: string;
+  /** Time to load in ms */
+  loadTimeMs: number;
+  /** Whether loaded from cache */
+  fromCache: boolean;
+  /** Device used */
+  device: "webgpu" | "cpu" | "wasm";
+  /** Whether load succeeded */
+  success: boolean;
+  /** Error message if failed */
+  error?: string;
+};
+type ConcurrencyConfig = {
+  /** Maximum concurrent generation requests (default: 1 for LLM) */
+  maxConcurrent?: number;
+  /** Request timeout in ms (default: 300000 = 5 min) */
+  timeout?: number;
+};
 //#endregion
-export { TTSModelConfig as A, SessionStats as C, StreamingTranscriptionOptions as D, SpeakResult as E, TranscribeResult as M, TranscribeSegment as N, StreamingTranscriptionSession as O, VoiceInfo as P, SearchResult as S, SpeakOptions as T, ModelSource as _, FallbackConfig as a, ProgressInfo as b, GerbilConfig as c, ImageInput as d, JsonOptions as f, ModelConfig as g, LoadTTSOptions as h, EmbedResult as i, TranscribeOptions as j, SystemInfo as k, GerbilModelSettings as l, LoadSTTOptions as m, CacheConfig as n, GenerateOptions as o, LoadOptions as p, EmbedOptions as r, GenerateResult as s, AudioChunk as t, GerbilProviderSettings as u, ModelStats as v, SimilarityResult as w, STTModelConfig as x, PreloadOptions as y };
-//# sourceMappingURL=types-evP8RShr.d.mts.map
+export { SpeakResult as A, PreloadOptions as C, SessionStats as D, SearchResult as E, TelemetryConfig as F, TranscribeOptions as I, TranscribeResult as L, StreamingTranscriptionSession as M, SystemInfo as N, SimilarityResult as O, TTSModelConfig as P, TranscribeSegment as R, ModelStats as S, STTModelConfig as T, LoadSTTOptions as _, EmbedResult as a, ModelLoadEvent as b, GenerateEvent as c, GerbilConfig as d, GerbilModelSettings as f, LoadOptions as g, JsonOptions as h, EmbedOptions as i, StreamingTranscriptionOptions as j, SpeakOptions as k, GenerateOptions as l, ImageInput as m, CacheConfig as n, ErrorContext as o, GerbilProviderSettings as p, ConcurrencyConfig as r, FallbackConfig as s, AudioChunk as t, GenerateResult as u, LoadTTSOptions as v, ProgressInfo as w, ModelSource as x, ModelConfig as y, VoiceInfo as z };
+//# sourceMappingURL=types-6uG8lC7u.d.mts.map

package/dist/types-6uG8lC7u.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-6uG8lC7u.d.mts","names":[],"sources":["../src/core/types.ts"],"sourcesContent":[],"mappings":";;;;AAyBY,KAfA,WAAA,GAeW;EASX,EAAA,EAAA,MAAA;EAWA,IAAA,EAAA,MAAA;EAmCA,WAAA,EAAA,MAAc;EA8Bd,IAAA,EAAA,MAAA;EAkBA,aAAA,EAAY,MAAA;EAQZ,gBAAW,EAAA,OAAA;EAWX,YAAA,EAAA,OAAY;EAWZ;EAkBA,cAAW,CAAA,EAAA,OAAA;EAcX;EAOA,iBAAY,CAAA,EAAA,MAAA;EAYZ,MAAA,EAAA,MAAA,GAAY,QAAA,GAAA,KAAA,GAAA,SAAA,GAAA,OAAA,GAAA,OAAA;CAWd;AAGG,KAtMD,WAAA,GAsMC;EAGC,IAAA,EAAA,SAAA,GAAA,aAAA,GAAA,OAAA;EAGE,IAAA,EAAA,MAAA;CAAiB;AAGrB,KAtMA,UAAA,GAsMW;EAiBX;EAqBA,MAAA,EAAA,MAAA;EAUA;EAOA,GAAA,CAAA,EAAA,MAAA;AAyBZ,CAAA;AAWY,KAtRA,eAAA,GAsRsB;EAYtB;EAeA,SAAA,CAAA,EAAA,MAAc;EAmBd;EAWA,WAAA,CAAA,EAAU,MAAA;EAWV;EAaA,IAAA,CAAA,EAAA,MAAA;EAWA;EAiBA,IAAA,CAAA,EAAA,MAAA;EASA;EASA,aAAA,CAAA,EAAA,MAAgB,EAAA;EAahB;EAWA,MAAA,CAAA,EAAA,MAAA;EAeA;EAES,QAAA,CAAA,EAAA,OAAA;EAEN;EAID,OAAA,CAAA,EAAA,CAAA,KAAA,EAAA,MAAA,EAAA,GAAA,IAAA;EAAO;EAqBT,MAAA,CAAA,EA/bD,UA+bgB,EAAA;EAKJ;EAMH,KAAA,CAAA,EAAA,OAAA;EAAgB;EAKZ,QAAA,CAAA,EAAA,MAAA;CAAc;AAQ1B,KA9cA,cAAA,GA8ca;EAeb;EAEA,IAAA,EAAA,MAAA;EAmBA;;;;;;;;;;;;;;;KApdA;;UAEF,CAAA,CAAE,QAAQ;;;;;;;;KAgBR,YAAA;;;;;;KAQA,WAAA;;;;;;;;KAWA,YAAA;;;;;;;;KAWA,gBAAA;;;;;;;;;;KAkBA,WAAA;;sBAEU;;;;;;;;KAYV,cAAA;;sBAEU;;;;KAKV,YAAA;;;;;;;KAYA,YAAA;;;;;;;;UAWF;;aAGG;;cAGC;;gBAGE;;KAGJ,WAAA;;;;;;;;;;;;KAiBA,cAAA;;;;;;;;;;;;KAqBA,YAAA;;;;;;;;;KAUA,UAAA;;;;;;KAOA,UAAA;;SAEH;;;;;;;;;;;;;;;;;;KAuBG,mBAAA;;;;;;;;KAWA,sBAAA;;;;;;KAYA,SAAA;;;;;;;;;;;;;;KAeA,cAAA;;;;;;;;;;;;UAYF;;;;;;KAOE,YAAA;;;;;;sBAMU;;yBAEG;;KAGb,UAAA;;WAED;;;;;;;;KASC,WAAA;;SAEH;;;;;;;;;;KAWG,cAAA;;sBAEU;;;;KASV,cAAA;;;;;;;;;;;;;;;;KAiBA,iBAAA;;;;;;sBAMU;;KAGV,iBAAA;;;;;;;;KASA,gBAAA;;;;;;aAMC;;;;;;KAOD,cAAA;;sBAEU;;;;KASV,6BAAA;;;;;;;;;;;;;;KAeA,6BAAA;;qBAES;;eAEN;;;;cAID;;;;;;;;;;;;;;;;KAqBF,eAAA;;;;;uBAKW;;;;;oBAMH,gBAAgB;;;;wBAKZ;;;;;;KAQZ,aAAA;;;;UAIF;;;;;;;;;;KAWE,YAAA,GAAe;KAEf,cAAA;;;;;;;;;;;;;;KAmBA,iBAAA"}

package/docs/architecture/overview.md

@@ -20,6 +20,8 @@ Responsibilities:
 - Generation orchestration
 - Streaming coordination
 - Session statistics
+- Request queue (concurrency control)
+- Telemetry hooks for observability
 
 ### 2. Model Registry (`src/core/models.ts`)
 

package/docs/observability.md

@@ -0,0 +1,230 @@
+# Production Observability
+
+Gerbil includes built-in support for production observability through telemetry hooks and request queuing.
+
+## Telemetry Hooks
+
+Configure telemetry hooks to integrate with Sentry, DataDog, or any monitoring system:
+
+```typescript
+import { Gerbil } from "@tryhamster/gerbil";
+import * as Sentry from "@sentry/node";
+
+const g = new Gerbil({
+  telemetry: {
+    // Called on any error (model load, generation, etc.)
+    onError: (error, context) => {
+      Sentry.captureException(error, {
+        extra: context,
+        tags: { operation: context.operation },
+      });
+    },
+
+    // Called after successful generation
+    onGenerate: (event) => {
+      console.log(`Generated ${event.result.tokensGenerated} tokens`);
+      // Track in your metrics system
+      metrics.histogram("gerbil.tokens_generated", event.result.tokensGenerated);
+      metrics.histogram("gerbil.tokens_per_second", event.result.tokensPerSecond);
+    },
+
+    // Called after model loading (success or failure)
+    onModelLoad: (event) => {
+      if (event.success) {
+        console.log(`Loaded ${event.modelId} in ${event.loadTimeMs}ms on ${event.device}`);
+      } else {
+        console.error(`Failed to load ${event.modelId}: ${event.error}`);
+      }
+    },
+
+    // Called when requests wait in queue (>100ms)
+    onQueueWait: (waitTimeMs) => {
+      metrics.histogram("gerbil.queue_wait_ms", waitTimeMs);
+    },
+  },
+});
+```
+
+### Telemetry Events
+
+#### `onError(error, context)`
+
+Called whenever an error occurs during Gerbil operations.
+
+```typescript
+type ErrorContext = {
+  operation: "generate" | "load" | "embed" | "speak" | "transcribe" | "json";
+  modelId?: string;
+  extra?: Record<string, unknown>;
+};
+```
+
+#### `onGenerate(event)`
+
+Called after successful text generation.
+
+```typescript
+type GenerateEvent = {
+  modelId: string;
+  result: GenerateResult;
+  cached: boolean;
+  queueTimeMs?: number; // Only if waited >100ms
+};
+```
+
+#### `onModelLoad(event)`
+
+Called after model loading completes (success or failure).
+
+```typescript
+type ModelLoadEvent = {
+  modelId: string;
+  loadTimeMs: number;
+  fromCache: boolean;
+  device: "webgpu" | "cpu" | "wasm";
+  success: boolean;
+  error?: string;
+};
+```
+
+#### `onQueueWait(waitTimeMs)`
+
+Called when a request waits in the queue for more than 100ms. Useful for detecting congestion.
+
+## Request Queue
+
+Gerbil uses a request queue to prevent GPU OOM errors under concurrent load. LLM inference can only run one request at a time on the GPU.
+
+### Default Behavior
+
+- **Concurrency**: 1 (single request at a time)
+- **Timeout**: 5 minutes (300,000ms)
+- Requests are processed in FIFO order
+- Timeout errors are thrown if exceeded
+
+### Custom Configuration
+
+```typescript
+const g = new Gerbil({
+  concurrency: {
+    maxConcurrent: 1,       // Max parallel requests (default: 1)
+    timeout: 300_000,       // Request timeout in ms (default: 5 min)
+  },
+});
+```
+
+### Why Queue?
+
+LLM inference on GPU is:
+1. **Memory-bound**: Models consume most of GPU VRAM
+2. **Non-concurrent**: Running multiple inferences simultaneously causes OOM
+3. **Variable duration**: Generation time depends on output length
+
+The queue ensures:
+- Predictable memory usage
+- No OOM crashes under load
+- Fair request ordering
+
+## Rate Limiting
+
+Gerbil does **not** include rate limiting. This is intentional—rate limiting is best handled at the application layer using middleware specific to your framework:
+
+```typescript
+// Express
+import rateLimit from "express-rate-limit";
+import { gerbil } from "@tryhamster/gerbil/express";
+
+app.use("/ai", rateLimit({ windowMs: 60000, max: 10 }));
+app.use("/ai", gerbil());
+
+// Next.js
+import { Ratelimit } from "@upstash/ratelimit";
+import { Redis } from "@upstash/redis";
+
+const ratelimit = new Ratelimit({
+  redis: Redis.fromEnv(),
+  limiter: Ratelimit.slidingWindow(10, "60s"),
+});
+
+export async function POST(req: Request) {
+  const ip = req.headers.get("x-forwarded-for") ?? "anonymous";
+  const { success } = await ratelimit.limit(ip);
+  if (!success) return Response.json({ error: "Rate limited" }, { status: 429 });
+  
+  // Continue with Gerbil...
+}
+```
+
+## Example: Full Production Setup
+
+```typescript
+import { Gerbil } from "@tryhamster/gerbil";
+import * as Sentry from "@sentry/node";
+
+Sentry.init({ dsn: process.env.SENTRY_DSN });
+
+const g = new Gerbil({
+  model: "qwen3-0.6b",
+  
+  telemetry: {
+    onError: (error, context) => {
+      Sentry.captureException(error, { extra: context });
+    },
+    
+    onGenerate: ({ result, queueTimeMs }) => {
+      // Log slow generations
+      if (result.totalTime > 10000) {
+        console.warn(`Slow generation: ${result.totalTime}ms`);
+      }
+      
+      // Track queue congestion
+      if (queueTimeMs && queueTimeMs > 5000) {
+        Sentry.captureMessage("High queue wait time", {
+          level: "warning",
+          extra: { queueTimeMs },
+        });
+      }
+    },
+    
+    onModelLoad: (event) => {
+      if (!event.success) {
+        Sentry.captureMessage(`Model load failed: ${event.error}`, {
+          level: "error",
+          extra: event,
+        });
+      }
+    },
+  },
+  
+  concurrency: {
+    maxConcurrent: 1,
+    timeout: 120_000, // 2 minute timeout for your use case
+  },
+});
+
+// Preload model on startup
+await g.loadModel();
+console.log("Gerbil ready for production");
+```
+
+## Health Checks
+
+For production deployments, implement a health check endpoint:
+
+```typescript
+// Express
+app.get("/health", async (req, res) => {
+  try {
+    const info = g.getInfo();
+    res.json({
+      status: "ok",
+      model: info.model?.id,
+      device: info.device.backend,
+      ready: info.device.status === "ready",
+    });
+  } catch (error) {
+    res.status(503).json({ status: "error", message: String(error) });
+  }
+});
+```
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryhamster/gerbil",
-  "version": "1.0.0-rc.23",
+  "version": "1.0.0-rc.25",
   "description": "Local LLM inference for Node.js. GPU-accelerated. Zero config. Works standalone or with Vercel AI SDK.",
   "type": "module",
   "main": "dist/index.mjs",
@@ -72,6 +72,7 @@
     "cli-progress": "^3.12.0",
     "commander": "^12.1.0",
     "ora": "^8.0.1",
+    "p-queue": "^9.0.1",
     "puppeteer-core": "^24.31.0",
     "react": "^19.0.0",
     "webgpu": "^0.3.8",
@@ -127,11 +128,9 @@
   "devDependencies": {
     "@ai-sdk/provider": "^2.0.0",
     "@biomejs/biome": "^2.3.8",
-    "@huggingface/transformers": "^3.8.0",
-    "kokoro-js": "^1.2.1",
-    "onnxruntime-web": "^1.21.0-dev.20250114-228dd16893",
     "@changesets/changelog-github": "^0.5.1",
     "@changesets/cli": "^2.28.1",
+    "@huggingface/transformers": "^3.8.0",
     "@types/cli-progress": "^3.11.6",
     "@types/express": "^4.17.21",
     "@types/ink-big-text": "^1.2.4",
@@ -145,7 +144,9 @@
     "ink-select-input": "^6.2.0",
     "ink-spinner": "^5.0.0",
     "ink-text-input": "^6.0.0",
+    "kokoro-js": "^1.2.1",
     "lefthook": "^2.0.5",
+    "onnxruntime-web": "^1.21.0-dev.20250114-228dd16893",
     "tsdown": "^0.17.0-beta.3",
     "tsx": "^4.15.0",
     "typescript": "^5.4.5",

package/dist/gerbil-DJygY0sJ.d.mts.map

@@ -1 +0,0 @@
-{"version":3,"file":"gerbil-DJygY0sJ.d.mts","names":[],"sources":["../src/core/gerbil.ts"],"sourcesContent":[],"mappings":";;;;AA0OsB,cAlBT,MAAA,CAkBS;EAiBC,QAAA,SAAA;EAIa,QAAA,SAAA;EA0Be,QAAA,KAAA;EAAmB,QAAA,QAAA;EA8SpD,QAAA,YAAA;EAwDsB,QAAA,WAAA;EAsCO,iBAAA,MAAA;EAAsB,QAAA,KAAA;EAiDxB,QAAA,SAAA;EAkBO,QAAA,aAAA;EAAsB,QAAA,WAAA;EA4BxB,QAAA,SAAA;EAkBO,QAAA,WAAA;EAAsB,QAAA,aAAA;EA4Bf,WAAA,CAAA,MAAA,CAAA,EAxkB1C,YAwkB0C;EAmBnD,OAAA,UAAA,CAAA,CAAA,EA1kBU,WA0kBV,EAAA;EACR,OAAA,QAAA,CAAA,OAAA,EAAA,MAAA,CAAA,EAvkB+B,WAukB/B,GAAA,SAAA;EAuEU;;;;;;;;;;;;;;;;;;EAooB8B,SAAA,CAAA,OAAA,CAAA,EAAA,MAAA,EAAA,OAAA,CAAA,EAxvCM,WAwvCN,CAAA,EAxvCyB,OAwvCzB,CAAA,IAAA,CAAA;EAA4B;;;;EAmDpE,QAAA,eAAA;EAkCQ;;;EAgCA,QAAA,CAAA,CAAA,EAAA,OAAA;EAAY;;;EAgCZ,cAAA,CAAA,CAAA,EAAA,OAAA;EA4DY;;;EA4B0B,YAAA,CAAA,CAAA,EAvrCjC,WAurCiC,GAAA,IAAA;EAiBd;;;EAmBxB,aAAA,CAAA,CAAA,EAAA,QAAA,GAAA,KAAA,GAAA,MAAA;EACO;;;EAQJ,QAAA,CAAA,CAAA,EAAA,MAAA;EAiCZ;;;EA8B6D,qBAAA,CAAA,CAAA,EAAA;IAkBN,IAAA,EAAA,MAAA;IAAiB,MAAA,EAAA,MAAA;IA8BjE,IAAA,EAAA,MAAA;IAAe,OAAA,EAAA,MAAA;EACb,CAAA;EACA;;;EAkCA,kBAAA,CAAA,CAAA,EAAA,IAAA;EAAR;;;;;;;;;;;;EA2Q2C,aAAA,CAAA,OAAA,EAAA,MAAA,CAAA,EA1kDR,OA0kDQ,CAAA,OAAA,CAAA;EAAR;;;;;;;;;;;;;;;;;;0CApiDO,iBAAsB;;;;iCAiDxB;;;;;;;yCAkBO,iBAAsB;;;;iCA4BxB;;;;;;;yCAkBO,iBAAsB;;;;uCA4Bf;;;;;;;+CAmBnD,iBACR;;;;;;;;;;;;eAuEU;;;;;;qBAYY;;;;;;;oBAUD;;;;;;;;;gBAWJ;;;;;;+CAW0B;;;;;;;;;;;;;;;qCAyBN,kBAAuB,QAAQ;;;;;;;mCAsN5D,kBACR,uBAAuB;;;;;;;;;;;;;mCA0Sa,YAAY,KAAK,QAAQ;;;;gCAqC7B,eAAoB,QAAQ;;;;wCAyBpB,eAAoB,QAAQ;;;;;;;;;;;;;;;;;;;;qDAkD5D,eACR,QADyB,gBAAA;;;;;;;;;;;;;;oDAmCjB;;MACR,QADoB,YAAA;;;;;;;;;;;mEAgCZ;;MACR,QADoB,YAAA;;;;cAyBX;;;;aAOD;;;;;;;;;;;;;;;;;;;;;;oBA4DY;;MAA2C;;;;4BA4BlC,iBAAiB;;;;;;;;;;;;gCAiBd,eAAoB,QAAQ;;;;;;;;;;;;;sCAmBpD,eACR,eAAe,YAAY;;;;gBAQhB;;;;;;;;;;;;;;;;mBAgCS,QACrB;;;;;;;;;;;;;;;;;;;;sCA8BuC,iBAAsB;;;;8CAkBN,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;oBA8BjE,eAAe,sBACb,oBACR,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;yCAiCA,gCACR,QAAQ;;;;mBAQY,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAiD5B,QAAQ;;;;2BAmCoB;;;;;iCAiBI;;;;;;qBA6EV;;;;;+BAUU;;;;;;;;cAQvB;;;;;;;;;;;;;;;0BA2BkB;;;;;;;;2CAkBiB;;;;;kCAkBT,QAAQ;;;;;;;;;;;;;;wCAuBF;;;;oCAgBJ"}