@tryhamster/gerbil 1.11.3 → 1.12.0

package/README.md

@@ -121,6 +121,84 @@ Reports steady-state decode tok/s, time-to-first-token, and the device it ran on
 For a copy-pasteable version see [`examples/benchmark.ts`](./examples/benchmark.ts)
 (and the rest of [`examples/`](./examples/)).
 
+### Batched decode & per-lane adapters (experimental, Node/Dawn)
+
+On Node the engine can decode many sequences in one lockstep batch — and each lane
+can run a **different LoRA adapter** over one shared base (SGMV-style). Both are
+flag-gated and token-exact: every lane's output matches the same prompt run solo.
+
+```bash
+GERBIL_BATCH=4 GERBIL_BATCH_LORA=1 node my-batch-script.mjs
+```
+
+```typescript
+const engine = await WebGPUEngine.create({ repo: "mlx-community/Qwen3.5-0.8B-4bit" });
+
+// Register adapters once — their factors are packed into shared GPU buffers,
+// and any number of lanes can run them concurrently.
+await engine.registerAdapter("pirate", "you/qwen3.5-0.8b-pirate-lora");
+await engine.registerAdapter("legal", "you/qwen3.5-0.8b-legal-lora");
+
+const results = await engine.generateBatch(
+  [promptA, promptB, promptC, promptD],
+  {
+    sampling: { temperature: 0 },
+    // Per-lane adapter selection: null = bare base model.
+    adapters: ["pirate", "pirate", "legal", null],
+  },
+);
+
+// The continuous-batching scheduler takes an adapter per request — mixed
+// adapters, queueing, and slot reuse stay token-exact vs solo runs:
+const scheduler = engine.createScheduler();
+const reply = await scheduler.submit(prompt, {
+  sampling: { temperature: 0 },
+  adapter: "legal",
+});
+
+// And a swarm fans ONE input out across tuned members concurrently (MoTA
+// mode B — one batched pass, default plan broadcasts, default reduce returns
+// a member-keyed map):
+const swarm = await engine.createSwarm({
+  members: { pirate: "you/pirate-lora", legal: "you/legal-lora", base: null },
+});
+const answers = await swarm.run(prompt, { sampling: { temperature: 0 } });
+```
+
+| Env flag | Effect |
+|----------|--------|
+| `GERBIL_BATCH=N` | Fixed lockstep batch of N lanes (`generateBatch`), greedy decode |
+| `GERBIL_BATCH_LORA=1` | Enables `registerAdapter`, the per-lane `adapters` option, per-request `adapter` on `scheduler.submit`, and `createSwarm` |
+| `GERBIL_GEMM_FAST=1` | Relaxed-parity f16-accumulate GEMM tier (faster aggregate) |
+| `GERBIL_DECODE_WINDOW=K` | Records K decode steps per GPU submit (2–8; Vulkan-targeted) |
+
+### Swarm quickstart
+
+A swarm is a set of named specialists: LoRA adapters tuned on one shared base
+model, plus optionally the bare base itself. All members decode together in one
+batched pass on one GPU, so asking four specialists costs about as much as
+asking the slowest one, not four times as much.
+
+Try it with the runnable example (Node with the native GPU path):
+
+```bash
+node scripts/engine/example-swarm.mjs
+```
+
+With no configuration it finds tuned adapters in your local caches, or falls
+back to generated demo adapters so you can see the mechanics either way. Point
+it at your own tuned adapters like this:
+
+```bash
+GERBIL_SWARM_MEMBERS="support=hf:you/support-lora,legal=hf:you/legal-lora" \
+  node scripts/engine/example-swarm.mjs
+```
+
+The example shows both patterns: routing different prompts to different
+members, and broadcasting one input to every member with `swarm.run()`. In
+your own scripts, batched adapter decode needs `GERBIL_BATCH=N` and
+`GERBIL_BATCH_LORA=1` set (the example sets them for you).
+
 ## React Quickstart
 
 `useEngine` (from `@tryhamster/gerbil/hooks`) owns the full engine lifecycle —
@@ -469,6 +547,7 @@ via `WebGPUEngine.create({ repo })`.
 |-------|------|-------|
 | **Qwen3.5-0.8B** | `mlx-community/Qwen3.5-0.8B-4bit` | Default text model; vision-capable (`Qwen/Qwen3.5-0.8B` for the ViT) |
 | **Qwen3.5-2B** | `Qwen/Qwen3.5-2B` | Higher quality; 262k context; multimodal (vision-capable) |
+| **Qwen3.5-4B** | `mlx-community/Qwen3.5-4B-4bit` | Highest quality that fits browser WebGPU (~2.5 GB INT4); 262k context; vision-capable; thinking |
 | **LFM2.5-350M** | `LiquidAI/LFM2.5-350M` | Hybrid conv/attention, very fast, ~199 MB q4 |
 | **Gemma 4 E2B** | `mlx-community/gemma-4-e2b-it-4bit` | PLE CPU-streamed; vision-capable |
 

package/dist/{architectures-DmZMEFsA.mjs → architectures-BH_z6k9d.mjs}

@@ -5826,11 +5826,12 @@ function generateQwen3_5Graph(rawConfig, dtype, groupSizeOverride, kvDtype, _emb
 	const kv_dim = num_kv_heads * head_dim;
 	const layerTypes = textConfig.layer_types ?? [];
 	const fullAttentionInterval = textConfig.full_attention_interval ?? 4;
-	const mamba_num_heads = textConfig.linear_num_key_heads ?? 16;
+	const mamba_num_k_heads = textConfig.linear_num_key_heads ?? 16;
+	const mamba_num_v_heads = textConfig.linear_num_value_heads ?? mamba_num_k_heads;
 	const mamba_key_dim = textConfig.linear_key_head_dim ?? 128;
 	const mamba_val_dim = textConfig.linear_value_head_dim ?? 128;
-	const mamba_qkv_dim = mamba_num_heads * (mamba_key_dim + mamba_key_dim + mamba_val_dim);
-	const mamba_out_dim = mamba_num_heads * mamba_val_dim;
+	const mamba_qkv_dim = 2 * mamba_num_k_heads * mamba_key_dim + mamba_num_v_heads * mamba_val_dim;
+	const mamba_out_dim = mamba_num_v_heads * mamba_val_dim;
 	const conv_kernel_dim = textConfig.linear_conv_kernel_dim ?? 4;
 	const ropeParams = textConfig.rope_parameters;
 	const rope_base = ropeParams?.rope_theta ?? textConfig.rope_theta ?? 1e7;
@@ -6450,13 +6451,13 @@ function generateQwen3_5Graph(rawConfig, dtype, groupSizeOverride, kvDtype, _emb
 		});
 		addTensor({
 			name: aProjOut,
-			shape: ["T", mamba_num_heads],
+			shape: ["T", mamba_num_v_heads],
 			dtype: "f32",
 			storage: "activation"
 		});
 		addTensor({
 			name: bProjOut,
-			shape: ["T", mamba_num_heads],
+			shape: ["T", mamba_num_v_heads],
 			dtype: "f32",
 			storage: "activation"
 		});
@@ -6467,8 +6468,8 @@ function generateQwen3_5Graph(rawConfig, dtype, groupSizeOverride, kvDtype, _emb
 			storage: "activation"
 		});
 		addLinearOp(`${prefix}_qkv_proj`, norm1Out, inProjQKV_W, [mamba_qkv_dim, hidden_size], qkvOut, hidden_size, mamba_qkv_dim);
-		addLinearOp(`${prefix}_a_proj`, norm1Out, inProjA_W, [mamba_num_heads, hidden_size], aProjOut, hidden_size, mamba_num_heads);
-		addLinearOp(`${prefix}_b_proj`, norm1Out, inProjB_W, [mamba_num_heads, hidden_size], bProjOut, hidden_size, mamba_num_heads);
+		addLinearOp(`${prefix}_a_proj`, norm1Out, inProjA_W, [mamba_num_v_heads, hidden_size], aProjOut, hidden_size, mamba_num_v_heads);
+		addLinearOp(`${prefix}_b_proj`, norm1Out, inProjB_W, [mamba_num_v_heads, hidden_size], bProjOut, hidden_size, mamba_num_v_heads);
 		addLinearOp(`${prefix}_z_proj`, norm1Out, inProjZ_W, [mamba_out_dim, hidden_size], zProjOut, hidden_size, mamba_out_dim);
 		const conv1dW = CANONICAL_KEYS.mambaConv1d(i);
 		addTensor({
@@ -6522,14 +6523,14 @@ function generateQwen3_5Graph(rawConfig, dtype, groupSizeOverride, kvDtype, _emb
 		const dtBiasW = CANONICAL_KEYS.mambaDtBias(i);
 		addTensor({
 			name: aLogW,
-			shape: [mamba_num_heads],
+			shape: [mamba_num_v_heads],
 			dtype: "f32",
 			storage: "constant",
 			safetensorsKey: aLogW
 		});
 		addTensor({
 			name: dtBiasW,
-			shape: [mamba_num_heads],
+			shape: [mamba_num_v_heads],
 			dtype: "f32",
 			storage: "constant",
 			safetensorsKey: dtBiasW
@@ -6538,7 +6539,7 @@ function generateQwen3_5Graph(rawConfig, dtype, groupSizeOverride, kvDtype, _emb
 		addTensor({
 			name: ssmState,
 			shape: [
-				mamba_num_heads,
+				mamba_num_v_heads,
 				mamba_key_dim,
 				mamba_val_dim
 			],
@@ -6565,7 +6566,8 @@ function generateQwen3_5Graph(rawConfig, dtype, groupSizeOverride, kvDtype, _emb
 			],
 			outputs: [ssmOut],
 			attributes: {
-				num_heads: mamba_num_heads,
+				num_heads: mamba_num_v_heads,
+				num_k_heads: mamba_num_k_heads,
 				key_dim: mamba_key_dim,
 				val_dim: mamba_val_dim,
 				qkv_dim: mamba_qkv_dim,
@@ -6767,4 +6769,4 @@ function isArchitectureSupported(name) {
 
 //#endregion
 export { kaniSinTensor as A, computeKaniPositions as C, kaniAttentionLayerIndices as D, generateNanoCodecDecoderGraph as E, CANONICAL_KEYS as F, DTYPE_BYTES as I, GEMMA4_VIS_KEYS as L, parseKaniConfig as M, DEFAULT_GROUP_SIZE as N, kaniCosTensor as O, quantizeInt4 as P, createDefaultHFKeyMapper as R, buildKaniLayerCosSin as S, generateKaniTtsGraph as T, moonshineEncoderFrames as _, PARLER_DAC_LATENT_DIM as a, KANI_START_OF_HUMAN as b, PARLER_SAMPLE_RATE as c, generateParlerEncoderGraph as d, parseParlerConfig as f, generateMoonshineEncoderGraph as g, generateMoonshineDecoderGraph as h, PARLER_DAC_DECODER_DIM as i, nanoCodecWeightMap as j, kaniLayerAlpha as k, buildT5RelativeBias as l, MOONSHINE_REMAINING_WORK as m, isArchitectureSupported as n, PARLER_DECODER_RATES as o, revertDelayPattern as p, PARLER_BOS_TOKEN_ID as r, PARLER_EOS_TOKEN_ID as s, generateGraph as t, generateParlerDecoderGraph as u, parseMoonshineConfig as v, foldNanoCodecWeightNorm as w, audioTokensToCodes as x, KANI_END_OF_HUMAN as y };
-//# sourceMappingURL=architectures-DmZMEFsA.mjs.map
+//# sourceMappingURL=architectures-BH_z6k9d.mjs.map