@tryhamster/gerbil 1.8.0 → 1.10.0

package/dist/{moonshine-stt-9wc6t11v.mjs → moonshine-stt-BYbYwSz2.mjs}

@@ -8673,6 +8673,120 @@ const convTranspose1dDepthwiseSpec = {
 		]);
 	}
 };
+const LORA_WG = 64;
+const WGSL_LORA_DOWN = `\
+struct Params { M: u32, IN: u32, R: u32, pad: u32 };
+@group(0) @binding(0) var<storage, read> X: array<f32>;
+@group(0) @binding(1) var<storage, read> A: array<f32>;
+@group(0) @binding(2) var<storage, read_write> T: array<f32>;
+@group(0) @binding(3) var<storage, read> params: Params;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(global_invocation_id) gid: vec3u,
+  @builtin(num_workgroups) nwg: vec3u,
+) {
+  let idx = gid.y * (nwg.x * 64u) + gid.x;
+  let total = params.M * params.R;
+  if (idx >= total) { return; }
+  let m = idx / params.R;
+  let k = idx % params.R;
+  let xBase = m * params.IN;
+  let aBase = k * params.IN;
+  var sum: f32 = 0.0;
+  for (var i: u32 = 0u; i < params.IN; i = i + 1u) {
+    sum = sum + X[xBase + i] * A[aBase + i];
+  }
+  T[idx] = sum;
+}
+`;
+const WGSL_LORA_UP = `\
+struct Params { M: u32, R: u32, OUT: u32, scaling: f32 };
+@group(0) @binding(0) var<storage, read> T: array<f32>;
+@group(0) @binding(1) var<storage, read> B: array<f32>;
+@group(0) @binding(2) var<storage, read_write> Y: array<f32>;
+@group(0) @binding(3) var<storage, read> params: Params;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(global_invocation_id) gid: vec3u,
+  @builtin(num_workgroups) nwg: vec3u,
+) {
+  let idx = gid.y * (nwg.x * 64u) + gid.x;
+  let total = params.M * params.OUT;
+  if (idx >= total) { return; }
+  let m = idx / params.OUT;
+  let o = idx % params.OUT;
+  let tBase = m * params.R;
+  let bBase = o * params.R;
+  var sum: f32 = 0.0;
+  for (var k: u32 = 0u; k < params.R; k = k + 1u) {
+    sum = sum + T[tBase + k] * B[bBase + k];
+  }
+  Y[idx] = Y[idx] + params.scaling * sum;
+}
+`;
+/** M (row count) for a LoRA dispatch = first dim of the shape-source activation. */
+function loraRows(op, resolvedShapes) {
+	return resolvedShapes[op.inputs[0]]?.[0] ?? 1;
+}
+/**
+* LoRA down-projection: tmp[M,R] = X · Aᵀ. `op.inputs[0]` names the input
+* activation (the shape source); attributes carry IN and R.
+*/
+const LORA_DOWN_SPEC = {
+	shaderCode: WGSL_LORA_DOWN,
+	entryPoint: "main",
+	bindings: [
+		{ type: "storage-read" },
+		{ type: "storage-read" },
+		{ type: "storage-read-write" },
+		{ type: "uniform" }
+	],
+	getDispatchSize(op, resolvedShapes) {
+		const m = loraRows(op, resolvedShapes);
+		const r = op.attributes.R;
+		return dispatch2D(cdiv(m * r, LORA_WG));
+	},
+	buildParams(op, resolvedShapes) {
+		return buildUniformBuffer([
+			loraRows(op, resolvedShapes),
+			op.attributes.IN,
+			op.attributes.R,
+			0
+		]);
+	}
+};
+/**
+* LoRA up-projection + accumulate: Y[M,OUT] += scaling · (tmp · Bᵀ).
+* `op.inputs[0]` names the shape-source activation; attributes carry R, OUT and
+* the f32 scaling (alpha/r, or alpha/√r for rslora).
+*/
+const LORA_UP_SPEC = {
+	shaderCode: WGSL_LORA_UP,
+	entryPoint: "main",
+	bindings: [
+		{ type: "storage-read" },
+		{ type: "storage-read" },
+		{ type: "storage-read-write" },
+		{ type: "uniform" }
+	],
+	getDispatchSize(op, resolvedShapes) {
+		const m = loraRows(op, resolvedShapes);
+		const out = op.attributes.OUT;
+		return dispatch2D(cdiv(m * out, LORA_WG));
+	},
+	buildParams(op, resolvedShapes) {
+		const m = loraRows(op, resolvedShapes);
+		const buf = /* @__PURE__ */ new ArrayBuffer(16);
+		const view = new DataView(buf);
+		view.setUint32(0, m, true);
+		view.setUint32(4, op.attributes.R, true);
+		view.setUint32(8, op.attributes.OUT, true);
+		view.setFloat32(12, op.attributes.scaling, true);
+		return buf;
+	}
+};
 const KERNEL_REGISTRY = {
 	Embedding: embeddingSpec,
 	EmbeddingInt4: embeddingInt4Spec,
@@ -8787,6 +8901,16 @@ var Executor = class Executor {
 	dispatchEntries = [];
 	/** Dispatch entries for decode (M=1): uses K-parallel matvec kernels. */
 	decodeEntries = [];
+	/** Per-node decode entries captured BEFORE fusion (the overlay base). */
+	decodeEntriesUnfused = [];
+	/** True while a runtime adapter is applied. */
+	loraActive = false;
+	/** Decode entries (unfused base + spliced LoRA dispatches). */
+	decodeEntriesLoRA = [];
+	/** Prefill entries (base + spliced LoRA dispatches). */
+	prefillEntriesLoRA = [];
+	/** GPU buffers owned by the current adapter (A/B factors + tmp); freed on swap. */
+	loraBuffers = [];
 	/** Argmax dispatch entry (created in initBindGroups). */
 	argmaxEntry = null;
 	/** True when running on Safari/WebKit (needs multi-encoder submit). */
@@ -8985,6 +9109,7 @@ var Executor = class Executor {
 				});
 			} else this.decodeEntries.push(prefillEntry);
 		}
+		this.decodeEntriesUnfused = this.decodeEntries.slice();
 		this.fuseSwiGLUDecodeEntries();
 		this.fuseDualMatVecDecodeEntries();
 		this.fuseDualKVCacheAppendEntries();
@@ -9023,6 +9148,164 @@ var Executor = class Executor {
 			usage: 12
 		});
 	}
+	/** True while a runtime LoRA overlay is applied. */
+	get hasRuntimeAdapter() {
+		return this.loraActive;
+	}
+	/** Decode entries in effect for the current adapter state. */
+	get activeDecodeEntries() {
+		return this.loraActive ? this.decodeEntriesLoRA : this.decodeEntries;
+	}
+	/**
+	* Apply a runtime LoRA overlay on the STATIC base. For each delta whose target
+	* maps to a base linear (MatMulInt4 / MatMul) with matching dims, uploads the
+	* f32 A/B factors and splices two low-rank dispatches — `tmp = A·x`, then
+	* `y += scaling·B·tmp` — right after that node in both the decode and prefill
+	* paths, so the corrected output is live before any consumer reads it.
+	*
+	* Rebuilds the overlay from scratch each call (this is how adapters hot-swap);
+	* the base weight buffers and bind groups are never touched or re-uploaded.
+	* Pass an empty list (or call {@link clearRuntimeLoRA}) to drop the overlay.
+	*/
+	applyRuntimeLoRA(deltas) {
+		this.clearRuntimeLoRA();
+		const overlay = /* @__PURE__ */ new Map();
+		let applied = 0;
+		let skipped = 0;
+		for (const d of deltas) {
+			if (d.fanInFanOut) {
+				console.warn(`[executor] runtime LoRA: fan_in_fan_out target ${d.key} unsupported; skipping.`);
+				skipped++;
+				continue;
+			}
+			const node = this.graph.nodes.find((n) => {
+				if (n.opType !== "MatMulInt4" && n.opType !== "MatMul") return false;
+				const w = n.inputs[1];
+				if (!w) return false;
+				return (w.endsWith(".q") ? w.slice(0, -2) : w) === d.key;
+			});
+			if (!node) {
+				skipped++;
+				continue;
+			}
+			const K = node.attributes.K;
+			const N = node.attributes.N;
+			if (K !== d.inFeatures || N !== d.outFeatures) {
+				console.warn(`[executor] runtime LoRA: dim mismatch on ${d.key} (base ${N}×${K}, delta ${d.outFeatures}×${d.inFeatures}); skipping.`);
+				skipped++;
+				continue;
+			}
+			const xName = node.inputs[0];
+			const yName = node.outputs[0];
+			const xBuf = this.getBuffer(xName);
+			const yBuf = this.getBuffer(yName);
+			if (!xBuf || !yBuf) {
+				skipped++;
+				continue;
+			}
+			const aBuf = createStorageBuffer(this.ctx, `lora_A_${node.id}`, d.A.byteLength, d.A);
+			const bBuf = createStorageBuffer(this.ctx, `lora_B_${node.id}`, d.B.byteLength, d.B);
+			const tmpBuf = createStorageBuffer(this.ctx, `lora_tmp_${node.id}`, this.maxSeqLen * d.r * 4);
+			this.loraBuffers.push(aBuf, bBuf, tmpBuf);
+			overlay.set(node.id, this.buildLoraPair(node.id, xName, xBuf, yBuf, aBuf, bBuf, tmpBuf, d));
+			applied++;
+		}
+		if (applied === 0) {
+			this.loraActive = false;
+			return {
+				applied,
+				skipped
+			};
+		}
+		this.decodeEntriesLoRA = this.spliceLoraOverlay(this.decodeEntriesUnfused, overlay);
+		this.prefillEntriesLoRA = this.spliceLoraOverlay(this.dispatchEntries, overlay);
+		this.loraActive = true;
+		return {
+			applied,
+			skipped
+		};
+	}
+	/** Drop the runtime LoRA overlay and free its GPU buffers. Base is untouched. */
+	clearRuntimeLoRA() {
+		this.loraActive = false;
+		this.decodeEntriesLoRA = [];
+		this.prefillEntriesLoRA = [];
+		for (const b of this.loraBuffers) try {
+			b.destroy();
+		} catch {}
+		this.loraBuffers = [];
+	}
+	/** Build the [down, up] low-rank dispatch pair for one adapted node. */
+	buildLoraPair(nodeId, xName, xBuf, yBuf, aBuf, bBuf, tmpBuf, d) {
+		const shapes = this.resolveShapes(1);
+		const downNode = {
+			id: `lora_down_${nodeId}`,
+			opType: "MatMul",
+			inputs: [xName],
+			outputs: [],
+			attributes: {
+				IN: d.inFeatures,
+				R: d.r
+			}
+		};
+		const downPipeline = getOrCreatePipeline(this.ctx, `lora_down_${nodeId}`, LORA_DOWN_SPEC.shaderCode, LORA_DOWN_SPEC.entryPoint);
+		const downUniform = createUniformBuffer(this.ctx, `lora_down_u_${nodeId}`, LORA_DOWN_SPEC.buildParams(downNode, shapes));
+		const downBind = createBindGroup(this.ctx, downPipeline, [
+			{ buffer: xBuf },
+			{ buffer: aBuf },
+			{ buffer: tmpBuf },
+			{ buffer: downUniform }
+		], `lora_down_bg_${nodeId}`);
+		const upNode = {
+			id: `lora_up_${nodeId}`,
+			opType: "MatMul",
+			inputs: [xName],
+			outputs: [],
+			attributes: {
+				R: d.r,
+				OUT: d.outFeatures,
+				scaling: d.scaling
+			}
+		};
+		const upPipeline = getOrCreatePipeline(this.ctx, `lora_up_${nodeId}`, LORA_UP_SPEC.shaderCode, LORA_UP_SPEC.entryPoint);
+		const upUniform = createUniformBuffer(this.ctx, `lora_up_u_${nodeId}`, LORA_UP_SPEC.buildParams(upNode, shapes));
+		const upBind = createBindGroup(this.ctx, upPipeline, [
+			{ buffer: tmpBuf },
+			{ buffer: bBuf },
+			{ buffer: yBuf },
+			{ buffer: upUniform }
+		], `lora_up_bg_${nodeId}`);
+		this.loraBuffers.push(downUniform, upUniform);
+		return [{
+			nodeId: downNode.id,
+			node: downNode,
+			spec: LORA_DOWN_SPEC,
+			pipeline: downPipeline,
+			bindGroup: downBind,
+			uniformBuffer: downUniform,
+			lastParamsBytes: null,
+			lastDispatchSize: null
+		}, {
+			nodeId: upNode.id,
+			node: upNode,
+			spec: LORA_UP_SPEC,
+			pipeline: upPipeline,
+			bindGroup: upBind,
+			uniformBuffer: upUniform,
+			lastParamsBytes: null,
+			lastDispatchSize: null
+		}];
+	}
+	/** Copy a base entry list, splicing each node's LoRA pair in right after it. */
+	spliceLoraOverlay(base, overlay) {
+		const out = [];
+		for (const entry of base) {
+			out.push(entry);
+			const pair = overlay.get(entry.nodeId);
+			if (pair) out.push(...pair);
+		}
+		return out;
+	}
 	/**
 	* Run a forward pass. Uses matvec kernels for M=1 (decode), tiled for M>1 (prefill).
 	*/
@@ -9030,7 +9313,7 @@ var Executor = class Executor {
 		const T = inputIds.length;
 		const resolvedShapes = this.resolveShapes(T);
 		const runtimeContext = { seqPos: this.seqPos };
-		const entries = T === 1 ? this.decodeEntries : this.dispatchEntries;
+		const entries = T === 1 ? this.loraActive ? this.decodeEntriesLoRA : this.decodeEntries : this.loraActive ? this.prefillEntriesLoRA : this.dispatchEntries;
 		this.ctx.device.queue.writeBuffer(this.inputIdsBuffer, 0, inputIds);
 		await this.streamPleRows(inputIds);
 		const dispatchSizes = new Array(entries.length);
@@ -9434,10 +9717,11 @@ var Executor = class Executor {
 		};
 		const resolvedShapes = this.resolveShapes(T);
 		const runtimeContext = { seqPos: this.seqPos };
+		const decodeEntries = this.activeDecodeEntries;
 		this.ctx.device.queue.writeBuffer(this.inputIdsBuffer, 0, inputIds);
-		const argmaxDispatchSizes = new Array(this.decodeEntries.length);
-		for (let i = 0; i < this.decodeEntries.length; i++) {
-			const entry = this.decodeEntries[i];
+		const argmaxDispatchSizes = new Array(decodeEntries.length);
+		for (let i = 0; i < decodeEntries.length; i++) {
+			const entry = decodeEntries[i];
 			const paramsData = entry.spec.buildParams(entry.node, resolvedShapes, runtimeContext);
 			const paramsView = new Uint8Array(paramsData);
 			let changed = !entry.lastParamsBytes || entry.lastParamsBytes.length !== paramsView.length;
@@ -9468,13 +9752,13 @@ var Executor = class Executor {
 			mark("uniformMs");
 			const group = this.webkitGroupSize;
 			const perOp = budget && group === 1 ? globalThis.__webkitBudgetByOp ??= {} : null;
-			for (let start = 0; start < this.decodeEntries.length; start += group) {
+			for (let start = 0; start < decodeEntries.length; start += group) {
 				const enc = this.ctx.device.createCommandEncoder();
-				const end = Math.min(start + group, this.decodeEntries.length);
+				const end = Math.min(start + group, decodeEntries.length);
 				for (let i = start; i < end; i++) {
 					const p = enc.beginComputePass();
-					p.setPipeline(this.decodeEntries[i].pipeline);
-					p.setBindGroup(0, this.decodeEntries[i].bindGroup);
+					p.setPipeline(decodeEntries[i].pipeline);
+					p.setBindGroup(0, decodeEntries[i].bindGroup);
 					p.dispatchWorkgroups(...argmaxDispatchSizes[i]);
 					p.end();
 				}
@@ -9484,7 +9768,7 @@ var Executor = class Executor {
 					const t0 = perOp ? performance.now() : 0;
 					await this.ctx.device.queue.onSubmittedWorkDone();
 					if (perOp) {
-						const op = this.decodeEntries[start].node.opType;
+						const op = decodeEntries[start].node.opType;
 						const cell = perOp[op] ??= {
 							ms: 0,
 							n: 0
@@ -9510,8 +9794,8 @@ var Executor = class Executor {
 		} else {
 			const encoder = this.ctx.device.createCommandEncoder({ label: "argmax" });
 			const pass = encoder.beginComputePass({ label: "am_pass" });
-			for (let i = 0; i < this.decodeEntries.length; i++) {
-				const entry = this.decodeEntries[i];
+			for (let i = 0; i < decodeEntries.length; i++) {
+				const entry = decodeEntries[i];
 				pass.setPipeline(entry.pipeline);
 				pass.setBindGroup(0, entry.bindGroup);
 				pass.dispatchWorkgroups(...argmaxDispatchSizes[i]);
@@ -9683,6 +9967,10 @@ var Executor = class Executor {
 			entry.lastParamsBytes = null;
 			entry.lastDispatchSize = null;
 		}
+		for (const entry of [...this.decodeEntriesLoRA, ...this.prefillEntriesLoRA]) {
+			entry.lastParamsBytes = null;
+			entry.lastDispatchSize = null;
+		}
 	}
 	/**
 	* Diagnostic: dispatch ONLY the first kernel (EmbeddingInt4) using the
@@ -14496,5 +14784,5 @@ function selectGraphWeights(graph, weights) {
 }
 
 //#endregion
-export { getOrCreatePipeline as C, destroyBuffers as S, verifyGPU as T, MATMUL_BIAS_F16C_SPEC as _, loadMoonshine as a, createStorageBuffer as b, quantizeBackboneInt4 as c, Tokenizer as d, applyLoRAToStore as f, KERNEL_REGISTRY as g, Executor as h, loadModel as i, quantizeKaniBackbone as l, fetchAdapter as m, MoonshineEncoderExecutor as n, loadOuteTTS as o, buildLoRADeltas as p, loadKaniTTS as r, loadParlerTTS as s, MoonshineSTT as t, remapPrunedToken as u, clearPipelineCache as v, initGPU as w, createUniformBuffer as x, createBindGroup as y };
-//# sourceMappingURL=moonshine-stt-9wc6t11v.mjs.map
+export { destroyBuffers as C, verifyGPU as E, createUniformBuffer as S, initGPU as T, KERNEL_REGISTRY as _, loadModel as a, createBindGroup as b, loadParlerTTS as c, remapPrunedToken as d, Tokenizer as f, Executor as g, fetchAdapter as h, loadKaniTTS as i, quantizeBackboneInt4 as l, buildLoRADeltas as m, MoonshineEncoderExecutor as n, loadMoonshine as o, applyLoRAToStore as p, createKeyMapperForArch as r, loadOuteTTS as s, MoonshineSTT as t, quantizeKaniBackbone as u, MATMUL_BIAS_F16C_SPEC as v, getOrCreatePipeline as w, createStorageBuffer as x, clearPipelineCache as y };
+//# sourceMappingURL=moonshine-stt-BYbYwSz2.mjs.map