@elizaos/app-core 2.0.0-beta.2 → 2.0.3-beta.2

package/scripts/kernel-patches/vulkan-kernels.mjs

@@ -1,8 +1,8 @@
-// Vulkan kernel-shipment + dispatch wiring for the v0.4.0-milady fork.
+// Vulkan kernel-shipment staging for the elizaOS/llama.cpp fork (v1.0.0-eliza).
 //
 // What this module does:
 //
-//   1. Copies the eight verified standalone .comp shaders from
+//   1. Copies the nine verified standalone .comp shaders from
 //      packages/inference/vulkan/ into the fork at
 //      ggml/src/ggml-vulkan/vulkan-shaders/<name>.comp. The fork's CMakeLists
 //      uses `file(GLOB CONFIGURE_DEPENDS ${input_dir}/*.comp)` to discover
@@ -12,44 +12,56 @@
 //      ggml-vulkan-shaders.hpp) is handled by the patch in
 //      vulkan-dispatch-patches/01-vulkan-shaders-gen.patch.
 //
-//   2. Applies the two unified-anchor patches under
+//   2. Applies the two shared-anchor staging patches under
 //      vulkan-dispatch-patches/:
-//        - 01-vulkan-shaders-gen.patch — adds 8 string_to_spv() registrations
+//        - 01-vulkan-shaders-gen.patch — adds 9 string_to_spv() registrations
 //          at the bottom of process_shaders().
-//        - 02-ggml-vulkan-pipelines.patch — extends vk_device_struct with 8
-//          pipeline slots and adds 8 ggml_vk_create_pipeline() calls at the
-//          bottom of ggml_vk_load_shaders(). End result: each milady SPV blob
+//        - 02-ggml-vulkan-pipelines.patch — extends vk_device_struct with 9
+//          pipeline slots and adds 9 ggml_vk_create_pipeline() calls at the
+//          bottom of ggml_vk_load_shaders(). End result: each eliza SPV blob
 //          is referenced at link time and `nm libggml-vulkan.so | grep
-//          milady_` shows the new symbols.
+//          eliza_` shows the new symbols.
 //
-//      Patches are idempotent: each carries a `MILADY-VK-DISPATCH-PATCH-V1`
+//   3. Adds Vulkan graph dispatch for the Eliza-1 attention score ops:
+//        - GGML_OP_ATTN_SCORE_QJL
+//        - GGML_OP_ATTN_SCORE_TBQ   (TBQ3_0, TBQ4_0, TBQ3_TCQ)
+//        - GGML_OP_ATTN_SCORE_POLAR
+//
+//      These routes bind the standalone eliza pipelines directly. They are
+//      intentionally conservative: q/pk/dst must be contiguous row tensors and
+//      the current route only advertises support for the single-batch shape
+//      used by the runtime smoke (ne[2]/ne[3] == 1, except pk head fanout).
+//
+//      Patches are idempotent: each carries a `ELIZA-VK-DISPATCH-PATCH-V1`
 //      sentinel; if the sentinel is already present in the target file, the
 //      hunk is skipped (re-running the build is safe).
 //
-// Out of scope (deliberate, mirrors metal-kernels.mjs's same staged approach):
+// Out of scope:
 //
-//   * Op-level dispatch wiring. The 8 standalones use bespoke push-constant
-//     layouts and bind sets that do NOT plug into the existing
-//     vk_op_binary_push_constants / vk_mat_vec_push_constants paths. The
-//     follow-up patch needs to introduce a milady-native dispatch entrypoint
-//     for GGML_OP_ATTN_SCORE_QJL (already declared in ggml.h:563 as a
-//     CPU-only op) and add per-op routing for QJL/Polar K-cache score and
-//     mul_mv. Until that lands, the kernels live as live, named pipelines
-//     inside libggml-vulkan.so but are NOT yet selected at runtime — symbol
-//     audit passes, end-to-end attention dispatch does not.
+//   * Generic mat-vec / get-rows replacement for every GGML call site. The
+//     runtime patch wires the attention-score graph routes needed by the
+//     Eliza-1 local voice/text path. Broader type-aware branches in
+//     ggml_vk_get_dequantize_mul_mat_vec() stay in a dedicated follow-up because those
+//     paths use different bind-set conventions from the standalone kernels.
 //
-//   * Type-aware case branches in ggml_vk_get_dequantize_mul_mat_vec() etc.
-//     Same reason as above: the existing dequant/mul_mat_vec paths assume a
-//     uniform bind-set that the milady kernels intentionally do not match.
+//   * Batched ne[2]/ne[3] graph shapes. The source patch refuses to advertise
+//     those shapes until a dedicated graph smoke covers offsets and fanout.
 
 import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
+import {
+  patchGgmlTbqPolarAttnOps,
+  STANDALONE_REFERENCE_DIR,
+} from "./metal-kernels.mjs";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-const STANDALONE_VULKAN_DIR = path.resolve(
+// packages/app-core/scripts/kernel-patches/  →  plugin-local-inference/native/vulkan/
+// Older workstreams staged these under packages/inference/vulkan; the current
+// native plugin owns the verified standalone shader sources.
+const LEGACY_STANDALONE_VULKAN_DIR = path.resolve(
   __dirname,
   "..",
   "..",
@@ -57,6 +69,20 @@ const STANDALONE_VULKAN_DIR = path.resolve(
   "inference",
   "vulkan",
 );
+const PLUGIN_STANDALONE_VULKAN_DIR = path.resolve(
+  __dirname,
+  "..",
+  "..",
+  "..",
+  "..",
+  "plugins",
+  "plugin-local-inference",
+  "native",
+  "vulkan",
+);
+const STANDALONE_VULKAN_DIR = fs.existsSync(LEGACY_STANDALONE_VULKAN_DIR)
+  ? LEGACY_STANDALONE_VULKAN_DIR
+  : PLUGIN_STANDALONE_VULKAN_DIR;
 
 const PATCHES_DIR = path.resolve(__dirname, "vulkan-dispatch-patches");
 
@@ -68,11 +94,47 @@ export const VULKAN_KERNEL_FILES = [
   "qjl_get_rows.comp",
   "qjl_mul_mv.comp",
   "polar.comp",
+  "polar_preht.comp",
   "polar_get_rows.comp",
 ];
 
-const SHADER_SENTINEL = "// MILADY-VK-DISPATCH-PATCH-V1";
-const PATCH_SENTINEL = "MILADY-VK-DISPATCH-PATCH-V1";
+// Multi-block-per-workgroup standalone variants (turbo3_multi.comp etc.). One
+// SPV family per kernel; the blocks/tokens-per-workgroup count is a SPIR-V
+// specialization constant (constant_id 0) the consumer sets at pipeline-create
+// time, so the same blob tunes per device without recompilation. They are
+// verified by `make -C packages/inference/verify vulkan-verify-multiblock`.
+// Staged into the fork alongside the single-block kernels; the runtime routes
+// non-voice / long-context scoring (large n_kv) through them with a
+// device-tuned spec constant (BLOCKS_PER_WG / TOKENS_PER_WG = 4 default).
+export const VULKAN_MULTIBLOCK_KERNEL_FILES = [
+  "turbo3_multi.comp",
+  "turbo4_multi.comp",
+  "turbo3_tcq_multi.comp",
+  "qjl_multi.comp",
+];
+
+// Fused-attention compute shaders (QJL-K score + V-mix, online softmax, score
+// never materialised). One workgroup per (q_head); q_pos plus causal
+// q_pos_base are push constants.
+// Verified standalone by `make -C packages/inference/verify vulkan-verify-fused`.
+// Staged into the fork. Runtime dispatch currently wires the TBQ3-V
+// GGML_OP_FUSED_ATTN_QJL_TBQ path; the Polar V-mix shader remains
+// standalone-only until a distinct graph op/API and CPU reference exist.
+export const VULKAN_FUSED_KERNEL_FILES = [
+  "fused_attn_qjl_tbq.comp",
+  "fused_attn_qjl_polar.comp",
+];
+
+// Everything staged into the fork's vulkan-shaders/ directory.
+const VULKAN_ALL_STAGED_FILES = [
+  ...VULKAN_KERNEL_FILES,
+  ...VULKAN_MULTIBLOCK_KERNEL_FILES,
+  ...VULKAN_FUSED_KERNEL_FILES,
+];
+
+const SHADER_SENTINEL = "// ELIZA-VK-DISPATCH-PATCH-V1";
+const _PATCH_SENTINEL = "ELIZA-VK-DISPATCH-PATCH-V1";
+const RUNTIME_SENTINEL = "// ELIZA-VK-RUNTIME-DISPATCH-V1";
 
 const PATCH_TARGETS = [
   {
@@ -87,18 +149,13 @@ const PATCH_TARGETS = [
   },
   {
     file: "02-ggml-vulkan-pipelines.patch",
-    target: path.posix.join(
-      "ggml",
-      "src",
-      "ggml-vulkan",
-      "ggml-vulkan.cpp",
-    ),
+    target: path.posix.join("ggml", "src", "ggml-vulkan", "ggml-vulkan.cpp"),
   },
 ];
 
 function assertStandalonesPresent() {
   const missing = [];
-  for (const name of VULKAN_KERNEL_FILES) {
+  for (const name of VULKAN_ALL_STAGED_FILES) {
     const src = path.join(STANDALONE_VULKAN_DIR, name);
     if (!fs.existsSync(src)) {
       missing.push(src);
@@ -118,7 +175,7 @@ function assertStandalonesPresent() {
 
 function copyStandalonesIntoFork(cacheDir, { dryRun }) {
   // vulkan-shaders/ is the directory the upstream CMakeLists uses for
-  // file(GLOB CONFIGURE_DEPENDS *.comp) — dropping our 8 files there causes
+  // file(GLOB CONFIGURE_DEPENDS *.comp) — dropping our files there causes
   // glslc to compile them automatically as part of the existing per-shader
   // add_custom_command pipeline. The string_to_spv() registration patch
   // (01-vulkan-shaders-gen.patch) wires the resulting .spv bytes into
@@ -138,7 +195,7 @@ function copyStandalonesIntoFork(cacheDir, { dryRun }) {
     );
   }
   const copied = [];
-  for (const name of VULKAN_KERNEL_FILES) {
+  for (const name of VULKAN_ALL_STAGED_FILES) {
     const src = path.join(STANDALONE_VULKAN_DIR, name);
     const dst = path.join(targetDir, name);
     if (dryRun) {
@@ -149,7 +206,7 @@ function copyStandalonesIntoFork(cacheDir, { dryRun }) {
       // human inspecting the fork tree can see the file came from us.
       const stamped =
         `${SHADER_SENTINEL} — staged from packages/inference/vulkan/${name} by\n` +
-        `// build-llama-cpp-dflash.mjs. Frozen — do not edit in fork.\n` +
+        `// build-llama-cpp-mtp.mjs. Frozen — do not edit in fork.\n` +
         text;
       fs.writeFileSync(dst, stamped, "utf8");
     }
@@ -169,15 +226,25 @@ function parsePatchFile(filePath) {
   let injectLines = [];
   for (const line of lines) {
     if (line.startsWith("ANCHOR")) {
-      cur = { anchor: line.replace(/^ANCHOR\s+/, ""), sentinel: null, inject: null };
+      cur = {
+        anchor: line.replace(/^ANCHOR\s+/, ""),
+        sentinel: null,
+        inject: null,
+      };
     } else if (line.startsWith("SENTINEL")) {
-      if (!cur) throw new Error(`[vulkan-kernels] SENTINEL before ANCHOR in ${filePath}`);
+      if (!cur)
+        throw new Error(
+          `[vulkan-kernels] SENTINEL before ANCHOR in ${filePath}`,
+        );
       cur.sentinel = line.replace(/^SENTINEL\s+/, "").trim();
     } else if (line === "---INJECT-BEGIN---") {
       inInject = true;
       injectLines = [];
     } else if (line === "---INJECT-END---") {
-      if (!cur) throw new Error(`[vulkan-kernels] INJECT-END without ANCHOR in ${filePath}`);
+      if (!cur)
+        throw new Error(
+          `[vulkan-kernels] INJECT-END without ANCHOR in ${filePath}`,
+        );
       cur.inject = injectLines.join("\n");
       hunks.push(cur);
       cur = null;
@@ -202,7 +269,7 @@ function parsePatchFile(filePath) {
 function applyHunk(text, hunk, ctx) {
   // Use the first non-blank line of the inject block as the per-hunk
   // sentinel. Each inject block in our patches starts with a unique
-  // `// MILADY-VK-DISPATCH-PATCH-V1 BEGIN — <description>` comment, so
+  // `// ELIZA-VK-DISPATCH-PATCH-V1 BEGIN — <description>` comment, so
   // first-non-blank identifies it precisely.
   const firstLine = hunk.inject.split("\n").find((l) => l.trim().length > 0);
   if (!firstLine) {
@@ -240,7 +307,13 @@ function applyPatches(cacheDir, { dryRun }) {
       console.log(
         `[vulkan-kernels] (dry-run) would apply ${hunks.length} hunk(s) from ${file} to ${target}`,
       );
-      results.push({ file, target, hunks: hunks.length, applied: 0, skipped: hunks.length });
+      results.push({
+        file,
+        target,
+        hunks: hunks.length,
+        applied: 0,
+        skipped: hunks.length,
+      });
       continue;
     }
     let text = fs.readFileSync(targetPath, "utf8");
@@ -249,7 +322,8 @@ function applyPatches(cacheDir, { dryRun }) {
     for (const hunk of hunks) {
       const r = applyHunk(text, hunk, target);
       text = r.text;
-      if (r.applied) applied++; else skipped++;
+      if (r.applied) applied++;
+      else skipped++;
     }
     fs.writeFileSync(targetPath, text, "utf8");
     results.push({ file, target, hunks: hunks.length, applied, skipped });
@@ -257,25 +331,702 @@ function applyPatches(cacheDir, { dryRun }) {
   return results;
 }
 
-// Public entry point used by build-llama-cpp-dflash.mjs.
-export function patchVulkanKernels(cacheDir, { dryRun = false, target = null } = {}) {
+function ensureLineAfter(text, anchor, line, ctx) {
+  if (text.includes(line)) return { text, changed: false };
+  if (!text.includes(anchor)) {
+    throw new Error(
+      `[vulkan-kernels] repair anchor not found in ${ctx}: ${anchor}`,
+    );
+  }
+  return {
+    text: text.replace(anchor, `${anchor}\n${line}`),
+    changed: true,
+  };
+}
+
+function repairPolarPrehtShaderRegistration(cacheDir, { dryRun }) {
+  const targetPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-vulkan",
+    "vulkan-shaders",
+    "vulkan-shaders-gen.cpp",
+  );
+  const anchor = `    string_to_spv("eliza_polar",          "polar.comp",          {});`;
+  const line = `    string_to_spv("eliza_polar_preht",    "polar_preht.comp",    {});`;
+  const original = fs.readFileSync(targetPath, "utf8");
+  const repaired = ensureLineAfter(original, anchor, line, targetPath);
+  if (repaired.changed && !dryRun) {
+    fs.writeFileSync(targetPath, repaired.text, "utf8");
+  }
+  return {
+    target: targetPath,
+    changed: repaired.changed && !dryRun,
+    wouldChange: repaired.changed,
+  };
+}
+
+function repairPolarPrehtPipeline(cacheDir, { dryRun }) {
+  const targetPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-vulkan",
+    "ggml-vulkan.cpp",
+  );
+  let text = fs.readFileSync(targetPath, "utf8");
+  let changed = false;
+
+  {
+    const r = ensureLineAfter(
+      text,
+      `    vk_pipeline pipeline_eliza_polar;`,
+      `    vk_pipeline pipeline_eliza_polar_preht;`,
+      targetPath,
+    );
+    text = r.text;
+    changed = changed || r.changed;
+  }
+  {
+    const r = ensureLineAfter(
+      text,
+      `    ggml_vk_create_pipeline(device, device->pipeline_eliza_polar,          "eliza_polar",          eliza_polar_len,          eliza_polar_data,          "main", 3, 6 * sizeof(uint32_t), {1, 1, 1}, {}, 1);`,
+      `    ggml_vk_create_pipeline(device, device->pipeline_eliza_polar_preht,    "eliza_polar_preht",    eliza_polar_preht_len,    eliza_polar_preht_data,    "main", 3, 6 * sizeof(uint32_t), {1, 1, 1}, {}, 1);`,
+      targetPath,
+    );
+    text = r.text;
+    changed = changed || r.changed;
+  }
+
+  if (changed && !dryRun) {
+    fs.writeFileSync(targetPath, text, "utf8");
+  }
+  return {
+    target: targetPath,
+    changed: changed && !dryRun,
+    wouldChange: changed,
+  };
+}
+
+function repairFusedAttnPipelinePushRanges(cacheDir, { dryRun }) {
+  const targetPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-vulkan",
+    "ggml-vulkan.cpp",
+  );
+  let text = fs.readFileSync(targetPath, "utf8");
+  const original = text;
+  text = text.replace(
+    `"eliza_fused_attn_qjl_tbq",   eliza_fused_attn_qjl_tbq_len,   eliza_fused_attn_qjl_tbq_data,   "main", 4, 6 * sizeof(uint32_t),`,
+    `"eliza_fused_attn_qjl_tbq",   eliza_fused_attn_qjl_tbq_len,   eliza_fused_attn_qjl_tbq_data,   "main", 4, 8 * sizeof(uint32_t),`,
+  );
+  text = text.replace(
+    `"eliza_fused_attn_qjl_polar", eliza_fused_attn_qjl_polar_len, eliza_fused_attn_qjl_polar_data, "main", 4, 7 * sizeof(uint32_t),`,
+    `"eliza_fused_attn_qjl_polar", eliza_fused_attn_qjl_polar_len, eliza_fused_attn_qjl_polar_data, "main", 4, 9 * sizeof(uint32_t),`,
+  );
+  const changed = text !== original;
+  if (changed && !dryRun) {
+    fs.writeFileSync(targetPath, text, "utf8");
+  }
+  return {
+    target: targetPath,
+    changed: changed && !dryRun,
+    wouldChange: changed,
+  };
+}
+
+function extractTcqCodebookSource() {
+  const referencePath = path.join(STANDALONE_REFERENCE_DIR, "turbo_kernels.c");
+  const source = fs.readFileSync(referencePath, "utf8");
+  const match = source.match(
+    /const float ELIZA_TURBO3_TCQ_CODEBOOK\[512\]\s*=\s*\{([\s\S]*?)\};/,
+  );
+  if (!match) {
+    throw new Error(
+      `[vulkan-runtime-dispatch] could not extract TCQ codebook from ${referencePath}`,
+    );
+  }
+  return match[1].trim();
+}
+
+function patchVulkanRuntimeDispatch(cacheDir, { dryRun }) {
+  // TBQ and Polar attention-score graph constructors are backend-neutral ggml
+  // surface area. The Metal patch already owns the idempotent ggml.h/ggml.c
+  // mutation; reuse it here so a fresh Vulkan-only build can compile the same
+  // graph ops instead of depending on a previous Metal build having dirtied the
+  // cached fork.
+  const ggmlOps = patchGgmlTbqPolarAttnOps(cacheDir, { dryRun });
+
+  const vulkanPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-vulkan",
+    "ggml-vulkan.cpp",
+  );
+  const original = fs.readFileSync(vulkanPath, "utf8");
+  let patched = original;
+
+  // 02-ggml-vulkan-pipelines.patch historically created eliza_polar with a
+  // 3*u32 push-constant range. Runtime graph dispatch needs per-head output and
+  // K-head byte offsets so it can bind the whole tensor and avoid illegal
+  // unaligned descriptor offsets. Keep old cached checkouts repairable.
+  patched = patched.replace(
+    `"eliza_polar",          eliza_polar_len,          eliza_polar_data,          "main", 3, 3 * sizeof(uint32_t),`,
+    `"eliza_polar",          eliza_polar_len,          eliza_polar_data,          "main", 3, 6 * sizeof(uint32_t),`,
+  );
+
+  // Keep older already-sentinelled cached forks repairable after the fused
+  // attention push ABI grew causal/q_pos_base fields.
+  patched = patched.replace(
+    `struct eliza_vk_fused_attn_push {
+    uint32_t n_heads;
+    uint32_t n_kv_heads;
+    uint32_t n_tokens;
+    uint32_t q_pos;
+    uint32_t sm_scale_bits;
+    uint32_t kv_tile;
+};`,
+    `struct eliza_vk_fused_attn_push {
+    uint32_t n_heads;
+    uint32_t n_kv_heads;
+    uint32_t n_tokens;
+    uint32_t q_pos;
+    uint32_t sm_scale_bits;
+    uint32_t kv_tile;
+    uint32_t causal;
+    uint32_t q_pos_base;
+};`,
+  );
+  patched = patched.replace(
+    `    const uint32_t kv_tile    = (uint32_t) params[3];
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];`,
+    `    const uint32_t kv_tile    = (uint32_t) params[3];
+    const uint32_t causal     = (uint32_t) (params[4] != 0);
+    const uint32_t q_pos_base = (uint32_t) params[5];
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];`,
+  );
+  patched = patched.replace(
+    `    const uint32_t kv_tile    = 0u;
+    const uint32_t causal     = 0u;
+    const uint32_t q_pos_base = 0u;
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];`,
+    `    const uint32_t kv_tile    = (uint32_t) params[3];
+    const uint32_t causal     = (uint32_t) (params[4] != 0);
+    const uint32_t q_pos_base = (uint32_t) params[5];
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];`,
+  );
+  patched = patched.replace(
+    `            n_heads, n_kv_heads, n_tokens, (uint32_t) p, sm_bits, kv_tile,
+        };`,
+    `            n_heads, n_kv_heads, n_tokens, (uint32_t) p, sm_bits, kv_tile, causal, q_pos_base,
+        };`,
+  );
+
+  if (!patched.includes(RUNTIME_SENTINEL)) {
+    const contextAnchor = `    // for GGML_VK_PERF_LOGGER`;
+    if (!patched.includes(contextAnchor)) {
+      throw new Error(
+        `[vulkan-runtime-dispatch] context anchor not found in ${vulkanPath}`,
+      );
+    }
+    patched = patched.replace(
+      contextAnchor,
+      `    ${RUNTIME_SENTINEL}
+    // Persistent device-side TCQ codebook used by GGML_OP_ATTN_SCORE_TBQ when
+    // packed_k is GGML_TYPE_TBQ3_TCQ. Created lazily on first dispatch.
+    vk_buffer eliza_turbo3_tcq_codebook;
+
+${contextAnchor}`,
+    );
+
+    const codebook = extractTcqCodebookSource();
+    const helperAnchor = `static void ggml_vk_compute_forward(ggml_backend_vk_context* ctx, ggml_cgraph * cgraph, ggml_tensor* tensor, int tensor_idx, bool almost_ready);`;
+    if (!patched.includes(helperAnchor)) {
+      throw new Error(
+        `[vulkan-runtime-dispatch] compute-forward anchor not found in ${vulkanPath}`,
+      );
+    }
+    const helper = `${RUNTIME_SENTINEL}
+struct eliza_vk_qjl_score_push {
+    uint32_t n_heads;
+    uint32_t n_kv_heads;
+    uint32_t n_tokens;
+    uint32_t proj_dim;
+};
+
+struct eliza_vk_tbq_score_push {
+    uint32_t head_dim;
+    uint32_t n_kv;
+    uint32_t kv_stride_blocks;
+    uint32_t q_head;
+    uint32_t head_offset_bytes;
+};
+
+struct eliza_vk_polar_score_push {
+    uint32_t n_rows;
+    uint32_t head_dim;
+    uint32_t use_qjl;
+    uint32_t k_offset_bytes;
+    uint32_t q_offset;
+    uint32_t y_offset;
+};
+
+struct eliza_vk_fused_attn_push {
+    uint32_t n_heads;
+    uint32_t n_kv_heads;
+    uint32_t n_tokens;
+    uint32_t q_pos;
+    uint32_t sm_scale_bits;
+    uint32_t kv_tile;
+    uint32_t causal;
+    uint32_t q_pos_base;
+};
+
+// Long-context / non-voice scoring amortises the per-dispatch launch tax by
+// folding several KV indices (resp. tokens) into one workgroup via the
+// constant_id=0 spec constant (BLOCKS_PER_WG / TOKENS_PER_WG) baked into the
+// _multi pipeline at create time. Voice / small-n_kv stays single-block.
+//
+// The *fold factor* is device-policy: it is chosen by vendor in
+// ggml_vk_load_shaders (02-ggml-vulkan-pipelines.patch hunk 2) and stored on
+// device->eliza_vk_{tbq,qjl}_multiblock_factor. The dispatch here reads that
+// field so the grid divisor always matches whatever the pipeline was created
+// with. From vulkan_bench (VK_QUERY_TYPE_TIMESTAMP per-dispatch sweep over
+// {1,2,4,8,16}; vulkan_kopt_2026-05-11.json):
+//   * NVIDIA discrete (RTX 5080 Laptop): TBQ factor 16 (turbo3 386us→48us at
+//     4k, ~3.2x at 32k), QJL factor 8 (4742us→374us at 4k, ~8.6x at 32k).
+//   * Intel Arc/Xe iGPU (Mesa ANV 25.2.8): bandwidth-bound at n_kv>=512, both
+//     factors stay 4 — only the engage threshold matters. QJL engages from
+//     1024 tokens (the fold hoists the 256-wide q_sketch + sign vector out of
+//     the per-token loop, ~1.3x at 512 / ~1.8x at 4k); TBQ engages only at
+//     n_kv>=8192 (it's a wash at 512 and a slight regression at 4k on ANV).
+//   * Default / unprofiled (Adreno, Mali, AMD): conservative factor 4 and the
+//     same thresholds — safe everywhere. AMD wave64 wants its own sweep.
+// (The thresholds below are the same on every device; only the fold factor
+// diverges. The discrete-GPU thresholds being identical to the iGPU ones is
+// deliberate — on NVIDIA the _multi path is a win at *every* measured length
+// (e.g. turbo3 512: 83us single → 30us folded), so engaging from 1024/8192
+// only loses a little of the available speedup at the very small end.)
+static const int64_t  ELIZA_VK_QJL_MULTIBLOCK_THRESHOLD = 1024;
+static const int64_t  ELIZA_VK_TBQ_MULTIBLOCK_THRESHOLD = 8192;
+
+static const float k_eliza_tbq3_tcq_codebook[512] = {
+${codebook}
+};
+
+static vk_pipeline eliza_vk_pipeline_for_tbq(ggml_backend_vk_context * ctx, ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return ctx->device->pipeline_eliza_turbo3;
+        case GGML_TYPE_TBQ4_0:   return ctx->device->pipeline_eliza_turbo4;
+        case GGML_TYPE_TBQ3_TCQ: return ctx->device->pipeline_eliza_turbo3_tcq;
+        default: GGML_ABORT("eliza_vk_pipeline_for_tbq: unsupported type");
+    }
+}
+
+static vk_pipeline eliza_vk_pipeline_for_tbq_multi(ggml_backend_vk_context * ctx, ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return ctx->device->pipeline_eliza_turbo3_multi;
+        case GGML_TYPE_TBQ4_0:   return ctx->device->pipeline_eliza_turbo4_multi;
+        case GGML_TYPE_TBQ3_TCQ: return ctx->device->pipeline_eliza_turbo3_tcq_multi;
+        default: GGML_ABORT("eliza_vk_pipeline_for_tbq_multi: unsupported type");
+    }
+}
+
+static vk_buffer eliza_vk_turbo3_tcq_codebook(ggml_backend_vk_context * ctx) {
+    if (ctx->eliza_turbo3_tcq_codebook == nullptr) {
+        ctx->eliza_turbo3_tcq_codebook = ggml_vk_create_buffer_device(ctx->device, sizeof(k_eliza_tbq3_tcq_codebook));
+        ggml_vk_buffer_write(ctx->eliza_turbo3_tcq_codebook, 0, k_eliza_tbq3_tcq_codebook, sizeof(k_eliza_tbq3_tcq_codebook));
+    }
+    return ctx->eliza_turbo3_tcq_codebook;
+}
+
+static void ggml_vk_eliza_attn_score_qjl(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+    const ggml_tensor * q  = dst->src[0];
+    const ggml_tensor * pk = dst->src[1];
+    GGML_ASSERT(q != nullptr && pk != nullptr);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(pk->type == GGML_TYPE_QJL1_256);
+    GGML_ASSERT(dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0] == 256 && pk->ne[0] == 128);
+    GGML_ASSERT(q->ne[2] == 1 && q->ne[3] == 1 && pk->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(dst));
+
+    const uint32_t n_heads    = (uint32_t) q->ne[1];
+    const uint32_t n_kv_heads = (uint32_t) ((const int32_t *) dst->op_params)[0];
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];
+    GGML_ASSERT(n_kv_heads > 0 && (n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(dst->ne[0] == (int64_t) n_tokens && dst->ne[1] == (int64_t) n_heads);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(GGML_TYPE_QJL1_256, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_tokens * pk->nb[1]);
+
+    const bool multi = (int64_t) n_tokens >= ELIZA_VK_QJL_MULTIBLOCK_THRESHOLD;
+    const uint32_t qjl_fold = ctx->device->eliza_vk_qjl_multiblock_factor;
+    vk_pipeline pipeline = multi ? ctx->device->pipeline_eliza_qjl_multi
+                                 : ctx->device->pipeline_eliza_qjl;
+    const uint32_t grid_y = multi ? (n_tokens + qjl_fold - 1u) / qjl_fold : n_tokens;
+    ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
+    const eliza_vk_qjl_score_push pc = { n_heads, n_kv_heads, n_tokens, 256u };
+    ggml_vk_dispatch_pipeline(
+        ctx, subctx, pipeline,
+        { ggml_vk_tensor_subbuffer(ctx, q), ggml_vk_tensor_subbuffer(ctx, pk), ggml_vk_tensor_subbuffer(ctx, dst) },
+        pc, { n_heads, grid_y, 1 });
+}
+
+static void ggml_vk_eliza_attn_score_tbq(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+    const ggml_tensor * q  = dst->src[0];
+    const ggml_tensor * pk = dst->src[1];
+    GGML_ASSERT(q != nullptr && pk != nullptr);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(pk->type == GGML_TYPE_TBQ3_0 || pk->type == GGML_TYPE_TBQ4_0 || pk->type == GGML_TYPE_TBQ3_TCQ);
+    GGML_ASSERT(dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0] == 128 && pk->ne[0] == 128);
+    GGML_ASSERT(q->ne[2] == 1 && q->ne[3] == 1 && pk->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(dst));
+
+    const uint32_t n_heads    = (uint32_t) q->ne[1];
+    const uint32_t n_kv_heads = (uint32_t) ((const int32_t *) dst->op_params)[0];
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];
+    const uint32_t gqa        = n_heads / n_kv_heads;
+    const uint32_t blocks_per_kv = (uint32_t) (pk->ne[0] / ggml_blck_size(pk->type));
+    GGML_ASSERT(n_kv_heads > 0 && (n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(dst->ne[0] == (int64_t) n_tokens && dst->ne[1] == (int64_t) n_heads);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(pk->type, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_tokens * pk->nb[1]);
+
+    const bool multi = (int64_t) n_tokens >= ELIZA_VK_TBQ_MULTIBLOCK_THRESHOLD;
+    const uint32_t tbq_fold = ctx->device->eliza_vk_tbq_multiblock_factor;
+    vk_pipeline pipeline = multi ? eliza_vk_pipeline_for_tbq_multi(ctx, pk->type)
+                                 : eliza_vk_pipeline_for_tbq(ctx, pk->type);
+    const uint32_t grid_x = multi ? (n_tokens + tbq_fold - 1u) / tbq_fold : n_tokens;
+    const vk_subbuffer q_buf   = ggml_vk_tensor_subbuffer(ctx, q);
+    const vk_subbuffer pk_buf  = ggml_vk_tensor_subbuffer(ctx, pk);
+    const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
+    const bool is_tcq = pk->type == GGML_TYPE_TBQ3_TCQ;
+    const vk_subbuffer codebook_buf = is_tcq ? ggml_vk_subbuffer(ctx, eliza_vk_turbo3_tcq_codebook(ctx)) : vk_subbuffer{};
+
+    ggml_pipeline_request_descriptor_sets(ctx, pipeline, n_heads);
+    for (uint32_t h = 0; h < n_heads; ++h) {
+        const uint32_t h_k = h / gqa;
+        const uint64_t head_offset = (uint64_t) h_k * (uint64_t) pk->nb[2];
+        GGML_ASSERT(head_offset <= UINT32_MAX);
+        const eliza_vk_tbq_score_push pc = {
+            128u,
+            n_tokens,
+            blocks_per_kv,
+            h,
+            (uint32_t) head_offset,
+        };
+        if (is_tcq) {
+            ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { q_buf, pk_buf, dst_buf, codebook_buf }, pc, { grid_x, 1, 1 });
+        } else {
+            ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { q_buf, pk_buf, dst_buf }, pc, { grid_x, 1, 1 });
+        }
+    }
+}
+
+static void ggml_vk_eliza_attn_score_polar(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+    const ggml_tensor * q  = dst->src[0];
+    const ggml_tensor * pk = dst->src[1];
+    GGML_ASSERT(q != nullptr && pk != nullptr);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(pk->type == GGML_TYPE_Q4_POLAR);
+    GGML_ASSERT(dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0] == 128 && pk->ne[0] == 128);
+    GGML_ASSERT(q->ne[2] == 1 && q->ne[3] == 1 && pk->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(dst));
+
+    const int32_t * params = (const int32_t *) dst->op_params;
+    const uint32_t n_heads    = (uint32_t) q->ne[1];
+    const uint32_t n_kv_heads = (uint32_t) params[0];
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];
+    const uint32_t use_qjl    = (uint32_t) (params[1] != 0);
+    const uint32_t gqa        = n_heads / n_kv_heads;
+    GGML_ASSERT(n_kv_heads > 0 && (n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(dst->ne[0] == (int64_t) n_tokens && dst->ne[1] == (int64_t) n_heads);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(GGML_TYPE_Q4_POLAR, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_tokens * pk->nb[1]);
+
+    vk_pipeline pipeline = ctx->device->pipeline_eliza_polar;
+    const vk_subbuffer pk_buf  = ggml_vk_tensor_subbuffer(ctx, pk);
+    const vk_subbuffer q_buf   = ggml_vk_tensor_subbuffer(ctx, q);
+    const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
+
+    ggml_pipeline_request_descriptor_sets(ctx, pipeline, n_heads);
+    for (uint32_t h = 0; h < n_heads; ++h) {
+        const uint32_t h_k = h / gqa;
+        const uint64_t k_offset = (uint64_t) h_k * (uint64_t) pk->nb[2];
+        const uint64_t q_offset = ((uint64_t) h * (uint64_t) q->nb[1]) / sizeof(float);
+        const uint64_t y_offset = ((uint64_t) h * (uint64_t) dst->nb[1]) / sizeof(float);
+        GGML_ASSERT(k_offset <= UINT32_MAX && q_offset <= UINT32_MAX && y_offset <= UINT32_MAX);
+        const eliza_vk_polar_score_push pc = {
+            n_tokens,
+            128u,
+            use_qjl,
+            (uint32_t) k_offset,
+            (uint32_t) q_offset,
+            (uint32_t) y_offset,
+        };
+        ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { pk_buf, q_buf, dst_buf }, pc, { n_tokens, 1, 1 });
+    }
+}
+
+// GGML_OP_FUSED_ATTN_QJL_TBQ — fused QJL-K score + TBQ3-V mix, online softmax,
+// the per-token score vector never materialised. Mirrors the C reference
+// eliza_fused_attn_qjl_tbq3() / fused_attn_qjl_tbq_ref() and the op contract
+// reports/porting/2026-05-11/fused-attn-op-contract.md §3/§6:
+//   src[0] = q          F32      [proj_dim=256, n_heads, n_q_pos, ne3]  (pre-projected QJL sketch)
+//   src[1] = packed_k   QJL1_256 [head_dim=128, n_kv, n_kv_heads, ne3]  (nb[1] == 34)
+//   src[2] = packed_v   TBQ3_0   [head_dim=128, n_kv, n_kv_heads, ne3]  (nb[1] == 56, 4 chunks/token)
+//   dst    = out        F32      [head_dim=128, n_heads, n_q_pos, ne3]
+//   op_params[0] = n_kv_heads, [1] = sm_scale (float bits), [2] = v_use_qjl (TBQ ignores it),
+//   [3] = kv_tile, [4] = causal, [5] = q_pos_base
+// One workgroup per (q_head); q_pos is a push constant, so n_q_pos > 1 is a
+// loop of dispatches (decode = 1). Conservative shape (ne3 == 1) matching the
+// other eliza graph routes; the unfused score → softmax → V-mix path covers
+// the wider shapes (AGENTS.md §3 — no silent degradation).
+static void ggml_vk_eliza_fused_attn_qjl_tbq(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+    const ggml_tensor * q  = dst->src[0];
+    const ggml_tensor * pk = dst->src[1];
+    const ggml_tensor * pv = dst->src[2];
+    GGML_ASSERT(q != nullptr && pk != nullptr && pv != nullptr);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(pk->type == GGML_TYPE_QJL1_256);
+    GGML_ASSERT(pv->type == GGML_TYPE_TBQ3_0);
+    GGML_ASSERT(dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0] == 256 && pk->ne[0] == 128 && pv->ne[0] == 128 && dst->ne[0] == 128);
+    GGML_ASSERT(q->ne[3] == 1 && pk->ne[3] == 1 && pv->ne[3] == 1 && dst->ne[3] == 1);
+    GGML_ASSERT(ggml_is_contiguous(q));
+    GGML_ASSERT(ggml_is_contiguous(pk));
+    GGML_ASSERT(ggml_is_contiguous(pv));
+    GGML_ASSERT(ggml_is_contiguous(dst));
+
+    const int32_t * params = (const int32_t *) dst->op_params;
+    const uint32_t n_heads    = (uint32_t) q->ne[1];
+    const uint32_t n_kv_heads = (uint32_t) params[0];
+    const uint32_t sm_bits    = (uint32_t) params[1];
+    const uint32_t kv_tile    = (uint32_t) params[3];
+    const uint32_t causal     = (uint32_t) (params[4] != 0);
+    const uint32_t q_pos_base = (uint32_t) params[5];
+    const uint32_t n_tokens   = (uint32_t) pk->ne[1];
+    const int64_t  n_q_pos    = q->ne[2];
+    GGML_ASSERT(n_kv_heads > 0 && (n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads && pv->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(dst->ne[1] == (int64_t) n_heads && dst->ne[2] == n_q_pos);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(GGML_TYPE_QJL1_256, 128));
+    GGML_ASSERT(pv->nb[1] == ggml_row_size(GGML_TYPE_TBQ3_0, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_tokens * pk->nb[1]);
+    GGML_ASSERT(pv->nb[2] == (size_t) n_tokens * pv->nb[1]);
+
+    vk_pipeline pipeline = ctx->device->pipeline_eliza_fused_attn_qjl_tbq;
+    const vk_subbuffer q_buf   = ggml_vk_tensor_subbuffer(ctx, q);
+    const vk_subbuffer pk_buf  = ggml_vk_tensor_subbuffer(ctx, pk);
+    const vk_subbuffer pv_buf  = ggml_vk_tensor_subbuffer(ctx, pv);
+    const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
+
+    ggml_pipeline_request_descriptor_sets(ctx, pipeline, (uint32_t) n_q_pos);
+    for (int64_t p = 0; p < n_q_pos; ++p) {
+        const eliza_vk_fused_attn_push pc = {
+            n_heads, n_kv_heads, n_tokens, (uint32_t) p, sm_bits, kv_tile, causal, q_pos_base,
+        };
+        ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { q_buf, pk_buf, pv_buf, dst_buf }, pc, { n_heads, 1, 1 });
+    }
+}
+
+`;
+    patched = patched.replace(helperAnchor, helper + helperAnchor);
+
+    const switchAnchor = `    case GGML_OP_FLASH_ATTN_EXT:
+        ggml_vk_flash_attn(ctx, compute_ctx, src0, src1, src2, src3, node->src[4], node);
+
+        break;`;
+    if (!patched.includes(switchAnchor)) {
+      throw new Error(
+        `[vulkan-runtime-dispatch] graph switch anchor not found in ${vulkanPath}`,
+      );
+    }
+    patched = patched.replace(
+      switchAnchor,
+      `    case GGML_OP_ATTN_SCORE_QJL:
+        ggml_vk_eliza_attn_score_qjl(ctx, compute_ctx, node);
+
+        break;
+    case GGML_OP_ATTN_SCORE_TBQ:
+        ggml_vk_eliza_attn_score_tbq(ctx, compute_ctx, node);
+
+        break;
+    case GGML_OP_ATTN_SCORE_POLAR:
+        ggml_vk_eliza_attn_score_polar(ctx, compute_ctx, node);
+
+        break;
+    case GGML_OP_FUSED_ATTN_QJL_TBQ:
+        ggml_vk_eliza_fused_attn_qjl_tbq(ctx, compute_ctx, node);
+
+        break;
+
+${switchAnchor}`,
+    );
+
+    const supportsAnchor = `        case GGML_OP_FLASH_ATTN_EXT:
+            {`;
+    if (!patched.includes(supportsAnchor)) {
+      throw new Error(
+        `[vulkan-runtime-dispatch] supports_op anchor not found in ${vulkanPath}`,
+      );
+    }
+    const supports = `        case GGML_OP_ATTN_SCORE_QJL:
+            return op->type == GGML_TYPE_F32 &&
+                   op->src[0] != nullptr &&
+                   op->src[1] != nullptr &&
+                   op->src[0]->type == GGML_TYPE_F32 &&
+                   op->src[1]->type == GGML_TYPE_QJL1_256 &&
+                   op->src[0]->ne[0] == 256 &&
+                   op->src[1]->ne[0] == 128 &&
+                   op->src[0]->ne[2] == 1 &&
+                   op->src[0]->ne[3] == 1 &&
+                   op->src[1]->ne[3] == 1 &&
+                   op->ne[2] == 1 &&
+                   op->ne[3] == 1 &&
+                   ggml_is_contiguous_rows(op) &&
+                   ggml_is_contiguous_rows(op->src[0]) &&
+                   ggml_is_contiguous_rows(op->src[1]);
+        case GGML_OP_ATTN_SCORE_TBQ:
+            return op->type == GGML_TYPE_F32 &&
+                   op->src[0] != nullptr &&
+                   op->src[1] != nullptr &&
+                   op->src[0]->type == GGML_TYPE_F32 &&
+                   (op->src[1]->type == GGML_TYPE_TBQ3_0 ||
+                    op->src[1]->type == GGML_TYPE_TBQ4_0 ||
+                    op->src[1]->type == GGML_TYPE_TBQ3_TCQ) &&
+                   op->src[0]->ne[0] == 128 &&
+                   op->src[1]->ne[0] == 128 &&
+                   op->src[0]->ne[2] == 1 &&
+                   op->src[0]->ne[3] == 1 &&
+                   op->src[1]->ne[3] == 1 &&
+                   op->ne[2] == 1 &&
+                   op->ne[3] == 1 &&
+                   ggml_is_contiguous_rows(op) &&
+                   ggml_is_contiguous_rows(op->src[0]) &&
+                   ggml_is_contiguous_rows(op->src[1]);
+        case GGML_OP_ATTN_SCORE_POLAR:
+            return op->type == GGML_TYPE_F32 &&
+                   op->src[0] != nullptr &&
+                   op->src[1] != nullptr &&
+                   op->src[0]->type == GGML_TYPE_F32 &&
+                   op->src[1]->type == GGML_TYPE_Q4_POLAR &&
+                   op->src[0]->ne[0] == 128 &&
+                   op->src[1]->ne[0] == 128 &&
+                   op->src[0]->ne[2] == 1 &&
+                   op->src[0]->ne[3] == 1 &&
+                   op->src[1]->ne[3] == 1 &&
+                   op->ne[2] == 1 &&
+                   op->ne[3] == 1 &&
+                   ggml_is_contiguous_rows(op) &&
+                   ggml_is_contiguous_rows(op->src[0]) &&
+                   ggml_is_contiguous_rows(op->src[1]);
+        case GGML_OP_FUSED_ATTN_QJL_TBQ:
+            return op->type == GGML_TYPE_F32 &&
+                   op->src[0] != nullptr &&
+                   op->src[1] != nullptr &&
+                   op->src[2] != nullptr &&
+                   op->src[0]->type == GGML_TYPE_F32 &&
+                   op->src[1]->type == GGML_TYPE_QJL1_256 &&
+                   op->src[2]->type == GGML_TYPE_TBQ3_0 &&
+                   op->src[0]->ne[0] == 256 &&
+                   op->src[1]->ne[0] == 128 &&
+                   op->src[2]->ne[0] == 128 &&
+                   op->ne[0] == 128 &&
+                   op->src[0]->ne[3] == 1 &&
+                   op->src[1]->ne[3] == 1 &&
+                   op->src[2]->ne[3] == 1 &&
+                   op->ne[3] == 1 &&
+                   ggml_is_contiguous(op) &&
+                   ggml_is_contiguous(op->src[0]) &&
+                   ggml_is_contiguous(op->src[1]) &&
+                   ggml_is_contiguous(op->src[2]);
+
+`;
+    patched = patched.replace(supportsAnchor, supports + supportsAnchor);
+  }
+
+  if (patched !== original && !dryRun) {
+    fs.writeFileSync(vulkanPath, patched, "utf8");
+  }
+  return {
+    ggmlOps,
+    path: vulkanPath,
+    changed: patched !== original && !dryRun,
+    sentinel: RUNTIME_SENTINEL,
+  };
+}
+
+// Public entry point used by build-llama-cpp-mtp.mjs.
+export function patchVulkanKernels(
+  cacheDir,
+  { dryRun = false, target = null } = {},
+) {
   if (!cacheDir || !fs.existsSync(cacheDir)) {
     throw new Error(`[vulkan-kernels] cacheDir does not exist: ${cacheDir}`);
   }
   assertStandalonesPresent();
   const copied = copyStandalonesIntoFork(cacheDir, { dryRun });
   const patchResults = applyPatches(cacheDir, { dryRun });
+  const prehtRegistration = repairPolarPrehtShaderRegistration(cacheDir, {
+    dryRun,
+  });
+  const prehtPipeline = repairPolarPrehtPipeline(cacheDir, { dryRun });
+  const fusedAttnPipeline = repairFusedAttnPipelinePushRanges(cacheDir, {
+    dryRun,
+  });
+  const runtimeDispatch = patchVulkanRuntimeDispatch(cacheDir, { dryRun });
   console.log(
-    `[vulkan-kernels] ${dryRun ? "(dry-run) " : ""}staged ${copied.length} standalone Vulkan shaders into vulkan-shaders/ ` +
-      `and applied ${patchResults.length} dispatch patches:`,
+    `[vulkan-kernels] ${dryRun ? "(dry-run) " : ""}target=${target ?? "unknown"} staged ${copied.length} standalone Vulkan shaders into vulkan-shaders/ ` +
+      `and applied ${patchResults.length} shader/pipeline patches plus runtime graph dispatch:`,
   );
   for (const r of patchResults) {
     console.log(
       `[vulkan-kernels]   ${r.file} → ${r.target}: ${r.applied} hunk(s) applied, ${r.skipped} idempotent-skipped`,
     );
   }
-  // Note: target arg is currently unused. AGENTS.md §3 enforcement (no
-  // milady-missing vulkan binary) is now done at build-llama-cpp-dflash.mjs
-  // post-build via the requiredKernels audit on the SPV blob list.
-  return { copied, patchResults };
+  console.log(
+    `[vulkan-kernels] runtime graph dispatch patch: ${runtimeDispatch.changed ? "patched" : "already-present/dry-run"} (${runtimeDispatch.path})`,
+  );
+  console.log(
+    `[vulkan-kernels] polar_preht registration repair: ${prehtRegistration.wouldChange ? (dryRun ? "would-patch" : "patched") : "already-present"} (${prehtRegistration.target})`,
+  );
+  console.log(
+    `[vulkan-kernels] polar_preht pipeline repair: ${prehtPipeline.wouldChange ? (dryRun ? "would-patch" : "patched") : "already-present"} (${prehtPipeline.target})`,
+  );
+  console.log(
+    `[vulkan-kernels] fused_attn push-range repair: ${fusedAttnPipeline.wouldChange ? (dryRun ? "would-patch" : "patched") : "already-present"} (${fusedAttnPipeline.target})`,
+  );
+  // AGENTS.md §3 enforcement (no eliza-missing vulkan binary) is done at
+  // build-llama-cpp-mtp.mjs post-build via the requiredKernels audit.
+  console.log(
+    `[vulkan-kernels] runtime-ready evidence still requires ` +
+      `make -C packages/inference/verify vulkan-dispatch-smoke on a native Vulkan build.`,
+  );
+  return {
+    copied,
+    patchResults,
+    prehtRegistration,
+    prehtPipeline,
+    fusedAttnPipeline,
+    runtimeDispatch,
+    runtimeReady: "source-patched-pending-smoke",
+    requiredGraphSmoke:
+      "make -C packages/inference/verify vulkan-dispatch-smoke",
+  };
 }