@elizaos/app-core 2.0.0-beta.3 → 2.0.11-beta.6

package/scripts/kernel-patches/metal-kernels.mjs

@@ -1,43 +1,37 @@
-// Real Metal kernel-shipment helpers — replace the v0.4.0-milady decorative
-// log no-ops in build-llama-cpp-dflash.mjs.
+// Real Metal kernel-shipment helpers — replace the fork's decorative
+// log no-ops in build-llama-cpp-mtp.mjs.
 //
 // What this module does:
 //
-//   1. Copies the five verified standalone Metal shaders from
+//   1. Copies the required and optimization standalone Metal shaders from
 //      packages/inference/metal/ into the fork's tree at
-//      ggml/src/ggml-metal/milady-shipped/<kernel>.metal. The standalones are
+//      ggml/src/ggml-metal/eliza-shipped/<kernel>.metal. The standalones are
 //      self-contained TUs (only #include <metal_stdlib>; their own structs,
 //      constants, kernel symbols), so they compile as independent .air files.
 //
 //   2. Patches ggml/src/ggml-metal/CMakeLists.txt so both Metal packaging
 //      branches build each standalone shader into its own .air via
 //      `xcrun metal -c` and merge all .air files (the original ggml-metal.air
-//      plus the five milady .air files) into one default.metallib.
+//      plus the five eliza .air files) into one default.metallib.
 //
 //   The original CMake snippet pipes `xcrun metal | xcrun metallib`. We
 //   replace that with explicit per-source compilation + a final merge step,
-//   keyed by a `# MILADY-KERNEL-PATCH-V1` sentinel so the patch is idempotent.
+//   keyed by a `# ELIZA-KERNEL-PATCH-V1` sentinel so the patch is idempotent.
 //
 //   3. Hard-throws on any error — missing files, missing anchor in
 //      CMakeLists.txt, fs failures. Per AGENTS.md §3, the build must exit
 //      non-zero rather than silently produce a kernel-missing artifact.
 //
-// What this module deliberately does NOT do (out of scope for v0.4.0-milady):
+// Runtime dispatch status:
 //
-//   * Wire dispatch sites in ggml-metal-ops.cpp / ggml-metal-device.m. The
-//     fork has zero existing dispatch entries for the milady quant types in
-//     the Metal backend (CUDA has them; Metal does not). Adding dispatch
-//     requires substantial fork-internals changes spanning case-statements
-//     per `GGML_TYPE_*`. Once kernels are in the metallib, follow-up work
-//     can add the dispatch wiring and the kernels become reachable. Until
-//     then the kernels ship as live symbols inside default.metallib but are
-//     not yet selected by the runtime — the symbol-presence audit (`nm`,
-//     `strings default.metallib`) passes, the dispatch audit does not.
+//   * QJL, Turbo3, Turbo3_TCQ, and PolarQuant are wired through dedicated
+//     attention-score graph ops. The dispatch smoke links against the built
+//     fork and numerically verifies graph execution selects the shipped Metal
+//     kernels.
 //
-//   * Convert the EMBED_LIBRARY path used by iOS targets to embed compiled
-//     metallib bytes rather than concatenated Metal source. This avoids
-//     duplicate declarations between ggml-metal.metal + standalones and lets
-//     iOS load the same multi-TU kernel set as desktop.
+//   * Turbo4 is wired through the same dedicated attention-score graph op.
+//     The shipped shader consumes the fork's real GGML_TYPE_TBQ4_0 layout:
+//     four 32-wide block_tbq4_0 records per 128-row (72 bytes).
 
 import fs from "node:fs";
 import path from "node:path";
@@ -46,8 +40,10 @@ import { fileURLToPath } from "node:url";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-// packages/app-core/scripts/kernel-patches/  →  packages/inference/metal/
-const STANDALONE_METAL_DIR = path.resolve(
+// packages/app-core/scripts/kernel-patches/  →  plugin-local-inference/native/metal/
+// Older workstreams staged these under packages/inference/metal; the current
+// native plugin owns the verified standalone shader sources.
+const LEGACY_STANDALONE_METAL_DIR = path.resolve(
   __dirname,
   "..",
   "..",
@@ -55,25 +51,124 @@ const STANDALONE_METAL_DIR = path.resolve(
   "inference",
   "metal",
 );
+const PLUGIN_STANDALONE_METAL_DIR = path.resolve(
+  __dirname,
+  "..",
+  "..",
+  "..",
+  "..",
+  "plugins",
+  "plugin-local-inference",
+  "native",
+  "metal",
+);
+const STANDALONE_METAL_DIR = fs.existsSync(LEGACY_STANDALONE_METAL_DIR)
+  ? LEGACY_STANDALONE_METAL_DIR
+  : PLUGIN_STANDALONE_METAL_DIR;
+
+// Reference C kernels (TCQ codebook source) — same restructure drift: older
+// workstreams kept these under packages/inference/reference; the native plugin
+// now owns them.
+const LEGACY_STANDALONE_REFERENCE_DIR = path.resolve(
+  __dirname,
+  "..",
+  "..",
+  "..",
+  "inference",
+  "reference",
+);
+const PLUGIN_STANDALONE_REFERENCE_DIR = path.resolve(
+  __dirname,
+  "..",
+  "..",
+  "..",
+  "..",
+  "plugins",
+  "plugin-local-inference",
+  "native",
+  "reference",
+);
+export const STANDALONE_REFERENCE_DIR = fs.existsSync(
+  LEGACY_STANDALONE_REFERENCE_DIR,
+)
+  ? LEGACY_STANDALONE_REFERENCE_DIR
+  : PLUGIN_STANDALONE_REFERENCE_DIR;
 
 // Map: standalone-shader-filename → in-fork relative path (under cacheDir).
 // Each standalone is copied verbatim — its content is not edited. Per agent
-// contract: the 13 standalone shaders are verified and must not be touched.
+// contract, verified shader math lives under packages/inference/metal/ and the
+// fork copy is a generated shipping copy.
+//
+// Apple M4 Max verification on 2026-05-11:
+//   * metal-verify: turbo3, turbo4, turbo3_tcq, qjl, polar, polar+QJL,
+//     polar_preht, polar_preht+QJL all 8/8 PASS.
+//   * metal-verify-multiblock: TurboQuant/QJL multi-block variants PASS.
+//   * metal-verify-fused: fused_attn_qjl_tbq and fused_attn_qjl_polar PASS.
+//
+// Runtime dispatch is flipped only for the graph ops listed in
+// METAL_RUNTIME_DISPATCH_GATES. The QJL+TBQ fused-attention kernel is wired
+// through a dedicated graph op; the QJL+Polar fused variant remains shipped
+// and standalone-verified only until it gets a separate graph contract.
 export const METAL_KERNEL_FILES = [
   "turbo3.metal",
   "turbo4.metal",
   "turbo3_tcq.metal",
   "qjl.metal",
+  "qjl_set_rows.metal",
   "polar.metal",
+  "polar_preht.metal",
+  "fused_attn_qjl_tbq.metal",
+  "fused_attn_qjl_polar.metal",
 ];
 
-const SENTINEL = "# MILADY-KERNEL-PATCH-V1";
-const SENTINEL_EMBED = "# MILADY-KERNEL-EMBED-PATCH-V1";
-const SENTINEL_EMBED_LOADER = "// MILADY-EMBEDDED-METALLIB-LOADER-V1";
-const SENTINEL_QJL_ATTN = "// MILADY-QJL-ATTN-DISPATCH-V1";
+export const METAL_RUNTIME_DISPATCH_GATES = {
+  turbo3: {
+    status: "runtime-ready",
+    runtimeReady: true,
+    graphOp: "GGML_OP_ATTN_SCORE_TBQ",
+    smokeTarget: "dispatch-smoke",
+  },
+  turbo4: {
+    status: "runtime-ready",
+    runtimeReady: true,
+    graphOp: "GGML_OP_ATTN_SCORE_TBQ",
+    smokeTarget: "dispatch-smoke",
+  },
+  turbo3_tcq: {
+    status: "runtime-ready",
+    runtimeReady: true,
+    graphOp: "GGML_OP_ATTN_SCORE_TBQ",
+    smokeTarget: "dispatch-smoke",
+  },
+  qjl_full: {
+    status: "runtime-ready",
+    runtimeReady: true,
+    graphOp: "GGML_OP_ATTN_SCORE_QJL",
+    smokeTarget: "dispatch-smoke",
+  },
+  polarquant: {
+    status: "runtime-ready",
+    runtimeReady: true,
+    graphOp: "GGML_OP_ATTN_SCORE_POLAR",
+    smokeTarget: "dispatch-smoke",
+  },
+  fused_attn_qjl_tbq: {
+    status: "runtime-ready",
+    runtimeReady: true,
+    graphOp: "GGML_OP_FUSED_ATTN_QJL_TBQ",
+    smokeTarget: "dispatch-smoke",
+  },
+};
+
+const SENTINEL = "# ELIZA-KERNEL-PATCH-V1";
+const SENTINEL_EMBED = "# ELIZA-KERNEL-EMBED-PATCH-V1";
+const SENTINEL_EMBED_LOADER = "// ELIZA-EMBEDDED-METALLIB-LOADER-V1";
+const SENTINEL_QJL_ATTN = "// ELIZA-QJL-ATTN-DISPATCH-V1";
+const SENTINEL_QJL_SET_ROWS = "// ELIZA-QJL-SET-ROWS-V1";
+const SENTINEL_TBQ_POLAR_ATTN = "// ELIZA-TBQ-POLAR-ATTN-DISPATCH-V1";
 
 function inForkRelpath(name) {
-  return path.posix.join("ggml", "src", "ggml-metal", "milady-shipped", name);
+  return path.posix.join("ggml", "src", "ggml-metal", "eliza-shipped", name);
 }
 
 // Verify all standalones exist and are non-empty before any fs writes — we
@@ -99,19 +194,23 @@ function assertStandalonesPresent() {
 }
 
 // Copy each standalone .metal into the fork at
-// ggml/src/ggml-metal/milady-shipped/<name>.metal, overwriting any prior copy
+// ggml/src/ggml-metal/eliza-shipped/<name>.metal, overwriting any prior copy
 // so the canonical source-of-truth is always the verified standalone.
 //
-// We deliberately overwrite the fork's stale ggml/src/ggml-metal/milady-kernels/
-// content if it exists, but we write into a sibling milady-shipped/ directory
-// so the patch is self-contained and the original (un-wired) milady-kernels/
+// We deliberately overwrite the fork's stale ggml/src/ggml-metal/eliza-kernels/
+// content if it exists, but we write into a sibling eliza-shipped/ directory
+// so the patch is self-contained and the original (un-wired) eliza-kernels/
 // drafts remain visible for diff-archaeology if a future agent wants them.
 function copyStandalonesIntoFork(cacheDir, { dryRun }) {
-  const targetDir = path.join(cacheDir, "ggml", "src", "ggml-metal", "milady-shipped");
+  const targetDir = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "eliza-shipped",
+  );
   if (dryRun) {
-    console.log(
-      `[metal-kernels] (dry-run) mkdir -p ${targetDir}`,
-    );
+    console.log(`[metal-kernels] (dry-run) mkdir -p ${targetDir}`);
   } else {
     fs.mkdirSync(targetDir, { recursive: true });
   }
@@ -128,7 +227,7 @@ function copyStandalonesIntoFork(cacheDir, { dryRun }) {
       // in-fork draft.
       const stamped =
         `// ${SENTINEL} — copied verbatim from packages/inference/metal/${name}\n` +
-        `// at build time by build-llama-cpp-dflash.mjs. Do not edit in place;\n` +
+        `// at build time by build-llama-cpp-mtp.mjs. Do not edit in place;\n` +
         `// edit the standalone source and rerun the build.\n` +
         text;
       fs.writeFileSync(dst, stamped, "utf8");
@@ -159,17 +258,17 @@ function patchMetalCMakeLists(cacheDir, { dryRun }) {
   let patched = original;
   let changed = false;
 
-  const miladyAirLinesForSdk = (sdkExpr) =>
+  const elizaAirLinesForSdk = (sdkExpr) =>
     METAL_KERNEL_FILES.map((name) => {
       const stem = name.replace(/\.metal$/, "");
-      return `        COMMAND xcrun -sdk ${sdkExpr} metal \${XC_FLAGS} -c \${CMAKE_CURRENT_SOURCE_DIR}/milady-shipped/${name} -o \${CMAKE_CURRENT_BINARY_DIR}/${stem}.air`;
+      return `        COMMAND xcrun -sdk ${sdkExpr} metal \${XC_FLAGS} -c \${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/${name} -o \${CMAKE_CURRENT_BINARY_DIR}/${stem}.air`;
     }).join("\n");
-  const miladyAirInputs = METAL_KERNEL_FILES.map((name) => {
+  const elizaAirInputs = METAL_KERNEL_FILES.map((name) => {
     const stem = name.replace(/\.metal$/, "");
     return `\${CMAKE_CURRENT_BINARY_DIR}/${stem}.air`;
   }).join(" ");
-  const miladyDepends = METAL_KERNEL_FILES.map(
-    (name) => `\${CMAKE_CURRENT_SOURCE_DIR}/milady-shipped/${name}`,
+  const elizaDepends = METAL_KERNEL_FILES.map(
+    (name) => `\${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/${name}`,
   ).join(" ");
 
   if (!patched.includes(SENTINEL_EMBED)) {
@@ -180,7 +279,7 @@ function patchMetalCMakeLists(cacheDir, { dryRun }) {
       embedStart === -1
         ? -1
         : patched.indexOf(
-            "\n\n    target_sources(ggml-metal PRIVATE \"${METALLIB_EMBED_ASM}\")",
+            '\n\n    target_sources(ggml-metal PRIVATE "${METALLIB_EMBED_ASM}")',
             embedStart,
           );
     if (embedStart === -1 || embedEnd === -1) {
@@ -189,11 +288,11 @@ function patchMetalCMakeLists(cacheDir, { dryRun }) {
           `the fork's GGML_METAL_EMBED_LIBRARY branch changed shape and the patch must be revisited.`,
       );
     }
-    const embedAirLines = miladyAirLinesForSdk("${METAL_SDK}");
+    const embedAirLines = elizaAirLinesForSdk("${METAL_SDK}");
     const embedReplacement = `    # ${SENTINEL_EMBED}
     # Build a compiled default.metallib for embedded-library targets (iOS).
     # The upstream path embedded concatenated Metal source and JIT-compiled it
-    # at runtime. That cannot include the milady standalones because the source
+    # at runtime. That cannot include the eliza standalones because the source
     # TUs intentionally redeclare block_* structs/constants that already exist
     # in ggml-common.h. Compile each TU separately, merge into one metallib,
     # and embed the binary metallib bytes instead.
@@ -217,31 +316,29 @@ function patchMetalCMakeLists(cacheDir, { dryRun }) {
 
     add_custom_command(
         OUTPUT "\${METALLIB_EMBED_ASM}"
-        COMMAND echo "Embedding Metal library (compiled metallib + milady-shipped kernels)"
+        COMMAND echo "Embedding Metal library (compiled metallib + eliza-shipped kernels)"
         COMMAND sed -e "/__embed_ggml-common.h__/r \${METALLIB_COMMON}"       -e "/__embed_ggml-common.h__/d"         < "\${METALLIB_SOURCE}"           > "\${METALLIB_SOURCE_EMBED_TMP}"
         COMMAND sed -e "/\\#include \\"ggml-metal-impl.h\\"/r \${METALLIB_IMPL}" -e "/\\#include \\"ggml-metal-impl.h\\"/d" < "\${METALLIB_SOURCE_EMBED_TMP}" > "\${METALLIB_SOURCE_EMBED}"
         COMMAND xcrun -sdk \${METAL_SDK} metal \${XC_FLAGS} -DGGML_METAL_EMBED_LIBRARY=1 -c "\${METALLIB_SOURCE_EMBED}" -o "\${METALLIB_EMBED_AIR}"
 ${embedAirLines}
-        COMMAND xcrun -sdk \${METAL_SDK} metallib "\${METALLIB_EMBED_AIR}" ${miladyAirInputs} -o "\${METALLIB_EMBED_BINARY}"
+        COMMAND xcrun -sdk \${METAL_SDK} metallib "\${METALLIB_EMBED_AIR}" ${elizaAirInputs} -o "\${METALLIB_EMBED_BINARY}"
         COMMAND echo ".section __DATA,__ggml_metallib"          >  "\${METALLIB_EMBED_ASM}"
         COMMAND echo ".globl _ggml_metallib_start"              >> "\${METALLIB_EMBED_ASM}"
         COMMAND echo "_ggml_metallib_start:"                    >> "\${METALLIB_EMBED_ASM}"
         COMMAND echo .incbin "\\"\${METALLIB_EMBED_BINARY}\\""    >> "\${METALLIB_EMBED_ASM}"
         COMMAND echo ".globl _ggml_metallib_end"                >> "\${METALLIB_EMBED_ASM}"
         COMMAND echo "_ggml_metallib_end:"                      >> "\${METALLIB_EMBED_ASM}"
-        DEPENDS ../ggml-common.h ggml-metal.metal ggml-metal-impl.h ${miladyDepends}
+        DEPENDS ../ggml-common.h ggml-metal.metal ggml-metal-impl.h ${elizaDepends}
         COMMENT "Generate assembly for embedded compiled Metal library"
         VERBATIM
     )`;
     patched =
-      patched.slice(0, embedStart) +
-      embedReplacement +
-      patched.slice(embedEnd);
+      patched.slice(0, embedStart) + embedReplacement + patched.slice(embedEnd);
     changed = true;
   }
 
   // The exact block we replace. This pipe pattern has been stable in the
-  // milady-ai/llama.cpp fork for the entire v0.4.x line; if the upstream
+  // elizaOS/llama.cpp fork for the entire v0.4.x line; if the upstream
   // ever rewrites it we want to fail loudly rather than silently no-op.
   if (!patched.includes(SENTINEL)) {
     const anchor = `    add_custom_command(
@@ -261,9 +358,9 @@ ${embedAirLines}
       );
     }
 
-    const miladyAirLines = miladyAirLinesForSdk("macosx");
+    const elizaAirLines = elizaAirLinesForSdk("macosx");
     const replacement = `    # ${SENTINEL}
-    # Build ggml-metal.metal AND each milady standalone shader into its own
+    # Build ggml-metal.metal AND each eliza standalone shader into its own
     # .air file, then merge all .air files into a single default.metallib.
     # The standalones are self-contained TUs (only #include <metal_stdlib>;
     # define their own block_*, constants, kernel functions) so they do not
@@ -271,17 +368,46 @@ ${embedAirLines}
     add_custom_command(
         OUTPUT \${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
         COMMAND xcrun -sdk macosx metal \${XC_FLAGS} -c \${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o \${CMAKE_CURRENT_BINARY_DIR}/ggml-metal.air
-${miladyAirLines}
-        COMMAND xcrun -sdk macosx metallib \${CMAKE_CURRENT_BINARY_DIR}/ggml-metal.air ${miladyAirInputs} -o \${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
+${elizaAirLines}
+        COMMAND xcrun -sdk macosx metallib \${CMAKE_CURRENT_BINARY_DIR}/ggml-metal.air ${elizaAirInputs} -o \${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
         COMMAND rm -f \${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
         COMMAND rm -f \${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal
-        DEPENDS ggml-metal.metal \${METALLIB_COMMON} ${miladyDepends}
-        COMMENT "Compiling Metal kernels (ggml-metal + milady-shipped: ${METAL_KERNEL_FILES.join(", ")})"
+        DEPENDS ggml-metal.metal \${METALLIB_COMMON} ${elizaDepends}
+        COMMENT "Compiling Metal kernels (ggml-metal + eliza-shipped: ${METAL_KERNEL_FILES.join(", ")})"
         )`;
     patched = patched.replace(anchor, replacement);
     changed = true;
   }
 
+  if (
+    patched.includes(SENTINEL) &&
+    patched.includes(SENTINEL_EMBED) &&
+    !patched.includes("qjl_set_rows.metal")
+  ) {
+    patched = patched
+      .replaceAll(
+        "COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS} -c ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl.metal -o ${CMAKE_CURRENT_BINARY_DIR}/qjl.air",
+        "COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS} -c ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl.metal -o ${CMAKE_CURRENT_BINARY_DIR}/qjl.air\n        COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS} -c ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl_set_rows.metal -o ${CMAKE_CURRENT_BINARY_DIR}/qjl_set_rows.air",
+      )
+      .replaceAll(
+        "COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl.metal -o ${CMAKE_CURRENT_BINARY_DIR}/qjl.air",
+        "COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl.metal -o ${CMAKE_CURRENT_BINARY_DIR}/qjl.air\n        COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl_set_rows.metal -o ${CMAKE_CURRENT_BINARY_DIR}/qjl_set_rows.air",
+      )
+      .replaceAll(
+        "${CMAKE_CURRENT_BINARY_DIR}/qjl.air ${CMAKE_CURRENT_BINARY_DIR}/polar.air",
+        "${CMAKE_CURRENT_BINARY_DIR}/qjl.air ${CMAKE_CURRENT_BINARY_DIR}/qjl_set_rows.air ${CMAKE_CURRENT_BINARY_DIR}/polar.air",
+      )
+      .replaceAll(
+        "${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl.metal ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/polar.metal",
+        "${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl.metal ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/qjl_set_rows.metal ${CMAKE_CURRENT_SOURCE_DIR}/eliza-shipped/polar.metal",
+      )
+      .replaceAll(
+        "qjl.metal, polar.metal",
+        "qjl.metal, qjl_set_rows.metal, polar.metal",
+      );
+    changed = true;
+  }
+
   if (patched === original) {
     return { changed: false, path: cmakePath };
   }
@@ -336,7 +462,7 @@ function patchEmbeddedMetallibLoader(cacheDir, { dryRun }) {
         // The build patch embeds compiled default.metallib bytes here, not
         // Metal source. Loading with newLibraryWithData keeps iOS on the same
         // multi-TU kernel set as desktop and avoids duplicate declarations
-        // between ggml-metal.metal and the milady standalone shaders.
+        // between ggml-metal.metal and the eliza standalone shaders.
         const NSUInteger metallib_len = (NSUInteger)(ggml_metallib_end - ggml_metallib_start);
         dispatch_data_t metallib_data = dispatch_data_create(ggml_metallib_start, metallib_len, nil, DISPATCH_DATA_DESTRUCTOR_DEFAULT);
         library = [device newLibraryWithData:metallib_data error:&error];
@@ -347,16 +473,24 @@ function patchEmbeddedMetallibLoader(cacheDir, { dryRun }) {
 #else`;
   const patched = original.replace(anchor, replacement);
   if (patched === original) {
-    throw new Error("[metal-kernels] embedded loader replace produced no change");
+    throw new Error(
+      "[metal-kernels] embedded loader replace produced no change",
+    );
   }
   if (!dryRun) fs.writeFileSync(deviceMPath, patched, "utf8");
   return { changed: !dryRun, path: deviceMPath };
 }
 
-const SENTINEL_DISPATCH = "// MILADY-DISPATCH-V1";
+const SENTINEL_DISPATCH = "// ELIZA-DISPATCH-V1";
 
 function patchMetalQjlAttnHeader(cacheDir, { dryRun }) {
-  const headerPath = path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.h");
+  const headerPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-device.h",
+  );
   const original = fs.readFileSync(headerPath, "utf8");
   if (original.includes(SENTINEL_QJL_ATTN)) {
     return { changed: false, path: headerPath };
@@ -386,10 +520,22 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_scor
 }
 
 function patchMetalQjlAttnDeviceCpp(cacheDir, { dryRun }) {
-  const cppPath = path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.cpp");
+  const cppPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-device.cpp",
+  );
   const original = fs.readFileSync(cppPath, "utf8");
   if (original.includes(SENTINEL_QJL_ATTN)) {
-    return { changed: false, path: cppPath };
+    const upgraded = original.replace(
+      'const char * name = "kernel_attn_score_qjl1_256";',
+      'const char * name = "kernel_attn_score_qjl1_256_multi";',
+    );
+    if (upgraded !== original && !dryRun)
+      fs.writeFileSync(cppPath, upgraded, "utf8");
+    return { changed: upgraded !== original && !dryRun, path: cppPath };
   }
   const anchor = `ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin(ggml_metal_library_t lib, const ggml_tensor * op, int32_t n_fuse) {`;
   if (!original.includes(anchor)) {
@@ -399,7 +545,7 @@ function patchMetalQjlAttnDeviceCpp(cacheDir, { dryRun }) {
   }
   const helper = `${SENTINEL_QJL_ATTN}
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_qjl(ggml_metal_library_t lib) {
-    const char * name = "kernel_attn_score_qjl1_256";
+    const char * name = "kernel_attn_score_qjl1_256_multi";
     ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
     if (!res.pipeline) {
         // Standalone shipped shader: it declares no Metal function constants,
@@ -424,7 +570,13 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_qjl(g
 }
 
 function patchMetalQjlAttnOpsHeader(cacheDir, { dryRun }) {
-  const headerPath = path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-ops.h");
+  const headerPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-ops.h",
+  );
   const original = fs.readFileSync(headerPath, "utf8");
   if (original.includes(SENTINEL_QJL_ATTN)) {
     return { changed: false, path: headerPath };
@@ -444,10 +596,98 @@ int ggml_metal_op_attn_score_qjl  (ggml_metal_op_t ctx, int idx);`;
 }
 
 function patchMetalQjlAttnOpsCpp(cacheDir, { dryRun }) {
-  const opsPath = path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-ops.cpp");
+  const opsPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-ops.cpp",
+  );
   const original = fs.readFileSync(opsPath, "utf8");
   if (original.includes(SENTINEL_QJL_ATTN)) {
-    return { changed: false, path: opsPath };
+    let upgraded = original;
+    if (!upgraded.includes("#include <cstdlib>")) {
+      upgraded = upgraded.replace(
+        `#include <cmath>
+`,
+        `#include <cmath>
+#include <cstdlib>
+`,
+      );
+    }
+    upgraded = upgraded.replace(
+      `struct eliza_qjl_score_args {
+    uint32_t n_heads;
+    uint32_t n_kv_heads;
+    uint32_t n_tokens;
+    uint32_t proj_dim;
+};`,
+      `struct eliza_qjl_score_args {
+    uint32_t n_heads;
+    uint32_t n_kv_heads;
+    uint32_t n_tokens;
+    uint32_t proj_dim;
+    uint32_t tokens_per_threadgroup;
+};`,
+    );
+    if (!upgraded.includes("static inline uint32_t eliza_env_u32")) {
+      upgraded = upgraded.replace(
+        `static inline ggml_metal_buffer_id eliza_metal_buffer_offset(ggml_metal_buffer_id id, size_t extra) {
+    id.offs += extra;
+    return id;
+}
+`,
+        `static inline ggml_metal_buffer_id eliza_metal_buffer_offset(ggml_metal_buffer_id id, size_t extra) {
+    id.offs += extra;
+    return id;
+}
+
+static inline uint32_t eliza_env_u32(const char * name, uint32_t fallback, uint32_t min_value, uint32_t max_value) {
+    const char * raw = std::getenv(name);
+    if (raw == nullptr || raw[0] == '\\0') {
+        return fallback;
+    }
+    char * end = nullptr;
+    const unsigned long parsed = std::strtoul(raw, &end, 10);
+    if (end == raw || *end != '\\0' || parsed < min_value || parsed > max_value) {
+        GGML_LOG_WARN("%s: ignoring invalid %s=%s (expected %u..%u)\\n",
+                      __func__, name, raw, min_value, max_value);
+        return fallback;
+    }
+    return (uint32_t) parsed;
+}
+`,
+      );
+    }
+    upgraded = upgraded.replace(
+      `        /* n_tokens   = */ n_tokens,
+        /* proj_dim   = */ 256u,
+    };`,
+      `        /* n_tokens   = */ n_tokens,
+        /* proj_dim   = */ 256u,
+        // M4 Max 2026-05-11 sweeps show N=4/8/16/32 trade median vs p99.
+        // Keep N=32 as the tail-latency-biased default until per-device
+        // autotuning can persist a device-specific table.
+        /* tokens_per_threadgroup = */ eliza_env_u32("ELIZA_METAL_QJL_TOKENS_PER_TG", 64u, 1u, 64u),
+    };`,
+    );
+    upgraded = upgraded.replace(
+      `            ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_heads, (int) n_tokens, 1, 32, 1, 1);`,
+      `            const int token_groups = (int) ((n_tokens + args.tokens_per_threadgroup - 1u) / args.tokens_per_threadgroup);
+            ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_heads, token_groups, 1, 32, 1, 1);`,
+    );
+    upgraded = upgraded.replace(
+      `    GGML_ASSERT(q->ne[0]  == 256);
+    GGML_ASSERT(pk->ne[0] == 128);`,
+      `    GGML_ASSERT(q->ne[0]  == 256);
+    GGML_ASSERT(pk->ne[0] == 128);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(op));`,
+    );
+    if (upgraded !== original && !dryRun)
+      fs.writeFileSync(opsPath, upgraded, "utf8");
+    return { changed: upgraded !== original && !dryRun, path: opsPath };
   }
 
   const funcAnchor = `static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {`;
@@ -457,18 +697,34 @@ function patchMetalQjlAttnOpsCpp(cacheDir, { dryRun }) {
     );
   }
   const opFunc = `${SENTINEL_QJL_ATTN}
-struct milady_qjl_score_args {
+struct eliza_qjl_score_args {
     uint32_t n_heads;
     uint32_t n_kv_heads;
     uint32_t n_tokens;
     uint32_t proj_dim;
+    uint32_t tokens_per_threadgroup;
 };
 
-static inline ggml_metal_buffer_id milady_metal_buffer_offset(ggml_metal_buffer_id id, size_t extra) {
+static inline ggml_metal_buffer_id eliza_metal_buffer_offset(ggml_metal_buffer_id id, size_t extra) {
     id.offs += extra;
     return id;
 }
 
+static inline uint32_t eliza_env_u32(const char * name, uint32_t fallback, uint32_t min_value, uint32_t max_value) {
+    const char * raw = std::getenv(name);
+    if (raw == nullptr || raw[0] == '\\0') {
+        return fallback;
+    }
+    char * end = nullptr;
+    const unsigned long parsed = std::strtoul(raw, &end, 10);
+    if (end == raw || *end != '\\0' || parsed < min_value || parsed > max_value) {
+        GGML_LOG_WARN("%s: ignoring invalid %s=%s (expected %u..%u)\\n",
+                      __func__, name, raw, min_value, max_value);
+        return fallback;
+    }
+    return (uint32_t) parsed;
+}
+
 int ggml_metal_op_attn_score_qjl(ggml_metal_op_t ctx, int idx) {
     ggml_tensor * op = ctx->node(idx);
 
@@ -485,6 +741,9 @@ int ggml_metal_op_attn_score_qjl(ggml_metal_op_t ctx, int idx) {
     GGML_ASSERT(op->type == GGML_TYPE_F32);
     GGML_ASSERT(q->ne[0]  == 256);
     GGML_ASSERT(pk->ne[0] == 128);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(op));
 
     const uint32_t n_heads     = (uint32_t) q->ne[1];
     const uint32_t n_kv_heads  = (uint32_t) ((const int32_t *) op->op_params)[0];
@@ -503,11 +762,15 @@ int ggml_metal_op_attn_score_qjl(ggml_metal_op_t ctx, int idx) {
     GGML_ASSERT(pk->nb[1] == ggml_row_size(GGML_TYPE_QJL1_256, 128));
     GGML_ASSERT(pk->nb[2] == (size_t) n_tokens * pk->nb[1]);
 
-    milady_qjl_score_args args = {
+    eliza_qjl_score_args args = {
         /* n_heads    = */ n_heads,
         /* n_kv_heads = */ n_kv_heads,
         /* n_tokens   = */ n_tokens,
         /* proj_dim   = */ 256u,
+        // M4 Max 2026-05-11 sweeps show N=4/8/16/32 trade median vs p99.
+        // Keep N=32 as the tail-latency-biased default until per-device
+        // autotuning can persist a device-specific table.
+        /* tokens_per_threadgroup = */ eliza_env_u32("ELIZA_METAL_QJL_TOKENS_PER_TG", 64u, 1u, 64u),
     };
 
     auto pipeline = ggml_metal_library_get_pipeline_attn_score_qjl(lib);
@@ -524,18 +787,132 @@ int ggml_metal_op_attn_score_qjl(ggml_metal_op_t ctx, int idx) {
         const size_t pk_i3 = (size_t) i3 * pk->nb[3];
         const size_t dst_i3 = (size_t) i3 * op->nb[3];
         for (int64_t ib = 0; ib < n_batch; ++ib) {
-            ggml_metal_encoder_set_buffer(enc, milady_metal_buffer_offset(q_base,  q_i3  + (size_t) ib * q->nb[2]),  0);
-            ggml_metal_encoder_set_buffer(enc, milady_metal_buffer_offset(pk_base, pk_i3),                          1);
-            ggml_metal_encoder_set_buffer(enc, milady_metal_buffer_offset(dst_base, dst_i3 + (size_t) ib * op->nb[2]), 2);
-            ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_heads, (int) n_tokens, 1, 32, 1, 1);
+            ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(q_base,  q_i3  + (size_t) ib * q->nb[2]),  0);
+            ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pk_base, pk_i3),                          1);
+            ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(dst_base, dst_i3 + (size_t) ib * op->nb[2]), 2);
+            const int token_groups = (int) ((n_tokens + args.tokens_per_threadgroup - 1u) / args.tokens_per_threadgroup);
+            ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_heads, token_groups, 1, 32, 1, 1);
         }
     }
 
     return 1;
 }
 
+struct eliza_fused_attn_qjl_tbq_args {
+    uint32_t head_dim;
+    uint32_t proj_dim;
+    uint32_t n_heads;
+    uint32_t n_kv_heads;
+    uint32_t n_q_pos;
+    uint32_t n_kv;
+    uint32_t kv_tile;
+    uint32_t v_use_qjl;
+    float    scale;
+    uint32_t causal;
+    uint32_t q_pos_base;
+};
+
+int ggml_metal_op_fused_attn_qjl_tbq(ggml_metal_op_t ctx, int idx) {
+    ggml_tensor * op = ctx->node(idx);
+
+    ggml_metal_library_t lib = ctx->lib;
+    ggml_metal_encoder_t enc = ctx->enc;
+
+    const ggml_tensor * q  = op->src[0];
+    const ggml_tensor * pk = op->src[1];
+    const ggml_tensor * pv = op->src[2];
+
+    GGML_ASSERT(q  != nullptr);
+    GGML_ASSERT(pk != nullptr);
+    GGML_ASSERT(pv != nullptr);
+    GGML_ASSERT(q->type  == GGML_TYPE_F32);
+    GGML_ASSERT(pk->type == GGML_TYPE_QJL1_256);
+    GGML_ASSERT(pv->type == GGML_TYPE_TBQ3_0);
+    GGML_ASSERT(op->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0]  == 256);
+    GGML_ASSERT(pk->ne[0] == 128);
+    GGML_ASSERT(pv->ne[0] == 128);
+    GGML_ASSERT(op->ne[0] == 128);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(pv));
+    GGML_ASSERT(ggml_is_contiguous_rows(op));
+
+    const int32_t * params = (const int32_t *) op->op_params;
+    const uint32_t n_kv_heads = (uint32_t) params[0];
+    union { int32_t i; float f; } scale_bits;
+    scale_bits.i = params[1];
+
+    const uint32_t n_heads = (uint32_t) q->ne[1];
+    const uint32_t n_q_pos = (uint32_t) q->ne[2];
+    const uint32_t n_kv    = (uint32_t) pk->ne[1];
+    const int64_t  ne3     = q->ne[3];
+
+    GGML_ASSERT(n_kv_heads > 0);
+    GGML_ASSERT((n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(pv->ne[1] == (int64_t) n_kv);
+    GGML_ASSERT(pv->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(pk->ne[3] == ne3);
+    GGML_ASSERT(pv->ne[3] == ne3);
+    GGML_ASSERT(op->ne[1] == (int64_t) n_heads);
+    GGML_ASSERT(op->ne[2] == (int64_t) n_q_pos);
+    GGML_ASSERT(op->ne[3] == ne3);
+    GGML_ASSERT(q->nb[1] == (size_t) q->ne[0] * ggml_type_size(q->type));
+    GGML_ASSERT(q->nb[2] == (size_t) n_heads * q->nb[1]);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(GGML_TYPE_QJL1_256, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_kv * pk->nb[1]);
+    GGML_ASSERT(pv->nb[1] == ggml_row_size(GGML_TYPE_TBQ3_0, 128));
+    GGML_ASSERT(pv->nb[2] == (size_t) n_kv * pv->nb[1]);
+    GGML_ASSERT(op->nb[1] == (size_t) op->ne[0] * ggml_type_size(op->type));
+    GGML_ASSERT(op->nb[2] == (size_t) n_heads * op->nb[1]);
+
+    eliza_fused_attn_qjl_tbq_args args = {
+        /* head_dim   = */ 128u,
+        /* proj_dim   = */ 256u,
+        /* n_heads    = */ n_heads,
+        /* n_kv_heads = */ n_kv_heads,
+        /* n_q_pos    = */ n_q_pos,
+        /* n_kv       = */ n_kv,
+        /* kv_tile    = */ (uint32_t) params[3],
+        /* v_use_qjl  = */ (uint32_t) params[2],
+        /* scale      = */ scale_bits.f,
+        /* causal     = */ (uint32_t) params[4],
+        /* q_pos_base = */ (uint32_t) params[5],
+    };
+
+    auto pipeline = ggml_metal_library_get_pipeline_fused_attn_qjl_tbq(lib);
+
+    const ggml_metal_buffer_id q_base   = ggml_metal_get_buffer_id(q);
+    const ggml_metal_buffer_id pk_base  = ggml_metal_get_buffer_id(pk);
+    const ggml_metal_buffer_id pv_base  = ggml_metal_get_buffer_id(pv);
+    const ggml_metal_buffer_id dst_base = ggml_metal_get_buffer_id(op);
+
+    ggml_metal_encoder_set_pipeline(enc, pipeline);
+    ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 4);
+
+    for (int64_t i3 = 0; i3 < ne3; ++i3) {
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(q_base,   (size_t) i3 * q->nb[3]),  0);
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pk_base,  (size_t) i3 * pk->nb[3]), 1);
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pv_base,  (size_t) i3 * pv->nb[3]), 2);
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(dst_base, (size_t) i3 * op->nb[3]), 3);
+        ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_heads, (int) n_q_pos, 1, 32, 1, 1);
+    }
+
+    return 1;
+}
+
 `;
   let patched = original.replace(funcAnchor, opFunc + funcAnchor);
+  if (!patched.includes("#include <cstdlib>")) {
+    patched = patched.replace(
+      `#include <cmath>
+`,
+      `#include <cmath>
+#include <cstdlib>
+`,
+    );
+  }
 
   const switchAnchor = `        case GGML_OP_FLASH_ATTN_EXT:
             {
@@ -557,10 +934,27 @@ int ggml_metal_op_attn_score_qjl(ggml_metal_op_t ctx, int idx) {
 }
 
 function patchMetalQjlAttnSupportsOp(cacheDir, { dryRun }) {
-  const deviceMPath = path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.m");
+  const deviceMPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-device.m",
+  );
   const original = fs.readFileSync(deviceMPath, "utf8");
   if (original.includes(SENTINEL_QJL_ATTN)) {
-    return { changed: false, path: deviceMPath };
+    const upgraded = original.replace(
+      `                op->src[0]->ne[0] == 256 &&
+                op->src[1]->ne[0] == 128;`,
+      `                op->src[0]->ne[0] == 256 &&
+                op->src[1]->ne[0] == 128 &&
+                ggml_is_contiguous_rows(op) &&
+                ggml_is_contiguous_rows(op->src[0]) &&
+                ggml_is_contiguous_rows(op->src[1]);`,
+    );
+    if (upgraded !== original && !dryRun)
+      fs.writeFileSync(deviceMPath, upgraded, "utf8");
+    return { changed: upgraded !== original && !dryRun, path: deviceMPath };
   }
   const anchor = `        case GGML_OP_FLASH_ATTN_EXT:
             // for new head sizes, add checks here`;
@@ -578,7 +972,10 @@ function patchMetalQjlAttnSupportsOp(cacheDir, { dryRun }) {
                 op->src[0]->type == GGML_TYPE_F32 &&
                 op->src[1]->type == GGML_TYPE_QJL1_256 &&
                 op->src[0]->ne[0] == 256 &&
-                op->src[1]->ne[0] == 128;
+                op->src[1]->ne[0] == 128 &&
+                ggml_is_contiguous_rows(op) &&
+                ggml_is_contiguous_rows(op->src[0]) &&
+                ggml_is_contiguous_rows(op->src[1]);
 ${anchor}`;
   const patched = original.replace(anchor, insert);
   if (!dryRun) fs.writeFileSync(deviceMPath, patched, "utf8");
@@ -594,45 +991,1237 @@ function patchMetalQjlAttnDispatch(cacheDir, { dryRun }) {
   return { header, deviceCpp, opsHeader, opsCpp, supportsOp };
 }
 
-export function patchMetalDispatch(cacheDir, { dryRun = false } = {}) {
-  const patchedFiles = [
-    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.h"),
-    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.cpp"),
-    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-ops.cpp"),
-  ].filter((file) => {
-    try {
-      return fs.readFileSync(file, "utf8").includes(SENTINEL_DISPATCH);
-    } catch {
-      return false;
+function readTcqCodebookLiteral() {
+  const referencePath = path.join(STANDALONE_REFERENCE_DIR, "turbo_kernels.c");
+  const source = fs.readFileSync(referencePath, "utf8");
+  const match = source.match(
+    /const float ELIZA_TURBO3_TCQ_CODEBOOK\[512\]\s*=\s*\{([\s\S]*?)\};/,
+  );
+  if (!match) {
+    throw new Error(
+      `[metal-tbq-polar-attn] could not extract TCQ codebook from ${referencePath}`,
+    );
+  }
+  return match[1].trim();
+}
+
+export function patchGgmlTbqPolarAttnOps(cacheDir, { dryRun }) {
+  const headerPath = path.join(cacheDir, "ggml", "include", "ggml.h");
+  const cPath = path.join(cacheDir, "ggml", "src", "ggml.c");
+  let changed = false;
+
+  const headerOriginal = fs.readFileSync(headerPath, "utf8");
+  let header = headerOriginal;
+  if (!header.includes(SENTINEL_TBQ_POLAR_ATTN)) {
+    const enumAnchor = `        GGML_OP_ATTN_SCORE_QJL, // QJL 1-bit packed-K attention score (CPU-only)
+        GGML_OP_FUSED_ATTN_QJL_TBQ, // fused QJL-K + TBQ-V attention (CPU-only)`;
+    if (!header.includes(enumAnchor)) {
+      throw new Error(
+        `[metal-tbq-polar-attn] ggml.h op enum anchor not found at ${headerPath}`,
+      );
     }
-  });
+    header = header.replace(
+      enumAnchor,
+      `        GGML_OP_ATTN_SCORE_QJL, // QJL 1-bit packed-K attention score
+        GGML_OP_ATTN_SCORE_TBQ, // ${SENTINEL_TBQ_POLAR_ATTN} TurboQuant packed-K attention score
+        GGML_OP_ATTN_SCORE_POLAR, // PolarQuant packed-K attention score
+        GGML_OP_FUSED_ATTN_QJL_TBQ, // fused QJL-K + TBQ-V attention (CPU-only)`,
+    );
+    const declAnchor = `    GGML_API struct ggml_tensor * ggml_attn_score_qjl(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * q,
+            struct ggml_tensor  * packed_k,
+            int                   n_kv_heads);`;
+    if (!header.includes(declAnchor)) {
+      throw new Error(
+        `[metal-tbq-polar-attn] ggml.h QJL declaration anchor not found at ${headerPath}`,
+      );
+    }
+    header = header.replace(
+      declAnchor,
+      `${declAnchor}
 
-  const message =
-    "[metal-dispatch] NOT wiring generic Metal GGML dispatch for milady " +
-    "QJL/Polar/TBQ kernels. The standalone kernels use bespoke attention/" +
-    "projection contracts that do not match generic MUL_MAT/GET_ROWS. " +
-    "Build output is symbol-shipped only until dedicated ATTN_SCORE and " +
-    "bundle-aware graph ops land.";
-  if (patchedFiles.length > 0) {
-    const detail =
-      `${message} Found an older unsafe MILADY-DISPATCH-V1 patch in:\n` +
-      `  ${patchedFiles.join("\n  ")}\n` +
-      "Use a clean milady-llama-cpp checkout/cache before producing artifacts.";
-    if (!dryRun) {
-      throw new Error(detail);
+    // ${SENTINEL_TBQ_POLAR_ATTN}
+    // TurboQuant packed-K attention score.
+    // q: F32 [128, n_heads, n_batch, ne3]
+    // packed_k: TBQ3_0/TBQ4_0/TBQ3_TCQ [128, n_kv_tokens, n_kv_heads, ne3]
+    // output: F32 [n_kv_tokens, n_heads, n_batch, ne3]
+    GGML_API struct ggml_tensor * ggml_attn_score_tbq(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * q,
+            struct ggml_tensor  * packed_k,
+            int                   n_kv_heads);
+
+    // PolarQuant packed-K attention score.
+    // q: F32 [128, n_heads, n_batch, ne3]
+    // packed_k: Q4_POLAR [128, n_kv_tokens, n_kv_heads, ne3]
+    // use_qjl mirrors the PolarQuant GGUF residual flag.
+    // output: F32 [n_kv_tokens, n_heads, n_batch, ne3]
+    GGML_API struct ggml_tensor * ggml_attn_score_polar(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * q,
+            struct ggml_tensor  * packed_k,
+            int                   n_kv_heads,
+            bool                  use_qjl);
+
+    // PolarQuant packed-K attention score with pre-Hadamarded query.
+    // q_preht MUST contain H*q for each query head, where H is the same
+    // unnormalised 128-point Walsh-Hadamard transform used by the PolarQuant
+    // decoder. This is faster than ggml_attn_score_polar() because the backend
+    // can use dot(H*x, q) == dot(x, H*q) and avoid one Hadamard per K row.
+    GGML_API struct ggml_tensor * ggml_attn_score_polar_preht(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * q_preht,
+            struct ggml_tensor  * packed_k,
+            int                   n_kv_heads,
+            bool                  use_qjl);`,
+    );
+    changed = true;
+  }
+
+  if (header !== headerOriginal && !dryRun)
+    fs.writeFileSync(headerPath, header, "utf8");
+
+  const cOriginal = fs.readFileSync(cPath, "utf8");
+  let c = cOriginal;
+  c = c.replace(
+    `    int32_t params[2];
+    params[0] = n_kv_heads;
+    union { float f; int32_t i; } scale_bits;
+    scale_bits.f = sm_scale;
+    params[1] = scale_bits.i;
+    ggml_set_op_params(result, params, sizeof(params));`,
+    `    int32_t params[6] = { 0 };
+    params[0] = n_kv_heads;
+    union { float f; int32_t i; } scale_bits;
+    scale_bits.f = sm_scale;
+    params[1] = scale_bits.i;
+    // Reserved for backend fused dispatch ABI: [2] v_use_qjl, [3] kv_tile,
+    // [4] causal, [5] q_pos_base. The public constructor preserves the
+    // existing non-causal CPU semantics by default.
+    params[5] = n_kv_tokens >= q->ne[2] ? (int32_t) (n_kv_tokens - q->ne[2]) : 0;
+    ggml_set_op_params(result, params, sizeof(params));`,
+  );
+  if (!c.includes(SENTINEL_TBQ_POLAR_ATTN)) {
+    c = c.replace(
+      `    "ATTN_SCORE_QJL",
+    "FUSED_ATTN_QJL_TBQ",`,
+      `    "ATTN_SCORE_QJL",
+    "ATTN_SCORE_TBQ",
+    "ATTN_SCORE_POLAR",
+    "FUSED_ATTN_QJL_TBQ",`,
+    );
+    c = c.replace(
+      `    "attn_score_qjl(q, packed_k)",
+    "fused_attn_qjl_tbq(q, packed_k, packed_v)",`,
+      `    "attn_score_qjl(q, packed_k)",
+    "attn_score_tbq(q, packed_k)",
+    "attn_score_polar(q, packed_k)",
+    "fused_attn_qjl_tbq(q, packed_k, packed_v)",`,
+    );
+    if (!c.includes(`    [GGML_TYPE_TBQ3_TCQ] = {`)) {
+      c = c.replace(
+        `    [GGML_TYPE_QJL1_256] = {`,
+        `    [GGML_TYPE_TBQ3_TCQ] = {
+        .type_name                = "tbq3_tcq",
+        .blck_size                = QK_TBQ3_TCQ,
+        .type_size                = sizeof(block_tbq3_tcq),
+        .is_quantized             = true,
+    },
+    [GGML_TYPE_QJL1_256] = {`,
+      );
     }
-    console.warn(detail);
-  } else {
-    console.log(`${dryRun ? "(dry-run) " : ""}${message}`);
+    c = c.replaceAll(
+      `static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");`,
+      `static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99");`,
+    );
+    const implAnchor = `// ggml_fused_attn_qjl_tbq
+//`;
+    if (!c.includes(implAnchor)) {
+      throw new Error(
+        `[metal-tbq-polar-attn] ggml.c implementation anchor not found at ${cPath}`,
+      );
+    }
+    const impl = `// ${SENTINEL_TBQ_POLAR_ATTN}
+// ggml_attn_score_tbq
+//
+struct ggml_tensor * ggml_attn_score_tbq(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * q,
+        struct ggml_tensor  * packed_k,
+        int                   n_kv_heads) {
+    GGML_ASSERT(q != NULL);
+    GGML_ASSERT(packed_k != NULL);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(packed_k->type == GGML_TYPE_TBQ3_0 ||
+                packed_k->type == GGML_TYPE_TBQ4_0 ||
+                packed_k->type == GGML_TYPE_TBQ3_TCQ);
+    GGML_ASSERT(q->ne[0] == 128);
+    GGML_ASSERT(packed_k->ne[0] == 128);
+
+    const int64_t n_heads     = q->ne[1];
+    const int64_t n_kv_tokens = packed_k->ne[1];
+
+    GGML_ASSERT(n_kv_heads > 0);
+    GGML_ASSERT((n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(packed_k->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(packed_k->ne[3] == q->ne[3]);
+
+    const int64_t ne[4] = { n_kv_tokens, n_heads, q->ne[2], q->ne[3] };
+    struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+    int32_t params[1] = { n_kv_heads };
+    ggml_set_op_params(result, params, sizeof(params));
+
+    result->op     = GGML_OP_ATTN_SCORE_TBQ;
+    result->src[0] = q;
+    result->src[1] = packed_k;
+
+    return result;
+}
+
+// ggml_attn_score_polar_impl
+//
+static struct ggml_tensor * ggml_attn_score_polar_impl(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * q,
+        struct ggml_tensor  * packed_k,
+        int                   n_kv_heads,
+        bool                  use_qjl,
+        bool                  q_preht) {
+    GGML_ASSERT(q != NULL);
+    GGML_ASSERT(packed_k != NULL);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(packed_k->type == GGML_TYPE_Q4_POLAR);
+    GGML_ASSERT(q->ne[0] == 128);
+    GGML_ASSERT(packed_k->ne[0] == 128);
+
+    const int64_t n_heads     = q->ne[1];
+    const int64_t n_kv_tokens = packed_k->ne[1];
+
+    GGML_ASSERT(n_kv_heads > 0);
+    GGML_ASSERT((n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(packed_k->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(packed_k->ne[3] == q->ne[3]);
+
+    const int64_t ne[4] = { n_kv_tokens, n_heads, q->ne[2], q->ne[3] };
+    struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+    int32_t params[3] = { n_kv_heads, use_qjl ? 1 : 0, q_preht ? 1 : 0 };
+    ggml_set_op_params(result, params, sizeof(params));
+
+    result->op     = GGML_OP_ATTN_SCORE_POLAR;
+    result->src[0] = q;
+    result->src[1] = packed_k;
+
+    return result;
+}
+
+// ggml_attn_score_polar
+//
+struct ggml_tensor * ggml_attn_score_polar(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * q,
+        struct ggml_tensor  * packed_k,
+        int                   n_kv_heads,
+        bool                  use_qjl) {
+    return ggml_attn_score_polar_impl(ctx, q, packed_k, n_kv_heads, use_qjl, false);
+}
+
+// ggml_attn_score_polar_preht
+//
+struct ggml_tensor * ggml_attn_score_polar_preht(
+        struct ggml_context * ctx,
+        struct ggml_tensor  * q_preht,
+        struct ggml_tensor  * packed_k,
+        int                   n_kv_heads,
+        bool                  use_qjl) {
+    return ggml_attn_score_polar_impl(ctx, q_preht, packed_k, n_kv_heads, use_qjl, true);
+}
+
+`;
+    c = c.replace(implAnchor, impl + implAnchor);
+    if (!c.includes("ATTN_SCORE_TBQ") || !c.includes("attn_score_polar")) {
+      throw new Error(
+        `[metal-tbq-polar-attn] ggml.c patch did not add expected op names at ${cPath}`,
+      );
+    }
+    changed = true;
   }
-  const qjlAttn = patchMetalQjlAttnDispatch(cacheDir, { dryRun });
+
+  if (c !== cOriginal && !dryRun) fs.writeFileSync(cPath, c, "utf8");
+  return { changed: changed && !dryRun, headerPath, cPath };
+}
+
+function patchMetalTbqPolarDeviceHeader(cacheDir, { dryRun }) {
+  const headerPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-device.h",
+  );
+  const original = fs.readFileSync(headerPath, "utf8");
+  if (original.includes(SENTINEL_TBQ_POLAR_ATTN)) {
+    const anchor = `struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_polar_preht(
+        ggml_metal_library_t lib);`;
+    const addition = `${anchor}
+
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fused_attn_qjl_tbq(
+        ggml_metal_library_t lib);`;
+    const patched = original.includes(
+      "ggml_metal_library_get_pipeline_fused_attn_qjl_tbq",
+    )
+      ? original
+      : original.replace(anchor, addition);
+    if (patched !== original && !dryRun)
+      fs.writeFileSync(headerPath, patched, "utf8");
+    return { changed: patched !== original && !dryRun, path: headerPath };
+  }
+  const anchor = `${SENTINEL_QJL_ATTN}
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_qjl(
+        ggml_metal_library_t lib);`;
+  if (!original.includes(anchor)) {
+    throw new Error(
+      `[metal-tbq-polar-attn] device.h QJL pipeline anchor not found at ${headerPath}`,
+    );
+  }
+  const patched = original.replace(
+    anchor,
+    `${anchor}
+
+${SENTINEL_TBQ_POLAR_ATTN}
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_tbq(
+        ggml_metal_library_t lib,
+        enum ggml_type        type);
+
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_polar(
+        ggml_metal_library_t lib);
+
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_polar_preht(
+        ggml_metal_library_t lib);
+
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fused_attn_qjl_tbq(
+        ggml_metal_library_t lib);`,
+  );
+  if (!dryRun) fs.writeFileSync(headerPath, patched, "utf8");
+  return { changed: !dryRun, path: headerPath };
+}
+
+function patchMetalTbqPolarDeviceCpp(cacheDir, { dryRun }) {
+  const cppPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-device.cpp",
+  );
+  const original = fs.readFileSync(cppPath, "utf8");
+  if (original.includes(SENTINEL_TBQ_POLAR_ATTN)) {
+    const anchor = `ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin(ggml_metal_library_t lib, const ggml_tensor * op, int32_t n_fuse) {`;
+    const helper = `ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fused_attn_qjl_tbq(ggml_metal_library_t lib) {
+    const char * name = "kernel_fused_attn_qjl_tbq3_f32";
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+    }
+    if (!res.pipeline) {
+        GGML_LOG_ERROR("fused_attn_qjl_tbq: kernel '%s' missing from default.metallib\\n", name);
+        GGML_ABORT("fused_attn_qjl_tbq: pipeline compile failed");
+    }
+    res.nr0 = 1;
+    res.nr1 = 1;
+    res.nsg = 1;
+    res.smem = 0;
+    return res;
+}
+
+`;
+    const patched = original.includes(
+      "ggml_metal_library_get_pipeline_fused_attn_qjl_tbq",
+    )
+      ? original
+      : original.replace(anchor, helper + anchor);
+    if (patched !== original && !dryRun)
+      fs.writeFileSync(cppPath, patched, "utf8");
+    return { changed: patched !== original && !dryRun, path: cppPath };
+  }
+  const anchor = `ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin(ggml_metal_library_t lib, const ggml_tensor * op, int32_t n_fuse) {`;
+  if (!original.includes(anchor)) {
+    throw new Error(
+      `[metal-tbq-polar-attn] device.cpp pipeline anchor not found at ${cppPath}`,
+    );
+  }
+  const helper = `${SENTINEL_TBQ_POLAR_ATTN}
+static const char * eliza_metal_tbq_kernel_name(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return "kernel_turbo3_dot_multi";
+        case GGML_TYPE_TBQ4_0:   return "kernel_turbo4_dot_multi";
+        case GGML_TYPE_TBQ3_TCQ: return "kernel_turbo3_tcq_dot_multi";
+        default: GGML_ABORT("unsupported TurboQuant attention score type");
+    }
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_tbq(ggml_metal_library_t lib, ggml_type type) {
+    const char * name = eliza_metal_tbq_kernel_name(type);
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+    }
+    if (!res.pipeline) {
+        GGML_LOG_ERROR("attn_score_tbq: kernel '%s' missing from default.metallib\\n", name);
+        GGML_ABORT("attn_score_tbq: pipeline compile failed");
+    }
+    res.nr0 = 1;
+    res.nr1 = 1;
+    res.nsg = 1;
+    res.smem = 0;
+    return res;
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_polar(ggml_metal_library_t lib) {
+    const char * name = "kernel_mul_mv_q4_polar_f32";
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+    }
+    if (!res.pipeline) {
+        GGML_LOG_ERROR("attn_score_polar: kernel '%s' missing from default.metallib\\n", name);
+        GGML_ABORT("attn_score_polar: pipeline compile failed");
+    }
+    res.nr0 = 1;
+    res.nr1 = 1;
+    res.nsg = 1;
+    res.smem = 0;
+    return res;
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_attn_score_polar_preht(ggml_metal_library_t lib) {
+    const char * name = "kernel_attn_score_q4_polar_preht_f32";
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+    }
+    if (!res.pipeline) {
+        GGML_LOG_ERROR("attn_score_polar_preht: kernel '%s' missing from default.metallib\\n", name);
+        GGML_ABORT("attn_score_polar_preht: pipeline compile failed");
+    }
+    res.nr0 = 1;
+    res.nr1 = 1;
+    res.nsg = 1;
+    res.smem = 0;
+    return res;
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fused_attn_qjl_tbq(ggml_metal_library_t lib) {
+    const char * name = "kernel_fused_attn_qjl_tbq3_f32";
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+    }
+    if (!res.pipeline) {
+        GGML_LOG_ERROR("fused_attn_qjl_tbq: kernel '%s' missing from default.metallib\\n", name);
+        GGML_ABORT("fused_attn_qjl_tbq: pipeline compile failed");
+    }
+    res.nr0 = 1;
+    res.nr1 = 1;
+    res.nsg = 1;
+    res.smem = 0;
+    return res;
+}
+
+`;
+  const patched = original.replace(anchor, helper + anchor);
+  if (!dryRun) fs.writeFileSync(cppPath, patched, "utf8");
+  return { changed: !dryRun, path: cppPath };
+}
+
+function patchMetalTbqPolarOpsHeader(cacheDir, { dryRun }) {
+  const headerPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-ops.h",
+  );
+  const original = fs.readFileSync(headerPath, "utf8");
+  if (original.includes(SENTINEL_TBQ_POLAR_ATTN)) {
+    const anchor = `int ggml_metal_op_attn_score_polar(ggml_metal_op_t ctx, int idx);`;
+    const addition = `${anchor}
+int ggml_metal_op_fused_attn_qjl_tbq(ggml_metal_op_t ctx, int idx);`;
+    const patched = original.includes("ggml_metal_op_fused_attn_qjl_tbq")
+      ? original
+      : original.replace(anchor, addition);
+    if (patched !== original && !dryRun)
+      fs.writeFileSync(headerPath, patched, "utf8");
+    return { changed: patched !== original && !dryRun, path: headerPath };
+  }
+  const anchor = `${SENTINEL_QJL_ATTN}
+int ggml_metal_op_attn_score_qjl  (ggml_metal_op_t ctx, int idx);`;
+  if (!original.includes(anchor)) {
+    throw new Error(
+      `[metal-tbq-polar-attn] ops.h QJL declaration anchor not found at ${headerPath}`,
+    );
+  }
+  const patched = original.replace(
+    anchor,
+    `${anchor}
+${SENTINEL_TBQ_POLAR_ATTN}
+int ggml_metal_op_attn_score_tbq  (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_attn_score_polar(ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_fused_attn_qjl_tbq(ggml_metal_op_t ctx, int idx);`,
+  );
+  if (!dryRun) fs.writeFileSync(headerPath, patched, "utf8");
+  return { changed: !dryRun, path: headerPath };
+}
+
+function patchMetalTbqPolarOpsCpp(cacheDir, { dryRun }) {
+  const opsPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-ops.cpp",
+  );
+  const original = fs.readFileSync(opsPath, "utf8");
+  if (original.includes(SENTINEL_TBQ_POLAR_ATTN)) {
+    let patched = original.replace(
+      "case GGML_TYPE_TBQ4_0:   return 1u;",
+      "case GGML_TYPE_TBQ4_0:   return 4u;",
+    );
+    if (!patched.includes("eliza_tbq_blocks_per_threadgroup")) {
+      patched = patched.replace(
+        `static inline uint32_t eliza_tbq_blocks_per_row(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return 4u;
+        case GGML_TYPE_TBQ4_0:   return 4u;
+        case GGML_TYPE_TBQ3_TCQ: return 1u;
+        default: GGML_ABORT("unsupported TurboQuant attention score type");
+    }
+}
+`,
+        `static inline uint32_t eliza_tbq_blocks_per_row(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return 4u;
+        case GGML_TYPE_TBQ4_0:   return 4u;
+        case GGML_TYPE_TBQ3_TCQ: return 1u;
+        default: GGML_ABORT("unsupported TurboQuant attention score type");
+    }
+}
+
+static inline uint32_t eliza_tbq_blocks_per_threadgroup(ggml_type type) {
+    // M4 Max multiblock/autotune bench best medians (2026-05-12):
+    //   TBQ3=16, TBQ4=8, TBQ3_TCQ=32. Voice-mode policy can still force N=1
+    //   at a higher scheduler layer when barge-in latency dominates.
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return eliza_env_u32("ELIZA_METAL_TBQ3_BLOCKS_PER_TG", 16u, 1u, 64u);
+        case GGML_TYPE_TBQ4_0:   return eliza_env_u32("ELIZA_METAL_TBQ4_BLOCKS_PER_TG", 8u, 1u, 64u);
+        case GGML_TYPE_TBQ3_TCQ: return eliza_env_u32("ELIZA_METAL_TBQ3_TCQ_BLOCKS_PER_TG", 32u, 1u, 64u);
+        default: GGML_ABORT("unsupported TurboQuant attention score type");
+    }
+}
+`,
+      );
+    }
+    patched = patched.replace(
+      "/* blocks_per_threadgroup = */ 8u,",
+      "/* blocks_per_threadgroup = */ eliza_tbq_blocks_per_threadgroup(ktype),",
+    );
+    if (!patched.includes("ggml_metal_op_fused_attn_qjl_tbq")) {
+      const funcAnchor = `static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {`;
+      patched = patched.replace(
+        funcAnchor,
+        `int ggml_metal_op_fused_attn_qjl_tbq(ggml_metal_op_t ctx, int idx) {
+    ggml_tensor * op = ctx->node(idx);
+
+    ggml_metal_library_t lib = ctx->lib;
+    ggml_metal_encoder_t enc = ctx->enc;
+
+    const ggml_tensor * q  = op->src[0];
+    const ggml_tensor * pk = op->src[1];
+    const ggml_tensor * pv = op->src[2];
+
+    GGML_ASSERT(q  != nullptr);
+    GGML_ASSERT(pk != nullptr);
+    GGML_ASSERT(pv != nullptr);
+    GGML_ASSERT(q->type  == GGML_TYPE_F32);
+    GGML_ASSERT(pk->type == GGML_TYPE_QJL1_256);
+    GGML_ASSERT(pv->type == GGML_TYPE_TBQ3_0);
+    GGML_ASSERT(op->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0]  == 256);
+    GGML_ASSERT(pk->ne[0] == 128);
+    GGML_ASSERT(pv->ne[0] == 128);
+    GGML_ASSERT(op->ne[0] == 128);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(pv));
+    GGML_ASSERT(ggml_is_contiguous_rows(op));
+
+    const int32_t * params = (const int32_t *) op->op_params;
+    const uint32_t n_kv_heads = (uint32_t) params[0];
+    union { int32_t i; float f; } scale_bits;
+    scale_bits.i = params[1];
+
+    const uint32_t n_heads = (uint32_t) q->ne[1];
+    const uint32_t n_q_pos = (uint32_t) q->ne[2];
+    const uint32_t n_kv    = (uint32_t) pk->ne[1];
+    const int64_t  ne3     = q->ne[3];
+
+    GGML_ASSERT(n_kv_heads > 0);
+    GGML_ASSERT((n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(pv->ne[1] == (int64_t) n_kv);
+    GGML_ASSERT(pv->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(pk->ne[3] == ne3);
+    GGML_ASSERT(pv->ne[3] == ne3);
+    GGML_ASSERT(op->ne[1] == (int64_t) n_heads);
+    GGML_ASSERT(op->ne[2] == (int64_t) n_q_pos);
+    GGML_ASSERT(op->ne[3] == ne3);
+    GGML_ASSERT(q->nb[1] == (size_t) q->ne[0] * ggml_type_size(q->type));
+    GGML_ASSERT(q->nb[2] == (size_t) n_heads * q->nb[1]);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(GGML_TYPE_QJL1_256, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_kv * pk->nb[1]);
+    GGML_ASSERT(pv->nb[1] == ggml_row_size(GGML_TYPE_TBQ3_0, 128));
+    GGML_ASSERT(pv->nb[2] == (size_t) n_kv * pv->nb[1]);
+    GGML_ASSERT(op->nb[1] == (size_t) op->ne[0] * ggml_type_size(op->type));
+    GGML_ASSERT(op->nb[2] == (size_t) n_heads * op->nb[1]);
+
+    eliza_fused_attn_qjl_tbq_args args = {
+        /* head_dim   = */ 128u,
+        /* proj_dim   = */ 256u,
+        /* n_heads    = */ n_heads,
+        /* n_kv_heads = */ n_kv_heads,
+        /* n_q_pos    = */ n_q_pos,
+        /* n_kv       = */ n_kv,
+        /* kv_tile    = */ (uint32_t) params[3],
+        /* v_use_qjl  = */ (uint32_t) params[2],
+        /* scale      = */ scale_bits.f,
+        /* causal     = */ (uint32_t) params[4],
+        /* q_pos_base = */ (uint32_t) params[5],
+    };
+
+    auto pipeline = ggml_metal_library_get_pipeline_fused_attn_qjl_tbq(lib);
+
+    const ggml_metal_buffer_id q_base   = ggml_metal_get_buffer_id(q);
+    const ggml_metal_buffer_id pk_base  = ggml_metal_get_buffer_id(pk);
+    const ggml_metal_buffer_id pv_base  = ggml_metal_get_buffer_id(pv);
+    const ggml_metal_buffer_id dst_base = ggml_metal_get_buffer_id(op);
+
+    ggml_metal_encoder_set_pipeline(enc, pipeline);
+    ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 4);
+
+    for (int64_t i3 = 0; i3 < ne3; ++i3) {
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(q_base,   (size_t) i3 * q->nb[3]),  0);
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pk_base,  (size_t) i3 * pk->nb[3]), 1);
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pv_base,  (size_t) i3 * pv->nb[3]), 2);
+        ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(dst_base, (size_t) i3 * op->nb[3]), 3);
+        ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_heads, (int) n_q_pos, 1, 32, 1, 1);
+    }
+
+    return 1;
+}
+
+${funcAnchor}`,
+      );
+    }
+    if (!patched.includes("case GGML_OP_FUSED_ATTN_QJL_TBQ:")) {
+      patched = patched.replace(
+        `        case GGML_OP_ATTN_SCORE_POLAR:
+            {
+                n_fuse = ggml_metal_op_attn_score_polar(ctx, idx);
+            } break;`,
+        `        case GGML_OP_ATTN_SCORE_POLAR:
+            {
+                n_fuse = ggml_metal_op_attn_score_polar(ctx, idx);
+            } break;
+        case GGML_OP_FUSED_ATTN_QJL_TBQ:
+            {
+                n_fuse = ggml_metal_op_fused_attn_qjl_tbq(ctx, idx);
+            } break;`,
+      );
+    }
+    if (patched !== original && !dryRun)
+      fs.writeFileSync(opsPath, patched, "utf8");
+    return { changed: patched !== original && !dryRun, path: opsPath };
+  }
+  const tcqCodebook = readTcqCodebookLiteral();
+  const funcAnchor = `static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {`;
+  if (!original.includes(funcAnchor)) {
+    throw new Error(
+      `[metal-tbq-polar-attn] ops.cpp encode anchor not found at ${opsPath}`,
+    );
+  }
+  const opFuncs = `${SENTINEL_TBQ_POLAR_ATTN}
+struct eliza_tbq_score_args {
+    uint32_t head_dim;
+    uint32_t n_kv;
+    uint32_t kv_stride_blocks;
+    uint32_t q_head;
+    uint32_t head_offset_bytes;
+    uint32_t blocks_per_threadgroup;
+};
+
+struct eliza_polar_score_args {
+    uint32_t n_rows;
+    uint32_t head_dim;
+    uint32_t use_qjl;
+};
+
+struct eliza_polar_preht_score_args {
+    uint32_t head_dim;
+    uint32_t n_kv;
+    uint32_t kv_stride_blocks;
+    uint32_t q_head;
+    uint32_t head_offset_bytes;
+    uint32_t use_qjl;
+};
+
+static const float k_eliza_tbq3_tcq_codebook[512] = {
+${tcqCodebook}
+};
+
+static inline uint32_t eliza_tbq_blocks_per_row(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return 4u;
+        case GGML_TYPE_TBQ4_0:   return 4u;
+        case GGML_TYPE_TBQ3_TCQ: return 1u;
+        default: GGML_ABORT("unsupported TurboQuant attention score type");
+    }
+}
+
+static inline uint32_t eliza_tbq_blocks_per_threadgroup(ggml_type type) {
+    // M4 Max multiblock/autotune bench best medians (2026-05-12):
+    //   TBQ3=16, TBQ4=8, TBQ3_TCQ=32. Voice-mode policy can still force N=1
+    //   at a higher scheduler layer when barge-in latency dominates.
+    switch (type) {
+        case GGML_TYPE_TBQ3_0:   return eliza_env_u32("ELIZA_METAL_TBQ3_BLOCKS_PER_TG", 16u, 1u, 64u);
+        case GGML_TYPE_TBQ4_0:   return eliza_env_u32("ELIZA_METAL_TBQ4_BLOCKS_PER_TG", 8u, 1u, 64u);
+        case GGML_TYPE_TBQ3_TCQ: return eliza_env_u32("ELIZA_METAL_TBQ3_TCQ_BLOCKS_PER_TG", 32u, 1u, 64u);
+        default: GGML_ABORT("unsupported TurboQuant attention score type");
+    }
+}
+
+int ggml_metal_op_attn_score_tbq(ggml_metal_op_t ctx, int idx) {
+    ggml_tensor * op = ctx->node(idx);
+
+    ggml_metal_library_t lib = ctx->lib;
+    ggml_metal_encoder_t enc = ctx->enc;
+
+    const ggml_tensor * q  = op->src[0];
+    const ggml_tensor * pk = op->src[1];
+    const ggml_type ktype = pk->type;
+
+    GGML_ASSERT(q  != nullptr);
+    GGML_ASSERT(pk != nullptr);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(ktype == GGML_TYPE_TBQ3_0 || ktype == GGML_TYPE_TBQ4_0 || ktype == GGML_TYPE_TBQ3_TCQ);
+    GGML_ASSERT(op->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0]  == 128);
+    GGML_ASSERT(pk->ne[0] == 128);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(op));
+
+    const uint32_t n_heads     = (uint32_t) q->ne[1];
+    const uint32_t n_kv_heads  = (uint32_t) ((const int32_t *) op->op_params)[0];
+    const uint32_t n_tokens    = (uint32_t) pk->ne[1];
+    const int64_t  n_batch     = q->ne[2];
+    const int64_t  ne3         = q->ne[3];
+
+    GGML_ASSERT(n_kv_heads > 0);
+    GGML_ASSERT((n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(pk->ne[3] == ne3);
+    GGML_ASSERT(op->ne[0] == (int64_t) n_tokens);
+    GGML_ASSERT(op->ne[1] == (int64_t) n_heads);
+    GGML_ASSERT(op->ne[2] == n_batch);
+    GGML_ASSERT(op->ne[3] == ne3);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(ktype, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_tokens * pk->nb[1]);
+
+    eliza_tbq_score_args args = {
+        /* head_dim = */ 128u,
+        /* n_kv = */ n_tokens,
+        /* kv_stride_blocks = */ eliza_tbq_blocks_per_row(ktype),
+        /* q_head = */ 0u,
+        /* head_offset_bytes = */ 0u,
+        /* blocks_per_threadgroup = */ eliza_tbq_blocks_per_threadgroup(ktype),
+    };
+
+    auto pipeline = ggml_metal_library_get_pipeline_attn_score_tbq(lib, ktype);
+
+    const ggml_metal_buffer_id q_base   = ggml_metal_get_buffer_id(q);
+    const ggml_metal_buffer_id pk_base  = ggml_metal_get_buffer_id(pk);
+    const ggml_metal_buffer_id dst_base = ggml_metal_get_buffer_id(op);
+    const uint32_t gqa = n_heads / n_kv_heads;
+
+    ggml_metal_encoder_set_pipeline(enc, pipeline);
+    if (ktype == GGML_TYPE_TBQ3_TCQ) {
+        ggml_metal_encoder_set_bytes(enc, (void *) k_eliza_tbq3_tcq_codebook, sizeof(k_eliza_tbq3_tcq_codebook), 3);
+        ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 4);
+    } else {
+        ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 3);
+    }
+
+    const int token_groups = (int) ((n_tokens + args.blocks_per_threadgroup - 1u) / args.blocks_per_threadgroup);
+    for (int64_t i3 = 0; i3 < ne3; ++i3) {
+        const size_t q_i3   = (size_t) i3 * q->nb[3];
+        const size_t pk_i3  = (size_t) i3 * pk->nb[3];
+        const size_t dst_i3 = (size_t) i3 * op->nb[3];
+        for (int64_t ib = 0; ib < n_batch; ++ib) {
+            for (uint32_t h = 0; h < n_heads; ++h) {
+                const uint32_t h_k = h / gqa;
+                ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(q_base,   q_i3  + (size_t) ib * q->nb[2]  + (size_t) h   * q->nb[1]),  0);
+                ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pk_base,  pk_i3 + (size_t) h_k * pk->nb[2]), 1);
+                ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(dst_base, dst_i3 + (size_t) ib * op->nb[2] + (size_t) h   * op->nb[1]), 2);
+                ggml_metal_encoder_dispatch_threadgroups(enc, token_groups, 1, 1, 32, 1, 1);
+            }
+        }
+    }
+
+    return 1;
+}
+
+int ggml_metal_op_attn_score_polar(ggml_metal_op_t ctx, int idx) {
+    ggml_tensor * op = ctx->node(idx);
+
+    ggml_metal_library_t lib = ctx->lib;
+    ggml_metal_encoder_t enc = ctx->enc;
+
+    const ggml_tensor * q  = op->src[0];
+    const ggml_tensor * pk = op->src[1];
+
+    GGML_ASSERT(q  != nullptr);
+    GGML_ASSERT(pk != nullptr);
+    GGML_ASSERT(q->type == GGML_TYPE_F32);
+    GGML_ASSERT(pk->type == GGML_TYPE_Q4_POLAR);
+    GGML_ASSERT(op->type == GGML_TYPE_F32);
+    GGML_ASSERT(q->ne[0]  == 128);
+    GGML_ASSERT(pk->ne[0] == 128);
+    GGML_ASSERT(ggml_is_contiguous_rows(q));
+    GGML_ASSERT(ggml_is_contiguous_rows(pk));
+    GGML_ASSERT(ggml_is_contiguous_rows(op));
+
+    const int32_t * params = (const int32_t *) op->op_params;
+    const uint32_t n_heads     = (uint32_t) q->ne[1];
+    const uint32_t n_kv_heads  = (uint32_t) params[0];
+    const uint32_t n_tokens    = (uint32_t) pk->ne[1];
+    const uint32_t use_qjl     = (uint32_t) (params[1] != 0);
+    const uint32_t q_preht     = (uint32_t) (params[2] != 0);
+    const int64_t  n_batch     = q->ne[2];
+    const int64_t  ne3         = q->ne[3];
+
+    GGML_ASSERT(n_kv_heads > 0);
+    GGML_ASSERT((n_heads % n_kv_heads) == 0);
+    GGML_ASSERT(pk->ne[2] == (int64_t) n_kv_heads);
+    GGML_ASSERT(pk->ne[3] == ne3);
+    GGML_ASSERT(op->ne[0] == (int64_t) n_tokens);
+    GGML_ASSERT(op->ne[1] == (int64_t) n_heads);
+    GGML_ASSERT(op->ne[2] == n_batch);
+    GGML_ASSERT(op->ne[3] == ne3);
+    GGML_ASSERT(pk->nb[1] == ggml_row_size(GGML_TYPE_Q4_POLAR, 128));
+    GGML_ASSERT(pk->nb[2] == (size_t) n_tokens * pk->nb[1]);
+
+    const ggml_metal_buffer_id q_base   = ggml_metal_get_buffer_id(q);
+    const ggml_metal_buffer_id pk_base  = ggml_metal_get_buffer_id(pk);
+    const ggml_metal_buffer_id dst_base = ggml_metal_get_buffer_id(op);
+    const uint32_t gqa = n_heads / n_kv_heads;
+
+    if (q_preht != 0u) {
+        auto pipeline = ggml_metal_library_get_pipeline_attn_score_polar_preht(lib);
+        ggml_metal_encoder_set_pipeline(enc, pipeline);
+
+        for (int64_t i3 = 0; i3 < ne3; ++i3) {
+            const size_t q_i3   = (size_t) i3 * q->nb[3];
+            const size_t pk_i3  = (size_t) i3 * pk->nb[3];
+            const size_t dst_i3 = (size_t) i3 * op->nb[3];
+            for (int64_t ib = 0; ib < n_batch; ++ib) {
+                ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(q_base,   q_i3  + (size_t) ib * q->nb[2]), 0);
+                ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pk_base,  pk_i3),                         1);
+                ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(dst_base, dst_i3 + (size_t) ib * op->nb[2]), 2);
+                for (uint32_t h = 0; h < n_heads; ++h) {
+                    const uint32_t h_k = h / gqa;
+                    eliza_polar_preht_score_args args = {
+                        /* head_dim = */ 128u,
+                        /* n_kv = */ n_tokens,
+                        /* kv_stride_blocks = */ 1u,
+                        /* q_head = */ h,
+                        /* head_offset_bytes = */ (uint32_t) ((size_t) h_k * pk->nb[2]),
+                        /* use_qjl = */ use_qjl,
+                    };
+                    ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 3);
+                    ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_tokens, 1, 1, 32, 1, 1);
+                }
+            }
+        }
+    } else {
+        eliza_polar_score_args args = {
+            /* n_rows = */ n_tokens,
+            /* head_dim = */ 128u,
+            /* use_qjl = */ use_qjl,
+        };
+
+        auto pipeline = ggml_metal_library_get_pipeline_attn_score_polar(lib);
+
+        ggml_metal_encoder_set_pipeline(enc, pipeline);
+        ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 3);
+
+        for (int64_t i3 = 0; i3 < ne3; ++i3) {
+            const size_t q_i3   = (size_t) i3 * q->nb[3];
+            const size_t pk_i3  = (size_t) i3 * pk->nb[3];
+            const size_t dst_i3 = (size_t) i3 * op->nb[3];
+            for (int64_t ib = 0; ib < n_batch; ++ib) {
+                for (uint32_t h = 0; h < n_heads; ++h) {
+                    const uint32_t h_k = h / gqa;
+                    ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(pk_base,  pk_i3 + (size_t) h_k * pk->nb[2]), 0);
+                    ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(q_base,   q_i3  + (size_t) ib * q->nb[2]  + (size_t) h   * q->nb[1]),  1);
+                    ggml_metal_encoder_set_buffer(enc, eliza_metal_buffer_offset(dst_base, dst_i3 + (size_t) ib * op->nb[2] + (size_t) h   * op->nb[1]), 2);
+                    ggml_metal_encoder_dispatch_threadgroups(enc, (int) n_tokens, 1, 1, 32, 1, 1);
+                }
+            }
+        }
+    }
+
+    return 1;
+}
+
+`;
+  let patched = original.replace(funcAnchor, opFuncs + funcAnchor);
+  const switchAnchor = `        case GGML_OP_ATTN_SCORE_QJL:
+            {
+                n_fuse = ggml_metal_op_attn_score_qjl(ctx, idx);
+            } break;`;
+  if (!patched.includes(switchAnchor)) {
+    throw new Error(
+      `[metal-tbq-polar-attn] ops.cpp QJL switch anchor not found at ${opsPath}`,
+    );
+  }
+  patched = patched.replace(
+    switchAnchor,
+    `${switchAnchor}
+        case GGML_OP_ATTN_SCORE_TBQ:
+            {
+                n_fuse = ggml_metal_op_attn_score_tbq(ctx, idx);
+            } break;
+        case GGML_OP_ATTN_SCORE_POLAR:
+            {
+                n_fuse = ggml_metal_op_attn_score_polar(ctx, idx);
+            } break;
+        case GGML_OP_FUSED_ATTN_QJL_TBQ:
+            {
+                n_fuse = ggml_metal_op_fused_attn_qjl_tbq(ctx, idx);
+            } break;`,
+  );
+  if (!dryRun) fs.writeFileSync(opsPath, patched, "utf8");
+  return { changed: !dryRun, path: opsPath };
+}
+
+function patchMetalTbqPolarSupportsOp(cacheDir, { dryRun }) {
+  const deviceMPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-device.m",
+  );
+  const original = fs.readFileSync(deviceMPath, "utf8");
+  if (original.includes(SENTINEL_TBQ_POLAR_ATTN)) {
+    let patched = original.replace(
+      `(op->src[1]->type == GGML_TYPE_TBQ3_0 ||
+                 op->src[1]->type == GGML_TYPE_TBQ3_TCQ) &&`,
+      `(op->src[1]->type == GGML_TYPE_TBQ3_0 ||
+                 op->src[1]->type == GGML_TYPE_TBQ4_0 ||
+                 op->src[1]->type == GGML_TYPE_TBQ3_TCQ) &&`,
+    );
+    if (!patched.includes("case GGML_OP_FUSED_ATTN_QJL_TBQ:")) {
+      patched = patched.replace(
+        `        case GGML_OP_ATTN_SCORE_QJL:`,
+        `        case GGML_OP_FUSED_ATTN_QJL_TBQ:
+            {
+                const int32_t * params = (const int32_t *) op->op_params;
+                const int64_t n_kv_heads = params[0];
+                return has_simdgroup_reduction &&
+                    op->type == GGML_TYPE_F32 &&
+                    op->src[0] != NULL &&
+                    op->src[1] != NULL &&
+                    op->src[2] != NULL &&
+                    op->src[0]->type == GGML_TYPE_F32 &&
+                    op->src[1]->type == GGML_TYPE_QJL1_256 &&
+                    op->src[2]->type == GGML_TYPE_TBQ3_0 &&
+                    op->src[0]->ne[0] == 256 &&
+                    op->src[1]->ne[0] == 128 &&
+                    op->src[2]->ne[0] == 128 &&
+                    op->ne[0] == 128 &&
+                    n_kv_heads > 0 &&
+                    (op->src[0]->ne[1] % n_kv_heads) == 0 &&
+                    op->src[1]->ne[1] == op->src[2]->ne[1] &&
+                    op->src[1]->ne[2] == n_kv_heads &&
+                    op->src[2]->ne[2] == n_kv_heads &&
+                    op->src[1]->ne[3] == op->src[0]->ne[3] &&
+                    op->src[2]->ne[3] == op->src[0]->ne[3] &&
+                    op->ne[1] == op->src[0]->ne[1] &&
+                    op->ne[2] == op->src[0]->ne[2] &&
+                    op->ne[3] == op->src[0]->ne[3] &&
+                    ggml_is_contiguous_rows(op) &&
+                    ggml_is_contiguous_rows(op->src[0]) &&
+                    ggml_is_contiguous_rows(op->src[1]) &&
+                    ggml_is_contiguous_rows(op->src[2]);
+            }
+        case GGML_OP_ATTN_SCORE_QJL:`,
+      );
+    }
+    if (patched !== original && !dryRun)
+      fs.writeFileSync(deviceMPath, patched, "utf8");
+    return { changed: patched !== original && !dryRun, path: deviceMPath };
+  }
+  const anchor = `        case GGML_OP_ATTN_SCORE_QJL:
+            // ${SENTINEL_QJL_ATTN}`;
+  if (!original.includes(anchor)) {
+    throw new Error(
+      `[metal-tbq-polar-attn] supports_op QJL anchor not found at ${deviceMPath}`,
+    );
+  }
+  const insert = `        case GGML_OP_ATTN_SCORE_TBQ:
+            // ${SENTINEL_TBQ_POLAR_ATTN}
+            return has_simdgroup_reduction &&
+                op->type == GGML_TYPE_F32 &&
+                op->src[0] != NULL &&
+                op->src[1] != NULL &&
+                op->src[0]->type == GGML_TYPE_F32 &&
+                (op->src[1]->type == GGML_TYPE_TBQ3_0 ||
+                 op->src[1]->type == GGML_TYPE_TBQ4_0 ||
+                 op->src[1]->type == GGML_TYPE_TBQ3_TCQ) &&
+                op->src[0]->ne[0] == 128 &&
+                op->src[1]->ne[0] == 128 &&
+                ggml_is_contiguous_rows(op) &&
+                ggml_is_contiguous_rows(op->src[0]) &&
+                ggml_is_contiguous_rows(op->src[1]);
+        case GGML_OP_ATTN_SCORE_POLAR:
+            return has_simdgroup_reduction &&
+                op->type == GGML_TYPE_F32 &&
+                op->src[0] != NULL &&
+                op->src[1] != NULL &&
+                op->src[0]->type == GGML_TYPE_F32 &&
+                op->src[1]->type == GGML_TYPE_Q4_POLAR &&
+                op->src[0]->ne[0] == 128 &&
+                op->src[1]->ne[0] == 128 &&
+                ggml_is_contiguous_rows(op) &&
+                ggml_is_contiguous_rows(op->src[0]) &&
+                ggml_is_contiguous_rows(op->src[1]);
+        case GGML_OP_FUSED_ATTN_QJL_TBQ:
+            {
+                const int32_t * params = (const int32_t *) op->op_params;
+                const int64_t n_kv_heads = params[0];
+                return has_simdgroup_reduction &&
+                    op->type == GGML_TYPE_F32 &&
+                    op->src[0] != NULL &&
+                    op->src[1] != NULL &&
+                    op->src[2] != NULL &&
+                    op->src[0]->type == GGML_TYPE_F32 &&
+                    op->src[1]->type == GGML_TYPE_QJL1_256 &&
+                    op->src[2]->type == GGML_TYPE_TBQ3_0 &&
+                    op->src[0]->ne[0] == 256 &&
+                    op->src[1]->ne[0] == 128 &&
+                    op->src[2]->ne[0] == 128 &&
+                    op->ne[0] == 128 &&
+                    n_kv_heads > 0 &&
+                    (op->src[0]->ne[1] % n_kv_heads) == 0 &&
+                    op->src[1]->ne[1] == op->src[2]->ne[1] &&
+                    op->src[1]->ne[2] == n_kv_heads &&
+                    op->src[2]->ne[2] == n_kv_heads &&
+                    op->src[1]->ne[3] == op->src[0]->ne[3] &&
+                    op->src[2]->ne[3] == op->src[0]->ne[3] &&
+                    op->ne[1] == op->src[0]->ne[1] &&
+                    op->ne[2] == op->src[0]->ne[2] &&
+                    op->ne[3] == op->src[0]->ne[3] &&
+                    ggml_is_contiguous_rows(op) &&
+                    ggml_is_contiguous_rows(op->src[0]) &&
+                    ggml_is_contiguous_rows(op->src[1]) &&
+                    ggml_is_contiguous_rows(op->src[2]);
+            }
+${anchor}`;
+  const patched = original.replace(anchor, insert);
+  if (!dryRun) fs.writeFileSync(deviceMPath, patched, "utf8");
+  return { changed: !dryRun, path: deviceMPath };
+}
+
+function patchMetalTbqPolarAttnDispatch(cacheDir, { dryRun }) {
+  const ggmlOps = patchGgmlTbqPolarAttnOps(cacheDir, { dryRun });
+  const deviceHeader = patchMetalTbqPolarDeviceHeader(cacheDir, { dryRun });
+  const deviceCpp = patchMetalTbqPolarDeviceCpp(cacheDir, { dryRun });
+  const opsHeader = patchMetalTbqPolarOpsHeader(cacheDir, { dryRun });
+  const opsCpp = patchMetalTbqPolarOpsCpp(cacheDir, { dryRun });
+  const supportsOp = patchMetalTbqPolarSupportsOp(cacheDir, { dryRun });
+  return { ggmlOps, deviceHeader, deviceCpp, opsHeader, opsCpp, supportsOp };
+}
+
+function patchMetalQjlSetRowsSupportsOp(cacheDir, { dryRun }) {
+  const deviceMPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-device.m",
+  );
+  const original = fs.readFileSync(deviceMPath, "utf8");
+  if (original.includes(SENTINEL_QJL_SET_ROWS)) {
+    return { changed: false, path: deviceMPath };
+  }
+  const anchor = `                    case GGML_TYPE_IQ4_NL:
+                        return true;`;
+  if (!original.includes(anchor)) {
+    throw new Error(
+      `[metal-qjl-set-rows] supports_op SET_ROWS anchor not found at ${deviceMPath}`,
+    );
+  }
+  const patched = original.replace(
+    anchor,
+    `                    case GGML_TYPE_IQ4_NL:
+                    case GGML_TYPE_QJL1_256:
+                        // ${SENTINEL_QJL_SET_ROWS}
+                        return true;`,
+  );
+  if (!dryRun) fs.writeFileSync(deviceMPath, patched, "utf8");
+  return { changed: !dryRun, path: deviceMPath };
+}
+
+function patchMetalQjlSetRowsOps(cacheDir, { dryRun }) {
+  const opsPath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-metal",
+    "ggml-metal-ops.cpp",
+  );
+  const original = fs.readFileSync(opsPath, "utf8");
+  if (original.includes(SENTINEL_QJL_SET_ROWS)) {
+    return { changed: false, path: opsPath };
+  }
+  const anchor = `    const int32_t nk0 = ne0/ggml_blck_size(op->type);
+
+    int nth = 32; // SIMD width`;
+  if (!original.includes(anchor)) {
+    throw new Error(
+      `[metal-qjl-set-rows] op_set_rows anchor not found at ${opsPath}`,
+    );
+  }
+  const insert = `    const int32_t nk0 = ne0/ggml_blck_size(op->type);
+
+    if (op->type == GGML_TYPE_QJL1_256) {
+        // ${SENTINEL_QJL_SET_ROWS}
+        ggml_metal_kargs_set_rows args = {
+            /*.nk0  =*/ nk0,
+            /*.ne01 =*/ ne01,
+            /*.nb01 =*/ nb01,
+            /*.nb02 =*/ nb02,
+            /*.nb03 =*/ nb03,
+            /*.ne11 =*/ ne11,
+            /*.ne12 =*/ ne12,
+            /*.nb10 =*/ nb10,
+            /*.nb11 =*/ nb11,
+            /*.nb12 =*/ nb12,
+            /*.nb1  =*/ nb1,
+            /*.nb2  =*/ nb2,
+            /*.nb3  =*/ nb3,
+        };
+
+        ggml_metal_encoder_set_pipeline(enc, pipeline);
+        ggml_metal_encoder_set_bytes   (enc, &args, sizeof(args), 0);
+        ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
+        ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
+        ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op),         3);
+
+        ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, 32, 1, 1);
+
+        return 1;
+    }
+
+    int nth = 32; // SIMD width`;
+  const patched = original.replace(anchor, insert);
+  if (!dryRun) fs.writeFileSync(opsPath, patched, "utf8");
+  return { changed: !dryRun, path: opsPath };
+}
+
+function patchMetalQjlSetRows(cacheDir, { dryRun }) {
+  return {
+    supportsOp: patchMetalQjlSetRowsSupportsOp(cacheDir, { dryRun }),
+    ops: patchMetalQjlSetRowsOps(cacheDir, { dryRun }),
+  };
+}
+
+export function patchMetalDispatch(cacheDir, { dryRun = false } = {}) {
+  const patchedFiles = [
+    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.h"),
+    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.cpp"),
+    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-ops.cpp"),
+  ].filter((file) => {
+    try {
+      return fs.readFileSync(file, "utf8").includes(SENTINEL_DISPATCH);
+    } catch {
+      return false;
+    }
+  });
+
+  const message =
+    "[metal-dispatch] NOT wiring generic Metal GGML dispatch for eliza " +
+    "QJL/Polar/TBQ kernels. The standalone kernels use bespoke attention/" +
+    "projection contracts that do not match generic MUL_MAT/GET_ROWS. " +
+    "Dedicated graph ops are required for runtime-ready bits.";
+  if (patchedFiles.length > 0) {
+    const detail =
+      `${message} Found an older unsafe ELIZA-DISPATCH-V1 patch in:\n` +
+      `  ${patchedFiles.join("\n  ")}\n` +
+      "Use a clean eliza-llama-cpp checkout/cache before producing artifacts.";
+    if (!dryRun) {
+      throw new Error(detail);
+    }
+    console.warn(detail);
+  } else {
+    console.log(`${dryRun ? "(dry-run) " : ""}${message}`);
+  }
+  const qjlAttn = patchMetalQjlAttnDispatch(cacheDir, { dryRun });
+  const qjlAnchorsAlreadyPresent = [
+    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-device.h"),
+    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-ops.h"),
+    path.join(cacheDir, "ggml", "src", "ggml-metal", "ggml-metal-ops.cpp"),
+  ].every(
+    (file) =>
+      fs.existsSync(file) &&
+      fs.readFileSync(file, "utf8").includes(SENTINEL_QJL_ATTN),
+  );
+  const tbqPolarAttn =
+    dryRun && !qjlAnchorsAlreadyPresent
+      ? { deferredUntilQjlPatchWrites: true }
+      : patchMetalTbqPolarAttnDispatch(cacheDir, { dryRun });
+  const qjlSetRows = patchMetalQjlSetRows(cacheDir, { dryRun });
+  console.log(
+    `[metal-dispatch] ${dryRun ? "(dry-run) " : ""}wired dedicated GGML_OP_ATTN_SCORE_QJL dispatch via kernel_attn_score_qjl1_256_multi`,
+  );
+  console.log(
+    `[metal-dispatch] ${dryRun ? "(dry-run) " : ""}wired dedicated GGML_OP_ATTN_SCORE_TBQ / GGML_OP_ATTN_SCORE_POLAR dispatch via shipped TurboQuant and PolarQuant kernels` +
+      (tbqPolarAttn.deferredUntilQjlPatchWrites
+        ? " (deferred in dry-run until QJL patch writes anchors)"
+        : ""),
+  );
   console.log(
-    `[metal-dispatch] ${dryRun ? "(dry-run) " : ""}wired dedicated GGML_OP_ATTN_SCORE_QJL dispatch via kernel_attn_score_qjl1_256`,
+    `[metal-dispatch] ${dryRun ? "(dry-run) " : ""}runtime-ready gates: ` +
+      Object.entries(METAL_RUNTIME_DISPATCH_GATES)
+        .map(
+          ([key, gate]) =>
+            `${key}=${gate.runtimeReady ? "runtime-ready" : gate.status}`,
+        )
+        .join(", "),
   );
-  return { status: "qjl-attn-only", unsafePatchPresent: patchedFiles, qjlAttn };
+  return {
+    status: "attn-score-qjl-tbq-polar",
+    unsafePatchPresent: patchedFiles,
+    qjlAttn,
+    tbqPolarAttn,
+    qjlSetRows,
+  };
 }
 
-// Public entry point used by build-llama-cpp-dflash.mjs.
+// Public entry point used by build-llama-cpp-mtp.mjs.
 // Throws on any failure. Idempotent across runs.
 export function patchMetalKernels(cacheDir, { dryRun = false } = {}) {
   if (!cacheDir || !fs.existsSync(cacheDir)) {