@elizaos/app-core 2.0.0-beta.2 → 2.0.11-beta.5

package/scripts/kernel-patches/cpu-polar-kernels.mjs

@@ -0,0 +1,441 @@
+// PolarQuant pre-Hadamard-transposed (`_preht`) CPU dot wiring for the
+// elizaOS/llama.cpp fork (v1.0.0-eliza).
+//
+// What this module does (applied after `git reset --hard` on the cached
+// fork checkout, every build):
+//
+//   1. Mirrors the verified standalone PolarQuant `_preht` translation
+//      units from packages/native/plugins/polarquant-cpu/src/ over a new
+//      fork subdir ggml/src/ggml-cpu/polarquant/:
+//        - polar_dot_preht_ref.c   (scalar baseline, bit-exact)
+//        - polar_dot_preht_avx2.c  (AVX2 / AVX-VNNI body, __AVX2__ guard)
+//        - polar_dot_preht_neon.c  (NEON body, __ARM_NEON guard)
+//        - polar_cpu_features.h    (runtime cpuid / hwcap probe)
+//      …plus two generated files that adapt the standalone API to the
+//      fork's existing Polar symbol surface (the standalone library's
+//      polarquant.h / polar_dispatch.c re-define block_q4_polar /
+//      polar_qjl_signs / dequantize_row_q4_polar / ggml_vec_dot_q4_polar_q8_0,
+//      which the fork already owns — so we DON'T mirror those, we shim):
+//        - polarquant_preht.h           — pulls in the fork's
+//          ggml-common.h (block_q4_polar, QK_POLAR), polar_centroids.h
+//          (POLAR_Q4_CENTROIDS, POLAR_QJL_CORRECTION_MAGNITUDE,
+//          polar_qjl_signs) and simd-mappings.h (GGML_FP16_TO_FP32),
+//          then defines the one missing helper polar_fp16_to_fp32() and
+//          forward-declares the three `_preht` entry points.
+//        - polarquant_preht_dispatch.c  — a trimmed runtime dispatcher
+//          providing ggml_vec_dot_q4_polar_preht_f32() (picks the best
+//          available `_preht` body from polar_cpu_features.h) and the
+//          polarquant_active_simd() diagnostic string. It does NOT
+//          re-export dequantize_row_q4_polar / ggml_vec_dot_q4_polar_q8_0
+//          (those stay the fork's quants-polar.c definitions).
+//      The mirrored `_preht` TUs' `#include "polarquant/polarquant.h"`
+//      line is rewritten to `#include "polarquant_preht.h"` so they
+//      resolve against the shim, not the (absent) standalone header.
+//
+//   2. Regenerates the Polar source list inside
+//      ggml/src/ggml-cpu/CMakeLists.txt so the new files compile, adds
+//      `ggml-cpu/polarquant` to the ggml-cpu include path, and adds the
+//      POLARQUANT_HAVE_* compile-definitions the dispatcher keys off
+//      (POLARQUANT_HAVE_AVX2 / POLARQUANT_HAVE_AVXVNNI on x86,
+//      POLARQUANT_HAVE_NEON on ARM). The per-TU bodies still self-guard
+//      on __AVX2__ / __ARM_NEON so a host without the ISA gets a dead
+//      body, never an undefined symbol.
+//
+// The `_preht` entry points are NEW symbols (the fork has no
+// ggml_vec_dot_q4_polar_preht_f32 of any signature), so dropping these
+// files into the build introduces zero link-time collisions; and ggml.c
+// (ggml-base) does not reference them, so no Windows/Android ggml-base
+// link-list change is needed.
+//
+// Idempotent: each mutation carries a `ELIZA-CPU-POLAR-PREHT-V1`
+// sentinel.
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+function firstExistingPath(candidates) {
+  return (
+    candidates.find((candidate) => fs.existsSync(candidate)) ?? candidates[0]
+  );
+}
+
+const POLAR_CPU_SRC_DIR = firstExistingPath([
+  path.resolve(
+    __dirname,
+    "..",
+    "..",
+    "..",
+    "native",
+    "plugins",
+    "polarquant-cpu",
+  ),
+  path.resolve(__dirname, "..", "..", "..", "native-plugins", "polarquant-cpu"),
+]);
+
+const SENTINEL = "ELIZA-CPU-POLAR-PREHT-V1";
+
+// Standalone TUs mirrored verbatim (with the include rewrite below) plus
+// the self-contained cpu-feature header.
+const POLAR_PREHT_C_FILES = [
+  "polar_dot_preht_ref.c",
+  "polar_dot_preht_avx2.c",
+  "polar_dot_preht_neon.c",
+];
+const POLAR_PREHT_H_FILES = ["polar_cpu_features.h"];
+
+// The shim header that adapts the standalone `_preht` TUs to the fork's
+// existing Polar symbol surface. Written into ggml-cpu/polarquant/.
+const POLARQUANT_PREHT_H =
+  `/* polarquant_preht.h — fork shim for the standalone PolarQuant ` +
+  `_preht TUs.
+ *
+ * AUTO-GENERATED by packages/app-core/scripts/kernel-patches/cpu-polar-kernels.mjs.
+ * Do not edit in the fork checkout — the checkout is git-reset --hard'd
+ * on every build and this file is rewritten from the patcher.
+ *
+ * The standalone packages/native/plugins/polarquant-cpu kernel library
+ * ships its own polarquant.h that (re)defines block_q4_polar, QK_POLAR,
+ * POLAR_Q4_CENTROIDS, POLAR_QJL_CORRECTION_MAGNITUDE, polar_qjl_signs,
+ * polar_fp16_to_fp32 and the dispatchers. The fork ALREADY owns all of
+ * those except polar_fp16_to_fp32 (it uses GGML_FP16_TO_FP32). So the
+ * mirrored \`_preht\` TUs include this shim instead of polarquant.h: it
+ * re-exports the fork's definitions and supplies the one missing helper.
+ */
+#ifndef ELIZA_POLARQUANT_PREHT_H
+#define ELIZA_POLARQUANT_PREHT_H
+
+#define GGML_COMMON_DECL_C
+#include "ggml-common.h"        /* block_q4_polar, QK_POLAR, ggml_half */
+#include "ggml-cpu-impl.h"
+#include "simd-mappings.h"      /* GGML_FP16_TO_FP32 */
+#include "polar_centroids.h"    /* POLAR_Q4_CENTROIDS, POLAR_QJL_CORRECTION_MAGNITUDE, polar_qjl_signs */
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The standalone TUs call polar_fp16_to_fp32(block.d); block.d is a
+ * ggml_half in the fork's block_q4_polar. GGML_FP16_TO_FP32 is the
+ * fork's conversion (table-lookup or hardware F16C). */
+static inline float polar_fp16_to_fp32(ggml_half h) {
+    return GGML_FP16_TO_FP32(h);
+}
+
+/* Memoized +/-1 QJL sign vector. The standalone library ships this in
+ * polar_qjl.c, which we do NOT mirror (the fork already owns
+ * polar_qjl_signs via polar_centroids.h); the \`_preht\` TUs only need a
+ * cached pointer to it. The fill is deterministic, so a benign race
+ * between first callers writing identical bytes is harmless. */
+static inline const float * polar_qjl_signs_cached(void) {
+    static float s_signs[QK_POLAR];
+    static volatile int s_ready = 0;
+    if (!s_ready) {
+        polar_qjl_signs(s_signs);
+        s_ready = 1;
+    }
+    return s_signs;
+}
+
+/* The three pre-Hadamard-transposed dot entry points. \`q_preht\` MUST be
+ * the Walsh-Hadamard-transformed query for the matching head/chunk; the
+ * caller owns that contract (passing a raw query is a hard error
+ * upstream, never a silent fallback). Bit-exact relative to
+ * ggml_vec_dot_q4_polar_preht_f32_ref. */
+void ggml_vec_dot_q4_polar_preht_f32_ref(
+    int n, float * s, const block_q4_polar * x, const float * q_preht, int use_qjl);
+void ggml_vec_dot_q4_polar_preht_f32_avx2(
+    int n, float * s, const block_q4_polar * x, const float * q_preht, int use_qjl);
+void ggml_vec_dot_q4_polar_preht_f32_neon(
+    int n, float * s, const block_q4_polar * x, const float * q_preht, int use_qjl);
+
+/* Best-available pre-Hadamard dot on the running CPU. */
+void ggml_vec_dot_q4_polar_preht_f32(
+    int n, float * s, const block_q4_polar * x, const float * q_preht, int use_qjl);
+
+/* Active SIMD path string for the \`_preht\` dispatcher
+ * ("avxvnni" / "avx2" / "neon" / "ref"). */
+const char * polarquant_active_simd(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ELIZA_POLARQUANT_PREHT_H */
+`;
+
+// Trimmed runtime dispatcher: only ggml_vec_dot_q4_polar_preht_f32 +
+// polarquant_active_simd. It does NOT redefine dequantize_row_q4_polar /
+// ggml_vec_dot_q4_polar_q8_0 (those stay the fork's quants-polar.c).
+const POLARQUANT_PREHT_DISPATCH_C = `/* polarquant_preht_dispatch.c — runtime dispatch for the PolarQuant
+ * pre-Hadamard-transposed dot.
+ *
+ * AUTO-GENERATED by packages/app-core/scripts/kernel-patches/cpu-polar-kernels.mjs.
+ *
+ * Each \`_preht\` SIMD TU is compiled only on arches whose intrinsics
+ * exist (__AVX2__ on x86, __ARM_NEON on AArch64); the build sets
+ * POLARQUANT_HAVE_* so this dispatcher knows which symbols were linked.
+ * The actual choice is made at runtime from cpuid / hwcap (see
+ * polar_cpu_features.h) so an AVX-VNNI-capable binary still runs
+ * correctly on an AVX2-only host. The scalar ref body is always linked
+ * and is the exact baseline.
+ *
+ * This file deliberately does NOT define dequantize_row_q4_polar or
+ * ggml_vec_dot_q4_polar_q8_0 — the fork's ggml-cpu/quants-polar.c owns
+ * those. We only add the new \`_preht\` surface.
+ */
+#include "polarquant_preht.h"
+#include "polar_cpu_features.h"
+
+#if defined(__ARM_NEON) || defined(__ARM_NEON__)
+#  ifndef POLARQUANT_HAVE_NEON
+#    define POLARQUANT_HAVE_NEON 1
+#  endif
+#endif
+
+typedef enum {
+    POLAR_PREHT_SIMD_REF = 0,
+    POLAR_PREHT_SIMD_NEON,
+    POLAR_PREHT_SIMD_AVX2,
+    POLAR_PREHT_SIMD_AVXVNNI,
+} polar_preht_simd_t;
+
+static polar_preht_simd_t polar_preht_pick(void) {
+    polar_cpu_features_t f;
+    polar_detect_cpu(&f);
+#if defined(POLARQUANT_HAVE_AVXVNNI)
+    if (f.has_avx_vnni && f.has_avx2 && f.has_fma) return POLAR_PREHT_SIMD_AVXVNNI;
+#endif
+#if defined(POLARQUANT_HAVE_AVX2)
+    if (f.has_avx2 && f.has_fma) return POLAR_PREHT_SIMD_AVX2;
+#endif
+#if defined(POLARQUANT_HAVE_NEON)
+    if (f.has_neon) return POLAR_PREHT_SIMD_NEON;
+#endif
+    (void)f;
+    return POLAR_PREHT_SIMD_REF;
+}
+
+static polar_preht_simd_t polar_preht_simd(void) {
+    static polar_preht_simd_t cached = (polar_preht_simd_t)-1;
+    if (cached == (polar_preht_simd_t)-1) cached = polar_preht_pick();
+    return cached;
+}
+
+void ggml_vec_dot_q4_polar_preht_f32(
+    int n, float * s, const block_q4_polar * x, const float * q_preht, int use_qjl)
+{
+    switch (polar_preht_simd()) {
+#if defined(POLARQUANT_HAVE_NEON)
+        case POLAR_PREHT_SIMD_NEON:
+            ggml_vec_dot_q4_polar_preht_f32_neon(n, s, x, q_preht, use_qjl); return;
+#endif
+#if defined(POLARQUANT_HAVE_AVX2)
+        case POLAR_PREHT_SIMD_AVX2:
+        case POLAR_PREHT_SIMD_AVXVNNI:
+            ggml_vec_dot_q4_polar_preht_f32_avx2(n, s, x, q_preht, use_qjl); return;
+#endif
+        default:
+            ggml_vec_dot_q4_polar_preht_f32_ref(n, s, x, q_preht, use_qjl); return;
+    }
+}
+
+const char * polarquant_active_simd(void) {
+    switch (polar_preht_simd()) {
+        case POLAR_PREHT_SIMD_AVXVNNI: return "avxvnni";
+        case POLAR_PREHT_SIMD_AVX2:    return "avx2";
+        case POLAR_PREHT_SIMD_NEON:    return "neon";
+        default:                       return "ref";
+    }
+}
+`;
+
+function polarPrehtSourceListBlock() {
+  const lines = [];
+  for (const f of POLAR_PREHT_C_FILES) {
+    lines.push(`        ggml-cpu/polarquant/${f}`);
+  }
+  for (const f of POLAR_PREHT_H_FILES) {
+    lines.push(`        ggml-cpu/polarquant/${f}`);
+  }
+  lines.push("        ggml-cpu/polarquant/polarquant_preht.h");
+  lines.push("        ggml-cpu/polarquant/polarquant_preht_dispatch.c");
+  return lines.join("\n");
+}
+
+function escapeRe(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function mirrorPolarSources(cacheDir, { dryRun }) {
+  const srcRoot = path.join(POLAR_CPU_SRC_DIR, "src");
+  for (const f of [...POLAR_PREHT_C_FILES, ...POLAR_PREHT_H_FILES]) {
+    const src = path.join(srcRoot, f);
+    if (!fs.existsSync(src)) {
+      throw new Error(
+        `[cpu-polar-kernels] missing polarquant-cpu source: ${src}`,
+      );
+    }
+  }
+  const forkPolarDir = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-cpu",
+    "polarquant",
+  );
+  if (dryRun) {
+    console.log(
+      `[cpu-polar-kernels] (dry-run) would mirror ${POLAR_PREHT_C_FILES.length + POLAR_PREHT_H_FILES.length} polarquant ` +
+        `TUs + 2 generated shim files into ${forkPolarDir}`,
+    );
+    return;
+  }
+  fs.mkdirSync(forkPolarDir, { recursive: true });
+  for (const f of POLAR_PREHT_C_FILES) {
+    const body = fs.readFileSync(path.join(srcRoot, f), "utf8");
+    // Rewrite the standalone library include to the fork shim.
+    const rewritten = body.replace(
+      /#include "polarquant\/polarquant\.h"/g,
+      '#include "polarquant_preht.h"',
+    );
+    if (rewritten === body) {
+      throw new Error(
+        `[cpu-polar-kernels] ${f}: expected '#include "polarquant/polarquant.h"' to rewrite — ` +
+          `the standalone TU layout has changed.`,
+      );
+    }
+    fs.writeFileSync(path.join(forkPolarDir, f), rewritten, "utf8");
+  }
+  for (const f of POLAR_PREHT_H_FILES) {
+    fs.copyFileSync(path.join(srcRoot, f), path.join(forkPolarDir, f));
+  }
+  fs.writeFileSync(
+    path.join(forkPolarDir, "polarquant_preht.h"),
+    POLARQUANT_PREHT_H,
+    "utf8",
+  );
+  fs.writeFileSync(
+    path.join(forkPolarDir, "polarquant_preht_dispatch.c"),
+    POLARQUANT_PREHT_DISPATCH_C,
+    "utf8",
+  );
+}
+
+function patchGgmlCpuCMakeLists(cacheDir, { dryRun }) {
+  const cmakePath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-cpu",
+    "CMakeLists.txt",
+  );
+  if (!fs.existsSync(cmakePath)) {
+    throw new Error(`[cpu-polar-kernels] missing ${cmakePath}`);
+  }
+  const original = fs.readFileSync(cmakePath, "utf8");
+  let patched = original;
+
+  // (a) Add the polarquant/ TU list right after the fork's last Polar
+  // source line (ggml-cpu/fused-hadamard-polar-dot.c). The fresh checkout
+  // has that line; if we've already patched (sentinel present), re-normalise.
+  const prehtBlock = `        # ${SENTINEL} — PolarQuant pre-Hadamard-transposed dot TUs
+        # (mirrored from packages/native/plugins/polarquant-cpu + two
+        # generated fork-shim files). Each per-ISA body self-guards on
+        # __AVX2__ / __ARM_NEON; the dispatcher picks at runtime.
+${polarPrehtSourceListBlock()}
+`;
+  const anchorRe = /([ \t]*ggml-cpu\/fused-hadamard-polar-dot\.c\n)/;
+  if (!anchorRe.test(patched)) {
+    throw new Error(
+      `[cpu-polar-kernels] anchor 'ggml-cpu/fused-hadamard-polar-dot.c' not found in ${cmakePath}; ` +
+        `the fork layout has changed.`,
+    );
+  }
+  if (!patched.includes(`${SENTINEL} — PolarQuant pre-Hadamard`)) {
+    patched = patched.replace(anchorRe, `$1${prehtBlock}`);
+  } else {
+    // Re-normalise the block in case the file list changed: match from
+    // the sentinel comment through the last generated polarquant/ line
+    // (polarquant_preht_dispatch.c) and replace with the canonical block.
+    const sentinelBlockRe = new RegExp(
+      `[ \\t]*# ${escapeRe(SENTINEL)} — PolarQuant pre-Hadamard[\\s\\S]*?ggml-cpu\\/polarquant\\/polarquant_preht_dispatch\\.c\\n`,
+    );
+    if (!sentinelBlockRe.test(patched)) {
+      throw new Error(
+        `[cpu-polar-kernels] sentinel present in ${cmakePath} but the Polar _preht block is malformed`,
+      );
+    }
+    patched = patched.replace(sentinelBlockRe, prehtBlock);
+  }
+
+  // (b) Add ggml-cpu/polarquant to the include path + POLARQUANT_HAVE_*
+  // defines. Anchor on the QJL include-dir line installed by
+  // cpu-simd-kernels.mjs (it always runs before us in applyForkPatches).
+  // Fallback anchor: the original target_include_directories for ggml-cpu.
+  const qjlIncAnchor =
+    "    target_include_directories(${GGML_CPU_NAME} PRIVATE ggml-cpu/qjl/include ggml-cpu/qjl)";
+  const fallbackIncAnchor =
+    "    target_include_directories(${GGML_CPU_NAME} PRIVATE .)";
+  const incAnchor = patched.includes(qjlIncAnchor)
+    ? qjlIncAnchor
+    : patched.includes(fallbackIncAnchor)
+      ? fallbackIncAnchor
+      : null;
+  if (incAnchor === null) {
+    throw new Error(
+      `[cpu-polar-kernels] no usable target_include_directories anchor found in ${cmakePath}`,
+    );
+  }
+  const defsSentinel = `# ${SENTINEL} POLARQUANT include + HAVE_* defines`;
+  if (!patched.includes(defsSentinel)) {
+    const defsBlock = `${incAnchor}
+
+    ${defsSentinel}
+    # ggml-cpu/polarquant holds the PolarQuant pre-Hadamard-transposed
+    # dot TUs + the generated fork-shim header. The runtime dispatcher
+    # (polarquant_preht_dispatch.c) picks the best *available* SIMD path
+    # at run time from cpuid/hwcap, but can only call into a path whose
+    # TU was actually compiled — POLARQUANT_HAVE_* records which.
+    target_include_directories(\${GGML_CPU_NAME} PRIVATE ggml-cpu/polarquant)
+    if (GGML_SYSTEM_ARCH STREQUAL "x86")
+        target_compile_definitions(\${GGML_CPU_NAME} PRIVATE POLARQUANT_HAVE_AVX2=1)
+        if (GGML_AVX_VNNI)
+            target_compile_definitions(\${GGML_CPU_NAME} PRIVATE POLARQUANT_HAVE_AVXVNNI=1)
+        endif()
+    elseif (GGML_SYSTEM_ARCH STREQUAL "ARM")
+        target_compile_definitions(\${GGML_CPU_NAME} PRIVATE POLARQUANT_HAVE_NEON=1)
+    endif()`;
+    patched = patched.replace(incAnchor, defsBlock);
+  }
+
+  if (patched !== original && !dryRun) {
+    fs.writeFileSync(cmakePath, patched, "utf8");
+  }
+  return { path: cmakePath, changed: patched !== original };
+}
+
+export function patchCpuPolarKernels(cacheDir, { dryRun = false } = {}) {
+  if (!cacheDir || !fs.existsSync(cacheDir)) {
+    throw new Error(`[cpu-polar-kernels] cacheDir does not exist: ${cacheDir}`);
+  }
+  mirrorPolarSources(cacheDir, { dryRun });
+  const cmake = patchGgmlCpuCMakeLists(cacheDir, { dryRun });
+  console.log(
+    `[cpu-polar-kernels] ${dryRun ? "(dry-run) " : ""}mirrored ${POLAR_PREHT_C_FILES.length} ` +
+      `polarquant _preht TUs + cpu-features header + 2 generated shim files into ggml-cpu/polarquant/; ` +
+      `ggml-cpu/CMakeLists.txt ${cmake.changed ? "patched" : "already-current"} ` +
+      `(Polar _preht source list + POLARQUANT_HAVE_* defines).`,
+  );
+  return { cmake };
+}
+
+// Re-exported for parity with the QJL ggml-base link list; ggml.c does
+// not reference any `_preht` symbol, so this is empty (no Windows/Android
+// ggml-base link change needed). Kept so callers can stay symmetric.
+export const POLAR_GGML_BASE_LINK_FILES = [];

package/scripts/kernel-patches/cpu-simd-kernels.mjs

@@ -0,0 +1,253 @@
+// CPU SIMD kernel staging for the elizaOS/llama.cpp fork (v1.0.0-eliza) (Wave A1 wiring).
+//
+// What this module does:
+//
+//   1. Mirrors the verified standalone CPU SIMD translation units from
+//      packages/native/plugins/qjl-cpu/{src,include} over the fork's
+//      ggml/src/ggml-cpu/qjl/ directory. The fork's checkout is `git reset
+//      --hard`'d on every build, so this is the only place the new
+//      avx-vnni / dotprod / int8-sketch TUs (and the runtime-cpuid
+//      dispatcher rewrite) can land — direct edits to the fork don't
+//      persist. quants-qjl.c is fork-only ggml glue and is preserved.
+//
+//   2. Regenerates the QJL source list inside ggml/src/ggml-cpu/CMakeLists.txt
+//      so the new files compile, and adds the QJL_HAVE_* target
+//      compile-definitions the runtime dispatcher keys off (so a binary
+//      built with -DGGML_AVX_VNNI=ON dispatches to the AVX-VNNI int8
+//      score path, and an AArch64 build with +dotprod dispatches to the
+//      SDOT/UDOT one). The per-TU SIMD bodies self-guard on __AVX2__ /
+//      __AVXVNNI__ / __ARM_FEATURE_DOTPROD, so they're always listed and
+//      the global ARCH_FLAGS (e.g. -march=native) decide which bodies
+//      survive preprocessing.
+//
+// Idempotent: each mutation carries a `# ELIZA-CPU-SIMD-PATCH-V1`
+// sentinel; re-running the build is safe.
+//
+// Out of scope (documented in the agent report):
+//
+//   * Mirroring polarquant-cpu/ as a fork subdir. The fork's
+//     ggml-cpu/quants-polar.c + fused-q4-polar-dot*.c already define
+//     ggml_vec_dot_q4_polar_q8_0 (with a different signature) and
+//     ggml-base's ggml-quants.c already defines polar_hadamard_inplace /
+//     polar_qjl_signs / {de,}quantize_row_q4_polar_ref, so dropping
+//     polarquant-cpu's polar_dispatch.c / polar_hadamard.c / polar_qjl.c
+//     into the build would collide at link time. Wiring the new polar
+//     `_preht` AVX2/NEON path needs the fork's polar op surface rewired
+//     to the polarquant-cpu API first — a fork-source change, not a
+//     patcher.
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+function firstExistingPath(candidates) {
+  return (
+    candidates.find((candidate) => fs.existsSync(candidate)) ?? candidates[0]
+  );
+}
+
+const QJL_CPU_SRC_DIR = firstExistingPath([
+  path.resolve(__dirname, "..", "..", "..", "native", "plugins", "qjl-cpu"),
+  path.resolve(__dirname, "..", "..", "..", "native-plugins", "qjl-cpu"),
+]);
+
+const SENTINEL = "# ELIZA-CPU-SIMD-PATCH-V1";
+
+// The QJL kernel-library files mirrored into ggml-cpu/qjl/. quants-qjl.c is
+// deliberately NOT in this list — it is fork-only ggml ABI glue, not part of
+// the standalone kernel library, and must survive the mirror untouched.
+const QJL_SRC_FILES = [
+  "qjl_block.h",
+  "qjl_cpu_features.h",
+  "qjl_dispatch.c",
+  "qjl_projection.c",
+  "qjl_quantize_ref.c",
+  "qjl_quantize_avx2.c",
+  "qjl_quantize_neon.c",
+  "qjl_score_ref.c",
+  "qjl_score_i8_ref.c",
+  "qjl_score_avx2.c",
+  "qjl_score_avxvnni.c",
+  "qjl_score_neon.c",
+  "qjl_score_dotprod.c",
+];
+
+// The CMakeLists `GGML_CPU_SOURCES` lines that list the QJL kernel TUs. We
+// rewrite the whole block so adding/removing files is a single source of
+// truth (QJL_SRC_FILES above). quants-qjl.c stays first (it includes
+// ggml-common.h and pulls in the rest).
+function qjlSourceListBlock() {
+  const lines = ["        ggml-cpu/qjl/quants-qjl.c"];
+  for (const f of QJL_SRC_FILES) {
+    lines.push(`        ggml-cpu/qjl/${f}`);
+  }
+  return lines.join("\n");
+}
+
+function mirrorQjlSources(cacheDir, { dryRun }) {
+  const srcRoot = path.join(QJL_CPU_SRC_DIR, "src");
+  const incFile = path.join(QJL_CPU_SRC_DIR, "include", "qjl", "qjl.h");
+  const forkQjlDir = path.join(cacheDir, "ggml", "src", "ggml-cpu", "qjl");
+  const forkQjlIncDir = path.join(forkQjlDir, "include", "qjl");
+  if (!fs.existsSync(forkQjlDir)) {
+    throw new Error(
+      `[cpu-simd-kernels] expected fork ggml-cpu/qjl/ to exist: ${forkQjlDir}`,
+    );
+  }
+  for (const f of QJL_SRC_FILES) {
+    const src = path.join(srcRoot, f);
+    if (!fs.existsSync(src)) {
+      throw new Error(`[cpu-simd-kernels] missing qjl-cpu source: ${src}`);
+    }
+  }
+  if (!fs.existsSync(incFile)) {
+    throw new Error(`[cpu-simd-kernels] missing qjl public header: ${incFile}`);
+  }
+  if (dryRun) {
+    console.log(
+      `[cpu-simd-kernels] (dry-run) would mirror ${QJL_SRC_FILES.length} qjl TUs + qjl.h into ${forkQjlDir}`,
+    );
+    return QJL_SRC_FILES.length;
+  }
+  fs.mkdirSync(forkQjlIncDir, { recursive: true });
+  for (const f of QJL_SRC_FILES) {
+    fs.copyFileSync(path.join(srcRoot, f), path.join(forkQjlDir, f));
+  }
+  fs.copyFileSync(incFile, path.join(forkQjlIncDir, "qjl.h"));
+  return QJL_SRC_FILES.length;
+}
+
+function patchGgmlCpuCMakeLists(cacheDir, { dryRun }) {
+  const cmakePath = path.join(
+    cacheDir,
+    "ggml",
+    "src",
+    "ggml-cpu",
+    "CMakeLists.txt",
+  );
+  if (!fs.existsSync(cmakePath)) {
+    throw new Error(`[cpu-simd-kernels] missing ${cmakePath}`);
+  }
+  const original = fs.readFileSync(cmakePath, "utf8");
+  let patched = original;
+
+  // (a) Rewrite the QJL source list to the canonical set. The fork's stale
+  // block lists: quants-qjl.c, qjl_block.h, qjl_dispatch.c, qjl_projection.c,
+  // qjl_quantize_{ref,avx2,neon}.c, qjl_score_{ref,avx2,neon}.c. Match it
+  // (with or without the SENTINEL already present) and replace with the
+  // current set plus a sentinel comment.
+  const blockRe =
+    /([ \t]*ggml-cpu\/qjl\/quants-qjl\.c\n)(?:[ \t]*ggml-cpu\/qjl\/[^\n]+\n)+/;
+  const m = patched.match(blockRe);
+  if (!m) {
+    throw new Error(
+      `[cpu-simd-kernels] QJL source block not found in ${cmakePath}; ` +
+        `the fork layout has changed.`,
+    );
+  }
+  const replacementBlock = `        ${SENTINEL} — QJL kernel TUs mirrored from packages/native/plugins/qjl-cpu
+${qjlSourceListBlock()}
+`;
+  if (!patched.includes(`${SENTINEL} — QJL kernel TUs`)) {
+    patched = patched.replace(blockRe, replacementBlock);
+  } else {
+    // Sentinel present: still re-normalise the block in case QJL_SRC_FILES
+    // changed since the last run (the cached checkout is reset --hard'd
+    // anyway, but be defensive about partial states).
+    const sentinelBlockRe = new RegExp(
+      `[ \\t]*${SENTINEL.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} — QJL kernel TUs[^\\n]*\\n(?:[ \\t]*ggml-cpu\\/qjl\\/[^\\n]+\\n)+`,
+    );
+    patched = patched.replace(sentinelBlockRe, replacementBlock);
+  }
+
+  // (b) Add the QJL_HAVE_* runtime-dispatch defines. Inject right after the
+  // target_include_directories(... ggml-cpu/qjl/include ggml-cpu/qjl) line.
+  const incAnchor =
+    "    target_include_directories(${GGML_CPU_NAME} PRIVATE ggml-cpu/qjl/include ggml-cpu/qjl)";
+  if (!patched.includes(incAnchor)) {
+    throw new Error(
+      `[cpu-simd-kernels] QJL include-dir anchor not found in ${cmakePath}`,
+    );
+  }
+  const defsSentinel = `${SENTINEL} QJL_HAVE_* dispatch defines`;
+  if (!patched.includes(defsSentinel)) {
+    const defsBlock = `${incAnchor}
+
+    ${defsSentinel}
+    # The runtime QJL dispatcher (ggml-cpu/qjl/qjl_dispatch.c) picks the
+    # best *available* SIMD path at run time from cpuid/hwcap, but it can
+    # only call into a path whose TU was actually compiled. Expose the
+    # QJL_HAVE_* feature macros so the dispatcher knows which symbols were
+    # linked; the per-TU bodies still self-guard on __AVX2__ / __AVXVNNI__
+    # / __ARM_FEATURE_DOTPROD so a host without the ISA gets a dead body,
+    # never an undefined symbol.
+    if (GGML_SYSTEM_ARCH STREQUAL "x86")
+        target_compile_definitions(\${GGML_CPU_NAME} PRIVATE QJL_HAVE_AVX2=1)
+        if (GGML_AVX_VNNI)
+            # build-llama-cpp-mtp.mjs sets -DGGML_AVX_VNNI=ON only for
+            # native x86_64 hosts that report avx_vnni, which is exactly
+            # when the global ARCH_FLAGS make __AVXVNNI__ available.
+            target_compile_definitions(\${GGML_CPU_NAME} PRIVATE QJL_HAVE_AVXVNNI=1)
+        endif()
+    elseif (GGML_SYSTEM_ARCH STREQUAL "ARM")
+        target_compile_definitions(\${GGML_CPU_NAME} PRIVATE QJL_HAVE_NEON=1)
+        if (GGML_INTERNAL_DOTPROD OR GGML_USE_DOTPROD)
+            target_compile_definitions(\${GGML_CPU_NAME} PRIVATE QJL_HAVE_NEON_DOTPROD=1)
+        endif()
+    endif()`;
+    patched = patched.replace(incAnchor, defsBlock);
+  }
+
+  const linkSentinel = `${SENTINEL} QJL pthread link dependency`;
+  if (!patched.includes(linkSentinel)) {
+    const linkAnchor =
+      "    target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)";
+    if (!patched.includes(linkAnchor)) {
+      throw new Error(
+        `[cpu-simd-kernels] QJL pthread link anchor not found in ${cmakePath}`,
+      );
+    }
+    const linkBlock = `    ${linkSentinel}
+    target_link_libraries(\${GGML_CPU_NAME} PRIVATE Threads::Threads)
+
+${linkAnchor}`;
+    patched = patched.replace(linkAnchor, linkBlock);
+  }
+
+  if (patched !== original && !dryRun) {
+    fs.writeFileSync(cmakePath, patched, "utf8");
+  }
+  return { path: cmakePath, changed: patched !== original };
+}
+
+// Public entry point. Mirrors the QJL CPU SIMD TUs into the fork and wires
+// them into the ggml-cpu build. Returns a summary for the build log.
+export function patchCpuSimdKernels(cacheDir, { dryRun = false } = {}) {
+  if (!cacheDir || !fs.existsSync(cacheDir)) {
+    throw new Error(`[cpu-simd-kernels] cacheDir does not exist: ${cacheDir}`);
+  }
+  const mirrored = mirrorQjlSources(cacheDir, { dryRun });
+  const cmake = patchGgmlCpuCMakeLists(cacheDir, { dryRun });
+  console.log(
+    `[cpu-simd-kernels] ${dryRun ? "(dry-run) " : ""}mirrored ${mirrored} QJL kernel TUs + qjl.h into ggml-cpu/qjl/; ` +
+      `ggml-cpu/CMakeLists.txt ${cmake.changed ? "patched" : "already-current"} ` +
+      `(QJL source list + QJL_HAVE_* dispatch defines).`,
+  );
+  return { mirrored, cmake };
+}
+
+// Files the Windows / Android / Apple ggml-base link must also pull in (the
+// PE/COFF + `-undefined error` + lld-default `-z defs` link of a standalone
+// ggml-base needs every QJL symbol referenced by ggml.c resolved at link
+// time). quants-qjl.c (the ggml ABI glue) plus every kernel-library .c TU.
+// patchGgmlBaseForWindowsQjl() in build-llama-cpp-mtp.mjs imports this so
+// the two source lists stay in sync.
+export const QJL_GGML_BASE_LINK_FILES = [
+  "ggml-cpu/qjl/quants-qjl.c",
+  ...QJL_SRC_FILES.filter((f) => f.endsWith(".c")).map(
+    (f) => `ggml-cpu/qjl/${f}`,
+  ),
+];