@clutor/cli 0.1.0 → 0.1.1

package/c/src/kernel.c

@@ -0,0 +1,125 @@
+#include "kernel.h"
+#include <math.h>
+#include <string.h>
+
+#if !defined(CL_FORCE_SCALAR) && defined(__ARM_NEON)
+#include <arm_neon.h>
+#define CL_PATH_NEON 1
+#elif !defined(CL_FORCE_SCALAR) && defined(__AVX2__)
+#include <immintrin.h>
+#define CL_PATH_AVX2 1
+#endif
+
+const char *cl_kernel_name(void) {
+#if defined(CL_PATH_NEON) && defined(__ARM_FEATURE_DOTPROD) && !defined(CL_NO_DOTPROD)
+  return "neon+dotprod";
+#elif defined(CL_PATH_NEON)
+  return "neon";
+#elif defined(CL_PATH_AVX2)
+  return "avx2";
+#else
+  return "scalar";
+#endif
+}
+
+float cl_f16_to_f32(uint16_t h) {
+  uint32_t sign = (uint32_t)(h & 0x8000u) << 16, exp = (h >> 10) & 0x1f, man = h & 0x3ffu, bits;
+  if (exp == 0) {
+    if (man == 0) bits = sign;
+    else { /* subnormal: normalise */
+      int e = -1;
+      do { e++; man <<= 1; } while (!(man & 0x400u));
+      bits = sign | ((uint32_t)(127 - 15 - e) << 23) | ((man & 0x3ffu) << 13);
+    }
+  } else if (exp == 31) bits = sign | 0x7f800000u | (man << 13);
+  else bits = sign | ((exp + 112) << 23) | (man << 13);
+  float f;
+  memcpy(&f, &bits, sizeof f);
+  return f;
+}
+
+void cl_quantize_q8_0(const float *x, cl_block_q8_0 *y, int n) {
+  for (int b = 0; b < n / CL_QK; b++) {
+    float amax = 0;
+    for (int j = 0; j < CL_QK; j++) { float a = fabsf(x[b * CL_QK + j]); if (a > amax) amax = a; }
+    float d = amax / 127.0f, id = d != 0.0f ? 1.0f / d : 0.0f;
+    y[b].d = d;
+    for (int j = 0; j < CL_QK; j++) y[b].qs[j] = (int8_t)roundf(x[b * CL_QK + j] * id);
+  }
+}
+
+float cl_dot_q4_0_q8_0_scalar(int nblocks, const cl_block_q4_0 *w, const cl_block_q8_0 *a) {
+  float sum = 0;
+  for (int b = 0; b < nblocks; b++) {
+    int s = 0;
+    for (int j = 0; j < CL_QK / 2; j++) {
+      int lo = (w[b].qs[j] & 0x0f) - 8, hi = (w[b].qs[j] >> 4) - 8;
+      s += lo * a[b].qs[j] + hi * a[b].qs[j + CL_QK / 2];
+    }
+    sum += (float)s * (cl_f16_to_f32(w[b].d) * a[b].d);
+  }
+  return sum;
+}
+
+float cl_dot_q4_0_q8_0(int nblocks, const cl_block_q4_0 *w, const cl_block_q8_0 *a) {
+#if defined(CL_PATH_NEON)
+  const uint8x16_t m4 = vdupq_n_u8(0x0f);
+  const int8x16_t eight = vdupq_n_s8(8);
+  float sum = 0;
+  for (int b = 0; b < nblocks; b++) {
+    uint8x16_t raw = vld1q_u8(w[b].qs);
+    /* unpack: low nibbles are weights 0..15, high nibbles are weights 16..31; both shifted from 0..15 to -8..7 */
+    int8x16_t lo = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(raw, m4)), eight);
+    int8x16_t hi = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(raw, 4)), eight);
+    int8x16_t a0 = vld1q_s8(a[b].qs), a1 = vld1q_s8(a[b].qs + 16);
+#if defined(__ARM_FEATURE_DOTPROD) && !defined(CL_NO_DOTPROD)
+    int32x4_t p = vdotq_s32(vdotq_s32(vdupq_n_s32(0), lo, a0), hi, a1);
+#else
+    /* |weight| <= 8 and |activation| <= 127, so two products added in 16 bits cannot overflow (at most 2032) */
+    int16x8_t p0 = vmlal_s8(vmull_s8(vget_low_s8(lo), vget_low_s8(a0)), vget_high_s8(lo), vget_high_s8(a0));
+    int16x8_t p1 = vmlal_s8(vmull_s8(vget_low_s8(hi), vget_low_s8(a1)), vget_high_s8(hi), vget_high_s8(a1));
+    int32x4_t p = vpadalq_s16(vpaddlq_s16(p0), p1);
+#endif
+    sum += (float)vaddvq_s32(p) * (cl_f16_to_f32(w[b].d) * a[b].d);
+  }
+  return sum;
+#elif defined(CL_PATH_AVX2)
+  const __m128i m4 = _mm_set1_epi8(0x0f);
+  const __m256i eight = _mm256_set1_epi8(8), ones = _mm256_set1_epi16(1);
+  float sum = 0;
+  for (int b = 0; b < nblocks; b++) {
+    __m128i raw = _mm_loadu_si128((const __m128i *)(const void *)w[b].qs);
+    __m128i lo = _mm_and_si128(raw, m4);
+    __m128i hi = _mm_and_si128(_mm_srli_epi16(raw, 4), m4);
+    __m256i q = _mm256_sub_epi8(_mm256_inserti128_si256(_mm256_castsi128_si256(lo), hi, 1), eight); /* weights 0..31 as -8..7 */
+    __m256i av = _mm256_loadu_si256((const __m256i *)(const void *)a[b].qs);
+    /* maddubs wants unsigned x signed: use |q| and move q's sign onto the activation */
+    __m256i p16 = _mm256_maddubs_epi16(_mm256_sign_epi8(q, q), _mm256_sign_epi8(av, q));
+    __m256i p32 = _mm256_madd_epi16(p16, ones);
+    __m128i s = _mm_add_epi32(_mm256_castsi256_si128(p32), _mm256_extracti128_si256(p32, 1));
+    s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(1, 0, 3, 2)));
+    s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(2, 3, 0, 1)));
+    sum += (float)_mm_cvtsi128_si32(s) * (cl_f16_to_f32(w[b].d) * a[b].d);
+  }
+  return sum;
+#else
+  return cl_dot_q4_0_q8_0_scalar(nblocks, w, a);
+#endif
+}
+
+void cl_gemv_q4_0(const cl_block_q4_0 *w, int rows, int blocks_per_row, const cl_block_q8_0 *a, float *out) {
+  for (int r = 0; r < rows; r++) out[r] = cl_dot_q4_0_q8_0(blocks_per_row, w + (size_t)r * blocks_per_row, a);
+}
+
+uint64_t cl_q4_0_pass(const void *bytes, uint64_t n, const cl_block_q8_0 *act, int blocks_per_row) {
+  uint64_t nblocks = n / sizeof(cl_block_q4_0), rows = nblocks / (uint64_t)blocks_per_row, digest = 0;
+  const cl_block_q4_0 *w = (const cl_block_q4_0 *)bytes;
+  for (uint64_t r = 0; r < rows; r++) {
+    float v = cl_dot_q4_0_q8_0(blocks_per_row, w + r * (uint64_t)blocks_per_row, act);
+    if (!isfinite(v)) v = 0.0f; /* arbitrary bytes can hold an Inf or NaN scale */
+    uint32_t bits;
+    memcpy(&bits, &v, sizeof bits);
+    digest = digest * 1099511628211ull + bits;
+  }
+  return digest;
+}

package/c/src/kernel.h

@@ -0,0 +1,45 @@
+#ifndef CL_KERNEL_H
+#define CL_KERNEL_H
+#include <stdint.h>
+
+/*
+ * Q4_0 weights times Q8_0 activations: the inner loop of a quantized matrix-vector product.
+ *
+ * A Q4_0 block holds 32 weights as 4-bit numbers plus one half-precision scale (18 bytes). Weight j (0..15) is the low nibble of byte j and
+ * weight j+16 is the high nibble of the same byte; the value is (nibble - 8) * scale. This is ggml's layout, so a real Q4_0 tensor can be read
+ * as it is. The activations are quantized to int8 blocks of 32 with a float scale (our own format, 36 bytes), and the dot product is exact
+ * integer arithmetic inside a block, so the vector paths agree with the scalar path up to the order in which block sums are added.
+ *
+ * Compile-time paths: ARM NEON (with the dot-product instruction when the CPU target has it), AVX2, and portable C11. -DCL_FORCE_SCALAR picks
+ * the portable one on any machine; the tests use that to compare it with the vector path.
+ */
+#define CL_QK 32
+
+typedef struct { uint16_t d; uint8_t qs[16]; } cl_block_q4_0;   /* 18 bytes */
+typedef struct { float d; int8_t qs[CL_QK]; } cl_block_q8_0;    /* 36 bytes */
+_Static_assert(sizeof(cl_block_q4_0) == 18, "a Q4_0 block is 18 bytes");
+_Static_assert(sizeof(cl_block_q8_0) == 36, "a Q8_0 activation block is 36 bytes");
+
+/* The path this build uses: "neon+dotprod", "neon", "avx2" or "scalar". */
+const char *cl_kernel_name(void);
+
+float cl_f16_to_f32(uint16_t h);
+
+/* Quantizes n floats (a multiple of 32) to n/32 blocks. */
+void cl_quantize_q8_0(const float *x, cl_block_q8_0 *y, int n);
+
+/* The dot product of nblocks weight blocks with nblocks activation blocks. */
+float cl_dot_q4_0_q8_0(int nblocks, const cl_block_q4_0 *w, const cl_block_q8_0 *a);
+float cl_dot_q4_0_q8_0_scalar(int nblocks, const cl_block_q4_0 *w, const cl_block_q8_0 *a);
+
+/* rows x blocks_per_row weights times one activation vector of blocks_per_row blocks. */
+void cl_gemv_q4_0(const cl_block_q4_0 *w, int rows, int blocks_per_row, const cl_block_q8_0 *a, float *out);
+
+/*
+ * Runs the product over a buffer of streamed bytes as if it were Q4_0 rows of blocks_per_row blocks (a trailing partial row is ignored) and
+ * folds the finite results into a digest. Used by the streaming engine: the bytes of a Q4_K or IQ2 model are not Q4_0, so what is
+ * computed is not that model's maths; the point is that the loaded bytes go through a real quantized kernel.
+ */
+uint64_t cl_q4_0_pass(const void *bytes, uint64_t n, const cl_block_q8_0 *act, int blocks_per_row);
+
+#endif

package/c/src/main.c

@@ -0,0 +1,145 @@
+#define _GNU_SOURCE
+#include "gguf.h"
+#include "stream.h"
+#include "kernel.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define VERSION "0.1.0"
+
+static void usage(void) {
+  puts("clutor-engine " VERSION "\n"
+       "  info <model.gguf> [--json]       Read the header and print where every layer's bytes and experts live\n"
+       "  stream <model.gguf> [options]    Stream experts from disk through the cache for N tokens of SYNTHETIC routing and report what was measured\n"
+       "\n"
+       "stream options:\n"
+       "  --tokens N (200)  --warmup N (20)  --cache-mb N (256)  --stage N (twice the experts used)\n"
+       "  --policy lru|wtinylfu|adaptive (adaptive)   --window-pct N   --smart J,M,TAU | off (off)   --prefetch on|off (on)\n"
+       "  --dense auto|resident|ring (auto: resident when the dense weights fit in a quarter of RAM)\n  --io-threads N (2)  --seed N (1)  --locality X (1.0)  --predict-corr X (0.8)  --compute-us N (0)\n  --compute kernel|checksum (kernel: a Q4_0 x Q8_0 dot-product kernel over the loaded bytes; checksum: I/O only)\n"
+       "  --shift AT,LEN   scenario: measured tokens AT..AT+LEN-1 are uniform scan noise, the rest one tight fixed topic; --window N  hit rate and peak RSS every N tokens\n  --no-os-cache   ask the OS not to cache what is read, so reads come from the disk\n"
+       "  --json");
+}
+
+static const char *arg(int argc, char **argv, const char *name) {
+  for (int i = 0; i < argc - 1; i++) if (!strcmp(argv[i], name)) return argv[i + 1];
+  return NULL;
+}
+static int flag(int argc, char **argv, const char *name) { for (int i = 0; i < argc; i++) if (!strcmp(argv[i], name)) return 1; return 0; }
+static double num(int argc, char **argv, const char *name, double dflt) { const char *v = arg(argc, argv, name); return v ? atof(v) : dflt; }
+
+static int cmd_info(int argc, char **argv) {
+  if (argc < 1) { usage(); return 2; }
+  cl_model m; char err[256];
+  if (cl_model_open(&m, argv[0], err, sizeof err)) { fprintf(stderr, "%s\n", err); return 1; }
+  if (flag(argc, argv, "--json")) {
+    printf("{\"arch\":\"%s\",\"name\":\"%s\",\"layers\":%u,\"experts\":%u,\"expertsUsed\":%u,\"moeLayers\":%u,\"residentBytes\":%llu,\"maxDenseBytes\":%llu,\"maxExpertBytes\":%llu,\"totalBytes\":%llu,\"unsizedTensors\":%u,\"layerTable\":[",
+           m.arch, m.name, m.n_layers, m.n_experts, m.n_used, m.moe_layers, (unsigned long long)m.resident_bytes, (unsigned long long)m.max_dense_bytes, (unsigned long long)m.max_expert_bytes, (unsigned long long)m.total_bytes, m.unsized_tensors);
+    for (uint32_t l = 0; l < m.n_layers; l++) printf("%s{\"denseBytes\":%llu,\"expertBytes\":%llu,\"routerBytes\":%llu}", l ? "," : "", (unsigned long long)m.layers[l].dense_bytes, (unsigned long long)m.layers[l].expert_bytes, (unsigned long long)m.layers[l].router_bytes);
+    puts("]}");
+  } else {
+    printf("%s (%s): %u layers, %u experts (%u used), %u of them routed layers\n", m.name, m.arch, m.n_layers, m.n_experts, m.n_used, m.moe_layers);
+    printf("resident %.1f MiB, largest layer's dense weights %.1f MiB, one expert %.2f MiB, whole file %.2f GiB\n", m.resident_bytes / 1048576.0, m.max_dense_bytes / 1048576.0, m.max_expert_bytes / 1048576.0, m.total_bytes / 1073741824.0);
+  }
+  cl_model_close(&m);
+  return 0;
+}
+
+static void print_json(const cl_model *m, const cl_stream_opts *o, const cl_stream_result *r) {
+  const cl_stream_totals *t = &r->t;
+  double wall = t->wall_ns / 1e9, readS = t->read_ns_total / 1e9, wait = t->io_wait_ns / 1e9;
+  double overlap = readS > 0 ? 1.0 - wait / readS : 0;
+  if (overlap < 0) overlap = 0;
+  if (o->compute_mode == CL_COMPUTE_KERNEL) printf("{\"provenance\":{\"routing\":\"synthetic\",\"disk\":\"measured\",\"compute\":\"Q4_0 x Q8_0 dot-product kernel (%s) run over the loaded bytes, read as Q4_0 whatever they are: a real kernel, not this model's maths\"},", cl_kernel_name());
+  else printf("{\"provenance\":{\"routing\":\"synthetic\",\"disk\":\"measured\",\"compute\":\"checksum over the loaded bytes, not matrix maths\"},");
+  printf("\"model\":{\"name\":\"%s\",\"arch\":\"%s\",\"layers\":%u,\"experts\":%u,\"expertsUsed\":%u,\"expertBytes\":%llu},", m->name, m->arch, m->n_layers, m->n_experts, m->n_used, (unsigned long long)r->slot_bytes);
+  printf("\"config\":{\"policy\":\"%s\",\"cacheSlots\":%u,\"stageSlots\":%u,\"smartRoute\":%s,\"J\":%d,\"M\":%d,\"tau\":%g,\"denseMode\":\"%s\",\"densePreloadBytes\":%llu,\"prefetch\":%s,\"ioThreads\":%d,\"osCacheBypass\":%s,\"computeMode\":\"%s\",\"computeKernel\":\"%s\",\"tokens\":%d,\"warmup\":%d,\"seed\":%llu,\"locality\":%g,\"predictCorr\":%g},",
+         o->policy == CL_LRU ? "lru" : o->policy == CL_WTINYLFU ? "wtinylfu" : "adaptive", r->cache_slots, r->stage_slots, o->smart.enabled ? "true" : "false", o->smart.J, o->smart.M, o->smart.tau, r->dense_resident ? "resident" : "ring", (unsigned long long)r->dense_preload_bytes, o->prefetch ? "true" : "false", o->io_threads, o->os_cache_bypass ? "true" : "false", o->compute_mode == CL_COMPUTE_KERNEL ? "kernel" : "checksum", o->compute_mode == CL_COMPUTE_KERNEL ? cl_kernel_name() : "none", o->tokens, o->warmup, (unsigned long long)o->seed, o->locality, o->predict_corr);
+  printf("\"totals\":{\"accesses\":%llu,\"hits\":%llu,\"stageHits\":%llu,\"misses\":%llu,\"bypass\":%llu,\"hitRate\":%.4f,\"substituted\":%llu,\"droppedMass\":%.6f,\"maxDeltaP\":%.6f,\"admitRejects\":%llu,\"evictions\":%llu,",
+         (unsigned long long)t->accesses, (unsigned long long)t->hits, (unsigned long long)t->stage_hits, (unsigned long long)t->misses, (unsigned long long)t->bypass, t->accesses ? (double)t->hits / t->accesses : 0.0, (unsigned long long)t->substituted, t->dropped_mass, t->max_delta, (unsigned long long)t->admit_rejects, (unsigned long long)t->evictions);
+  printf("\"prefetchIssued\":%llu,\"prefetchLate\":%llu,\"prefetchWasted\":%llu,\"demandBytes\":%llu,\"prefetchBytes\":%llu,\"denseBytes\":%llu,",
+         (unsigned long long)t->prefetch_issued, (unsigned long long)t->prefetch_late, (unsigned long long)t->prefetch_wasted, (unsigned long long)t->demand_bytes, (unsigned long long)t->prefetch_bytes, (unsigned long long)t->dense_bytes);
+  printf("\"wallSeconds\":%.4f,\"ioWaitSeconds\":%.4f,\"computeSeconds\":%.4f,\"readSecondsTotal\":%.4f,\"overlap\":%.4f,\"tokensPerSecond\":%.3f,\"gbPerToken\":%.5f,\"missesPerToken\":%.3f,\"checksum\":\"%llx\"},",
+         wall, wait, t->compute_ns / 1e9, readS, overlap, wall > 0 ? o->tokens / wall : 0.0, o->tokens ? (double)(t->demand_bytes + t->prefetch_bytes + t->dense_bytes) / o->tokens / 1e9 : 0.0, o->tokens ? (double)t->misses / o->tokens : 0.0, (unsigned long long)t->checksum);
+  printf("\"rss\":{\"startBytes\":%llu,\"peakBytes\":%llu,\"predictedBytes\":%llu},", (unsigned long long)r->rss_start, (unsigned long long)r->rss_peak, (unsigned long long)r->rss_predicted);
+  printf("\"scenario\":{\"shiftAt\":%d,\"shiftLen\":%d},\"windows\":[", o->shift_at, o->shift_len);
+  for (int i = 0; i < r->n_windows; i++) printf("%s{\"tokEnd\":%d,\"accesses\":%llu,\"hits\":%llu,\"peakRssBytes\":%llu}", i ? "," : "", r->windows[i].tok_end, (unsigned long long)r->windows[i].accesses, (unsigned long long)r->windows[i].hits, (unsigned long long)r->windows[i].peak_rss);
+  printf("],");
+  printf("\"layers\":[");
+  for (uint32_t l = 0; l < m->n_layers; l++) {
+    const cl_layer_stats *s = &r->layers[l];
+    printf("%s{\"layer\":%u,\"accesses\":%llu,\"hits\":%llu,\"misses\":%llu,\"substituted\":%llu,\"ioWaitSeconds\":%.5f}", l ? "," : "", l, (unsigned long long)s->accesses, (unsigned long long)s->hits, (unsigned long long)s->misses, (unsigned long long)s->substituted, s->io_wait_ns / 1e9);
+  }
+  puts("]}");
+}
+
+static void print_text(const cl_model *m, const cl_stream_opts *o, const cl_stream_result *r) {
+  const cl_stream_totals *t = &r->t;
+  double wall = t->wall_ns / 1e9;
+  printf("%s: %u layers, %u experts (%u used), expert %.2f MiB\n", m->name, m->n_layers, m->n_experts, m->n_used, r->slot_bytes / 1048576.0);
+  printf("policy %s, cache %u experts (%.0f MiB) + %u staged, smart-route %s, prefetch %s, dense weights %s, %d io threads%s\n", o->policy == CL_LRU ? "lru" : o->policy == CL_WTINYLFU ? "wtinylfu" : "adaptive", r->cache_slots, r->cache_slots * (double)r->slot_bytes / 1048576.0, r->stage_slots, o->smart.enabled ? "on" : "off", o->prefetch ? "on" : "off", r->dense_resident ? "resident" : "ring (re-read every token)", o->io_threads, o->os_cache_bypass ? ", OS cache bypass requested" : "");
+  printf("MEASURED  disk reads: %.2f GB in %.2f s of read time; the compute thread waited %.2f s of %.2f s wall\n", (t->demand_bytes + t->prefetch_bytes + t->dense_bytes) / 1e9, t->read_ns_total / 1e9, t->io_wait_ns / 1e9, wall);
+  printf("MEASURED  cache: %.1f%% hits (%llu of %llu), %llu from prefetch, %llu admission rejects, %.1f misses per token\n", t->accesses ? 100.0 * t->hits / t->accesses : 0.0, (unsigned long long)t->hits, (unsigned long long)t->accesses, (unsigned long long)t->stage_hits, (unsigned long long)t->admit_rejects, o->tokens ? (double)t->misses / o->tokens : 0.0);
+  printf("MEASURED  prefetch: %llu issued, %llu arrived late, %llu never used\n", (unsigned long long)t->prefetch_issued, (unsigned long long)t->prefetch_late, (unsigned long long)t->prefetch_wasted);
+  if (o->smart.enabled) printf("MEASURED  smart-route: %llu substitutions, largest probability moved %.4f (limit %g), total %.4f\n", (unsigned long long)t->substituted, t->max_delta, o->smart.tau, t->dropped_mass);
+  if (o->compute_mode == CL_COMPUTE_KERNEL) printf("compute: Q4_0 x Q8_0 dot-product kernel (%s) over every loaded byte, %.2f s in total. The bytes are read as Q4_0 whatever their real type.\n", cl_kernel_name(), t->compute_ns / 1e9);
+  printf("passes: %d tokens in %.2f s = %.2f passes/s. This is a streaming pass over SYNTHETIC routing; the compute is %s, NOT a transformer, so this is NOT model tokens per second.\n", o->tokens, wall, wall > 0 ? o->tokens / wall : 0.0, o->compute_mode == CL_COMPUTE_KERNEL ? "a real quantized kernel on bytes that are not this model's weights" : "a checksum");
+}
+
+static int cmd_stream(int argc, char **argv) {
+  if (argc < 1) { usage(); return 2; }
+  cl_model m; char err[256];
+  if (cl_model_open(&m, argv[0], err, sizeof err)) { fprintf(stderr, "%s\n", err); return 1; }
+  cl_stream_opts o = {0};
+  o.tokens = (int)num(argc, argv, "--tokens", 200); o.warmup = (int)num(argc, argv, "--warmup", 20);
+  o.cache_bytes = (uint64_t)(num(argc, argv, "--cache-mb", 256) * 1048576.0);
+  o.stage_slots = (int)num(argc, argv, "--stage", 0);
+  const char *pol = arg(argc, argv, "--policy");
+  o.policy = !pol || !strcmp(pol, "adaptive") ? CL_ADAPTIVE : !strcmp(pol, "lru") ? CL_LRU : CL_WTINYLFU;
+  if (pol && strcmp(pol, "lru") && strcmp(pol, "wtinylfu") && strcmp(pol, "adaptive")) { fprintf(stderr, "--policy must be lru, wtinylfu or adaptive\n"); return 2; }
+  o.window_pct = (int)num(argc, argv, "--window-pct", 0);
+  const char *sm = arg(argc, argv, "--smart");
+  if (sm && strcmp(sm, "off")) {
+    int J, M; double tau;
+    if (sscanf(sm, "%d,%d,%lf", &J, &M, &tau) != 3 || J < 0 || M < 1 || tau < 0) { fprintf(stderr, "--smart wants J,M,TAU (for example 2,12,0.015) or off\n"); return 2; }
+    o.smart.enabled = 1; o.smart.J = J; o.smart.M = M; o.smart.tau = tau;
+  }
+  const char *dn = arg(argc, argv, "--dense");
+  o.dense_mode = !dn || !strcmp(dn, "auto") ? CL_DENSE_AUTO : !strcmp(dn, "resident") ? CL_DENSE_RESIDENT : !strcmp(dn, "ring") ? CL_DENSE_RING : -1;
+  if (o.dense_mode < 0) { fprintf(stderr, "--dense must be auto, resident or ring\n"); return 2; }
+  const char *pf = arg(argc, argv, "--prefetch");
+  o.prefetch = !(pf && !strcmp(pf, "off"));
+  o.io_threads = (int)num(argc, argv, "--io-threads", 2);
+  o.seed = (uint64_t)num(argc, argv, "--seed", 1);
+  o.locality = num(argc, argv, "--locality", 1.0);
+  o.predict_corr = num(argc, argv, "--predict-corr", 0.8);
+  o.compute_us = (int)num(argc, argv, "--compute-us", 0);
+  const char *cm = arg(argc, argv, "--compute");
+  o.compute_mode = !cm || !strcmp(cm, "kernel") ? CL_COMPUTE_KERNEL : !strcmp(cm, "checksum") ? CL_COMPUTE_CHECKSUM : -1;
+  if (o.compute_mode < 0) { fprintf(stderr, "--compute must be kernel or checksum\n"); return 2; }
+  o.window_tokens = (int)num(argc, argv, "--window", 0);
+  const char *sh = arg(argc, argv, "--shift");
+  if (sh) {
+    int a, b;
+    if (sscanf(sh, "%d,%d", &a, &b) != 2 || a < 0 || b < 1) { fprintf(stderr, "--shift wants AT,LEN (for example 80,10): measured tokens AT..AT+LEN-1 are uniform scan noise\n"); return 2; }
+    o.shift_at = a; o.shift_len = b;
+  }
+  o.os_cache_bypass = flag(argc, argv, "--no-os-cache");
+  if (o.tokens < 1 || o.tokens > 1000000 || o.warmup < 0 || o.predict_corr < 0 || o.predict_corr > 1) { fprintf(stderr, "bad --tokens, --warmup or --predict-corr\n"); return 2; }
+  cl_stream_result r;
+  int rc = cl_stream_run(&m, argv[0], &o, &r);
+  if (rc) { fprintf(stderr, "%s\n", r.error); cl_stream_result_free(&r); cl_model_close(&m); return 1; }
+  if (flag(argc, argv, "--json")) print_json(&m, &o, &r); else print_text(&m, &o, &r);
+  cl_stream_result_free(&r);
+  cl_model_close(&m);
+  return 0;
+}
+
+int main(int argc, char **argv) {
+  if (argc < 2) { usage(); return 2; }
+  if (!strcmp(argv[1], "--version")) { puts(VERSION); return 0; }
+  if (!strcmp(argv[1], "info")) return cmd_info(argc - 2, argv + 2);
+  if (!strcmp(argv[1], "stream")) return cmd_stream(argc - 2, argv + 2);
+  usage();
+  return strcmp(argv[1], "--help") ? 2 : 0;
+}

package/c/src/route.c

@@ -0,0 +1,52 @@
+#include "route.h"
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct { int e; double p; } ranked;
+static int by_p_desc(const void *a, const void *b) {
+  const ranked *x = a, *y = b;
+  if (x->p != y->p) return x->p < y->p ? 1 : -1;
+  return x->e - y->e; /* ties: lower expert id first, so routing is deterministic */
+}
+
+int cl_route(const float *logits, int E, int K, const cl_smart *smart, cl_resident_fn resident, void *ctx, cl_route_result *out) {
+  if (E <= 0 || E > CL_MAX_E || K <= 0 || K > CL_MAX_K || K > E) return -1;
+  ranked r[CL_MAX_E];
+  double mx = logits[0], sum = 0;
+  for (int i = 1; i < E; i++) if (logits[i] > mx) mx = logits[i];
+  for (int i = 0; i < E; i++) { r[i].e = i; r[i].p = exp((double)logits[i] - mx); sum += r[i].p; }
+  for (int i = 0; i < E; i++) r[i].p /= sum;
+  qsort(r, (size_t)E, sizeof r[0], by_p_desc);
+  memset(out, 0, sizeof *out);
+  out->k = K;
+  for (int i = 0; i < K; i++) out->selected[i] = r[i].e;
+  if (!smart || !smart->enabled || !resident) return 0;
+
+  int J = smart->J < 0 ? 0 : smart->J > K ? K : smart->J;
+  int M = smart->M < K ? K : smart->M > E ? E : smart->M;
+  char taken[CL_MAX_E] = {0};
+  int cold[CL_MAX_K], ncold = 0;
+  for (int i = 0; i < K; i++) {
+    if (i < J || resident(ctx, r[i].e)) taken[r[i].e] = 1;
+    else cold[ncold++] = i; /* rank of a true-top expert that would have to be read from disk */
+  }
+  for (int c = 0; c < ncold; c++) {
+    int rank = cold[c], pick = -1;
+    for (int j = 0; j < M; j++) { /* best-ranked resident expert not already used, within tau of the one it replaces */
+      if (taken[r[j].e] || j == rank) continue;
+      if (!resident(ctx, r[j].e)) continue;
+      if (r[rank].p - r[j].p > smart->tau) continue;
+      pick = j;
+      break;
+    }
+    if (pick < 0) { taken[r[rank].e] = 1; continue; }
+    double d = r[rank].p - r[pick].p;
+    taken[r[pick].e] = 1;
+    out->substituted++;
+    out->dropped_mass += d;
+    if (d > out->max_delta) out->max_delta = d;
+    out->selected[rank] = r[pick].e;
+  }
+  return 0;
+}

package/c/src/route.h

@@ -0,0 +1,27 @@
+/*
+ * Expert routing with optional cache awareness (SMART-ROUTE).
+ *
+ * Plain routing takes the K experts with the highest router probability. SMART-ROUTE keeps the true top-J, and for each remaining
+ * slot that would need an expert that is NOT in memory, substitutes an expert that IS in memory when it ranks within the top-M and
+ * its router probability is within `tau` of the one it replaces. With tau = infinity this is the unbounded "cache-aware routing"
+ * heuristic; with a small tau a substitution can only ever move a bounded amount of probability, which is the point.
+ */
+#ifndef CL_ROUTE_H
+#define CL_ROUTE_H
+#define CL_MAX_K 32
+#define CL_MAX_E 1024
+
+typedef struct { int enabled; int J, M; double tau; } cl_smart;
+
+typedef struct {
+  int selected[CL_MAX_K];
+  int k;
+  int substituted;      /* slots filled with a different expert than the router's own choice */
+  double dropped_mass;  /* sum over substitutions of (p of the expert replaced - p of the one used) */
+  double max_delta;     /* largest single such difference; never exceeds tau */
+} cl_route_result;
+
+typedef int (*cl_resident_fn)(void *ctx, int expert);
+/* Returns 0, or -1 if the arguments are out of range (E > CL_MAX_E, K > CL_MAX_K, K > E). */
+int cl_route(const float *logits, int E, int K, const cl_smart *smart, cl_resident_fn resident, void *ctx, cl_route_result *out);
+#endif