@clutor/cli 0.2.0 → 0.3.0

package/c/Makefile

@@ -44,22 +44,34 @@ test-avx2: build
 # Run on the default build and, to cover every kernel path, on the portable-C and NEON-without-dotprod builds.
 GOLDEN_GGUF = build/tiny_olmoe.gguf
 GOLDEN_Q3 = build/tiny_qwen3moe.gguf
+GOLDEN_GR = build/tiny_granitemoe.gguf
+GOLDEN_Q2 = build/tiny_qwen2.gguf
 golden-files: build
 	python3 test/tiny_olmoe.py write $(GOLDEN_GGUF)
 	python3 test/tiny_olmoe.py write $(GOLDEN_Q3) --arch qwen3moe
+	python3 test/tiny_olmoe.py write $(GOLDEN_GR) --arch granitemoe
+	python3 test/tiny_olmoe.py write $(GOLDEN_Q2) --arch qwen2
 golden: $(BIN) golden-files
 	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_GGUF)
 	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_Q3) --arch qwen3moe
+	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_GR) --arch granitemoe
+	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_Q2) --arch qwen2
 golden-paths: $(BIN) golden-files
 	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_GGUF)
 	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_Q3) --arch qwen3moe
+	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_GR) --arch granitemoe
+	python3 test/tiny_olmoe.py check $(BIN) $(GOLDEN_Q2) --arch qwen2
 	$(CC) $(PATH_FLAGS) -DCL_FORCE_SCALAR $(SRC) src/main.c -o build/engine-scalar $(LDLIBS)
 	python3 test/tiny_olmoe.py check build/engine-scalar $(GOLDEN_GGUF)
 	python3 test/tiny_olmoe.py check build/engine-scalar $(GOLDEN_Q3) --arch qwen3moe
+	python3 test/tiny_olmoe.py check build/engine-scalar $(GOLDEN_GR) --arch granitemoe
+	python3 test/tiny_olmoe.py check build/engine-scalar $(GOLDEN_Q2) --arch qwen2
 ifeq ($(shell uname -m),arm64)
 	$(CC) $(PATH_FLAGS) -mcpu=native -DCL_NO_DOTPROD $(SRC) src/main.c -o build/engine-neon $(LDLIBS)
 	python3 test/tiny_olmoe.py check build/engine-neon $(GOLDEN_GGUF)
 	python3 test/tiny_olmoe.py check build/engine-neon $(GOLDEN_Q3) --arch qwen3moe
+	python3 test/tiny_olmoe.py check build/engine-neon $(GOLDEN_GR) --arch granitemoe
+	python3 test/tiny_olmoe.py check build/engine-neon $(GOLDEN_Q2) --arch qwen2
 endif
 
 # Mutation fuzzing of the file readers under ASan + UBSan: corrupt a valid model many ways, expect clean errors and never a crash or hang.

package/c/src/forward.c

@@ -35,15 +35,28 @@ static float *to_float(const cl_w *w, size_t n) {
 }
 
 /* A metadata key of this model's architecture, for example "olmoe.block_count". */
+typedef struct { const char *name; int qk_mode, norm_topk, rope_pairs, expert_ff_key, scaled, dense, qkv_bias; } arch_spec;
+static const arch_spec ARCHS[] = {
+  /* name        qk-norm  renorm  pair-rope  expert_feed_forward_length  granite multipliers  dense  qkv bias */
+  {"olmoe",      1,       0,      0,         0,                          0,                   0,     0},
+  {"qwen3moe",   2,       1,      0,         1,                          0,                   0,     0},
+  {"granitemoe", 0,       1,      1,         0,                          1,                   0,     0},
+  {"qwen2",      0,       0,      0,         0,                          0,                   1,     1},
+};
+
 #define AKEY(buf, suffix) snprintf(buf, sizeof buf, "%s.%s", f->arch, suffix)
 #define REQ_U(suffix, dst) do { char k_[96]; uint64_t v_; AKEY(k_, suffix); if (!cl_meta_u64(&f->meta, k_, &v_)) { snprintf(err, errlen, "the model does not state %s", k_); goto fail; } dst = (int)v_; } while (0)
 
 int cl_fwd_open(cl_fwd *f, const char *path, int n_ctx, const cl_fwd_opts *opts, char *err, size_t errlen) {
   memset(f, 0, sizeof *f);
   if (cl_model_open(&f->model, path, err, errlen)) return 1;
-  if (strcmp(f->model.arch, "olmoe") && strcmp(f->model.arch, "qwen3moe")) { snprintf(err, errlen, "the forward pass supports the \"olmoe\" and \"qwen3moe\" architectures, not \"%s\"", f->model.arch); goto fail; }
-  snprintf(f->arch, sizeof f->arch, "%s", f->model.arch);
-  f->qk_per_head = f->norm_topk = !strcmp(f->arch, "qwen3moe");
+  const arch_spec *spec = NULL;
+  for (size_t i = 0; i < sizeof ARCHS / sizeof *ARCHS; i++) if (!strcmp(f->model.arch, ARCHS[i].name)) spec = &ARCHS[i];
+  if (!spec) { snprintf(err, errlen, "the forward pass supports the \"olmoe\", \"qwen3moe\", \"granitemoe\" and \"qwen2\" (dense) architectures, not \"%s\"", f->model.arch); goto fail; }
+  snprintf(f->arch, sizeof f->arch, "%s", spec->name);
+  f->qk_mode = spec->qk_mode; f->qk_per_head = spec->qk_mode == 2; f->norm_topk = spec->norm_topk; f->rope_pairs = spec->rope_pairs; f->dense = spec->dense; f->qkv_bias = spec->qkv_bias;
+  const int NU = spec->dense ? 1 : 0;   /* a dense model has no experts, but its single feed-forward needs the same scratch as one expert */
+  f->embed_scale = f->resid_scale = f->logit_scale = 1.0f; f->attn_scale = 0;
   int fd = open(path, O_RDONLY);
   struct stat st;
   if (fd < 0 || fstat(fd, &st)) { snprintf(err, errlen, "cannot open %s", path); if (fd >= 0) close(fd); goto fail; }
@@ -56,23 +69,33 @@ int cl_fwd_open(cl_fwd *f, const char *path, int n_ctx, const cl_fwd_opts *opts,
   REQ_U("attention.head_count", f->n_head);
   REQ_U("attention.head_count_kv", f->n_head_kv);
   { char k1[96], k2[96]; uint64_t v_; AKEY(k1, "expert_feed_forward_length"); AKEY(k2, "feed_forward_length");
-    if (cl_meta_u64(&f->meta, k1, &v_) || (!f->qk_per_head && cl_meta_u64(&f->meta, k2, &v_))) f->n_ff_exp = (int)v_; else { snprintf(err, errlen, "the model does not state its expert feed-forward length"); goto fail; } }   /* olmoe files store the per-expert width under the plain key */
+    if ((spec->expert_ff_key && cl_meta_u64(&f->meta, k1, &v_)) || (!spec->expert_ff_key && cl_meta_u64(&f->meta, k2, &v_))) f->n_ff_exp = (int)v_; else { snprintf(err, errlen, "the model does not state its expert feed-forward length"); goto fail; } }   /* olmoe files store the per-expert width under the plain key */
   REQ_U("context_length", f->n_ctx_max);
   f->n_layer = (int)f->model.n_layers; f->n_expert = (int)f->model.n_experts; f->n_used = (int)f->model.n_used;
   double d = 0;
   { char k[96];
     AKEY(k, "attention.layer_norm_rms_epsilon"); f->rms_eps = cl_meta_f64(&f->meta, k, &d) ? (float)d : 1e-5f;
     AKEY(k, "rope.freq_base"); f->rope_base = cl_meta_f64(&f->meta, k, &d) ? (float)d : 10000.0f; }
-  if (f->n_embd < 256 || f->n_head < 1 || f->n_head > f->n_embd || f->n_embd % f->n_head || f->n_layer < 1 || f->n_expert < 1 || f->n_used < 1 || f->n_used > f->n_expert || f->n_ff_exp < 1) {
+  if (f->n_embd < 256 || f->n_head < 1 || f->n_head > f->n_embd || f->n_embd % f->n_head || f->n_layer < 1 || (!f->dense && (f->n_expert < 1 || f->n_used < 1 || f->n_used > f->n_expert)) || f->n_ff_exp < 1) {
     snprintf(err, errlen, "implausible hyper-parameters (embedding %d, heads %d, layers %d, experts %d, used %d, expert width %d)", f->n_embd, f->n_head, f->n_layer, f->n_expert, f->n_used, f->n_ff_exp); goto fail;
   }
   f->head_dim = f->n_embd / f->n_head;
   { char k[96]; uint64_t v_; AKEY(k, "attention.key_length"); if (cl_meta_u64(&f->meta, k, &v_) && v_ >= 2 && v_ <= 1024 && v_ % 2 == 0) f->head_dim = (int)v_; }   /* qwen3moe: 128, not n_embd / n_head */
   if (f->n_head_kv < 1 || f->n_head_kv > f->n_head || f->n_head % f->n_head_kv) { snprintf(err, errlen, "implausible head counts (%d query heads, %d key/value heads)", f->n_head, f->n_head_kv); goto fail; }
+  { char k[96]; uint64_t v_; AKEY(k, "rope.dimension_count");
+    if (cl_meta_u64(&f->meta, k, &v_) && (int)v_ != f->head_dim) { snprintf(err, errlen, "partial rotary embeddings (%d of %d dimensions) are not implemented", (int)v_, f->head_dim); goto fail; } }
+  if (spec->scaled) {
+    char k[96]; double dv;
+    AKEY(k, "embedding_scale"); if (cl_meta_f64(&f->meta, k, &dv)) f->embed_scale = (float)dv;
+    AKEY(k, "residual_scale"); if (cl_meta_f64(&f->meta, k, &dv)) f->resid_scale = (float)dv;
+    AKEY(k, "logit_scale"); if (cl_meta_f64(&f->meta, k, &dv) && dv > 0) f->logit_scale = (float)dv;
+    AKEY(k, "attention.scale"); if (cl_meta_f64(&f->meta, k, &dv) && dv > 0) f->attn_scale = (float)dv;
+  }
   if (f->head_dim * f->n_head != f->n_embd && !f->qk_per_head) { snprintf(err, errlen, "head_dim * heads must equal the embedding width for this architecture"); goto fail; }
   f->n_q = f->n_head * f->head_dim; f->n_kvw = f->n_head_kv * f->head_dim;
+  if (f->attn_scale <= 0) f->attn_scale = 1.0f / sqrtf((float)f->head_dim);
   if (f->n_embd % 256 || f->n_ff_exp % 256 || f->n_q % 256) { snprintf(err, errlen, "embedding width %d / expert width %d / query width %d must be multiples of 256 for the K-quant kernels", f->n_embd, f->n_ff_exp, f->n_q); goto fail; }
-  if (f->n_used > 64 || f->n_expert > 4096) { snprintf(err, errlen, "unsupported expert counts"); goto fail; }
+  if (!f->dense && (f->n_used > 64 || f->n_expert > 4096)) { snprintf(err, errlen, "unsupported expert counts"); goto fail; }
   if (n_ctx > f->n_ctx_max) n_ctx = f->n_ctx_max;
   if (n_ctx < 1) n_ctx = 512;
   f->n_ctx_max = n_ctx;
@@ -80,7 +103,8 @@ int cl_fwd_open(cl_fwd *f, const char *path, int n_ctx, const cl_fwd_opts *opts,
   if (bind(f, &f->tok_embd, "token_embd.weight", -1, 2, err, errlen)) goto fail;
   f->n_vocab = f->tok_embd.rows;
   if (f->tok_embd.cols != f->n_embd || f->n_vocab < 1) { snprintf(err, errlen, "token_embd.weight is %d x %d, expected a vocabulary of rows by %d columns", f->tok_embd.rows, f->tok_embd.cols, f->n_embd); goto fail; }
-  if (bind(f, &f->output, "output.weight", -1, 2, err, errlen)) goto fail;
+  if (find(&f->model, "output.weight")) { if (bind(f, &f->output, "output.weight", -1, 2, err, errlen)) goto fail; }
+  else f->output = f->tok_embd;   /* tied embeddings: the output projection is the embedding matrix */
   cl_w on;
   if (bind(f, &on, "output_norm.weight", -1, 1, err, errlen)) goto fail;
   if (on.cols != f->n_embd || f->output.cols != f->n_embd || f->output.rows != f->n_vocab) { snprintf(err, errlen, "output tensors do not match the embedding width and vocabulary"); goto fail; }
@@ -90,44 +114,53 @@ int cl_fwd_open(cl_fwd *f, const char *path, int n_ctx, const cl_fwd_opts *opts,
   for (int l = 0; l < f->n_layer; l++) {
     cl_fwd_layer *y = &f->L[l];
     if (bind(f, &y->attn_norm, "attn_norm.weight", l, 1, err, errlen) || bind(f, &y->wq, "attn_q.weight", l, 2, err, errlen) || bind(f, &y->wk, "attn_k.weight", l, 2, err, errlen) ||
-        bind(f, &y->wv, "attn_v.weight", l, 2, err, errlen) || bind(f, &y->wo, "attn_output.weight", l, 2, err, errlen) || bind(f, &y->qn, "attn_q_norm.weight", l, 1, err, errlen) ||
-        bind(f, &y->kn, "attn_k_norm.weight", l, 1, err, errlen) || bind(f, &y->ffn_norm, "ffn_norm.weight", l, 1, err, errlen) || bind(f, &y->router, "ffn_gate_inp.weight", l, 2, err, errlen) ||
-        bind(f, &y->gate, "ffn_gate_exps.weight", l, 3, err, errlen) || bind(f, &y->up, "ffn_up_exps.weight", l, 3, err, errlen) || bind(f, &y->down, "ffn_down_exps.weight", l, 3, err, errlen)) goto fail;
+        bind(f, &y->wv, "attn_v.weight", l, 2, err, errlen) || bind(f, &y->wo, "attn_output.weight", l, 2, err, errlen) || (f->qk_mode && bind(f, &y->qn, "attn_q_norm.weight", l, 1, err, errlen)) ||
+        (f->qk_mode && bind(f, &y->kn, "attn_k_norm.weight", l, 1, err, errlen)) || bind(f, &y->ffn_norm, "ffn_norm.weight", l, 1, err, errlen)) goto fail;
+    if (f->dense) {
+      if (bind(f, &y->gate, "ffn_gate.weight", l, 2, err, errlen) || bind(f, &y->up, "ffn_up.weight", l, 2, err, errlen) || bind(f, &y->down, "ffn_down.weight", l, 2, err, errlen)) goto fail;
+    } else if (bind(f, &y->router, "ffn_gate_inp.weight", l, 2, err, errlen) || bind(f, &y->gate, "ffn_gate_exps.weight", l, 3, err, errlen) || bind(f, &y->up, "ffn_up_exps.weight", l, 3, err, errlen) || bind(f, &y->down, "ffn_down_exps.weight", l, 3, err, errlen)) goto fail;
+    if (f->qkv_bias && (bind(f, &y->bq, "attn_q.bias", l, 1, err, errlen) || bind(f, &y->bk, "attn_k.bias", l, 1, err, errlen) || bind(f, &y->bv, "attn_v.bias", l, 1, err, errlen))) goto fail;
     const int N = f->n_embd;
     const int QN = f->qk_per_head ? f->head_dim : f->n_q, KN = f->qk_per_head ? f->head_dim : f->n_kvw;
-    if (y->attn_norm.cols != N || y->ffn_norm.cols != N || y->qn.cols != QN || y->kn.cols != KN ||
+    if (y->attn_norm.cols != N || y->ffn_norm.cols != N || (f->qk_mode && (y->qn.cols != QN || y->kn.cols != KN)) ||
         y->wq.rows != f->n_q || y->wq.cols != N || y->wk.rows != f->n_kvw || y->wk.cols != N || y->wv.rows != f->n_kvw || y->wv.cols != N || y->wo.rows != N || y->wo.cols != f->n_q ||
-        y->up.rows != f->n_ff_exp || y->up.cols != N || y->gate.cols != N) {
+        y->up.rows != f->n_ff_exp || y->up.cols != N || y->gate.cols != N || (f->qkv_bias && (y->bq.cols != f->n_q || y->bk.cols != f->n_kvw || y->bv.cols != f->n_kvw))) {
       snprintf(err, errlen, "layer %d: attention or norm tensor shapes do not match the embedding width %d", l, N); goto fail;
     }
-    char nm[200];
-    snprintf(nm, sizeof nm, "blk.%d.ffn_gate_exps.weight", l);
-    const cl_tensor *tg = find(&f->model, nm);
-    if (y->router.rows != f->n_expert || y->router.cols != f->n_embd || (int)tg->dims[2] != f->n_expert || y->up.cols != N || y->gate.rows != f->n_ff_exp || y->down.rows != f->n_embd || y->down.cols != f->n_ff_exp) {
-      snprintf(err, errlen, "layer %d: tensor shapes do not match the hyper-parameters", l); goto fail;
+    if (f->dense) {
+      if (y->gate.rows != f->n_ff_exp || y->down.rows != f->n_embd || y->down.cols != f->n_ff_exp) { snprintf(err, errlen, "layer %d: feed-forward shapes do not match the hyper-parameters", l); goto fail; }
+    } else {
+      char nm[200];
+      snprintf(nm, sizeof nm, "blk.%d.ffn_gate_exps.weight", l);
+      const cl_tensor *tg = find(&f->model, nm);
+      if (y->router.rows != f->n_expert || y->router.cols != f->n_embd || (int)tg->dims[2] != f->n_expert || y->up.cols != N || y->gate.rows != f->n_ff_exp || y->down.rows != f->n_embd || y->down.cols != f->n_ff_exp) {
+        snprintf(err, errlen, "layer %d: tensor shapes do not match the hyper-parameters", l); goto fail;
+      }
+      y->gate_slab = (uint64_t)y->gate.rows * cl_row_bytes(y->gate.type, (uint64_t)y->gate.cols);
+      y->up_slab = (uint64_t)y->up.rows * cl_row_bytes(y->up.type, (uint64_t)y->up.cols);
+      y->down_slab = (uint64_t)y->down.rows * cl_row_bytes(y->down.type, (uint64_t)y->down.cols);
     }
-    y->gate_slab = (uint64_t)y->gate.rows * cl_row_bytes(y->gate.type, (uint64_t)y->gate.cols);
-    y->up_slab = (uint64_t)y->up.rows * cl_row_bytes(y->up.type, (uint64_t)y->up.cols);
-    y->down_slab = (uint64_t)y->down.rows * cl_row_bytes(y->down.type, (uint64_t)y->down.cols);
     y->attn_norm_f = to_float(&y->attn_norm, (size_t)f->n_embd); y->ffn_norm_f = to_float(&y->ffn_norm, (size_t)f->n_embd);
-    y->qn_f = to_float(&y->qn, (size_t)QN); y->kn_f = to_float(&y->kn, (size_t)KN);
-    y->router_f = malloc((size_t)f->n_expert * f->n_embd * sizeof(float));
-    if (!y->attn_norm_f || !y->ffn_norm_f || !y->qn_f || !y->kn_f || !y->router_f) { snprintf(err, errlen, "out of memory"); goto fail; }
+    if (f->qk_mode) { y->qn_f = to_float(&y->qn, (size_t)QN); y->kn_f = to_float(&y->kn, (size_t)KN); }
+    if (f->qkv_bias) { y->bq_f = to_float(&y->bq, (size_t)f->n_q); y->bk_f = to_float(&y->bk, (size_t)f->n_kvw); y->bv_f = to_float(&y->bv, (size_t)f->n_kvw); }
+    if (!f->dense) y->router_f = malloc((size_t)f->n_expert * f->n_embd * sizeof(float));
+    if (!y->attn_norm_f || !y->ffn_norm_f || (f->qk_mode && (!y->qn_f || !y->kn_f)) || (!f->dense && !y->router_f) || (f->qkv_bias && (!y->bq_f || !y->bk_f || !y->bv_f))) { snprintf(err, errlen, "out of memory"); goto fail; }
     for (int e = 0; e < f->n_expert; e++) cl_dequant_row(y->router.type, y->router.p + (size_t)e * cl_row_bytes(y->router.type, (uint64_t)f->n_embd), y->router_f + (size_t)e * f->n_embd, f->n_embd);
   }
   size_t kvn = (size_t)f->n_layer * f->n_ctx_max * f->n_kvw;
   size_t E = (size_t)f->n_embd;
   f->kc = calloc(kvn, sizeof(float)); f->vc = calloc(kvn, sizeof(float));
   f->x = calloc(E, 4); f->xn = calloc(E, 4); f->q = calloc((size_t)f->n_q, 4); f->k = calloc((size_t)f->n_kvw, 4); f->v = calloc((size_t)f->n_kvw, 4); f->att = calloc((size_t)f->n_q, 4);
-  f->scores = calloc((size_t)f->n_ctx_max, 4); f->moe = calloc(E, 4); f->hg = calloc((size_t)f->n_ff_exp * f->n_used, 4); f->hu = calloc((size_t)f->n_ff_exp * f->n_used, 4); f->hd = calloc(E * (size_t)f->n_used, 4);
-  f->rl = calloc((size_t)f->n_expert, 4); f->rlogit = calloc((size_t)f->n_expert, 4);
-  f->xq_ffk = calloc((size_t)f->n_used, sizeof(cl_xq));
+  f->scores = calloc((size_t)f->n_ctx_max, 4); f->moe = calloc(E, 4); f->hg = calloc((size_t)f->n_ff_exp * (size_t)(f->n_used + NU), 4); f->hu = calloc((size_t)f->n_ff_exp * (size_t)(f->n_used + NU), 4); f->hd = calloc(E * (size_t)(f->n_used + NU), 4);
+  f->rl = calloc((size_t)f->n_expert + 1, 4); f->rlogit = calloc((size_t)f->n_expert + 1, 4);
+  f->xq_ffk = calloc((size_t)f->n_used + 1, sizeof(cl_xq));
   if (!f->xq_ffk || !cl_xq_init(&f->xq_e, f->n_embd) || !cl_xq_init(&f->xq_q, f->n_q) || !cl_xq_init(&f->xq_ff, f->n_ff_exp)) { snprintf(err, errlen, "out of memory"); goto fail; }
   for (int k = 0; k < f->n_used; k++) if (!cl_xq_init(&f->xq_ffk[k], f->n_ff_exp)) { snprintf(err, errlen, "out of memory"); goto fail; }
   if (!f->kc || !f->vc || !f->x || !f->xn || !f->q || !f->k || !f->v || !f->att || !f->scores || !f->moe || !f->hg || !f->hu || !f->hd || !f->rl || !f->rlogit) { snprintf(err, errlen, "out of memory"); goto fail; }
+  if (f->dense && opts && (opts->cache_bytes || opts->smart.enabled)) { snprintf(err, errlen, "this model is dense (it has no experts), so there is nothing to stream or substitute: --cache-mb and --smart apply to mixture-of-experts models only"); goto fail; }
   { long pages = sysconf(_SC_PHYS_PAGES), psz = sysconf(_SC_PAGE_SIZE);
     if ((!opts || !opts->cache_bytes) && pages > 0 && psz > 0 && f->map_size > (size_t)pages * (size_t)psz * 8 / 10) {
-      snprintf(err, errlen, "the model file (%.1f GiB) is larger than 80%% of this machine's memory (%.1f GiB): mapping it whole would thrash; stream its experts with --cache-mb N", f->map_size / 1073741824.0, (double)pages * (double)psz / 1073741824.0); goto fail;
+      snprintf(err, errlen, "the model file (%.1f GiB) is larger than 80%% of this machine's memory (%.1f GiB): mapping it whole would thrash; %s", f->map_size / 1073741824.0, (double)pages * (double)psz / 1073741824.0, f->dense ? "a dense model has no experts to stream, so it needs enough memory for the whole file" : "stream its experts with --cache-mb N"); goto fail;
     } }
   if (opts && opts->smart.enabled && !opts->cache_bytes) { snprintf(err, errlen, "--smart needs --cache-mb: it substitutes experts that are already in the cache"); goto fail; }
   if (opts && opts->cache_bytes) {
@@ -159,7 +192,7 @@ fail:
 }
 
 void cl_fwd_close(cl_fwd *f) {
-  if (f->L) for (int l = 0; l < f->n_layer; l++) { free(f->L[l].attn_norm_f); free(f->L[l].ffn_norm_f); free(f->L[l].qn_f); free(f->L[l].kn_f); free(f->L[l].router_f); }
+  if (f->L) for (int l = 0; l < f->n_layer; l++) { free(f->L[l].attn_norm_f); free(f->L[l].ffn_norm_f); free(f->L[l].bq_f); free(f->L[l].bk_f); free(f->L[l].bv_f); free(f->L[l].qn_f); free(f->L[l].kn_f); free(f->L[l].router_f); }
   free(f->L); free(f->out_norm_f);
   free(f->kc); free(f->vc); free(f->x); free(f->xn); free(f->q); free(f->k); free(f->v); free(f->att); free(f->scores); free(f->moe); free(f->hg); free(f->hu); free(f->hd); free(f->rl); free(f->rlogit);
   cl_xq_free(&f->xq_e); cl_xq_free(&f->xq_q); cl_xq_free(&f->xq_ff);
@@ -181,6 +214,15 @@ static void rmsnorm(float *out, const float *x, const float *w, int n, float eps
   for (int i = 0; i < n; i++) out[i] = x[i] * s * w[i];
 }
 
+/* llama.cpp's NORM rope: rotate adjacent pairs (v[2i], v[2i+1]) */
+static void rope_pairs(float *v, int hd, int pos, float base) {
+  for (int i = 0; i < hd / 2; i++) {
+    float theta = (float)pos * powf(base, -2.0f * (float)i / (float)hd), c = cosf(theta), s = sinf(theta);
+    float a = v[2 * i], b = v[2 * i + 1];
+    v[2 * i] = a * c - b * s; v[2 * i + 1] = a * s + b * c;
+  }
+}
+
 static void rope_neox(float *v, int hd, int pos, float base) {
   int half = hd / 2;
   for (int i = 0; i < half; i++) {
@@ -202,6 +244,7 @@ int cl_fwd_step(cl_fwd *f, int token, float *logits, cl_fwd_route *route, char *
   if (f->pos >= f->n_ctx_max) { snprintf(err, errlen, "the context is full (%d tokens)", f->n_ctx_max); return 1; }
   const int E = f->n_embd, H = f->n_head, KVH = f->n_head_kv, hd = f->head_dim, pos = f->pos, KW = f->n_kvw, GROUP = f->n_head / f->n_head_kv;
   cl_dequant_row(f->tok_embd.type, f->tok_embd.p + (size_t)token * cl_row_bytes(f->tok_embd.type, (uint64_t)E), f->x, E);
+  if (f->embed_scale != 1.0f) for (int i = 0; i < E; i++) f->x[i] *= f->embed_scale;
   if (f->pred_sel) for (int i = 0; i < f->n_layer * f->n_used; i++) f->pred_sel[i] = -1;
   if (f->xs && f->prefetch) for (int k = 0; k < f->prefetch_n; k++) { cl_xs_prefetch(f->xs, 0, f->prev_sel[k]); f->pred_sel[k] = f->prev_sel[k]; }   /* layer 0, as the previous token chose it (nothing else is known yet) */
   for (int l = 0; l < f->n_layer; l++) {
@@ -211,18 +254,23 @@ int cl_fwd_step(cl_fwd *f, int token, float *logits, cl_fwd_route *route, char *
     rmsnorm(f->xn, f->x, y->attn_norm_f, E, f->rms_eps);
     cl_xq_quantize(&f->xq_e, f->xn);
     { cl_mv j[3] = {mv(&y->wq, f->xn, &f->xq_e, f->q), mv(&y->wk, f->xn, &f->xq_e, f->k), mv(&y->wv, f->xn, &f->xq_e, f->v)}; cl_matvec_batch(j, 3); }
-    if (f->qk_per_head) {   /* qwen3moe: each head's vector is normalised on its own, with one weight shared by all heads */
+    if (f->qkv_bias) {   /* qwen2: the projections carry biases */
+      for (int i = 0; i < f->n_q; i++) f->q[i] += y->bq_f[i];
+      for (int i = 0; i < f->n_kvw; i++) { f->k[i] += y->bk_f[i]; f->v[i] += y->bv_f[i]; }
+    }
+    if (f->qk_mode == 0) { /* granitemoe: no QK-norm */ }
+    else if (f->qk_per_head) {   /* qwen3moe: each head's vector is normalised on its own, with one weight shared by all heads */
       for (int h = 0; h < H; h++) rmsnorm(f->q + h * hd, f->q + h * hd, y->qn_f, hd, f->rms_eps);
       for (int h = 0; h < KVH; h++) rmsnorm(f->k + h * hd, f->k + h * hd, y->kn_f, hd, f->rms_eps);
     } else {   /* olmoe: over the whole projected vector, before it is split into heads */
       rmsnorm(f->q, f->q, y->qn_f, f->n_q, f->rms_eps);
       rmsnorm(f->k, f->k, y->kn_f, f->n_kvw, f->rms_eps);
     }
-    for (int h = 0; h < H; h++) rope_neox(f->q + h * hd, hd, pos, f->rope_base);
-    for (int h = 0; h < KVH; h++) rope_neox(f->k + h * hd, hd, pos, f->rope_base);
+    for (int h = 0; h < H; h++) (f->rope_pairs ? rope_pairs : rope_neox)(f->q + h * hd, hd, pos, f->rope_base);
+    for (int h = 0; h < KVH; h++) (f->rope_pairs ? rope_pairs : rope_neox)(f->k + h * hd, hd, pos, f->rope_base);
     float *kc = f->kc + ((size_t)l * f->n_ctx_max + pos) * KW, *vc = f->vc + ((size_t)l * f->n_ctx_max + pos) * KW;
     memcpy(kc, f->k, (size_t)KW * 4); memcpy(vc, f->v, (size_t)KW * 4);
-    const float scale = 1.0f / sqrtf((float)hd);
+    const float scale = f->attn_scale;
     for (int h = 0; h < H; h++) {
       const float *qh = f->q + h * hd;
       const int kvo = (h / GROUP) * hd;   /* grouped-query attention: GROUP query heads share one key/value head */
@@ -246,7 +294,19 @@ int cl_fwd_step(cl_fwd *f, int token, float *logits, cl_fwd_route *route, char *
     }
     cl_xq_quantize(&f->xq_q, f->att);
     { cl_mv j = mv(&y->wo, f->att, &f->xq_q, f->hd); cl_matvec_batch(&j, 1); }
-    for (int i = 0; i < E; i++) f->x[i] += f->hd[i];
+    for (int i = 0; i < E; i++) f->x[i] += f->resid_scale * f->hd[i];
+
+    if (f->dense) {   /* one plain SwiGLU feed-forward instead of a router and experts */
+      const int FF = f->n_ff_exp;
+      rmsnorm(f->xn, f->x, y->ffn_norm_f, E, f->rms_eps);
+      cl_xq_quantize(&f->xq_e, f->xn);
+      { cl_mv j[2] = {mv(&y->gate, f->xn, &f->xq_e, f->hg), mv(&y->up, f->xn, &f->xq_e, f->hu)}; cl_matvec_batch(j, 2); }
+      for (int i = 0; i < FF; i++) f->hg[i] = f->hg[i] / (1.0f + expf(-f->hg[i])) * f->hu[i];
+      cl_xq_quantize(&f->xq_ff, f->hg);
+      { cl_mv j = mv(&y->down, f->hg, &f->xq_ff, f->hd); cl_matvec_batch(&j, 1); }
+      for (int i = 0; i < E; i++) f->x[i] += f->resid_scale * f->hd[i];
+      continue;
+    }
 
     /* router-ahead: apply the NEXT layer's router to this layer's hidden state (the residual stream changes slowly) and start reading what it picks,
      * so the reads overlap this layer's experts */
@@ -344,11 +404,12 @@ int cl_fwd_step(cl_fwd *f, int token, float *logits, cl_fwd_route *route, char *
       if (f->xs) cl_xs_release(f->xs, hnd[k]);
     }
     if (f->prev_sel) memcpy(f->prev_sel + l * f->n_used, sel, (size_t)f->n_used * sizeof(int));
-    for (int i = 0; i < E; i++) f->x[i] += f->moe[i];
+    for (int i = 0; i < E; i++) f->x[i] += f->resid_scale * f->moe[i];
   }
   rmsnorm(f->xn, f->x, f->out_norm_f, E, f->rms_eps);
   cl_xq_quantize(&f->xq_e, f->xn);
   { cl_mv j = mv(&f->output, f->xn, &f->xq_e, logits); cl_matvec_batch(&j, 1); }
+  if (f->logit_scale != 1.0f) for (int i = 0; i < f->n_vocab; i++) logits[i] /= f->logit_scale;
   f->pos++;
   return 0;
 }

package/c/src/forward.h

@@ -1,5 +1,5 @@
 /*
- * A real forward pass for two mixture-of-experts architectures (llama.cpp's "olmoe" and "qwen3moe"): token embedding, RMSNorm, attention with QK-norm and
+ * A real forward pass for two mixture-of-experts architectures (llama.cpp's "olmoe", "qwen3moe" and "granitemoe"): token embedding, RMSNorm, attention with QK-norm and
  * NeoX rotary embeddings (grouped-query for qwen3moe), a softmax router over every expert, the top-k experts' SwiGLU feed-forwards, and the output projection.
  * Weights are read straight from the memory-mapped GGUF and dequantized block by block; the arithmetic is float32.
  *
@@ -18,7 +18,9 @@
 typedef struct { const uint8_t *p; uint32_t type; int rows, cols; } cl_w;   /* rows x cols weights of `type`, row-major (cols is the input width) */
 
 typedef struct {
-  cl_w attn_norm, wq, wk, wv, wo, qn, kn, ffn_norm, router, gate, up, down; /* gate/up/down are the fused expert tensors */
+  cl_w attn_norm, wq, wk, wv, wo, qn, kn, ffn_norm, router, gate, up, down; /* gate/up/down are the fused expert tensors, or (dense models) the plain feed-forward matrices */
+  cl_w bq, bk, bv;                          /* qwen2: biases of the query, key and value projections */
+  float *bq_f, *bk_f, *bv_f;
   float *attn_norm_f, *qn_f, *kn_f, *ffn_norm_f, *router_f;
   uint64_t gate_slab, up_slab, down_slab;   /* bytes of one expert in each fused tensor */
 } cl_fwd_layer;
@@ -27,11 +29,16 @@ typedef struct {
   cl_model model;
   cl_meta meta;
   uint8_t *map; size_t map_size;
-  char arch[16];           /* "olmoe" or "qwen3moe" */
+  char arch[16];           /* "olmoe", "qwen3moe" or "granitemoe" */
   int n_layer, n_embd, n_head, n_head_kv, head_dim, n_ff_exp, n_expert, n_used, n_vocab, n_ctx_max;
   int n_q, n_kvw;          /* width of the query projection (n_head * head_dim) and of the key / value projections (n_head_kv * head_dim) */
-  int qk_per_head;         /* qwen3moe: RMSNorm over each head_dim vector with one shared weight; olmoe: over the whole projected vector */
-  int norm_topk;           /* qwen3moe renormalises the chosen experts' weights to sum to 1; olmoe uses the raw softmax probabilities */
+  int qk_mode;             /* 0: no QK-norm (granitemoe); 1: RMSNorm over the whole projected vector (olmoe); 2: over each head_dim vector with one shared weight (qwen3moe) */
+  int qk_per_head;         /* qk_mode == 2 */
+  int dense;               /* qwen2: no experts, every layer has one plain SwiGLU feed-forward of width n_ff_exp */
+  int qkv_bias;            /* the query, key and value projections have bias vectors */
+  int rope_pairs;          /* 1: rotate adjacent pairs (2i, 2i+1) (llama.cpp's NORM rope, granitemoe); 0: rotate the two halves (NEOX: olmoe, qwen3moe) */
+  float embed_scale, attn_scale, resid_scale, logit_scale; /* granite's multipliers; 1, 1/sqrt(head_dim), 1, 1 for the others */
+  int norm_topk;           /* qwen3moe and granitemoe renormalise the chosen experts' weights to sum to 1; olmoe uses the raw softmax probabilities */
   float rms_eps, rope_base;
   cl_w tok_embd, output;
   float *out_norm_f;

package/c/src/main.c

@@ -12,7 +12,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#define VERSION "0.2.0"
+#define VERSION "0.3.0"
 
 static void usage(void) {
   puts("clutor-engine " VERSION "\n"

package/c/src/tokenizer.c

@@ -25,7 +25,7 @@ struct cl_tok {
   smap vocab, merges;
   char *merge_buf;
   int bos, eos, add_bos;
-  int pre;                           /* 0: olmo (GPT-2 style), 1: qwen2 */
+  int pre;                           /* 0: olmo (GPT-2 style), 1: qwen2, 2: refact (every digit alone, then GPT-2 style) */
   int *special; int n_special;       /* ids of control / user-defined tokens, longest text first */
   int byte_cp[256];                  /* GPT-2 byte -> code point */
   int cp_byte[512];                  /* code point (< 512) -> byte, or -1 */
@@ -141,6 +141,11 @@ static size_t m_qwen2(const uint8_t *s, size_t i, size_t end) {
   return we - i;                                                           /* \s+ : a single space before text */
 }
 
+static size_t m_one_digit(const uint8_t *s, size_t i, size_t end) {   /* \p{N}: a single digit */
+  int cp, n = cp_at(s, i, end, &cp);
+  return is_digit(cp) ? (size_t)n : 0;
+}
+
 typedef size_t (*matcher)(const uint8_t *, size_t, size_t);
 typedef struct { size_t a, b; } seg;
 typedef struct { seg *v; size_t n, cap; } segs;
@@ -168,8 +173,12 @@ cl_tok *cl_tok_load(const cl_meta *m, char *err, size_t errlen) {
   cl_str mdl, pre;
   if (!cl_meta_str(m, "tokenizer.ggml.model", &mdl) || mdl.len != 4 || memcmp(mdl.p, "gpt2", 4)) { snprintf(err, errlen, "only the gpt2 (byte-level BPE) tokenizer is supported"); return NULL; }
   int pre_kind = -1;
-  if (cl_meta_str(m, "tokenizer.ggml.pre", &pre)) { if (pre.len == 4 && !memcmp(pre.p, "olmo", 4)) pre_kind = 0; else if (pre.len == 5 && !memcmp(pre.p, "qwen2", 5)) pre_kind = 1; }
-  if (pre_kind < 0) { snprintf(err, errlen, "only the \"olmo\" and \"qwen2\" pre-tokenizers are implemented"); return NULL; }
+  if (cl_meta_str(m, "tokenizer.ggml.pre", &pre)) {
+    if (pre.len == 4 && !memcmp(pre.p, "olmo", 4)) pre_kind = 0;
+    else if (pre.len == 5 && !memcmp(pre.p, "qwen2", 5)) pre_kind = 1;
+    else if (pre.len == 6 && !memcmp(pre.p, "refact", 6)) pre_kind = 2;
+  }
+  if (pre_kind < 0) { snprintf(err, errlen, "only the \"olmo\", \"qwen2\" and \"refact\" pre-tokenizers are implemented"); return NULL; }
   cl_tok *t = calloc(1, sizeof *t);
   if (!t) { snprintf(err, errlen, "out of memory"); return NULL; }
   t->pre = pre_kind;
@@ -246,7 +255,7 @@ static int bpe(const cl_tok *t, const uint8_t *s, size_t len, int32_t *out, int
 static int encode_plain(const cl_tok *t, const uint8_t *s, size_t len, int32_t *out, int cap, int *n) {
   segs a = {0}, b = {0}, c = {0}, d = {0};
   int rc = 1;
-  if (!segs_add(&a, 0, len) || !split(s, &a, &d, t->pre == 1 ? m_qwen2 : m_gpt2)) rc = 0;
+  if (!segs_add(&a, 0, len) || !(t->pre == 2 ? (split(s, &a, &b, m_one_digit) && split(s, &b, &d, m_gpt2)) : split(s, &a, &d, t->pre == 1 ? m_qwen2 : m_gpt2))) rc = 0;
   for (size_t i = 0; rc == 1 && i < d.n; i++) rc = bpe(t, s + d.v[i].a, d.v[i].b - d.v[i].a, out, cap, n);
   free(a.v); free(b.v); free(c.v); free(d.v);
   return rc;

package/c/src/tokenizer.h

@@ -1,6 +1,6 @@
 /*
  * Byte-level BPE tokenizer (GPT-2 family) built from a GGUF file's own tokenizer tables. The pre-tokenizers implemented are the ones llama.cpp
- * calls "olmo" and "qwen2". Letter, number and punctuation classes are exact for ASCII and approximate above U+007F (see tokenizer.c), so non-English
+ * calls "olmo", "qwen2" and "refact". Letter, number and punctuation classes are exact for ASCII and approximate above U+007F (see tokenizer.c), so non-English
  * text can tokenize differently from llama.cpp; `make verify-llama` checks agreement on a fixed set of prompts.
  */
 #ifndef CL_TOKENIZER_H

package/dist/bin.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   main
-} from "./chunk-2FBPYNHP.js";
+} from "./chunk-ODS6ND7T.js";
 import "./chunk-U5GHVJJO.js";
 
 // src/bin.ts

package/dist/{chunk-2FBPYNHP.js → chunk-ODS6ND7T.js}

@@ -382,7 +382,9 @@ async function listLocalModels(dir) {
 var GiB_ = 2 ** 30;
 var DOWNLOADS = [
   { id: "olmoe-1b-7b-q4_k_m", catalogId: "olmoe-1b-7b", repo: "allenai/OLMoE-1B-7B-0924-Instruct-GGUF", file: "olmoe-1b-7b-0924-instruct-q4_k_m.gguf", bytes: 4213512672, sha256: "8c310f1435a1222338fd2d3d974975be9cd908180b644bab0c2a94da1ac32f3f", note: "a real mixture-of-experts model (64 experts, 8 used) that fits in memory: the one `clutor generate` was validated on" },
+  { id: "granite-3.0-3b-a800m-q4_k_m", catalogId: "granite-3.0-3b-a800m", repo: "bartowski/granite-3.0-3b-a800m-instruct-GGUF", file: "granite-3.0-3b-a800m-instruct-Q4_K_M.gguf", bytes: 2059356e3, sha256: "aa6287bafd182d06a6d11b0d001458deec902e8c0522875fc20cde2f561ebf6d", note: "IBM Granite MoE (40 experts, 8 used, 1.9 GiB): the third architecture the engine can run with `clutor generate`" },
   { id: "qwen3-30b-a3b-q4_k_m", catalogId: "qwen3-30b-a3b", repo: "Qwen/Qwen3-30B-A3B-GGUF", file: "Qwen3-30B-A3B-Q4_K_M.gguf", bytes: 18556685824, sha256: "0d003f6662faee786ed5da3e31b29c978de5ae5d275c8794c606a7f3c01aa8f5", note: "a mixture-of-experts model (128 experts, 8 used) LARGER than 16 GiB of RAM: run it with `clutor generate ... --cache-mb 4096`" },
+  { id: "qwen2.5-1.5b-q4_k_m", catalogId: "qwen2.5-1.5b", repo: "Qwen/Qwen2.5-1.5B-Instruct-GGUF", file: "qwen2.5-1.5b-instruct-q4_k_m.gguf", bytes: 1117320736, sha256: "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e", note: "a small dense model (1.0 GiB): the fourth architecture the engine can run with `clutor generate`" },
   { id: "qwen2.5-coder-7b-q4_k_m", catalogId: "qwen2.5-coder-7b", repo: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", file: "qwen2.5-coder-7b-instruct-q4_k_m.gguf", bytes: 4683073536, sha256: "509287f78cb4d4cf6b3843734733b914b2c158e43e22a7f4bf5e963800894d3c", note: "a dense coding model for the agent (not a mixture of experts)" }
 ];
 var downloadUrl = (d) => `https://huggingface.co/${d.repo}/resolve/main/${d.file}`;
@@ -947,6 +949,7 @@ function createClutorServer(o) {
         if (e instanceof DecideError) throw new HttpError(e.status, e.message);
         throw e;
       }
+      await Promise.all(ready);
       const root = roots.values().next().value;
       const receipt = root ? await sealDecision(root, model, b.prompt, b.choices, result, startedAt) : null;
       return reply(200, { choice: result.choice, probabilities: result.probabilities, probabilityMassOnChoices: result.probabilityMassOnChoices, latencyMs: result.latencyMs, receipt });
@@ -2369,4 +2372,4 @@ export {
   main,
   startBackendIfConfigured
 };
-//# sourceMappingURL=chunk-2FBPYNHP.js.map
+//# sourceMappingURL=chunk-ODS6ND7T.js.map