@clutor/cli 0.1.0 → 0.2.0

package/c/src/tokenizer.c

@@ -0,0 +1,287 @@
+#include "tokenizer.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* ---- string -> int hash table (open addressing; keys are views into the mapped file or into our own buffer) ---- */
+typedef struct { const char *k; uint32_t len; int32_t v; } slot;
+typedef struct { slot *s; size_t cap; } smap;
+
+static uint64_t hash(const char *p, size_t n) { uint64_t h = 1469598103934665603ull; for (size_t i = 0; i < n; i++) { h ^= (uint8_t)p[i]; h *= 1099511628211ull; } return h; }
+static int smap_init(smap *m, size_t n) { m->cap = 1; while (m->cap < n * 2 + 2) m->cap <<= 1; m->s = calloc(m->cap, sizeof(slot)); return m->s != NULL; }
+static void smap_put(smap *m, const char *k, size_t len, int32_t v) {
+  size_t i = hash(k, len) & (m->cap - 1);
+  while (m->s[i].k) { if (m->s[i].len == len && !memcmp(m->s[i].k, k, len)) return; i = (i + 1) & (m->cap - 1); }   /* first definition wins, as in llama.cpp */
+  m->s[i].k = k; m->s[i].len = (uint32_t)len; m->s[i].v = v;
+}
+static int32_t smap_get(const smap *m, const char *k, size_t len) {
+  size_t i = hash(k, len) & (m->cap - 1);
+  while (m->s[i].k) { if (m->s[i].len == len && !memcmp(m->s[i].k, k, len)) return m->s[i].v; i = (i + 1) & (m->cap - 1); }
+  return -1;
+}
+
+struct cl_tok {
+  cl_str *tokens; int32_t *types; uint64_t n;
+  smap vocab, merges;
+  char *merge_buf;
+  int bos, eos, add_bos;
+  int pre;                           /* 0: olmo (GPT-2 style), 1: qwen2 */
+  int *special; int n_special;       /* ids of control / user-defined tokens, longest text first */
+  int byte_cp[256];                  /* GPT-2 byte -> code point */
+  int cp_byte[512];                  /* code point (< 512) -> byte, or -1 */
+};
+
+static int put_utf8(char *o, int cp) {
+  if (cp < 0x80) { o[0] = (char)cp; return 1; }
+  if (cp < 0x800) { o[0] = (char)(0xC0 | (cp >> 6)); o[1] = (char)(0x80 | (cp & 63)); return 2; }
+  o[0] = (char)(0xE0 | (cp >> 12)); o[1] = (char)(0x80 | ((cp >> 6) & 63)); o[2] = (char)(0x80 | (cp & 63)); return 3;
+}
+/* Decodes one code point; an invalid byte gives U+FFFD and length 1. */
+static int get_utf8(const uint8_t *p, size_t avail, int *cp) {
+  uint8_t c = p[0];
+  if (c < 0x80) { *cp = c; return 1; }
+  int n = c >= 0xF0 ? 4 : c >= 0xE0 ? 3 : c >= 0xC0 ? 2 : 0;
+  if (!n || (size_t)n > avail) { *cp = 0xFFFD; return 1; }
+  int v = c & (0x3F >> (n - 1));
+  for (int i = 1; i < n; i++) { if ((p[i] & 0xC0) != 0x80) { *cp = 0xFFFD; return 1; } v = (v << 6) | (p[i] & 63); }
+  *cp = v;
+  return n;
+}
+
+/* ---- character classes: exact for ASCII, approximate above it ---- */
+static int is_space(int c) { return c == ' ' || (c >= 9 && c <= 13) || c == 0x85 || c == 0xA0 || (c >= 0x2000 && c <= 0x200A) || c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000 || c == 0x1680; }
+static int is_digit(int c) { return (c >= '0' && c <= '9') || c == 0xB2 || c == 0xB3 || c == 0xB9 || (c >= 0xBC && c <= 0xBE) || (c >= 0x660 && c <= 0x669) || (c >= 0xFF10 && c <= 0xFF19); }
+static int is_punct(int c) {   /* Unicode P*: ASCII punctuation except the symbols $ + < = > ^ ` | ~ */
+  if (c < 0x80) return c && strchr("!\"#%&'()*,-./:;?@[\\]_{}", c) != NULL;
+  return c == 0xA1 || c == 0xA7 || c == 0xAB || c == 0xB6 || c == 0xB7 || c == 0xBB || c == 0xBF || (c >= 0x2010 && c <= 0x2027) || (c >= 0x2030 && c <= 0x205E) || (c >= 0x3001 && c <= 0x3003) || (c >= 0x3008 && c <= 0x3011) || (c >= 0xFF01 && c <= 0xFF0F && c != 0xFF04 && c != 0xFF0B);
+}
+static int is_letter(int c) {
+  if (c < 0x80) return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+  if (c == 0xAA || c == 0xB5 || c == 0xBA) return 1;
+  if (c < 0xC0 || c == 0xD7 || c == 0xF7) return 0;   /* Latin-1 symbols */
+  if (is_space(c) || is_digit(c) || is_punct(c)) return 0;
+  if ((c >= 0x2000 && c <= 0x2BFF) || (c >= 0x3000 && c <= 0x303F) || (c >= 0xFE00 && c <= 0xFE0F) || (c >= 0x1F000 && c <= 0x1FAFF) || (c >= 0xFF00 && c <= 0xFF0F)) return 0;   /* punctuation, symbols, emoji */
+  return 1;
+}
+
+/* ---- the three splitting patterns of the "olmo" pre-tokenizer; each returns the length of the match at s[i] (0 = no match) ---- */
+static int cp_at(const uint8_t *s, size_t i, size_t end, int *cp) { return get_utf8(s + i, end - i, cp); }
+
+static size_t run(const uint8_t *s, size_t i, size_t end, int (*in)(int), int negate_all_three) {
+  size_t j = i;
+  while (j < end) {
+    int cp, n = cp_at(s, j, end, &cp);
+    int ok = negate_all_three ? !(is_space(cp) || is_letter(cp) || is_digit(cp)) : in(cp);
+    if (!ok) break;
+    j += (size_t)n;
+  }
+  return j - i;
+}
+static size_t m_gpt2(const uint8_t *s, size_t i, size_t end) {   /* 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S) */
+  static const char *contr[] = {"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"};
+  for (size_t c = 0; c < sizeof contr / sizeof *contr; c++) { size_t n = strlen(contr[c]); if (end - i >= n && !memcmp(s + i, contr[c], n)) return n; }
+  size_t j = i;
+  if (s[j] == ' ') j++;
+  if (j < end) {
+    size_t n;
+    if ((n = run(s, j, end, is_letter, 0))) return j - i + n;
+    if ((n = run(s, j, end, is_digit, 0))) return j - i + n;
+    if ((n = run(s, j, end, NULL, 1))) return j - i + n;
+  }
+  /* \s+(?!\S): the longest whitespace run that is followed by whitespace or the end; a run followed by text gives back its last character */
+  size_t k = i, last = i;
+  while (k < end) { int cp, n = cp_at(s, k, end, &cp); if (!is_space(cp)) break; last = k; k += (size_t)n; }
+  if (k == i) return 0;
+  if (k == end) return k - i;
+  return last > i ? last - i : 0;
+}
+
+static int is_nl(int c) { return c == '\r' || c == '\n'; }
+static size_t ws_run_end(const uint8_t *s, size_t i, size_t end) {   /* end of the whitespace run starting at i */
+  size_t k = i;
+  while (k < end) { int cp, n = cp_at(s, k, end, &cp); if (!is_space(cp)) break; k += (size_t)n; }
+  return k;
+}
+/* Qwen2's pre-tokenizer: (?i:'s|'t|'re|'ve|'m|'ll|'d) | [^\r\n\p{L}\p{N}]?\p{L}+ | \p{N} | ?[^\s\p{L}\p{N}]+[\r\n]* | \s*[\r\n]+ | \s+(?!\S) | \s+ */
+static size_t m_qwen2(const uint8_t *s, size_t i, size_t end) {
+  static const char *contr[] = {"'s", "'t", "'m", "'d", "'re", "'ve", "'ll"};
+  for (size_t c = 0; c < sizeof contr / sizeof *contr; c++) {
+    size_t n = strlen(contr[c]);
+    if (end - i < n) continue;
+    size_t k = 0;
+    while (k < n && (s[i + k] == (uint8_t)contr[c][k] || (k > 0 && s[i + k] >= 'A' && s[i + k] <= 'Z' && s[i + k] + 32 == (uint8_t)contr[c][k]))) k++;
+    if (k == n) return n;
+  }
+  int cp, n0 = cp_at(s, i, end, &cp);
+  if (is_letter(cp)) return run(s, i, end, is_letter, 0);                 /* the optional leading character is absent */
+  if (!is_nl(cp) && !is_digit(cp)) {                                       /* one leading character that is not a letter, digit or newline, then letters */
+    size_t m = run(s, i + (size_t)n0, end, is_letter, 0);
+    if (m) return (size_t)n0 + m;
+  }
+  if (is_digit(cp)) return (size_t)n0;                                     /* digits are one token each */
+  size_t j = i;
+  if (s[j] == ' ') j++;
+  if (j < end) {
+    size_t m = run(s, j, end, NULL, 1);
+    if (m) {
+      size_t k = j + m;
+      while (k < end && (s[k] == '\r' || s[k] == '\n')) k++;
+      return k - i;
+    }
+  }
+  size_t we = ws_run_end(s, i, end);
+  if (we == i) return 0;
+  size_t last_nl = 0; int has_nl = 0;
+  for (size_t k = i; k < we;) { int c2, n = cp_at(s, k, end, &c2); if (is_nl(c2)) { last_nl = k + (size_t)n; has_nl = 1; } k += (size_t)n; }
+  if (has_nl) return last_nl - i;                                          /* \s*[\r\n]+ : up to the last newline of the run */
+  if (we == end) return we - i;                                            /* \s+(?!\S) at the end of the text */
+  size_t lastc = i;
+  for (size_t k = i; k < we;) { lastc = k; int c2, n = cp_at(s, k, end, &c2); k += (size_t)n; }
+  if (lastc > i) return lastc - i;                                         /* \s+(?!\S) gives the last space back to the next token */
+  return we - i;                                                           /* \s+ : a single space before text */
+}
+
+typedef size_t (*matcher)(const uint8_t *, size_t, size_t);
+typedef struct { size_t a, b; } seg;
+typedef struct { seg *v; size_t n, cap; } segs;
+static int segs_add(segs *l, size_t a, size_t b) {
+  if (a >= b) return 1;
+  if (l->n == l->cap) { l->cap = l->cap ? l->cap * 2 : 64; l->v = realloc(l->v, l->cap * sizeof(seg)); if (!l->v) return 0; }
+  l->v[l->n++] = (seg){a, b};
+  return 1;
+}
+/* Applies one pattern to every segment: matches become their own segments, and what lies between matches stays a segment for the next pattern. */
+static int split(const uint8_t *s, const segs *in, segs *out, matcher m) {
+  for (size_t k = 0; k < in->n; k++) {
+    size_t a = in->v[k].a, end = in->v[k].b, gap = a, i = a;
+    while (i < end) {
+      size_t n = m(s, i, end);
+      if (n) { if (!segs_add(out, gap, i) || !segs_add(out, i, i + n)) return 0; i += n; gap = i; }
+      else { int cp; i += (size_t)cp_at(s, i, end, &cp); }
+    }
+    if (!segs_add(out, gap, end)) return 0;
+  }
+  return 1;
+}
+
+cl_tok *cl_tok_load(const cl_meta *m, char *err, size_t errlen) {
+  cl_str mdl, pre;
+  if (!cl_meta_str(m, "tokenizer.ggml.model", &mdl) || mdl.len != 4 || memcmp(mdl.p, "gpt2", 4)) { snprintf(err, errlen, "only the gpt2 (byte-level BPE) tokenizer is supported"); return NULL; }
+  int pre_kind = -1;
+  if (cl_meta_str(m, "tokenizer.ggml.pre", &pre)) { if (pre.len == 4 && !memcmp(pre.p, "olmo", 4)) pre_kind = 0; else if (pre.len == 5 && !memcmp(pre.p, "qwen2", 5)) pre_kind = 1; }
+  if (pre_kind < 0) { snprintf(err, errlen, "only the \"olmo\" and \"qwen2\" pre-tokenizers are implemented"); return NULL; }
+  cl_tok *t = calloc(1, sizeof *t);
+  if (!t) { snprintf(err, errlen, "out of memory"); return NULL; }
+  t->pre = pre_kind;
+  t->tokens = cl_meta_str_array(m, "tokenizer.ggml.tokens", &t->n);
+  uint64_t nt = 0, nm = 0;
+  t->types = cl_meta_i32_array(m, "tokenizer.ggml.token_type", &nt);
+  cl_str *merges = cl_meta_str_array(m, "tokenizer.ggml.merges", &nm);
+  if (!t->tokens || !merges) { snprintf(err, errlen, "the model has no tokenizer tables"); free(merges); cl_tok_free(t); return NULL; }
+  if (!smap_init(&t->vocab, t->n) || !smap_init(&t->merges, nm)) { snprintf(err, errlen, "out of memory"); free(merges); cl_tok_free(t); return NULL; }
+  for (uint64_t i = 0; i < t->n; i++) smap_put(&t->vocab, t->tokens[i].p, t->tokens[i].len, (int32_t)i);
+  for (uint64_t i = 0; i < nm; i++) smap_put(&t->merges, merges[i].p, merges[i].len, (int32_t)i);   /* "left right": the rank is the line number */
+  free(merges);
+  uint64_t v;
+  t->bos = cl_meta_u64(m, "tokenizer.ggml.bos_token_id", &v) ? (int)v : -1;
+  t->eos = cl_meta_u64(m, "tokenizer.ggml.eos_token_id", &v) ? (int)v : -1;
+  t->add_bos = cl_meta_u64(m, "tokenizer.ggml.add_bos_token", &v) ? (int)v : 0;
+  int n = 0;
+  for (int i = 0; i < 512; i++) t->cp_byte[i] = -1;
+  for (int b = 0; b < 256; b++) {
+    int printable = (b >= 33 && b <= 126) || (b >= 161 && b <= 172) || (b >= 174 && b <= 255);
+    t->byte_cp[b] = printable ? b : 256 + n++;
+    t->cp_byte[t->byte_cp[b]] = b;
+  }
+  t->special = malloc((t->n + 1) * sizeof(int));
+  if (t->types && t->special) for (uint64_t i = 0; i < t->n && i < nt; i++) if ((t->types[i] == 3 || t->types[i] == 4) && t->tokens[i].len > 0) t->special[t->n_special++] = (int)i;
+  for (int a = 0; a < t->n_special; a++) for (int b = a + 1; b < t->n_special; b++)   /* longest first */
+    if (t->tokens[t->special[b]].len > t->tokens[t->special[a]].len) { int x = t->special[a]; t->special[a] = t->special[b]; t->special[b] = x; }
+  return t;
+}
+
+void cl_tok_free(cl_tok *t) {
+  if (!t) return;
+  free(t->tokens); free(t->types); free(t->vocab.s); free(t->merges.s); free(t->special);
+  free(t);
+}
+int cl_tok_vocab_size(const cl_tok *t) { return (int)t->n; }
+int cl_tok_bos(const cl_tok *t) { return t->bos; }
+int cl_tok_eos(const cl_tok *t) { return t->eos; }
+int cl_tok_add_bos(const cl_tok *t) { return t->add_bos; }
+
+/* Byte-pair-encodes one pre-token and appends its ids. */
+static int bpe(const cl_tok *t, const uint8_t *s, size_t len, int32_t *out, int cap, int *n) {
+  if (len == 0) return 1;
+  char *buf = malloc(len * 3 + 1);
+  size_t *st = malloc((len + 1) * sizeof(size_t));   /* start offset of each symbol in buf, plus the end */
+  if (!buf || !st) { free(buf); free(st); return 0; }
+  size_t pos = 0, ns = 0;
+  for (size_t i = 0; i < len; i++) { st[ns++] = pos; pos += (size_t)put_utf8(buf + pos, t->byte_cp[s[i]]); }
+  st[ns] = pos;
+  for (;;) {   /* merge the adjacent pair with the lowest rank until none is in the merge table */
+    int32_t best = -1; size_t bi = 0;
+    for (size_t i = 0; i + 1 < ns; i++) {
+      size_t la = st[i + 1] - st[i], lb = st[i + 2] - st[i + 1];
+      char key[256];
+      if (la + lb + 1 > sizeof key) continue;
+      memcpy(key, buf + st[i], la); key[la] = ' '; memcpy(key + la + 1, buf + st[i + 1], lb);
+      int32_t r = smap_get(&t->merges, key, la + 1 + lb);
+      if (r >= 0 && (best < 0 || r < best)) { best = r; bi = i; }
+    }
+    if (best < 0) break;
+    memmove(&st[bi + 1], &st[bi + 2], (ns - bi - 1) * sizeof(size_t));   /* drop the boundary between the pair */
+    ns--;
+  }
+  for (size_t i = 0; i < ns; i++) {
+    int32_t id = smap_get(&t->vocab, buf + st[i], st[i + 1] - st[i]);
+    if (id < 0) { free(buf); free(st); return 0; }
+    if (*n >= cap) { free(buf); free(st); return -1; }
+    out[(*n)++] = id;
+  }
+  free(buf); free(st);
+  return 1;
+}
+
+static int encode_plain(const cl_tok *t, const uint8_t *s, size_t len, int32_t *out, int cap, int *n) {
+  segs a = {0}, b = {0}, c = {0}, d = {0};
+  int rc = 1;
+  if (!segs_add(&a, 0, len) || !split(s, &a, &d, t->pre == 1 ? m_qwen2 : m_gpt2)) rc = 0;
+  for (size_t i = 0; rc == 1 && i < d.n; i++) rc = bpe(t, s + d.v[i].a, d.v[i].b - d.v[i].a, out, cap, n);
+  free(a.v); free(b.v); free(c.v); free(d.v);
+  return rc;
+}
+
+int cl_tok_encode(const cl_tok *t, const char *text, size_t len, int parse_special, int32_t *out, int cap) {
+  const uint8_t *s = (const uint8_t *)text;
+  int n = 0;
+  size_t from = 0, i = 0;
+  while (parse_special && i < len) {
+    int hit = -1;
+    for (int k = 0; k < t->n_special && hit < 0; k++) {
+      const cl_str *tk = &t->tokens[t->special[k]];
+      if (len - i >= tk->len && !memcmp(s + i, tk->p, tk->len)) hit = t->special[k];
+    }
+    if (hit < 0) { i++; continue; }
+    int rc = encode_plain(t, s + from, i - from, out, cap, &n);
+    if (rc != 1) return -1;
+    if (n >= cap) return -1;
+    out[n++] = hit;
+    i += t->tokens[hit].len; from = i;
+  }
+  return encode_plain(t, s + from, len - from, out, cap, &n) == 1 ? n : -1;
+}
+
+int cl_tok_piece(const cl_tok *t, int id, char *out, int cap) {
+  if (id < 0 || (uint64_t)id >= t->n) return 0;
+  const cl_str *tk = &t->tokens[id];
+  int ctl = t->types && (t->types[id] == 3 || t->types[id] == 4);
+  int w = 0;
+  for (size_t i = 0; i < tk->len && w < cap;) {
+    if (ctl) { out[w++] = tk->p[i++]; continue; }
+    int cp, n = get_utf8((const uint8_t *)tk->p + i, tk->len - i, &cp);
+    i += (size_t)n;
+    out[w++] = (char)(cp < 512 && t->cp_byte[cp] >= 0 ? t->cp_byte[cp] : '?');
+  }
+  return w;
+}

package/c/src/tokenizer.h

@@ -0,0 +1,25 @@
+/*
+ * Byte-level BPE tokenizer (GPT-2 family) built from a GGUF file's own tokenizer tables. The pre-tokenizers implemented are the ones llama.cpp
+ * calls "olmo" and "qwen2". Letter, number and punctuation classes are exact for ASCII and approximate above U+007F (see tokenizer.c), so non-English
+ * text can tokenize differently from llama.cpp; `make verify-llama` checks agreement on a fixed set of prompts.
+ */
+#ifndef CL_TOKENIZER_H
+#define CL_TOKENIZER_H
+#include "meta.h"
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct cl_tok cl_tok;
+
+cl_tok *cl_tok_load(const cl_meta *m, char *err, size_t errlen);
+void cl_tok_free(cl_tok *t);
+int cl_tok_vocab_size(const cl_tok *t);
+int cl_tok_bos(const cl_tok *t);
+int cl_tok_eos(const cl_tok *t);
+int cl_tok_add_bos(const cl_tok *t);
+/* Encodes text. With parse_special, control tokens such as <|endoftext|> written in the text become their single token. Returns the number of
+ * tokens written, or -1 when `cap` is too small. */
+int cl_tok_encode(const cl_tok *t, const char *text, size_t len, int parse_special, int32_t *out, int cap);
+/* The bytes of one token (control tokens give their text). Returns the length written, at most cap. */
+int cl_tok_piece(const cl_tok *t, int id, char *out, int cap);
+#endif

package/c/src/xs.c

@@ -0,0 +1,215 @@
+#define _GNU_SOURCE
+#include "xs.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+enum { S_EMPTY = 0, S_LOADING = 1, S_READY = 2 };
+typedef struct job { int slot, layer, expert, prefetch; struct job *next; } job;
+
+struct cl_xs {
+  int fd, n_layer, n_expert, n_used, bypass_os;
+  const cl_xs_layer *lay;
+  cl_xs_layer *lay_copy;
+  pthread_mutex_t mu;
+  pthread_cond_t cv_work, cv_done;
+  job *dq_head, *dq_tail, *pq_head, *pq_tail;
+  int shutdown, failed, nth;
+  pthread_t th[16];
+  cl_cache *cache;
+  uint32_t total;          /* cache + staging slots */
+  int nbypass, bypass_next;
+  uint64_t slot_bytes;
+  uint8_t *arena;
+  int *state;
+  cl_xs_stats st;
+};
+
+static uint64_t now_ns(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; }
+
+static void push_job(cl_xs *x, job *j, int demand) {
+  job **h = demand ? &x->dq_head : &x->pq_head, **t = demand ? &x->dq_tail : &x->pq_tail;
+  j->next = NULL;
+  if (*t) (*t)->next = j; else *h = j;
+  *t = j;
+  pthread_cond_signal(&x->cv_work);
+}
+static job *pop_job(cl_xs *x) {
+  job **h = x->dq_head ? &x->dq_head : &x->pq_head, **t = x->dq_head ? &x->dq_tail : &x->pq_tail;   /* demand before prefetch */
+  job *j = *h;
+  if (!j) return NULL;
+  *h = j->next;
+  if (!*h) *t = NULL;
+  return j;
+}
+
+static int read_full(cl_xs *x, uint8_t *dst, uint64_t len, uint64_t off) {
+  uint64_t done = 0;
+  while (done < len) {
+    ssize_t n = pread(x->fd, dst + done, (size_t)(len - done), (off_t)(off + done));
+    if (n < 0 && errno == EINTR) continue;
+    if (n <= 0) return -1;
+    done += (uint64_t)n;
+  }
+#if defined(__linux__)
+  if (x->bypass_os) posix_fadvise(x->fd, (off_t)off, (off_t)len, POSIX_FADV_DONTNEED);
+#endif
+  return 0;
+}
+
+static void *worker(void *arg) {
+  cl_xs *x = arg;
+  pthread_mutex_lock(&x->mu);
+  for (;;) {
+    job *j = pop_job(x);
+    if (!j) { if (x->shutdown) break; pthread_cond_wait(&x->cv_work, &x->mu); continue; }
+    pthread_mutex_unlock(&x->mu);
+    uint64_t t0 = now_ns();
+    const cl_xs_layer *L = &x->lay_copy[j->layer];
+    uint8_t *dst = x->arena + (size_t)j->slot * x->slot_bytes;
+    uint64_t at = 0, bytes = 0;
+    int rc = 0;
+    for (int i = 0; i < 3 && !rc; i++) { rc = read_full(x, dst + at, L->slab[i], L->off[i] + (uint64_t)j->expert * L->slab[i]); at += L->slab[i]; }
+    bytes = at;
+    uint64_t dt = now_ns() - t0;
+    pthread_mutex_lock(&x->mu);
+    x->st.read_ns += dt;
+    if (rc) x->failed = 1;
+    x->state[j->slot] = rc ? S_EMPTY : S_READY;
+    if (j->prefetch) { x->st.prefetch_bytes += bytes; cl_cache_unpin(x->cache, j->slot); } else x->st.demand_bytes += bytes;
+    pthread_cond_broadcast(&x->cv_done);
+    free(j);
+  }
+  pthread_mutex_unlock(&x->mu);
+  return NULL;
+}
+
+cl_xs *cl_xs_open(const char *path, const cl_xs_layer *layers, int n_layer, int n_expert, int n_used, uint64_t cache_bytes, int stage_slots,
+                  cl_policy policy, int io_threads, int os_cache_bypass, char *err, size_t errlen) {
+  cl_xs *x = calloc(1, sizeof *x);
+  if (!x) { snprintf(err, errlen, "out of memory"); return NULL; }
+  x->n_layer = n_layer; x->n_expert = n_expert; x->n_used = n_used; x->bypass_os = os_cache_bypass;
+  x->lay_copy = malloc((size_t)n_layer * sizeof *layers);
+  if (!x->lay_copy) { snprintf(err, errlen, "out of memory"); free(x); return NULL; }
+  memcpy(x->lay_copy, layers, (size_t)n_layer * sizeof *layers);
+  for (int l = 0; l < n_layer; l++) { uint64_t s = layers[l].slab[0] + layers[l].slab[1] + layers[l].slab[2]; if (s > x->slot_bytes) x->slot_bytes = s; }
+  x->fd = open(path, O_RDONLY);
+  if (x->fd < 0) { snprintf(err, errlen, "cannot open %s: %s", path, strerror(errno)); free(x->lay_copy); free(x); return NULL; }
+#ifdef F_NOCACHE
+  if (os_cache_bypass) fcntl(x->fd, F_NOCACHE, 1);
+#endif
+  uint32_t cslots = (uint32_t)(cache_bytes / x->slot_bytes);
+  if (cslots < (uint32_t)n_used) cslots = (uint32_t)n_used;   /* a layer's experts must fit at once */
+  uint32_t sslots = stage_slots < 0 ? 0 : (uint32_t)stage_slots;
+  x->cache = cl_cache_new(cslots, sslots, (uint32_t)n_layer * (uint32_t)n_expert, policy, 0);
+  x->total = x->cache ? cl_cache_total_slots(x->cache) : 0;   /* what the cache really manages (it may round) */
+  x->nbypass = n_used + 2;
+  size_t nbuf = (size_t)x->total + (size_t)x->nbypass;
+  x->arena = aligned_alloc(4096, (nbuf * x->slot_bytes + 4095) / 4096 * 4096);
+  x->state = calloc(nbuf, sizeof(int));
+  if (!x->cache || !x->arena || !x->state) { snprintf(err, errlen, "out of memory"); cl_xs_close(x); return NULL; }
+  pthread_mutex_init(&x->mu, NULL); pthread_cond_init(&x->cv_work, NULL); pthread_cond_init(&x->cv_done, NULL);
+  x->nth = io_threads < 1 ? 1 : io_threads > 16 ? 16 : io_threads;
+  for (int i = 0; i < x->nth; i++) pthread_create(&x->th[i], NULL, worker, x);
+  return x;
+}
+
+void cl_xs_close(cl_xs *x) {
+  if (!x) return;
+  if (x->nth) {
+    pthread_mutex_lock(&x->mu);
+    x->shutdown = 1;
+    pthread_cond_broadcast(&x->cv_work);
+    pthread_mutex_unlock(&x->mu);
+    for (int i = 0; i < x->nth; i++) pthread_join(x->th[i], NULL);
+  }
+  if (x->cache) cl_cache_free(x->cache);
+  if (x->fd > 0) close(x->fd);
+  free(x->arena); free(x->state); free(x->lay_copy); free(x);
+}
+
+uint64_t cl_xs_slot_bytes(const cl_xs *x) { return x->slot_bytes; }
+uint32_t cl_xs_cache_slots(const cl_xs *x) { return x->total ? cl_cache_total_slots(x->cache) : 0; }
+uint64_t cl_xs_buffer_bytes(const cl_xs *x) { return ((uint64_t)x->total + (uint64_t)x->nbypass) * x->slot_bytes + cl_cache_meta_bytes(x->cache); }
+
+int cl_xs_request(cl_xs *x, int layer, int expert) {
+  if (layer < 0 || layer >= x->n_layer || expert < 0 || expert >= x->n_expert) return -1;
+  pthread_mutex_lock(&x->mu);
+  int slot, r = cl_cache_access(x->cache, (uint32_t)(layer * x->n_expert + expert), &slot);
+  x->st.accesses++;
+  if (r == 1) x->st.hits++;
+  else if (r == 2) { x->st.stage_hits++; x->st.hits++; if (x->state[slot] != S_READY) x->st.prefetch_late++; }
+  else {
+    x->st.misses++;
+    if (r == -1) {   /* nothing could be evicted: read into a spare buffer that the cache does not manage */
+      slot = (int)x->total + (x->bypass_next++ % x->nbypass);
+      x->st.bypass++;
+    }
+    x->state[slot] = S_LOADING;
+    job *j = calloc(1, sizeof *j);
+    j->slot = slot; j->layer = layer; j->expert = expert; j->prefetch = 0;
+    push_job(x, j, 1);
+  }
+  pthread_mutex_unlock(&x->mu);
+  return slot;
+}
+
+const uint8_t *cl_xs_wait(cl_xs *x, int h) {
+  pthread_mutex_lock(&x->mu);
+  uint64_t t0 = 0;
+  int waited = 0;
+  while (x->state[h] != S_READY && !x->failed) {
+    if (!waited) { waited = 1; t0 = now_ns(); }
+    pthread_cond_wait(&x->cv_done, &x->mu);
+  }
+  if (waited) x->st.wait_ns += now_ns() - t0;
+  int ok = x->state[h] == S_READY;
+  pthread_mutex_unlock(&x->mu);
+  return ok ? x->arena + (size_t)h * x->slot_bytes : NULL;
+}
+
+void cl_xs_release(cl_xs *x, int h) {
+  if (h < 0) return;
+  pthread_mutex_lock(&x->mu);
+  if ((uint32_t)h < x->total) cl_cache_unpin(x->cache, h);
+  else x->state[h] = S_EMPTY;
+  pthread_mutex_unlock(&x->mu);
+}
+
+int cl_xs_resident(cl_xs *x, int layer, int expert) {
+  if (layer < 0 || layer >= x->n_layer || expert < 0 || expert >= x->n_expert) return 0;
+  pthread_mutex_lock(&x->mu);
+  int slot = cl_cache_find(x->cache, (uint32_t)(layer * x->n_expert + expert));
+  int r = slot >= 0 && x->state[slot] == S_READY;
+  pthread_mutex_unlock(&x->mu);
+  return r;
+}
+
+void cl_xs_prefetch(cl_xs *x, int layer, int expert) {
+  if (layer < 0 || layer >= x->n_layer || expert < 0 || expert >= x->n_expert) return;
+  pthread_mutex_lock(&x->mu);
+  int slot = cl_cache_stage(x->cache, (uint32_t)(layer * x->n_expert + expert));
+  if (slot >= 0) {
+    x->state[slot] = S_LOADING;
+    x->st.prefetch_issued++;
+    job *j = calloc(1, sizeof *j);
+    j->slot = slot; j->layer = layer; j->expert = expert; j->prefetch = 1;
+    push_job(x, j, 0);
+  }
+  pthread_mutex_unlock(&x->mu);
+}
+
+cl_xs_stats cl_xs_get_stats(cl_xs *x) {
+  pthread_mutex_lock(&x->mu);
+  cl_xs_stats s = x->st;
+  cl_cache_stats cs = cl_cache_get_stats(x->cache);
+  s.admit_rejects = cs.admit_rejects; s.evictions = cs.evictions; s.prefetch_wasted = cs.stage_wasted;
+  pthread_mutex_unlock(&x->mu);
+  return s;
+}
+void cl_xs_reset_stats(cl_xs *x) { pthread_mutex_lock(&x->mu); memset(&x->st, 0, sizeof x->st); pthread_mutex_unlock(&x->mu); }

package/c/src/xs.h

@@ -0,0 +1,39 @@
+/*
+ * Expert store: serves the experts of a real forward pass from disk through Clutor's cache, so resident memory is bounded by the cache
+ * budget and not by the model's size. A slot holds one expert's gate, up and down matrices back to back. Misses are read by a pool of I/O
+ * threads (pread, with the OS cache bypassed when asked); a prefetch fills a staging slot that the cache policy cannot be evicted by.
+ */
+#ifndef CL_XS_H
+#define CL_XS_H
+#include "cache.h"
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct { uint64_t off[3], slab[3]; } cl_xs_layer;   /* file offset of expert 0 and bytes per expert, for gate, up, down */
+
+typedef struct {
+  uint64_t accesses, hits, stage_hits, misses, bypass, prefetch_issued, prefetch_late, prefetch_wasted, admit_rejects, evictions;
+  uint64_t demand_bytes, prefetch_bytes, read_ns, wait_ns;
+} cl_xs_stats;
+
+typedef struct cl_xs cl_xs;
+
+cl_xs *cl_xs_open(const char *path, const cl_xs_layer *layers, int n_layer, int n_expert, int n_used, uint64_t cache_bytes, int stage_slots,
+                  cl_policy policy, int io_threads, int os_cache_bypass, char *err, size_t errlen);
+void cl_xs_close(cl_xs *x);
+uint64_t cl_xs_slot_bytes(const cl_xs *x);
+uint32_t cl_xs_cache_slots(const cl_xs *x);
+uint64_t cl_xs_buffer_bytes(const cl_xs *x);   /* memory the store holds for expert data and its tables */
+
+/* Starts getting an expert. Returns a handle (>= 0) or -1 on failure. Misses are read in the background. */
+int cl_xs_request(cl_xs *x, int layer, int expert);
+/* Blocks until the expert is in memory; returns its bytes (gate, up, down in that order) or NULL if the read failed. */
+const uint8_t *cl_xs_wait(cl_xs *x, int handle);
+void cl_xs_release(cl_xs *x, int handle);
+/* 1 when this expert is in memory right now (in the cache or fully read into staging), so asking for it costs no disk read. */
+int cl_xs_resident(cl_xs *x, int layer, int expert);
+/* A hint: read this expert into staging if there is room and it is not already here. */
+void cl_xs_prefetch(cl_xs *x, int layer, int expert);
+cl_xs_stats cl_xs_get_stats(cl_xs *x);
+void cl_xs_reset_stats(cl_xs *x);
+#endif

package/dist/bin.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   main
-} from "./chunk-6WAEQCCP.js";
+} from "./chunk-2FBPYNHP.js";
 import "./chunk-U5GHVJJO.js";
 
 // src/bin.ts
@@ -40,6 +40,7 @@ var prompt = (q) => new Promise((resolve) => {
 });
 main(process.argv.slice(2), {
   out: (l) => console.log(l),
+  write: (s) => void process.stdout.write(s),
   err: (l) => console.error(l),
   cwd: process.cwd(),
   ...process.stdin.isTTY ? { prompt } : {}

package/dist/bin.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { mkdirSync, readFileSync, appendFileSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport { createInterface, type Interface } from \"node:readline\";\nimport { main } from \"./cli\";\n\n// One readline interface for the whole process, so arrow-key history works across prompts and Ctrl-D ends an interactive session.\nlet rl: Interface | undefined;\nconst historyFile = join(homedir(), \".clutor\", \"history\");\nfunction readline(): Interface {\n  if (rl) return rl;\n  let history: string[] = [];\n  try { history = readFileSync(historyFile, \"utf8\").split(\"\\n\").filter(Boolean).slice(-500).reverse(); } catch { /* no history yet */ }\n  rl = createInterface({ input: process.stdin, output: process.stderr, history, historySize: 500 });\n  return rl;\n}\nconst prompt = (q: string): Promise<string> => new Promise((resolve) => {\n  const r = readline();\n  const onClose = (): void => resolve(\"/exit\");\n  r.once(\"close\", onClose);\n  r.question(q, (a) => {\n    r.off(\"close\", onClose);\n    if (a.trim()) { try { mkdirSync(join(homedir(), \".clutor\"), { recursive: true, mode: 0o700 }); appendFileSync(historyFile, `${a.replace(/\\n/g, \" \")}\\n`, { mode: 0o600 }); } catch { /* history is a convenience */ } }\n    resolve(a);\n  });\n});\n\nmain(process.argv.slice(2), {\n  out: (l) => console.log(l),\n  err: (l) => console.error(l),\n  cwd: process.cwd(),\n  ...(process.stdin.isTTY ? { prompt } : {}),\n}).then(\n  (code) => process.exit(code),\n  (e: unknown) => {\n    console.error(e instanceof Error ? e.message : String(e));\n    process.exit(1);\n  },\n);\n"],"mappings":";;;;;;;AACA,SAAS,WAAW,cAAc,sBAAsB;AACxD,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,uBAAuC;AAIhD,IAAI;AACJ,IAAM,cAAc,KAAK,QAAQ,GAAG,WAAW,SAAS;AACxD,SAAS,WAAsB;AAC7B,MAAI,GAAI,QAAO;AACf,MAAI,UAAoB,CAAC;AACzB,MAAI;AAAE,cAAU,aAAa,aAAa,MAAM,EAAE,MAAM,IAAI,EAAE,OAAO,OAAO,EAAE,MAAM,IAAI,EAAE,QAAQ;AAAA,EAAG,QAAQ;AAAA,EAAuB;AACpI,OAAK,gBAAgB,EAAE,OAAO,QAAQ,OAAO,QAAQ,QAAQ,QAAQ,SAAS,aAAa,IAAI,CAAC;AAChG,SAAO;AACT;AACA,IAAM,SAAS,CAAC,MAA+B,IAAI,QAAQ,CAAC,YAAY;AACtE,QAAM,IAAI,SAAS;AACnB,QAAM,UAAU,MAAY,QAAQ,OAAO;AAC3C,IAAE,KAAK,SAAS,OAAO;AACvB,IAAE,SAAS,GAAG,CAAC,MAAM;AACnB,MAAE,IAAI,SAAS,OAAO;AACtB,QAAI,EAAE,KAAK,GAAG;AAAE,UAAI;AAAE,kBAAU,KAAK,QAAQ,GAAG,SAAS,GAAG,EAAE,WAAW,MAAM,MAAM,IAAM,CAAC;AAAG,uBAAe,aAAa,GAAG,EAAE,QAAQ,OAAO,GAAG,CAAC;AAAA,GAAM,EAAE,MAAM,IAAM,CAAC;AAAA,MAAG,QAAQ;AAAA,MAAiC;AAAA,IAAE;AACtN,YAAQ,CAAC;AAAA,EACX,CAAC;AACH,CAAC;AAED,KAAK,QAAQ,KAAK,MAAM,CAAC,GAAG;AAAA,EAC1B,KAAK,CAAC,MAAM,QAAQ,IAAI,CAAC;AAAA,EACzB,KAAK,CAAC,MAAM,QAAQ,MAAM,CAAC;AAAA,EAC3B,KAAK,QAAQ,IAAI;AAAA,EACjB,GAAI,QAAQ,MAAM,QAAQ,EAAE,OAAO,IAAI,CAAC;AAC1C,CAAC,EAAE;AAAA,EACD,CAAC,SAAS,QAAQ,KAAK,IAAI;AAAA,EAC3B,CAAC,MAAe;AACd,YAAQ,MAAM,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,CAAC;AACxD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;","names":[]}
+{"version":3,"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { mkdirSync, readFileSync, appendFileSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport { createInterface, type Interface } from \"node:readline\";\nimport { main } from \"./cli\";\n\n// One readline interface for the whole process, so arrow-key history works across prompts and Ctrl-D ends an interactive session.\nlet rl: Interface | undefined;\nconst historyFile = join(homedir(), \".clutor\", \"history\");\nfunction readline(): Interface {\n  if (rl) return rl;\n  let history: string[] = [];\n  try { history = readFileSync(historyFile, \"utf8\").split(\"\\n\").filter(Boolean).slice(-500).reverse(); } catch { /* no history yet */ }\n  rl = createInterface({ input: process.stdin, output: process.stderr, history, historySize: 500 });\n  return rl;\n}\nconst prompt = (q: string): Promise<string> => new Promise((resolve) => {\n  const r = readline();\n  const onClose = (): void => resolve(\"/exit\");\n  r.once(\"close\", onClose);\n  r.question(q, (a) => {\n    r.off(\"close\", onClose);\n    if (a.trim()) { try { mkdirSync(join(homedir(), \".clutor\"), { recursive: true, mode: 0o700 }); appendFileSync(historyFile, `${a.replace(/\\n/g, \" \")}\\n`, { mode: 0o600 }); } catch { /* history is a convenience */ } }\n    resolve(a);\n  });\n});\n\nmain(process.argv.slice(2), {\n  out: (l) => console.log(l),\n  write: (s) => void process.stdout.write(s),\n  err: (l) => console.error(l),\n  cwd: process.cwd(),\n  ...(process.stdin.isTTY ? { prompt } : {}),\n}).then(\n  (code) => process.exit(code),\n  (e: unknown) => {\n    console.error(e instanceof Error ? e.message : String(e));\n    process.exit(1);\n  },\n);\n"],"mappings":";;;;;;;AACA,SAAS,WAAW,cAAc,sBAAsB;AACxD,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,uBAAuC;AAIhD,IAAI;AACJ,IAAM,cAAc,KAAK,QAAQ,GAAG,WAAW,SAAS;AACxD,SAAS,WAAsB;AAC7B,MAAI,GAAI,QAAO;AACf,MAAI,UAAoB,CAAC;AACzB,MAAI;AAAE,cAAU,aAAa,aAAa,MAAM,EAAE,MAAM,IAAI,EAAE,OAAO,OAAO,EAAE,MAAM,IAAI,EAAE,QAAQ;AAAA,EAAG,QAAQ;AAAA,EAAuB;AACpI,OAAK,gBAAgB,EAAE,OAAO,QAAQ,OAAO,QAAQ,QAAQ,QAAQ,SAAS,aAAa,IAAI,CAAC;AAChG,SAAO;AACT;AACA,IAAM,SAAS,CAAC,MAA+B,IAAI,QAAQ,CAAC,YAAY;AACtE,QAAM,IAAI,SAAS;AACnB,QAAM,UAAU,MAAY,QAAQ,OAAO;AAC3C,IAAE,KAAK,SAAS,OAAO;AACvB,IAAE,SAAS,GAAG,CAAC,MAAM;AACnB,MAAE,IAAI,SAAS,OAAO;AACtB,QAAI,EAAE,KAAK,GAAG;AAAE,UAAI;AAAE,kBAAU,KAAK,QAAQ,GAAG,SAAS,GAAG,EAAE,WAAW,MAAM,MAAM,IAAM,CAAC;AAAG,uBAAe,aAAa,GAAG,EAAE,QAAQ,OAAO,GAAG,CAAC;AAAA,GAAM,EAAE,MAAM,IAAM,CAAC;AAAA,MAAG,QAAQ;AAAA,MAAiC;AAAA,IAAE;AACtN,YAAQ,CAAC;AAAA,EACX,CAAC;AACH,CAAC;AAED,KAAK,QAAQ,KAAK,MAAM,CAAC,GAAG;AAAA,EAC1B,KAAK,CAAC,MAAM,QAAQ,IAAI,CAAC;AAAA,EACzB,OAAO,CAAC,MAAM,KAAK,QAAQ,OAAO,MAAM,CAAC;AAAA,EACzC,KAAK,CAAC,MAAM,QAAQ,MAAM,CAAC;AAAA,EAC3B,KAAK,QAAQ,IAAI;AAAA,EACjB,GAAI,QAAQ,MAAM,QAAQ,EAAE,OAAO,IAAI,CAAC;AAC1C,CAAC,EAAE;AAAA,EACD,CAAC,SAAS,QAAQ,KAAK,IAAI;AAAA,EAC3B,CAAC,MAAe;AACd,YAAQ,MAAM,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,CAAC;AACxD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;","names":[]}