@clutor/cli 0.3.0 → 0.3.1

package/c/src/main.c

@@ -12,7 +12,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#define VERSION "0.3.0"
+#define VERSION "0.3.1"
 
 static void usage(void) {
   puts("clutor-engine " VERSION "\n"
@@ -374,7 +374,7 @@ static int cmd_generate(int argc, char **argv) {
     int b = 0;
     for (int i = 1; i < f.n_vocab; i++) if (logits[i] > logits[b]) b = i;
     gen++;
-    if (b == cl_tok_eos(tok) || gen > want) break;
+    if (cl_tok_is_eog(tok, b) || gen > want) break;
     char piece[256];
     int pl = cl_tok_piece(tok, b, piece, sizeof piece);
     if (!quiet) { fwrite(piece, 1, (size_t)pl, stdout); fflush(stdout); }

package/c/src/tokenizer.c

@@ -219,6 +219,16 @@ int cl_tok_bos(const cl_tok *t) { return t->bos; }
 int cl_tok_eos(const cl_tok *t) { return t->eos; }
 int cl_tok_add_bos(const cl_tok *t) { return t->add_bos; }
 
+int cl_tok_is_eog(const cl_tok *t, int id) {
+  if (id < 0 || (uint64_t)id >= t->n) return 0;
+  if (id == t->eos) return 1;
+  if (!t->types || (t->types[id] != 3 && t->types[id] != 4)) return 0;   /* only control or user-defined tokens */
+  static const char *const eog[] = {"<|endoftext|>", "<|im_end|>", "<|eot_id|>", "<|end_of_text|>", "<|end|>", "<|eom_id|>", "<end_of_turn>", "</s>"};
+  const cl_str *tk = &t->tokens[id];
+  for (size_t i = 0; i < sizeof eog / sizeof *eog; i++) if (tk->len == strlen(eog[i]) && !memcmp(tk->p, eog[i], tk->len)) return 1;
+  return 0;
+}
+
 /* Byte-pair-encodes one pre-token and appends its ids. */
 static int bpe(const cl_tok *t, const uint8_t *s, size_t len, int32_t *out, int cap, int *n) {
   if (len == 0) return 1;

package/c/src/tokenizer.h

@@ -17,6 +17,8 @@ int cl_tok_vocab_size(const cl_tok *t);
 int cl_tok_bos(const cl_tok *t);
 int cl_tok_eos(const cl_tok *t);
 int cl_tok_add_bos(const cl_tok *t);
+/* True for a token that ends generation: the model's EOS, or a control token such as <|endoftext|>, <|im_end|> or <|eot_id|> (llama.cpp stops on all of these, and a model may emit one that is not its configured EOS). */
+int cl_tok_is_eog(const cl_tok *t, int id);
 /* Encodes text. With parse_special, control tokens such as <|endoftext|> written in the text become their single token. Returns the number of
  * tokens written, or -1 when `cap` is too small. */
 int cl_tok_encode(const cl_tok *t, const char *text, size_t len, int parse_special, int32_t *out, int cap);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clutor/cli",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "The clutor command and local server: plan whether a model fits your machine, inspect GGUF files, simulate expert streaming, run a sandboxed coding agent, and serve an OpenAI- and Ollama-compatible API on localhost in front of llama.cpp.",
   "license": "MIT",
   "type": "module",
@@ -13,6 +13,6 @@
   "exports": { ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" } },
   "files": ["dist", "c/src", "c/Makefile", "README.md", "LICENSE"],
   "engines": { "node": ">=20" },
-  "dependencies": { "@clutor/core": "^0.3.0", "zod": "^4.0.0" },
+  "dependencies": { "@clutor/core": "^0.3.1", "zod": "^4.0.0" },
   "scripts": { "prepack": "node ../../scripts/pack-engine.mjs", "postpack": "node ../../scripts/pack-engine.mjs clean", "build": "tsup", "typecheck": "tsc --noEmit", "test": "vitest run" }
 }