@lazarusrolando/crayoncli 0.2.0 → 0.3.0

package/.env.example

@@ -3,3 +3,4 @@ CRAYON_HOST=127.0.0.1
 CRAYON_PORT=8765
 CRAYON_PYTHON=python
 CRAYON_MAX_NEW_TOKENS=1024
+CRAYON_TEMPERATURE=0.7

package/README.md

@@ -1,38 +1,40 @@
-# CRAYON v0.2.0 — Local Gemma CLI
+# CRAYON v0.3.0 — Local Gemma Streaming
 
-Open-source Crayon-style CLI using Node.js/TypeScript for the agent interface and Google's Gemma model from Hugging Face for local inference.
+Real local token streaming for the CRAYON CLI.
 
-## Requirements
-- Node.js 20+
-- Python 3.11+
-- Sufficient RAM/VRAM for the selected Gemma model
-- Hugging Face access accepted for Gemma if required
+## Flow
+
+You → Node.js CLI → local FastAPI → Transformers TextIteratorStreamer → local Gemma → tokens → CLI
+
+Hugging Face is only the model repository. Inference is local.
 
 ## Install
 
 ```powershell
 npm install
-python -m pip install -r model/requirements.txt
+python -m pip install -r model\requirements.txt
 Copy-Item .env.example .env
 npm run build
-npm link
 ```
 
 ## Run
 
 ```powershell
-crayon
-crayon "explain this project"
+node dist/index.js
 ```
 
-The first run downloads the model through Transformers into the normal Hugging Face cache. Inference runs locally; CRAYON does not use the Hugging Face inference API.
+or:
 
-## Configuration
-
-```env
-CRAYON_MODEL=google/gemma-4-E2B-it
-CRAYON_HOST=127.0.0.1
-CRAYON_PORT=8765
-CRAYON_PYTHON=python
-CRAYON_MAX_NEW_TOKENS=1024
+```powershell
+node dist/index.js "Explain recursion in JavaScript"
 ```
+
+## What v0.3 adds
+
+- `TextIteratorStreamer` for local token streaming
+- background generation thread
+- FastAPI `StreamingResponse`
+- Node.js `ReadableStream` consumption
+- keeps `/generate` for non-streaming compatibility
+
+Streaming changes delivery timing, not the inference provider. The model still runs locally.

package/dist/index.js

@@ -1,45 +1,54 @@
-#!/usr/bin/env node
 import "dotenv/config";
-import { Command } from "commander";
-import chalk from "chalk";
+import readline from "node:readline";
 import { LocalGemma } from "./providers/local-gemma.js";
-import { createInterface } from "node:readline/promises";
-import { stdin as input, stdout as output } from "node:process";
-const program = new Command();
-program
-    .name("crayon")
-    .description("CRAYON — local Crayon-style coding agent")
-    .argument("[task]");
-program.action(async (task) => {
-    const gemma = new LocalGemma();
-    process.on("SIGINT", () => {
+const gemma = new LocalGemma({
+    python: process.env.CRAYON_PYTHON || "python",
+    model: process.env.CRAYON_MODEL || "google/gemma-4-E2B-it",
+    host: process.env.CRAYON_HOST || "127.0.0.1",
+    port: Number(process.env.CRAYON_PORT || 8765),
+    maxNewTokens: Number(process.env.CRAYON_MAX_NEW_TOKENS || 1024),
+    temperature: Number(process.env.CRAYON_TEMPERATURE || 0.7)
+});
+async function main() {
+    const oneShot = process.argv.slice(2).join(" ").trim();
+    await gemma.start();
+    if (oneShot) {
+        await gemma.generateStream(oneShot);
         gemma.stop();
-        process.exit(130);
-    });
-    console.log(chalk.cyan("\n╭──────────────────────────────────────────────╮"));
-    console.log(chalk.cyan("│              CRAYON LOCAL CODER                 │"));
-    console.log(chalk.cyan("│        Google Gemma · Local Inference        │"));
-    console.log(chalk.cyan("╰──────────────────────────────────────────────╯\n"));
-    if (!task) {
-        const rl = createInterface({ input, output });
+        return;
+    }
+    console.log("\n==========================================");
+    console.log("       CRAYON v0.3.0 - LOCAL STREAMING");
+    console.log("==========================================");
+    console.log("Gemma inference: LOCAL");
+    console.log("Hugging Face inference API: NOT USED");
+    console.log("Streaming: ENABLED\n");
+    console.log("Type /exit to quit.\n");
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout, prompt: "You: " });
+    const shutdown = () => { rl.close(); gemma.stop(); process.exit(0); };
+    process.on("SIGINT", shutdown);
+    rl.prompt();
+    rl.on("line", async (line) => {
+        const prompt = line.trim();
+        if (!prompt) {
+            rl.prompt();
+            return;
+        }
+        if (prompt === "/exit") {
+            shutdown();
+            return;
+        }
+        rl.pause();
         try {
-            while (true) {
-                const q = (await rl.question(chalk.green("> "))).trim();
-                if (!q || q === "exit" || q === "quit")
-                    break;
-                console.log(chalk.dim("Thinking...\n"));
-                console.log(await gemma.generate(q));
-                console.log();
-            }
+            await gemma.generateStream(prompt);
+        }
+        catch (error) {
+            console.error("\n[CRAYON ERROR]", error instanceof Error ? error.message : error);
         }
         finally {
-            rl.close();
-            gemma.stop();
+            rl.resume();
+            rl.prompt();
         }
-    }
-    else {
-        console.log(await gemma.generate(task));
-        gemma.stop();
-    }
-});
-await program.parseAsync();
+    });
+}
+main().catch(error => { console.error("[CRAYON FATAL]", error); gemma.stop(); process.exit(1); });

package/dist/providers/local-gemma.js

@@ -1,109 +1,79 @@
 import { spawn } from "node:child_process";
-import { setTimeout as delay } from "node:timers/promises";
 export class LocalGemma {
     process;
-    python;
-    host;
-    port;
-    model;
-    constructor(options = {}) {
-        this.python = options.python ?? process.env.CRAYON_PYTHON ?? "python";
-        this.host = options.host ?? process.env.CRAYON_HOST ?? "127.0.0.1";
-        this.port = options.port ?? Number(process.env.CRAYON_PORT ?? 8765);
-        this.model =
-            options.model ??
-                process.env.CRAYON_MODEL ??
-                "google/gemma-4-E2B-it";
-    }
-    url(path) {
-        return `http://${this.host}:${this.port}${path}`;
+    options;
+    baseUrl;
+    constructor(options) {
+        this.options = options;
+        this.baseUrl = `http://${options.host}:${options.port}`;
     }
     async start() {
-        // Check whether a server is already running.
-        try {
-            const response = await fetch(this.url("/health"));
-            if (response.ok) {
+        if (await this.isReady())
+            return;
+        console.log("[CRAYON] Starting local Gemma server...");
+        console.log(`[CRAYON] Model: ${this.options.model}`);
+        this.process = spawn(this.options.python, ["model/server.py"], {
+            stdio: ["ignore", "pipe", "pipe"],
+            env: { ...process.env, CRAYON_MODEL: this.options.model, CRAYON_HOST: this.options.host,
+                CRAYON_PORT: String(this.options.port) }
+        });
+        this.process.stdout?.on("data", (d) => process.stdout.write(`[Gemma] ${d.toString()}`));
+        this.process.stderr?.on("data", (d) => process.stderr.write(`[Gemma] ${d.toString()}`));
+        const deadline = Date.now() + 180000;
+        while (Date.now() < deadline) {
+            if (await this.isReady()) {
+                console.log("[CRAYON] Local Gemma server is ready.");
                 return;
             }
+            await new Promise(r => setTimeout(r, 1000));
+        }
+        throw new Error("Gemma server did not become ready within 180 seconds.");
+    }
+    async isReady() {
+        try {
+            return (await fetch(`${this.baseUrl}/health`)).ok;
         }
         catch {
-            // Server isn't running; start it below.
+            return false;
         }
-        console.log(`[CRAYON] Starting local Gemma server...`);
-        console.log(`[CRAYON] Model: ${this.model}`);
-        this.process = spawn(this.python, ["model/server.py"], {
-            stdio: ["ignore", "pipe", "pipe"],
-            env: {
-                ...process.env,
-                CRAYON_MODEL: this.model,
-                CRAYON_HOST: this.host,
-                CRAYON_PORT: String(this.port),
-            },
-        });
-        // stdout is available because we configured it as "pipe".
-        this.process.stdout?.on("data", (data) => {
-            process.stdout.write(`\x1b[90m[Gemma] ${data.toString()}\x1b[0m`);
-        });
-        // stderr is available because we configured it as "pipe".
-        this.process.stderr?.on("data", (data) => {
-            process.stderr.write(`\x1b[90m[Gemma] ${data.toString()}\x1b[0m`);
-        });
-        this.process.on("error", (error) => {
-            console.error("[CRAYON] Failed to start Gemma:", error);
-        });
-        this.process.on("exit", (code, signal) => {
-            if (code !== 0 && code !== null) {
-                console.error(`[CRAYON] Gemma server exited with code ${code}`);
-            }
-            if (signal) {
-                console.error(`[CRAYON] Gemma server stopped by signal ${signal}`);
-            }
+    }
+    async generateStream(prompt) {
+        const response = await fetch(`${this.baseUrl}/generate/stream`, {
+            method: "POST", headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ prompt, max_new_tokens: this.options.maxNewTokens, temperature: this.options.temperature })
         });
-        // Wait for FastAPI to become available.
-        for (let attempt = 0; attempt < 180; attempt++) {
-            try {
-                const response = await fetch(this.url("/health"));
-                if (response.ok) {
-                    console.log("[CRAYON] Gemma server is ready.");
-                    return;
-                }
-            }
-            catch {
-                // Still starting.
+        if (!response.ok)
+            throw new Error(`Gemma streaming error (${response.status}): ${await response.text()}`);
+        if (!response.body)
+            throw new Error("Gemma returned no streaming body.");
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+        process.stdout.write("\nCRAYON: ");
+        try {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                process.stdout.write(decoder.decode(value, { stream: true }));
             }
-            await delay(1000);
+            process.stdout.write(decoder.decode());
+            process.stdout.write("\n");
+        }
+        finally {
+            reader.releaseLock();
         }
-        this.stop();
-        throw new Error("Gemma server did not become ready within 180 seconds.");
     }
-    async generate(prompt, maxNewTokens = Number(process.env.CRAYON_MAX_NEW_TOKENS ?? 1024)) {
-        await this.start();
-        const response = await fetch(this.url("/generate"), {
-            method: "POST",
-            headers: {
-                "content-type": "application/json",
-            },
-            body: JSON.stringify({
-                prompt,
-                max_new_tokens: maxNewTokens,
-                temperature: 0.2,
-                top_p: 0.9,
-            }),
+    async generate(prompt) {
+        const response = await fetch(`${this.baseUrl}/generate`, {
+            method: "POST", headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ prompt, max_new_tokens: this.options.maxNewTokens, temperature: this.options.temperature })
         });
-        if (!response.ok) {
-            const errorText = await response.text();
-            throw new Error(`Gemma server returned HTTP ${response.status}: ${errorText}`);
-        }
-        const result = (await response.json());
-        return result.text;
+        if (!response.ok)
+            throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
+        return (await response.json()).text;
     }
     stop() {
-        if (!this.process) {
-            return;
-        }
-        if (!this.process.killed) {
+        if (this.process && !this.process.killed)
             this.process.kill();
-        }
-        this.process = undefined;
     }
 }

package/model/requirements.txt

@@ -1,6 +1,6 @@
-torch>=2.6
-transformers>=4.57
-accelerate>=1.10
-safetensors>=0.5
-fastapi>=0.115
-uvicorn[standard]>=0.34
+fastapi
+uvicorn[standard]
+transformers
+torch
+accelerate
+sentencepiece

package/model/server.py

@@ -1,159 +1,113 @@
 import os
+from threading import Thread
 import torch
-from fastapi import FastAPI, HTTPException
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
-MODEL_ID = os.getenv("CRAYON_MODEL", "google/gemma-4-E2B-it")
-HOST = os.getenv("CRAYON_HOST", "127.0.0.1")
-PORT = int(os.getenv("CRAYON_PORT", "8765"))
-
-print("=" * 60)
-print("[CRAYON] Local Gemma Server")
-print("=" * 60)
-print(f"[CRAYON] Model: {MODEL_ID}")
-print(f"[CRAYON] PyTorch: {torch.__version__}")
-print(f"[CRAYON] CUDA available: {torch.cuda.is_available()}")
-
-if torch.cuda.is_available():
-    print(f"[CRAYON] GPU: {torch.cuda.get_device_name(0)}")
-    print(f"[CRAYON] CUDA: {torch.version.cuda}")
-
-print("[CRAYON] Loading tokenizer...")
-
-tokenizer = AutoTokenizer.from_pretrained(
-    MODEL_ID
-)
-
-print("[CRAYON] Tokenizer loaded.")
-print("[CRAYON] Loading model...")
-
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    device_map="auto",
-    torch_dtype="auto",
-)
+MODEL_ID=os.getenv("CRAYON_MODEL","google/gemma-4-E2B-it")
+HOST=os.getenv("CRAYON_HOST","127.0.0.1")
+PORT=int(os.getenv("CRAYON_PORT","8765"))
 
+print(f"[CRAYON MODEL] Loading: {MODEL_ID}",flush=True)
+print(f"[CRAYON MODEL] CUDA available: {torch.cuda.is_available()}",flush=True)
+tokenizer=AutoTokenizer.from_pretrained(MODEL_ID)
+model=AutoModelForCausalLM.from_pretrained(MODEL_ID,device_map="auto",torch_dtype="auto")
 model.eval()
+print("[CRAYON MODEL] Model loaded.",flush=True)
+print(f"[CRAYON MODEL] Device: {model.device}",flush=True)
 
-print("[CRAYON] Model loaded successfully.")
-
-app = FastAPI(
-    title="CRAYON Local Gemma Server",
-    version="0.2.1",
-)
-
+app=FastAPI(title="CRAYON Local Gemma Server",version="0.3.0")
 
 class GenerateRequest(BaseModel):
-    prompt: str
-    max_new_tokens: int = 1024
-    temperature: float = 0.2
-    top_p: float = 0.9
-
+    prompt:str
+    max_new_tokens:int=1024
+    temperature:float=0.7
 
 @app.get("/health")
 def health():
-    return {
-        "ok": True,
-        "model": MODEL_ID,
-        "cuda": torch.cuda.is_available(),
-        "device": str(model.device),
-    }
-
-
-@app.post("/generate")
-def generate(req: GenerateRequest):
-
-    try:
-        print(f"[CRAYON] Request: {req.prompt[:100]}")
-
-        messages = [
-            {
-                "role": "user",
-                "content": req.prompt,
-            }
-        ]
-
-        # Create the model input using Gemma's chat template.
-        inputs = tokenizer.apply_chat_template(
-            messages,
-            add_generation_prompt=True,
-            tokenize=True,
-            return_tensors="pt",
-            return_dict=True,
-        )
-
-        # BatchEncoding -> tensors
-        input_ids = inputs["input_ids"]
-        attention_mask = inputs.get("attention_mask")
-
-        # Move tensors to the model's device.
-        input_ids = input_ids.to(model.device)
-
-        if attention_mask is not None:
-            attention_mask = attention_mask.to(model.device)
-
-        print(
-            f"[CRAYON] Input tokens: {input_ids.shape[-1]}"
-        )
-
-        generation_kwargs = {
-            "input_ids": input_ids,
-            "max_new_tokens": req.max_new_tokens,
-            "do_sample": req.temperature > 0,
-            "temperature": max(req.temperature, 1e-5),
-            "top_p": req.top_p,
-            "pad_token_id": tokenizer.eos_token_id,
-        }
-
-        if attention_mask is not None:
-            generation_kwargs["attention_mask"] = attention_mask
-
-        print("[CRAYON] Generating...")
-
-        with torch.inference_mode():
-
-            output = model.generate(
-                **generation_kwargs
+    return {"status":"ok","model":MODEL_ID,"device":str(model.device),
+            "cuda":torch.cuda.is_available(),"streaming":True,"inference":"local"}
+
+def build_inputs(prompt):
+    messages=[{"role":"user","content":prompt}]
+    inputs=tokenizer.apply_chat_template(messages,tokenize=True,add_generation_prompt=True,
+                                          return_tensors="pt",return_dict=True)
+    input_ids=inputs["input_ids"].to(model.device)
+    attention_mask=inputs.get("attention_mask")
+    if attention_mask is not None:
+        attention_mask=attention_mask.to(model.device)
+    return input_ids,attention_mask
+
+@app.post("/generate/stream")
+def generate_stream(request:GenerateRequest):
+    input_ids,attention_mask=build_inputs(request.prompt)
+    streamer=TextIteratorStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)
+    kwargs={"input_ids":input_ids,"streamer":streamer,
+            "max_new_tokens":request.max_new_tokens,
+            "do_sample":request.temperature>0}
+    if attention_mask is not None: kwargs["attention_mask"]=attention_mask
+    if request.temperature>0: kwargs["temperature"]=request.temperature
+
+    worker=Thread(target=model.generate,kwargs=kwargs,daemon=True)
+    worker.start()
+
+    def token_iterator():
+        generated_text = ""
+        chunk_count = 0
+
+        try:
+            for text in streamer:
+                chunk_count += 1
+                generated_text += text
+
+                # TextIteratorStreamer emits decoded text chunks. Re-tokenize
+                # accumulated output to report the generated-token count.
+                token_count = len(
+                    tokenizer.encode(
+                        generated_text,
+                        add_special_tokens=False,
+                    )
+                )
+
+                print(
+                    f"[CRAYON TOKEN] #{token_count}: {text!r}",
+                    flush=True,
+                )
+
+                yield text
+
+            final_token_count = len(
+                tokenizer.encode(
+                    generated_text,
+                    add_special_tokens=False,
+                )
             )
 
-        # Remove the original prompt tokens.
-        generated_tokens = output[
-            0,
-            input_ids.shape[-1]:
-        ]
-
-        text = tokenizer.decode(
-            generated_tokens,
-            skip_special_tokens=True,
-        )
-
-        print(
-            f"[CRAYON] Generated {len(generated_tokens)} tokens."
-        )
-
-        return {
-            "text": text,
-            "model": MODEL_ID,
-        }
-
-    except Exception as exc:
-
-        print("[CRAYON] Generation error:")
-        print(repr(exc))
-
-        raise HTTPException(
-            status_code=500,
-            detail=str(exc),
-        )
-
+            print(
+                f"[CRAYON STREAM] Generation complete | "
+                f"chunks={chunk_count} | "
+                f"generated_tokens={final_token_count}",
+                flush=True,
+            )
+        finally:
+            worker.join()
 
-if __name__ == "__main__":
+    return StreamingResponse(token_iterator(),media_type="text/plain; charset=utf-8",
+                             headers={"Cache-Control":"no-cache","X-CRAYON-Streaming":"true"})
 
+@app.post("/generate")
+def generate(request:GenerateRequest):
+    input_ids,attention_mask=build_inputs(request.prompt)
+    kwargs={"input_ids":input_ids,"max_new_tokens":request.max_new_tokens,
+            "do_sample":request.temperature>0}
+    if attention_mask is not None: kwargs["attention_mask"]=attention_mask
+    if request.temperature>0: kwargs["temperature"]=request.temperature
+    output=model.generate(**kwargs)
+    new_tokens=output[0][input_ids.shape[-1]:]
+    return {"text":tokenizer.decode(new_tokens,skip_special_tokens=True)}
+
+if __name__=="__main__":
     import uvicorn
-
-    uvicorn.run(
-        app,
-        host=HOST,
-        port=PORT,
-    )
+    uvicorn.run(app,host=HOST,port=PORT)

package/package.json

@@ -1,21 +1,18 @@
 {
     "name": "@lazarusrolando/crayoncli",
-    "version": "0.2.0",
+    "version": "0.3.0",
     "type": "module",
-    "bin": {
-        "crayon": "dist/index.js"
-    },
     "scripts": {
         "build": "tsc",
-        "start": "node dist/index.js"
+        "start": "node dist/index.js",
+        "dev": "tsx src/index.ts"
     },
     "dependencies": {
-        "chalk": "^5.4.1",
-        "commander": "^14.0.1",
-        "dotenv": "^17.2.1"
+        "dotenv": "^16.4.7"
     },
     "devDependencies": {
-        "@types/node": "^24.0.0",
-        "typescript": "^5.9.2"
+        "@types/node": "^22.10.5",
+        "tsx": "^4.19.2",
+        "typescript": "^5.7.2"
     }
 }

package/src/index.ts

@@ -1,52 +1,43 @@
-#!/usr/bin/env node
 import "dotenv/config";
-import { Command } from "commander";
-import chalk from "chalk";
+import readline from "node:readline";
 import { LocalGemma } from "./providers/local-gemma.js";
-import { createInterface } from "node:readline/promises";
-import { stdin as input, stdout as output } from "node:process";
 
-const program = new Command();
-
-program
-  .name("crayon")
-  .description("CRAYON — local Crayon-style coding agent")
-  .argument("[task]");
-
-program.action(async (task?: string) => {
-  const gemma = new LocalGemma();
-
-  process.on("SIGINT", () => {
-    gemma.stop();
-    process.exit(130);
-  });
-
-  console.log(chalk.cyan("\n╭──────────────────────────────────────────────╮"));
-  console.log(chalk.cyan("│              CRAYON LOCAL CODER                 │"));
-  console.log(chalk.cyan("│        Google Gemma · Local Inference        │"));
-  console.log(chalk.cyan("╰──────────────────────────────────────────────╯\n"));
-
-  if (!task) {
-    const rl = createInterface({input, output});
-
-    try {
-      while (true) {
-        const q = (await rl.question(chalk.green("> "))).trim();
-
-        if (!q || q === "exit" || q === "quit") break;
-
-        console.log(chalk.dim("Thinking...\n"));
-        console.log(await gemma.generate(q));
-        console.log();
-      }
-    } finally {
-      rl.close();
-      gemma.stop();
-    }
-  } else {
-    console.log(await gemma.generate(task));
-    gemma.stop();
-  }
+const gemma=new LocalGemma({
+  python:process.env.CRAYON_PYTHON||"python",
+  model:process.env.CRAYON_MODEL||"google/gemma-4-E2B-it",
+  host:process.env.CRAYON_HOST||"127.0.0.1",
+  port:Number(process.env.CRAYON_PORT||8765),
+  maxNewTokens:Number(process.env.CRAYON_MAX_NEW_TOKENS||1024),
+  temperature:Number(process.env.CRAYON_TEMPERATURE||0.7)
 });
 
-await program.parseAsync();
+async function main(){
+  const oneShot=process.argv.slice(2).join(" ").trim();
+  await gemma.start();
+
+  if(oneShot){ await gemma.generateStream(oneShot); gemma.stop(); return; }
+
+  console.log("\n==========================================");
+  console.log("       CRAYON v0.3.0 - LOCAL STREAMING");
+  console.log("==========================================");
+  console.log("Gemma inference: LOCAL");
+  console.log("Hugging Face inference API: NOT USED");
+  console.log("Streaming: ENABLED\n");
+  console.log("Type /exit to quit.\n");
+
+  const rl=readline.createInterface({input:process.stdin,output:process.stdout,prompt:"You: "});
+  const shutdown=()=>{rl.close();gemma.stop();process.exit(0);};
+  process.on("SIGINT",shutdown);
+  rl.prompt();
+
+  rl.on("line",async line=>{
+    const prompt=line.trim();
+    if(!prompt){rl.prompt();return;}
+    if(prompt==="/exit"){shutdown();return;}
+    rl.pause();
+    try{await gemma.generateStream(prompt);}
+    catch(error){console.error("\n[CRAYON ERROR]",error instanceof Error?error.message:error);}
+    finally{rl.resume();rl.prompt();}
+  });
+}
+main().catch(error=>{console.error("[CRAYON FATAL]",error);gemma.stop();process.exit(1);});

package/src/providers/local-gemma.ts

@@ -1,165 +1,78 @@
-import { spawn, type ChildProcess } from "node:child_process";
-import { setTimeout as delay } from "node:timers/promises";
+import { spawn, ChildProcess } from "node:child_process";
 
 export interface GemmaOptions {
-  python?: string;
-  host?: string;
-  port?: number;
-  model?: string;
+  python:string; model:string; host:string; port:number;
+  maxNewTokens:number; temperature:number;
 }
 
 export class LocalGemma {
-  private process?: ChildProcess;
+  private process?:ChildProcess;
+  private readonly options:GemmaOptions;
+  private readonly baseUrl:string;
 
-  private readonly python: string;
-  private readonly host: string;
-  private readonly port: number;
-  private readonly model: string;
-
-  constructor(options: GemmaOptions = {}) {
-    this.python = options.python ?? process.env.CRAYON_PYTHON ?? "python";
-    this.host = options.host ?? process.env.CRAYON_HOST ?? "127.0.0.1";
-    this.port = options.port ?? Number(process.env.CRAYON_PORT ?? 8765);
-    this.model =
-      options.model ??
-      process.env.CRAYON_MODEL ??
-      "google/gemma-4-E2B-it";
-  }
-
-  private url(path: string): string {
-    return `http://${this.host}:${this.port}${path}`;
+  constructor(options:GemmaOptions){
+    this.options=options;
+    this.baseUrl=`http://${options.host}:${options.port}`;
   }
 
-  async start(): Promise<void> {
-    // Check whether a server is already running.
-    try {
-      const response = await fetch(this.url("/health"));
-
-      if (response.ok) {
-        return;
-      }
-    } catch {
-      // Server isn't running; start it below.
-    }
-
-    console.log(`[CRAYON] Starting local Gemma server...`);
-    console.log(`[CRAYON] Model: ${this.model}`);
-
-    this.process = spawn(
-      this.python,
-      ["model/server.py"],
-      {
-        stdio: ["ignore", "pipe", "pipe"],
-        env: {
-          ...process.env,
-          CRAYON_MODEL: this.model,
-          CRAYON_HOST: this.host,
-          CRAYON_PORT: String(this.port),
-        },
-      },
-    );
+  async start():Promise<void>{
+    if(await this.isReady()) return;
+    console.log("[CRAYON] Starting local Gemma server...");
+    console.log(`[CRAYON] Model: ${this.options.model}`);
 
-    // stdout is available because we configured it as "pipe".
-    this.process.stdout?.on("data", (data: Buffer) => {
-      process.stdout.write(
-        `\x1b[90m[Gemma] ${data.toString()}\x1b[0m`,
-      );
+    this.process=spawn(this.options.python,["model/server.py"],{
+      stdio:["ignore","pipe","pipe"],
+      env:{...process.env,CRAYON_MODEL:this.options.model,CRAYON_HOST:this.options.host,
+           CRAYON_PORT:String(this.options.port)}
     });
 
-    // stderr is available because we configured it as "pipe".
-    this.process.stderr?.on("data", (data: Buffer) => {
-      process.stderr.write(
-        `\x1b[90m[Gemma] ${data.toString()}\x1b[0m`,
-      );
-    });
+    this.process.stdout?.on("data",(d:Buffer)=>process.stdout.write(`[Gemma] ${d.toString()}`));
+    this.process.stderr?.on("data",(d:Buffer)=>process.stderr.write(`[Gemma] ${d.toString()}`));
 
-    this.process.on("error", (error) => {
-      console.error("[CRAYON] Failed to start Gemma:", error);
-    });
+    const deadline=Date.now()+180000;
+    while(Date.now()<deadline){
+      if(await this.isReady()){ console.log("[CRAYON] Local Gemma server is ready."); return; }
+      await new Promise(r=>setTimeout(r,1000));
+    }
+    throw new Error("Gemma server did not become ready within 180 seconds.");
+  }
 
-    this.process.on("exit", (code, signal) => {
-      if (code !== 0 && code !== null) {
-        console.error(
-          `[CRAYON] Gemma server exited with code ${code}`,
-        );
-      }
+  async isReady():Promise<boolean>{
+    try{return (await fetch(`${this.baseUrl}/health`)).ok;}catch{return false;}
+  }
 
-      if (signal) {
-        console.error(
-          `[CRAYON] Gemma server stopped by signal ${signal}`,
-        );
-      }
+  async generateStream(prompt:string):Promise<void>{
+    const response=await fetch(`${this.baseUrl}/generate/stream`,{
+      method:"POST",headers:{"Content-Type":"application/json"},
+      body:JSON.stringify({prompt,max_new_tokens:this.options.maxNewTokens,temperature:this.options.temperature})
     });
-
-    // Wait for FastAPI to become available.
-    for (let attempt = 0; attempt < 180; attempt++) {
-      try {
-        const response = await fetch(this.url("/health"));
-
-        if (response.ok) {
-          console.log("[CRAYON] Gemma server is ready.");
-          return;
-        }
-      } catch {
-        // Still starting.
+    if(!response.ok) throw new Error(`Gemma streaming error (${response.status}): ${await response.text()}`);
+    if(!response.body) throw new Error("Gemma returned no streaming body.");
+
+    const reader=response.body.getReader();
+    const decoder=new TextDecoder();
+    process.stdout.write("\nCRAYON: ");
+    try{
+      while(true){
+        const {done,value}=await reader.read();
+        if(done) break;
+        process.stdout.write(decoder.decode(value,{stream:true}));
       }
-
-      await delay(1000);
-    }
-
-    this.stop();
-
-    throw new Error(
-      "Gemma server did not become ready within 180 seconds.",
-    );
+      process.stdout.write(decoder.decode());
+      process.stdout.write("\n");
+    }finally{reader.releaseLock();}
   }
 
-  async generate(
-    prompt: string,
-    maxNewTokens = Number(
-      process.env.CRAYON_MAX_NEW_TOKENS ?? 1024,
-    ),
-  ): Promise<string> {
-    await this.start();
-
-    const response = await fetch(this.url("/generate"), {
-      method: "POST",
-      headers: {
-        "content-type": "application/json",
-      },
-      body: JSON.stringify({
-        prompt,
-        max_new_tokens: maxNewTokens,
-        temperature: 0.2,
-        top_p: 0.9,
-      }),
+  async generate(prompt:string):Promise<string>{
+    const response=await fetch(`${this.baseUrl}/generate`,{
+      method:"POST",headers:{"Content-Type":"application/json"},
+      body:JSON.stringify({prompt,max_new_tokens:this.options.maxNewTokens,temperature:this.options.temperature})
     });
-
-    if (!response.ok) {
-      const errorText = await response.text();
-
-      throw new Error(
-        `Gemma server returned HTTP ${response.status}: ${errorText}`,
-      );
-    }
-
-    const result = (await response.json()) as {
-      text: string;
-      model?: string;
-    };
-
-    return result.text;
+    if(!response.ok) throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
+    return ((await response.json()) as {text:string}).text;
   }
 
-  stop(): void {
-    if (!this.process) {
-      return;
-    }
-
-    if (!this.process.killed) {
-      this.process.kill();
-    }
-
-    this.process = undefined;
+  stop():void{
+    if(this.process && !this.process.killed) this.process.kill();
   }
-}
+}

package/tsconfig.json

@@ -1,15 +1 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "outDir": "dist",
-    "rootDir": "src",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true
-  },
-  "include": [
-    "src"
-  ]
-}
+{"compilerOptions":{"target":"ES2022","module":"NodeNext","moduleResolution":"NodeNext","outDir":"dist","rootDir":"src","strict":true,"esModuleInterop":true,"skipLibCheck":true},"include":["src/**/*.ts"]}

package/.npm-global/crayon

@@ -1,16 +0,0 @@
-#!/bin/sh
-basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
-
-case `uname` in
-    *CYGWIN*|*MINGW*|*MSYS*)
-        if command -v cygpath > /dev/null 2>&1; then
-            basedir=`cygpath -w "$basedir"`
-        fi
-    ;;
-esac
-
-if [ -x "$basedir/node" ]; then
-  exec "$basedir/node"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" "$@"
-else 
-  exec node  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" "$@"
-fi

package/.npm-global/crayon.cmd

@@ -1,17 +0,0 @@
-@ECHO off
-GOTO start
-:find_dp0
-SET dp0=%~dp0
-EXIT /b
-:start
-SETLOCAL
-CALL :find_dp0
-
-IF EXIST "%dp0%\node.exe" (
-  SET "_prog=%dp0%\node.exe"
-) ELSE (
-  SET "_prog=node"
-  SET PATHEXT=%PATHEXT:;.JS;=;%
-)
-
-endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%"  "%dp0%\node_modules\Crayon-cli-local-gemma\dist\index.js" %*

package/.npm-global/crayon.ps1

@@ -1,28 +0,0 @@
-#!/usr/bin/env pwsh
-$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
-
-$exe=""
-if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
-  # Fix case when both the Windows and Linux builds of Node
-  # are installed in the same directory
-  $exe=".exe"
-}
-$ret=0
-if (Test-Path "$basedir/node$exe") {
-  # Support pipeline input
-  if ($MyInvocation.ExpectingInput) {
-    $input | & "$basedir/node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  } else {
-    & "$basedir/node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  }
-  $ret=$LASTEXITCODE
-} else {
-  # Support pipeline input
-  if ($MyInvocation.ExpectingInput) {
-    $input | & "node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  } else {
-    & "node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  }
-  $ret=$LASTEXITCODE
-}
-exit $ret