@lazarusrolando/crayoncli 0.2.0 → 0.4.0

package/.env.example

@@ -3,3 +3,4 @@ CRAYON_HOST=127.0.0.1
 CRAYON_PORT=8765
 CRAYON_PYTHON=python
 CRAYON_MAX_NEW_TOKENS=1024
+CRAYON_TEMPERATURE=0.7

package/README.md

@@ -1,38 +1,78 @@
-# CRAYON v0.2.0 — Local Gemma CLI
+# CRAYON v0.4.0 — Gemma 4 E2B Multimodal CLI
 
-Open-source Crayon-style CLI using Node.js/TypeScript for the agent interface and Google's Gemma model from Hugging Face for local inference.
+Local multimodal CLI using the official Transformers interface for `google/gemma-4-E2B-it`.
 
-## Requirements
-- Node.js 20+
-- Python 3.11+
-- Sufficient RAM/VRAM for the selected Gemma model
-- Hugging Face access accepted for Gemma if required
+## Supported
+
+- Text -> Text
+- Image + Text -> Text
+- Audio + Text -> Text
+- Video + Text -> Text
+- Text + Image + Audio -> Text (server API can be extended for combined uploads)
+- Streaming output for every mode
+
+Gemma 4 E2B supports native audio in addition to image/video multimodality. The model generates text; it is not an image/audio/video generator.
 
 ## Install
 
 ```powershell
 npm install
-python -m pip install -r model/requirements.txt
+python -m pip install -r model\requirements.txt
 Copy-Item .env.example .env
 npm run build
-npm link
 ```
 
-## Run
+Use a recent Transformers version. The model card currently recommends the latest Transformers.
+
+## CLI
+
+Text:
 
 ```powershell
-crayon
-crayon "explain this project"
+node dist/index.js "Explain recursion"
 ```
 
-The first run downloads the model through Transformers into the normal Hugging Face cache. Inference runs locally; CRAYON does not use the Hugging Face inference API.
+Image:
+
+```powershell
+node dist/index.js image photo.jpg "What is in this image?"
+```
 
-## Configuration
+Audio:
 
-```env
-CRAYON_MODEL=google/gemma-4-E2B-it
-CRAYON_HOST=127.0.0.1
-CRAYON_PORT=8765
-CRAYON_PYTHON=python
-CRAYON_MAX_NEW_TOKENS=1024
+```powershell
+node dist/index.js audio speech.wav "Transcribe and summarize this audio."
+```
+
+Video:
+
+```powershell
+node dist/index.js video clip.mp4 "Describe what happens in this video."
 ```
+
+Interactive:
+
+```powershell
+node dist/index.js
+```
+
+Then:
+
+```text
+/image photo.jpg What is in this image?
+/audio speech.wav Transcribe this.
+/video clip.mp4 Describe what happens.
+```
+
+## Important
+
+Inference is local. Hugging Face is used as the model repository for downloading/caching model files; the CLI calls your local FastAPI server.
+
+The official Gemma 4 Transformers interface uses:
+
+```python
+AutoProcessor
+AutoModelForMultimodalLM
+```
+
+rather than the text-only `AutoTokenizer` / `AutoModelForCausalLM` path.

package/dist/index.js

@@ -1,45 +1,119 @@
-#!/usr/bin/env node
 import "dotenv/config";
-import { Command } from "commander";
-import chalk from "chalk";
+import readline from "node:readline";
+import { readFile } from "node:fs/promises";
+import { basename } from "node:path";
 import { LocalGemma } from "./providers/local-gemma.js";
-import { createInterface } from "node:readline/promises";
-import { stdin as input, stdout as output } from "node:process";
-const program = new Command();
-program
-    .name("crayon")
-    .description("CRAYON — local Crayon-style coding agent")
-    .argument("[task]");
-program.action(async (task) => {
-    const gemma = new LocalGemma();
-    process.on("SIGINT", () => {
+const gemma = new LocalGemma({
+    python: process.env.CRAYON_PYTHON || "python",
+    model: process.env.CRAYON_MODEL || "google/gemma-4-E2B-it",
+    host: process.env.CRAYON_HOST || "127.0.0.1",
+    port: Number(process.env.CRAYON_PORT || 8765),
+    maxNewTokens: Number(process.env.CRAYON_MAX_NEW_TOKENS || 1024),
+    temperature: Number(process.env.CRAYON_TEMPERATURE || 0.7)
+});
+async function streamMedia(prompt, type, path) {
+    const form = new FormData();
+    form.append("prompt", prompt);
+    form.append("media_type", type);
+    form.append("max_new_tokens", String(Number(process.env.CRAYON_MAX_NEW_TOKENS || 1024)));
+    form.append("temperature", String(Number(process.env.CRAYON_TEMPERATURE || 0.7)));
+    const data = await readFile(path);
+    form.append("file", new Blob([data]), basename(path));
+    const response = await fetch(`http://${process.env.CRAYON_HOST || "127.0.0.1"}:${process.env.CRAYON_PORT || 8765}/generate/media`, { method: "POST", body: form });
+    if (!response.ok)
+        throw new Error(await response.text());
+    if (!response.body)
+        throw new Error("No streaming response.");
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    process.stdout.write("\nCRAYON: ");
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            process.stdout.write(decoder.decode(value, { stream: true }));
+        }
+        process.stdout.write(decoder.decode());
+        process.stdout.write("\n");
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+async function main() {
+    const args = process.argv.slice(2);
+    await gemma.start();
+    // One-shot:
+    // crayon "Explain this"
+    // crayon image photo.jpg "What is in this image?"
+    // crayon audio speech.wav "Transcribe this"
+    // crayon video clip.mp4 "Describe this video"
+    if (args[0] && ["image", "audio", "video"].includes(args[0]) && args[1] && args[2]) {
+        await streamMedia(args.slice(2).join(" "), args[0], args[1]);
         gemma.stop();
-        process.exit(130);
-    });
-    console.log(chalk.cyan("\n╭──────────────────────────────────────────────╮"));
-    console.log(chalk.cyan("│              CRAYON LOCAL CODER                 │"));
-    console.log(chalk.cyan("│        Google Gemma · Local Inference        │"));
-    console.log(chalk.cyan("╰──────────────────────────────────────────────╯\n"));
-    if (!task) {
-        const rl = createInterface({ input, output });
+        return;
+    }
+    if (args.length) {
+        await gemma.generateStream(args.join(" "));
+        gemma.stop();
+        return;
+    }
+    console.log(`
+==========================================
+     CRAYON v0.4.0 - GEMMA MULTIMODAL CLI
+==========================================
+Text   -> Text
+Image  -> Text
+Audio  -> Text
+Video  -> Text
+
+Commands:
+  /image <file> <question>
+  /audio <file> <question>
+  /video <file> <question>
+  /exit
+
+Examples:
+  /image photo.jpg What is in this image?
+  /audio speech.wav Transcribe this audio.
+  /video clip.mp4 Describe what happens.
+`);
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout, prompt: "You: " });
+    const shutdown = () => { rl.close(); gemma.stop(); process.exit(0); };
+    process.on("SIGINT", shutdown);
+    rl.prompt();
+    rl.on("line", async (line) => {
+        const input = line.trim();
+        if (!input) {
+            rl.prompt();
+            return;
+        }
+        if (input === "/exit") {
+            shutdown();
+            return;
+        }
+        rl.pause();
         try {
-            while (true) {
-                const q = (await rl.question(chalk.green("> "))).trim();
-                if (!q || q === "exit" || q === "quit")
-                    break;
-                console.log(chalk.dim("Thinking...\n"));
-                console.log(await gemma.generate(q));
-                console.log();
+            const parts = input.split(/\s+/);
+            const cmd = parts[0];
+            if ((cmd === "/image" || cmd === "/audio" || cmd === "/video") && parts.length >= 3) {
+                const type = cmd.slice(1);
+                const path = parts[1];
+                const prompt = parts.slice(2).join(" ");
+                await streamMedia(prompt, type, path);
+            }
+            else {
+                await gemma.generateStream(input);
             }
         }
+        catch (error) {
+            console.error("\n[CRAYON ERROR]", error instanceof Error ? error.message : error);
+        }
         finally {
-            rl.close();
-            gemma.stop();
+            rl.resume();
+            rl.prompt();
         }
-    }
-    else {
-        console.log(await gemma.generate(task));
-        gemma.stop();
-    }
-});
-await program.parseAsync();
+    });
+}
+main().catch(error => { console.error("[CRAYON FATAL]", error); gemma.stop(); process.exit(1); });

package/dist/providers/local-gemma.js

@@ -1,109 +1,74 @@
 import { spawn } from "node:child_process";
-import { setTimeout as delay } from "node:timers/promises";
 export class LocalGemma {
     process;
-    python;
-    host;
-    port;
-    model;
-    constructor(options = {}) {
-        this.python = options.python ?? process.env.CRAYON_PYTHON ?? "python";
-        this.host = options.host ?? process.env.CRAYON_HOST ?? "127.0.0.1";
-        this.port = options.port ?? Number(process.env.CRAYON_PORT ?? 8765);
-        this.model =
-            options.model ??
-                process.env.CRAYON_MODEL ??
-                "google/gemma-4-E2B-it";
-    }
-    url(path) {
-        return `http://${this.host}:${this.port}${path}`;
+    options;
+    baseUrl;
+    constructor(options) {
+        this.options = options;
+        this.baseUrl = `http://${options.host}:${options.port}`;
     }
     async start() {
-        // Check whether a server is already running.
-        try {
-            const response = await fetch(this.url("/health"));
-            if (response.ok) {
+        if (await this.isReady())
+            return;
+        console.log("[CRAYON] Starting local Gemma multimodal server...");
+        console.log(`[CRAYON] Model: ${this.options.model}`);
+        this.process = spawn(this.options.python, ["model/server.py"], {
+            stdio: ["ignore", "pipe", "pipe"],
+            env: { ...process.env, CRAYON_MODEL: this.options.model, CRAYON_HOST: this.options.host,
+                CRAYON_PORT: String(this.options.port) }
+        });
+        this.process.stdout?.on("data", (d) => process.stdout.write(`[Gemma] ${d.toString()}`));
+        this.process.stderr?.on("data", (d) => process.stderr.write(`[Gemma] ${d.toString()}`));
+        const deadline = Date.now() + 180000;
+        while (Date.now() < deadline) {
+            if (await this.isReady()) {
+                console.log("[CRAYON] Local multimodal Gemma server is ready.");
                 return;
             }
+            await new Promise(r => setTimeout(r, 1000));
+        }
+        throw new Error("Gemma server did not become ready within 180 seconds.");
+    }
+    async isReady() {
+        try {
+            return (await fetch(`${this.baseUrl}/health`)).ok;
         }
         catch {
-            // Server isn't running; start it below.
+            return false;
         }
-        console.log(`[CRAYON] Starting local Gemma server...`);
-        console.log(`[CRAYON] Model: ${this.model}`);
-        this.process = spawn(this.python, ["model/server.py"], {
-            stdio: ["ignore", "pipe", "pipe"],
-            env: {
-                ...process.env,
-                CRAYON_MODEL: this.model,
-                CRAYON_HOST: this.host,
-                CRAYON_PORT: String(this.port),
-            },
-        });
-        // stdout is available because we configured it as "pipe".
-        this.process.stdout?.on("data", (data) => {
-            process.stdout.write(`\x1b[90m[Gemma] ${data.toString()}\x1b[0m`);
-        });
-        // stderr is available because we configured it as "pipe".
-        this.process.stderr?.on("data", (data) => {
-            process.stderr.write(`\x1b[90m[Gemma] ${data.toString()}\x1b[0m`);
-        });
-        this.process.on("error", (error) => {
-            console.error("[CRAYON] Failed to start Gemma:", error);
-        });
-        this.process.on("exit", (code, signal) => {
-            if (code !== 0 && code !== null) {
-                console.error(`[CRAYON] Gemma server exited with code ${code}`);
-            }
-            if (signal) {
-                console.error(`[CRAYON] Gemma server stopped by signal ${signal}`);
-            }
-        });
-        // Wait for FastAPI to become available.
-        for (let attempt = 0; attempt < 180; attempt++) {
-            try {
-                const response = await fetch(this.url("/health"));
-                if (response.ok) {
-                    console.log("[CRAYON] Gemma server is ready.");
-                    return;
-                }
-            }
-            catch {
-                // Still starting.
+    }
+    async consumeStream(response) {
+        if (!response.ok)
+            throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
+        if (!response.body)
+            throw new Error("Gemma returned no stream.");
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+        process.stdout.write("\nCRAYON: ");
+        try {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                process.stdout.write(decoder.decode(value, { stream: true }));
             }
-            await delay(1000);
+            process.stdout.write(decoder.decode());
+            process.stdout.write("\n");
+        }
+        finally {
+            reader.releaseLock();
         }
-        this.stop();
-        throw new Error("Gemma server did not become ready within 180 seconds.");
     }
-    async generate(prompt, maxNewTokens = Number(process.env.CRAYON_MAX_NEW_TOKENS ?? 1024)) {
-        await this.start();
-        const response = await fetch(this.url("/generate"), {
-            method: "POST",
-            headers: {
-                "content-type": "application/json",
-            },
-            body: JSON.stringify({
-                prompt,
-                max_new_tokens: maxNewTokens,
-                temperature: 0.2,
-                top_p: 0.9,
-            }),
+    async generateStream(prompt) {
+        const response = await fetch(`${this.baseUrl}/generate/stream`, {
+            method: "POST", headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ prompt, max_new_tokens: this.options.maxNewTokens,
+                temperature: this.options.temperature })
         });
-        if (!response.ok) {
-            const errorText = await response.text();
-            throw new Error(`Gemma server returned HTTP ${response.status}: ${errorText}`);
-        }
-        const result = (await response.json());
-        return result.text;
+        await this.consumeStream(response);
     }
     stop() {
-        if (!this.process) {
-            return;
-        }
-        if (!this.process.killed) {
+        if (this.process && !this.process.killed)
             this.process.kill();
-        }
-        this.process = undefined;
     }
 }

package/model/requirements.txt

@@ -1,6 +1,9 @@
-torch>=2.6
-transformers>=4.57
-accelerate>=1.10
-safetensors>=0.5
-fastapi>=0.115
-uvicorn[standard]>=0.34
+fastapi
+uvicorn[standard]
+transformers
+torch
+accelerate
+sentencepiece
+torchvision
+librosa
+Pillow

package/model/server.py

@@ -1,159 +1,171 @@
 import os
+import json
+import base64
+import mimetypes
+import tempfile
+from pathlib import Path
+from threading import Thread
+
 import torch
-from fastapi import FastAPI, HTTPException
+from fastapi import FastAPI, UploadFile, File, Form, HTTPException
+from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
-from transformers import AutoTokenizer, AutoModelForCausalLM
-
-MODEL_ID = os.getenv("CRAYON_MODEL", "google/gemma-4-E2B-it")
-HOST = os.getenv("CRAYON_HOST", "127.0.0.1")
-PORT = int(os.getenv("CRAYON_PORT", "8765"))
-
-print("=" * 60)
-print("[CRAYON] Local Gemma Server")
-print("=" * 60)
-print(f"[CRAYON] Model: {MODEL_ID}")
-print(f"[CRAYON] PyTorch: {torch.__version__}")
-print(f"[CRAYON] CUDA available: {torch.cuda.is_available()}")
-
-if torch.cuda.is_available():
-    print(f"[CRAYON] GPU: {torch.cuda.get_device_name(0)}")
-    print(f"[CRAYON] CUDA: {torch.version.cuda}")
-
-print("[CRAYON] Loading tokenizer...")
+from transformers import AutoProcessor, AutoModelForMultimodalLM, TextIteratorStreamer
 
-tokenizer = AutoTokenizer.from_pretrained(
-    MODEL_ID
-)
+MODEL_ID=os.getenv("CRAYON_MODEL","google/gemma-4-E2B-it")
+HOST=os.getenv("CRAYON_HOST","127.0.0.1")
+PORT=int(os.getenv("CRAYON_PORT","8765"))
 
-print("[CRAYON] Tokenizer loaded.")
-print("[CRAYON] Loading model...")
+print(f"[CRAYON MODEL] Loading: {MODEL_ID}",flush=True)
+print(f"[CRAYON MODEL] CUDA available: {torch.cuda.is_available()}",flush=True)
 
-model = AutoModelForCausalLM.from_pretrained(
+processor=AutoProcessor.from_pretrained(MODEL_ID)
+model=AutoModelForMultimodalLM.from_pretrained(
     MODEL_ID,
+    dtype="auto",
     device_map="auto",
-    torch_dtype="auto",
 )
-
 model.eval()
 
-print("[CRAYON] Model loaded successfully.")
-
-app = FastAPI(
-    title="CRAYON Local Gemma Server",
-    version="0.2.1",
-)
-
-
-class GenerateRequest(BaseModel):
-    prompt: str
-    max_new_tokens: int = 1024
-    temperature: float = 0.2
-    top_p: float = 0.9
-
+print("[CRAYON MODEL] Multimodal model loaded.",flush=True)
+print(f"[CRAYON MODEL] Device: {model.device}",flush=True)
+
+app=FastAPI(title="CRAYON Local Gemma Multimodal Server",version="0.4.0")
+
+class TextRequest(BaseModel):
+    prompt:str
+    max_new_tokens:int=1024
+    temperature:float=0.7
+    enable_thinking:bool=False
+
+def prepare_messages(prompt, media_type=None, media_path=None):
+    content=[]
+    if media_type in ("image","video","audio"):
+        if media_type=="image":
+            content.append({"type":"image","path":media_path})
+        elif media_type=="video":
+            content.append({"type":"video","path":media_path})
+        elif media_type=="audio":
+            content.append({"type":"audio","path":media_path})
+    content.append({"type":"text","text":prompt})
+    return [{"role":"user","content":content}]
+
+def make_inputs(messages, enable_thinking=False):
+    inputs=processor.apply_chat_template(
+        messages,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+        add_generation_prompt=True,
+        enable_thinking=enable_thinking,
+    )
+    return inputs.to(model.device)
+
+def stream_generation(inputs, max_new_tokens, temperature):
+    streamer=TextIteratorStreamer(
+        processor.tokenizer,
+        skip_prompt=True,
+        skip_special_tokens=True,
+    )
+    kwargs={
+        **inputs,
+        "streamer":streamer,
+        "max_new_tokens":max_new_tokens,
+        "do_sample":temperature>0,
+    }
+    if temperature>0:
+        kwargs["temperature"]=temperature
+
+    worker=Thread(target=model.generate,kwargs=kwargs,daemon=True)
+    worker.start()
+
+    def iterator():
+        generated=""
+        count=0
+        try:
+            for text in streamer:
+                generated += text
+                count += 1
+                print(f"[CRAYON STREAM] chunk={count}: {text!r}",flush=True)
+                yield text
+            print(
+                f"[CRAYON STREAM] complete | chunks={count} | chars={len(generated)}",
+                flush=True
+            )
+        finally:
+            worker.join()
+    return iterator()
 
 @app.get("/health")
 def health():
     return {
-        "ok": True,
-        "model": MODEL_ID,
-        "cuda": torch.cuda.is_available(),
-        "device": str(model.device),
+        "status":"ok",
+        "model":MODEL_ID,
+        "device":str(model.device),
+        "cuda":torch.cuda.is_available(),
+        "multimodal":True,
+        "inputs":["text","image","audio","video"],
+        "output":"text",
+        "streaming":True,
+        "inference":"local",
     }
 
-
 @app.post("/generate")
-def generate(req: GenerateRequest):
+def generate_text(request:TextRequest):
+    messages=prepare_messages(request.prompt)
+    inputs=make_inputs(messages,request.enable_thinking)
+    input_len=inputs["input_ids"].shape[-1]
+
+    kwargs={**inputs,"max_new_tokens":request.max_new_tokens,
+            "do_sample":request.temperature>0}
+    if request.temperature>0: kwargs["temperature"]=request.temperature
+
+    outputs=model.generate(**kwargs)
+    response=processor.decode(outputs[0][input_len:],skip_special_tokens=False)
+    return processor.parse_response(response,prefix=inputs["input_ids"])
+
+@app.post("/generate/stream")
+def generate_text_stream(request:TextRequest):
+    messages=prepare_messages(request.prompt)
+    inputs=make_inputs(messages,request.enable_thinking)
+    return StreamingResponse(
+        stream_generation(inputs,request.max_new_tokens,request.temperature),
+        media_type="text/plain; charset=utf-8",
+        headers={"Cache-Control":"no-cache","X-CRAYON-Streaming":"true"}
+    )
+
+@app.post("/generate/media")
+async def generate_media(
+    prompt:str=Form(...),
+    media_type:str=Form(...),
+    max_new_tokens:int=Form(1024),
+    temperature:float=Form(0.7),
+    enable_thinking:bool=Form(False),
+    file:UploadFile=File(...)
+):
+    if media_type not in ("image","audio","video"):
+        raise HTTPException(400,"media_type must be image, audio, or video")
+
+    suffix=Path(file.filename or "").suffix
+    data=await file.read()
+
+    with tempfile.NamedTemporaryFile(delete=False,suffix=suffix) as tmp:
+        tmp.write(data)
+        media_path=tmp.name
 
     try:
-        print(f"[CRAYON] Request: {req.prompt[:100]}")
-
-        messages = [
-            {
-                "role": "user",
-                "content": req.prompt,
-            }
-        ]
-
-        # Create the model input using Gemma's chat template.
-        inputs = tokenizer.apply_chat_template(
-            messages,
-            add_generation_prompt=True,
-            tokenize=True,
-            return_tensors="pt",
-            return_dict=True,
-        )
-
-        # BatchEncoding -> tensors
-        input_ids = inputs["input_ids"]
-        attention_mask = inputs.get("attention_mask")
-
-        # Move tensors to the model's device.
-        input_ids = input_ids.to(model.device)
-
-        if attention_mask is not None:
-            attention_mask = attention_mask.to(model.device)
-
-        print(
-            f"[CRAYON] Input tokens: {input_ids.shape[-1]}"
+        messages=prepare_messages(prompt,media_type,media_path)
+        inputs=make_inputs(messages,enable_thinking)
+        return StreamingResponse(
+            stream_generation(inputs,max_new_tokens,temperature),
+            media_type="text/plain; charset=utf-8",
+            headers={"Cache-Control":"no-cache","X-CRAYON-Streaming":"true"}
         )
+    finally:
+        # The processor may consume the media before the response iterator starts.
+        # Keep the file until generation has completed by wrapping the iterator.
+        pass
 
-        generation_kwargs = {
-            "input_ids": input_ids,
-            "max_new_tokens": req.max_new_tokens,
-            "do_sample": req.temperature > 0,
-            "temperature": max(req.temperature, 1e-5),
-            "top_p": req.top_p,
-            "pad_token_id": tokenizer.eos_token_id,
-        }
-
-        if attention_mask is not None:
-            generation_kwargs["attention_mask"] = attention_mask
-
-        print("[CRAYON] Generating...")
-
-        with torch.inference_mode():
-
-            output = model.generate(
-                **generation_kwargs
-            )
-
-        # Remove the original prompt tokens.
-        generated_tokens = output[
-            0,
-            input_ids.shape[-1]:
-        ]
-
-        text = tokenizer.decode(
-            generated_tokens,
-            skip_special_tokens=True,
-        )
-
-        print(
-            f"[CRAYON] Generated {len(generated_tokens)} tokens."
-        )
-
-        return {
-            "text": text,
-            "model": MODEL_ID,
-        }
-
-    except Exception as exc:
-
-        print("[CRAYON] Generation error:")
-        print(repr(exc))
-
-        raise HTTPException(
-            status_code=500,
-            detail=str(exc),
-        )
-
-
-if __name__ == "__main__":
-
+if __name__=="__main__":
     import uvicorn
-
-    uvicorn.run(
-        app,
-        host=HOST,
-        port=PORT,
-    )
+    uvicorn.run(app,host=HOST,port=PORT)

package/package.json

@@ -1,21 +1,18 @@
 {
     "name": "@lazarusrolando/crayoncli",
-    "version": "0.2.0",
+    "version": "0.4.0",
     "type": "module",
-    "bin": {
-        "crayon": "dist/index.js"
-    },
     "scripts": {
         "build": "tsc",
-        "start": "node dist/index.js"
+        "start": "node dist/index.js",
+        "dev": "tsx src/index.ts"
     },
     "dependencies": {
-        "chalk": "^5.4.1",
-        "commander": "^14.0.1",
-        "dotenv": "^17.2.1"
+        "dotenv": "^16.4.7"
     },
     "devDependencies": {
-        "@types/node": "^24.0.0",
-        "typescript": "^5.9.2"
+        "@types/node": "^22.10.5",
+        "tsx": "^4.19.2",
+        "typescript": "^5.7.2"
     }
 }

package/src/index.ts

@@ -1,52 +1,122 @@
-#!/usr/bin/env node
 import "dotenv/config";
-import { Command } from "commander";
-import chalk from "chalk";
+import readline from "node:readline";
+import { readFile } from "node:fs/promises";
+import { basename } from "node:path";
 import { LocalGemma } from "./providers/local-gemma.js";
-import { createInterface } from "node:readline/promises";
-import { stdin as input, stdout as output } from "node:process";
 
-const program = new Command();
+const gemma=new LocalGemma({
+  python:process.env.CRAYON_PYTHON||"python",
+  model:process.env.CRAYON_MODEL||"google/gemma-4-E2B-it",
+  host:process.env.CRAYON_HOST||"127.0.0.1",
+  port:Number(process.env.CRAYON_PORT||8765),
+  maxNewTokens:Number(process.env.CRAYON_MAX_NEW_TOKENS||1024),
+  temperature:Number(process.env.CRAYON_TEMPERATURE||0.7)
+});
+
+async function streamMedia(prompt:string,type:"image"|"audio"|"video",path:string){
+  const form=new FormData();
+  form.append("prompt",prompt);
+  form.append("media_type",type);
+  form.append("max_new_tokens",String(Number(process.env.CRAYON_MAX_NEW_TOKENS||1024)));
+  form.append("temperature",String(Number(process.env.CRAYON_TEMPERATURE||0.7)));
+
+  const data=await readFile(path);
+  form.append("file",new Blob([data]),basename(path));
 
-program
-  .name("crayon")
-  .description("CRAYON — local Crayon-style coding agent")
-  .argument("[task]");
+  const response=await fetch(
+    `http://${process.env.CRAYON_HOST||"127.0.0.1"}:${process.env.CRAYON_PORT||8765}/generate/media`,
+    {method:"POST",body:form}
+  );
 
-program.action(async (task?: string) => {
-  const gemma = new LocalGemma();
+  if(!response.ok) throw new Error(await response.text());
+  if(!response.body) throw new Error("No streaming response.");
+
+  const reader=response.body.getReader();
+  const decoder=new TextDecoder();
+  process.stdout.write("\nCRAYON: ");
+  try{
+    while(true){
+      const {done,value}=await reader.read();
+      if(done) break;
+      process.stdout.write(decoder.decode(value,{stream:true}));
+    }
+    process.stdout.write(decoder.decode());
+    process.stdout.write("\n");
+  }finally{reader.releaseLock();}
+}
 
-  process.on("SIGINT", () => {
+async function main(){
+  const args=process.argv.slice(2);
+  await gemma.start();
+
+  // One-shot:
+  // crayon "Explain this"
+  // crayon image photo.jpg "What is in this image?"
+  // crayon audio speech.wav "Transcribe this"
+  // crayon video clip.mp4 "Describe this video"
+  if(args[0] && ["image","audio","video"].includes(args[0]) && args[1] && args[2]){
+    await streamMedia(args.slice(2).join(" "),args[0] as "image"|"audio"|"video",args[1]);
     gemma.stop();
-    process.exit(130);
-  });
+    return;
+  }
 
-  console.log(chalk.cyan("\n╭──────────────────────────────────────────────╮"));
-  console.log(chalk.cyan("│              CRAYON LOCAL CODER                 │"));
-  console.log(chalk.cyan("│        Google Gemma · Local Inference        │"));
-  console.log(chalk.cyan("╰──────────────────────────────────────────────╯\n"));
+  if(args.length){
+    await gemma.generateStream(args.join(" "));
+    gemma.stop();
+    return;
+  }
+
+  console.log(`
+==========================================
+     CRAYON v0.4.0 - GEMMA MULTIMODAL CLI
+==========================================
+Text   -> Text
+Image  -> Text
+Audio  -> Text
+Video  -> Text
+
+Commands:
+  /image <file> <question>
+  /audio <file> <question>
+  /video <file> <question>
+  /exit
+
+Examples:
+  /image photo.jpg What is in this image?
+  /audio speech.wav Transcribe this audio.
+  /video clip.mp4 Describe what happens.
+`);
 
-  if (!task) {
-    const rl = createInterface({input, output});
+  const rl=readline.createInterface({input:process.stdin,output:process.stdout,prompt:"You: "});
+  const shutdown=()=>{rl.close();gemma.stop();process.exit(0);};
+  process.on("SIGINT",shutdown);
+  rl.prompt();
 
-    try {
-      while (true) {
-        const q = (await rl.question(chalk.green("> "))).trim();
+  rl.on("line",async line=>{
+    const input=line.trim();
+    if(!input){rl.prompt();return;}
+    if(input==="/exit"){shutdown();return;}
+    rl.pause();
 
-        if (!q || q === "exit" || q === "quit") break;
+    try{
+      const parts=input.split(/\s+/);
+      const cmd=parts[0];
 
-        console.log(chalk.dim("Thinking...\n"));
-        console.log(await gemma.generate(q));
-        console.log();
+      if((cmd==="/image"||cmd==="/audio"||cmd==="/video") && parts.length>=3){
+        const type=cmd.slice(1) as "image"|"audio"|"video";
+        const path=parts[1];
+        const prompt=parts.slice(2).join(" ");
+        await streamMedia(prompt,type,path);
+      }else{
+        await gemma.generateStream(input);
       }
-    } finally {
-      rl.close();
-      gemma.stop();
+    }catch(error){
+      console.error("\n[CRAYON ERROR]",error instanceof Error?error.message:error);
+    }finally{
+      rl.resume();
+      rl.prompt();
     }
-  } else {
-    console.log(await gemma.generate(task));
-    gemma.stop();
-  }
-});
+  });
+}
 
-await program.parseAsync();
+main().catch(error=>{console.error("[CRAYON FATAL]",error);gemma.stop();process.exit(1);});

package/src/providers/local-gemma.ts

@@ -1,165 +1,75 @@
-import { spawn, type ChildProcess } from "node:child_process";
-import { setTimeout as delay } from "node:timers/promises";
+import { spawn, ChildProcess } from "node:child_process";
+import { createReadStream } from "node:fs";
+import { basename } from "node:path";
 
 export interface GemmaOptions {
-  python?: string;
-  host?: string;
-  port?: number;
-  model?: string;
+  python:string; model:string; host:string; port:number;
+  maxNewTokens:number; temperature:number;
 }
 
 export class LocalGemma {
-  private process?: ChildProcess;
+  private process?:ChildProcess;
+  private readonly options:GemmaOptions;
+  private readonly baseUrl:string;
 
-  private readonly python: string;
-  private readonly host: string;
-  private readonly port: number;
-  private readonly model: string;
-
-  constructor(options: GemmaOptions = {}) {
-    this.python = options.python ?? process.env.CRAYON_PYTHON ?? "python";
-    this.host = options.host ?? process.env.CRAYON_HOST ?? "127.0.0.1";
-    this.port = options.port ?? Number(process.env.CRAYON_PORT ?? 8765);
-    this.model =
-      options.model ??
-      process.env.CRAYON_MODEL ??
-      "google/gemma-4-E2B-it";
-  }
-
-  private url(path: string): string {
-    return `http://${this.host}:${this.port}${path}`;
+  constructor(options:GemmaOptions){
+    this.options=options;
+    this.baseUrl=`http://${options.host}:${options.port}`;
   }
 
-  async start(): Promise<void> {
-    // Check whether a server is already running.
-    try {
-      const response = await fetch(this.url("/health"));
+  async start():Promise<void>{
+    if(await this.isReady()) return;
+    console.log("[CRAYON] Starting local Gemma multimodal server...");
+    console.log(`[CRAYON] Model: ${this.options.model}`);
 
-      if (response.ok) {
-        return;
-      }
-    } catch {
-      // Server isn't running; start it below.
-    }
-
-    console.log(`[CRAYON] Starting local Gemma server...`);
-    console.log(`[CRAYON] Model: ${this.model}`);
-
-    this.process = spawn(
-      this.python,
-      ["model/server.py"],
-      {
-        stdio: ["ignore", "pipe", "pipe"],
-        env: {
-          ...process.env,
-          CRAYON_MODEL: this.model,
-          CRAYON_HOST: this.host,
-          CRAYON_PORT: String(this.port),
-        },
-      },
-    );
-
-    // stdout is available because we configured it as "pipe".
-    this.process.stdout?.on("data", (data: Buffer) => {
-      process.stdout.write(
-        `\x1b[90m[Gemma] ${data.toString()}\x1b[0m`,
-      );
-    });
-
-    // stderr is available because we configured it as "pipe".
-    this.process.stderr?.on("data", (data: Buffer) => {
-      process.stderr.write(
-        `\x1b[90m[Gemma] ${data.toString()}\x1b[0m`,
-      );
-    });
-
-    this.process.on("error", (error) => {
-      console.error("[CRAYON] Failed to start Gemma:", error);
-    });
-
-    this.process.on("exit", (code, signal) => {
-      if (code !== 0 && code !== null) {
-        console.error(
-          `[CRAYON] Gemma server exited with code ${code}`,
-        );
-      }
-
-      if (signal) {
-        console.error(
-          `[CRAYON] Gemma server stopped by signal ${signal}`,
-        );
-      }
+    this.process=spawn(this.options.python,["model/server.py"],{
+      stdio:["ignore","pipe","pipe"],
+      env:{...process.env,CRAYON_MODEL:this.options.model,CRAYON_HOST:this.options.host,
+           CRAYON_PORT:String(this.options.port)}
     });
+    this.process.stdout?.on("data",(d:Buffer)=>process.stdout.write(`[Gemma] ${d.toString()}`));
+    this.process.stderr?.on("data",(d:Buffer)=>process.stderr.write(`[Gemma] ${d.toString()}`));
 
-    // Wait for FastAPI to become available.
-    for (let attempt = 0; attempt < 180; attempt++) {
-      try {
-        const response = await fetch(this.url("/health"));
-
-        if (response.ok) {
-          console.log("[CRAYON] Gemma server is ready.");
-          return;
-        }
-      } catch {
-        // Still starting.
-      }
-
-      await delay(1000);
+    const deadline=Date.now()+180000;
+    while(Date.now()<deadline){
+      if(await this.isReady()){ console.log("[CRAYON] Local multimodal Gemma server is ready."); return; }
+      await new Promise(r=>setTimeout(r,1000));
     }
+    throw new Error("Gemma server did not become ready within 180 seconds.");
+  }
 
-    this.stop();
-
-    throw new Error(
-      "Gemma server did not become ready within 180 seconds.",
-    );
+  async isReady():Promise<boolean>{
+    try{return (await fetch(`${this.baseUrl}/health`)).ok;}catch{return false;}
   }
 
-  async generate(
-    prompt: string,
-    maxNewTokens = Number(
-      process.env.CRAYON_MAX_NEW_TOKENS ?? 1024,
-    ),
-  ): Promise<string> {
-    await this.start();
+  private async consumeStream(response:Response):Promise<void>{
+    if(!response.ok) throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
+    if(!response.body) throw new Error("Gemma returned no stream.");
+
+    const reader=response.body.getReader();
+    const decoder=new TextDecoder();
+    process.stdout.write("\nCRAYON: ");
+    try{
+      while(true){
+        const {done,value}=await reader.read();
+        if(done) break;
+        process.stdout.write(decoder.decode(value,{stream:true}));
+      }
+      process.stdout.write(decoder.decode());
+      process.stdout.write("\n");
+    }finally{reader.releaseLock();}
+  }
 
-    const response = await fetch(this.url("/generate"), {
-      method: "POST",
-      headers: {
-        "content-type": "application/json",
-      },
-      body: JSON.stringify({
-        prompt,
-        max_new_tokens: maxNewTokens,
-        temperature: 0.2,
-        top_p: 0.9,
-      }),
+  async generateStream(prompt:string):Promise<void>{
+    const response=await fetch(`${this.baseUrl}/generate/stream`,{
+      method:"POST",headers:{"Content-Type":"application/json"},
+      body:JSON.stringify({prompt,max_new_tokens:this.options.maxNewTokens,
+                           temperature:this.options.temperature})
     });
-
-    if (!response.ok) {
-      const errorText = await response.text();
-
-      throw new Error(
-        `Gemma server returned HTTP ${response.status}: ${errorText}`,
-      );
-    }
-
-    const result = (await response.json()) as {
-      text: string;
-      model?: string;
-    };
-
-    return result.text;
+    await this.consumeStream(response);
   }
 
-  stop(): void {
-    if (!this.process) {
-      return;
-    }
-
-    if (!this.process.killed) {
-      this.process.kill();
-    }
-
-    this.process = undefined;
+  stop():void{
+    if(this.process && !this.process.killed) this.process.kill();
   }
 }

package/tsconfig.json

@@ -1,15 +1,5 @@
 {
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "outDir": "dist",
-    "rootDir": "src",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true
-  },
-  "include": [
-    "src"
-  ]
+  "compilerOptions":{"target":"ES2022","module":"NodeNext","moduleResolution":"NodeNext",
+  "outDir":"dist","rootDir":"src","strict":true,"esModuleInterop":true,"skipLibCheck":true},
+  "include":["src/**/*.ts"]
 }

package/.npm-global/crayon

@@ -1,16 +0,0 @@
-#!/bin/sh
-basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
-
-case `uname` in
-    *CYGWIN*|*MINGW*|*MSYS*)
-        if command -v cygpath > /dev/null 2>&1; then
-            basedir=`cygpath -w "$basedir"`
-        fi
-    ;;
-esac
-
-if [ -x "$basedir/node" ]; then
-  exec "$basedir/node"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" "$@"
-else 
-  exec node  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" "$@"
-fi

package/.npm-global/crayon.cmd

@@ -1,17 +0,0 @@
-@ECHO off
-GOTO start
-:find_dp0
-SET dp0=%~dp0
-EXIT /b
-:start
-SETLOCAL
-CALL :find_dp0
-
-IF EXIST "%dp0%\node.exe" (
-  SET "_prog=%dp0%\node.exe"
-) ELSE (
-  SET "_prog=node"
-  SET PATHEXT=%PATHEXT:;.JS;=;%
-)
-
-endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%"  "%dp0%\node_modules\Crayon-cli-local-gemma\dist\index.js" %*

package/.npm-global/crayon.ps1

@@ -1,28 +0,0 @@
-#!/usr/bin/env pwsh
-$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
-
-$exe=""
-if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
-  # Fix case when both the Windows and Linux builds of Node
-  # are installed in the same directory
-  $exe=".exe"
-}
-$ret=0
-if (Test-Path "$basedir/node$exe") {
-  # Support pipeline input
-  if ($MyInvocation.ExpectingInput) {
-    $input | & "$basedir/node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  } else {
-    & "$basedir/node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  }
-  $ret=$LASTEXITCODE
-} else {
-  # Support pipeline input
-  if ($MyInvocation.ExpectingInput) {
-    $input | & "node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  } else {
-    & "node$exe"  "$basedir/node_modules/Crayon-cli-local-gemma/dist/index.js" $args
-  }
-  $ret=$LASTEXITCODE
-}
-exit $ret