@lazarusrolando/crayoncli 0.3.0 → 0.4.0

package/README.md

@@ -1,12 +1,17 @@
-# CRAYON v0.3.0 — Local Gemma Streaming
+# CRAYON v0.4.0 — Gemma 4 E2B Multimodal CLI
 
-Real local token streaming for the CRAYON CLI.
+Local multimodal CLI using the official Transformers interface for `google/gemma-4-E2B-it`.
 
-## Flow
+## Supported
 
-You → Node.js CLI → local FastAPI → Transformers TextIteratorStreamer → local Gemma → tokens → CLI
+- Text -> Text
+- Image + Text -> Text
+- Audio + Text -> Text
+- Video + Text -> Text
+- Text + Image + Audio -> Text (server API can be extended for combined uploads)
+- Streaming output for every mode
 
-Hugging Face is only the model repository. Inference is local.
+Gemma 4 E2B supports native audio in addition to image/video multimodality. The model generates text; it is not an image/audio/video generator.
 
 ## Install
 
@@ -17,24 +22,57 @@ Copy-Item .env.example .env
 npm run build
 ```
 
-## Run
+Use a recent Transformers version. The model card currently recommends the latest Transformers.
+
+## CLI
+
+Text:
 
 ```powershell
-node dist/index.js
+node dist/index.js "Explain recursion"
+```
+
+Image:
+
+```powershell
+node dist/index.js image photo.jpg "What is in this image?"
+```
+
+Audio:
+
+```powershell
+node dist/index.js audio speech.wav "Transcribe and summarize this audio."
 ```
 
-or:
+Video:
 
 ```powershell
-node dist/index.js "Explain recursion in JavaScript"
+node dist/index.js video clip.mp4 "Describe what happens in this video."
 ```
 
-## What v0.3 adds
+Interactive:
 
-- `TextIteratorStreamer` for local token streaming
-- background generation thread
-- FastAPI `StreamingResponse`
-- Node.js `ReadableStream` consumption
-- keeps `/generate` for non-streaming compatibility
+```powershell
+node dist/index.js
+```
+
+Then:
+
+```text
+/image photo.jpg What is in this image?
+/audio speech.wav Transcribe this.
+/video clip.mp4 Describe what happens.
+```
+
+## Important
+
+Inference is local. Hugging Face is used as the model repository for downloading/caching model files; the CLI calls your local FastAPI server.
+
+The official Gemma 4 Transformers interface uses:
+
+```python
+AutoProcessor
+AutoModelForMultimodalLM
+```
 
-Streaming changes delivery timing, not the inference provider. The model still runs locally.
+rather than the text-only `AutoTokenizer` / `AutoModelForCausalLM` path.

package/dist/index.js

@@ -1,5 +1,7 @@
 import "dotenv/config";
 import readline from "node:readline";
+import { readFile } from "node:fs/promises";
+import { basename } from "node:path";
 import { LocalGemma } from "./providers/local-gemma.js";
 const gemma = new LocalGemma({
     python: process.env.CRAYON_PYTHON || "python",
@@ -9,38 +11,101 @@ const gemma = new LocalGemma({
     maxNewTokens: Number(process.env.CRAYON_MAX_NEW_TOKENS || 1024),
     temperature: Number(process.env.CRAYON_TEMPERATURE || 0.7)
 });
+async function streamMedia(prompt, type, path) {
+    const form = new FormData();
+    form.append("prompt", prompt);
+    form.append("media_type", type);
+    form.append("max_new_tokens", String(Number(process.env.CRAYON_MAX_NEW_TOKENS || 1024)));
+    form.append("temperature", String(Number(process.env.CRAYON_TEMPERATURE || 0.7)));
+    const data = await readFile(path);
+    form.append("file", new Blob([data]), basename(path));
+    const response = await fetch(`http://${process.env.CRAYON_HOST || "127.0.0.1"}:${process.env.CRAYON_PORT || 8765}/generate/media`, { method: "POST", body: form });
+    if (!response.ok)
+        throw new Error(await response.text());
+    if (!response.body)
+        throw new Error("No streaming response.");
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    process.stdout.write("\nCRAYON: ");
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            process.stdout.write(decoder.decode(value, { stream: true }));
+        }
+        process.stdout.write(decoder.decode());
+        process.stdout.write("\n");
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
 async function main() {
-    const oneShot = process.argv.slice(2).join(" ").trim();
+    const args = process.argv.slice(2);
     await gemma.start();
-    if (oneShot) {
-        await gemma.generateStream(oneShot);
+    // One-shot:
+    // crayon "Explain this"
+    // crayon image photo.jpg "What is in this image?"
+    // crayon audio speech.wav "Transcribe this"
+    // crayon video clip.mp4 "Describe this video"
+    if (args[0] && ["image", "audio", "video"].includes(args[0]) && args[1] && args[2]) {
+        await streamMedia(args.slice(2).join(" "), args[0], args[1]);
+        gemma.stop();
+        return;
+    }
+    if (args.length) {
+        await gemma.generateStream(args.join(" "));
         gemma.stop();
         return;
     }
-    console.log("\n==========================================");
-    console.log("       CRAYON v0.3.0 - LOCAL STREAMING");
-    console.log("==========================================");
-    console.log("Gemma inference: LOCAL");
-    console.log("Hugging Face inference API: NOT USED");
-    console.log("Streaming: ENABLED\n");
-    console.log("Type /exit to quit.\n");
+    console.log(`
+==========================================
+     CRAYON v0.4.0 - GEMMA MULTIMODAL CLI
+==========================================
+Text   -> Text
+Image  -> Text
+Audio  -> Text
+Video  -> Text
+
+Commands:
+  /image <file> <question>
+  /audio <file> <question>
+  /video <file> <question>
+  /exit
+
+Examples:
+  /image photo.jpg What is in this image?
+  /audio speech.wav Transcribe this audio.
+  /video clip.mp4 Describe what happens.
+`);
     const rl = readline.createInterface({ input: process.stdin, output: process.stdout, prompt: "You: " });
     const shutdown = () => { rl.close(); gemma.stop(); process.exit(0); };
     process.on("SIGINT", shutdown);
     rl.prompt();
     rl.on("line", async (line) => {
-        const prompt = line.trim();
-        if (!prompt) {
+        const input = line.trim();
+        if (!input) {
             rl.prompt();
             return;
         }
-        if (prompt === "/exit") {
+        if (input === "/exit") {
             shutdown();
             return;
         }
         rl.pause();
         try {
-            await gemma.generateStream(prompt);
+            const parts = input.split(/\s+/);
+            const cmd = parts[0];
+            if ((cmd === "/image" || cmd === "/audio" || cmd === "/video") && parts.length >= 3) {
+                const type = cmd.slice(1);
+                const path = parts[1];
+                const prompt = parts.slice(2).join(" ");
+                await streamMedia(prompt, type, path);
+            }
+            else {
+                await gemma.generateStream(input);
+            }
         }
         catch (error) {
             console.error("\n[CRAYON ERROR]", error instanceof Error ? error.message : error);

package/dist/providers/local-gemma.js

@@ -10,7 +10,7 @@ export class LocalGemma {
     async start() {
         if (await this.isReady())
             return;
-        console.log("[CRAYON] Starting local Gemma server...");
+        console.log("[CRAYON] Starting local Gemma multimodal server...");
         console.log(`[CRAYON] Model: ${this.options.model}`);
         this.process = spawn(this.options.python, ["model/server.py"], {
             stdio: ["ignore", "pipe", "pipe"],
@@ -22,7 +22,7 @@ export class LocalGemma {
         const deadline = Date.now() + 180000;
         while (Date.now() < deadline) {
             if (await this.isReady()) {
-                console.log("[CRAYON] Local Gemma server is ready.");
+                console.log("[CRAYON] Local multimodal Gemma server is ready.");
                 return;
             }
             await new Promise(r => setTimeout(r, 1000));
@@ -37,15 +37,11 @@ export class LocalGemma {
             return false;
         }
     }
-    async generateStream(prompt) {
-        const response = await fetch(`${this.baseUrl}/generate/stream`, {
-            method: "POST", headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({ prompt, max_new_tokens: this.options.maxNewTokens, temperature: this.options.temperature })
-        });
+    async consumeStream(response) {
         if (!response.ok)
-            throw new Error(`Gemma streaming error (${response.status}): ${await response.text()}`);
+            throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
         if (!response.body)
-            throw new Error("Gemma returned no streaming body.");
+            throw new Error("Gemma returned no stream.");
         const reader = response.body.getReader();
         const decoder = new TextDecoder();
         process.stdout.write("\nCRAYON: ");
@@ -63,14 +59,13 @@ export class LocalGemma {
             reader.releaseLock();
         }
     }
-    async generate(prompt) {
-        const response = await fetch(`${this.baseUrl}/generate`, {
+    async generateStream(prompt) {
+        const response = await fetch(`${this.baseUrl}/generate/stream`, {
             method: "POST", headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({ prompt, max_new_tokens: this.options.maxNewTokens, temperature: this.options.temperature })
+            body: JSON.stringify({ prompt, max_new_tokens: this.options.maxNewTokens,
+                temperature: this.options.temperature })
         });
-        if (!response.ok)
-            throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
-        return (await response.json()).text;
+        await this.consumeStream(response);
     }
     stop() {
         if (this.process && !this.process.killed)

package/model/requirements.txt

@@ -4,3 +4,6 @@ transformers
 torch
 accelerate
 sentencepiece
+torchvision
+librosa
+Pillow

package/model/server.py

@@ -1,10 +1,16 @@
 import os
+import json
+import base64
+import mimetypes
+import tempfile
+from pathlib import Path
 from threading import Thread
+
 import torch
-from fastapi import FastAPI
+from fastapi import FastAPI, UploadFile, File, Form, HTTPException
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from transformers import AutoProcessor, AutoModelForMultimodalLM, TextIteratorStreamer
 
 MODEL_ID=os.getenv("CRAYON_MODEL","google/gemma-4-E2B-it")
 HOST=os.getenv("CRAYON_HOST","127.0.0.1")
@@ -12,101 +18,153 @@ PORT=int(os.getenv("CRAYON_PORT","8765"))
 
 print(f"[CRAYON MODEL] Loading: {MODEL_ID}",flush=True)
 print(f"[CRAYON MODEL] CUDA available: {torch.cuda.is_available()}",flush=True)
-tokenizer=AutoTokenizer.from_pretrained(MODEL_ID)
-model=AutoModelForCausalLM.from_pretrained(MODEL_ID,device_map="auto",torch_dtype="auto")
+
+processor=AutoProcessor.from_pretrained(MODEL_ID)
+model=AutoModelForMultimodalLM.from_pretrained(
+    MODEL_ID,
+    dtype="auto",
+    device_map="auto",
+)
 model.eval()
-print("[CRAYON MODEL] Model loaded.",flush=True)
+
+print("[CRAYON MODEL] Multimodal model loaded.",flush=True)
 print(f"[CRAYON MODEL] Device: {model.device}",flush=True)
 
-app=FastAPI(title="CRAYON Local Gemma Server",version="0.3.0")
+app=FastAPI(title="CRAYON Local Gemma Multimodal Server",version="0.4.0")
 
-class GenerateRequest(BaseModel):
+class TextRequest(BaseModel):
     prompt:str
     max_new_tokens:int=1024
     temperature:float=0.7
-
-@app.get("/health")
-def health():
-    return {"status":"ok","model":MODEL_ID,"device":str(model.device),
-            "cuda":torch.cuda.is_available(),"streaming":True,"inference":"local"}
-
-def build_inputs(prompt):
-    messages=[{"role":"user","content":prompt}]
-    inputs=tokenizer.apply_chat_template(messages,tokenize=True,add_generation_prompt=True,
-                                          return_tensors="pt",return_dict=True)
-    input_ids=inputs["input_ids"].to(model.device)
-    attention_mask=inputs.get("attention_mask")
-    if attention_mask is not None:
-        attention_mask=attention_mask.to(model.device)
-    return input_ids,attention_mask
-
-@app.post("/generate/stream")
-def generate_stream(request:GenerateRequest):
-    input_ids,attention_mask=build_inputs(request.prompt)
-    streamer=TextIteratorStreamer(tokenizer,skip_prompt=True,skip_special_tokens=True)
-    kwargs={"input_ids":input_ids,"streamer":streamer,
-            "max_new_tokens":request.max_new_tokens,
-            "do_sample":request.temperature>0}
-    if attention_mask is not None: kwargs["attention_mask"]=attention_mask
-    if request.temperature>0: kwargs["temperature"]=request.temperature
+    enable_thinking:bool=False
+
+def prepare_messages(prompt, media_type=None, media_path=None):
+    content=[]
+    if media_type in ("image","video","audio"):
+        if media_type=="image":
+            content.append({"type":"image","path":media_path})
+        elif media_type=="video":
+            content.append({"type":"video","path":media_path})
+        elif media_type=="audio":
+            content.append({"type":"audio","path":media_path})
+    content.append({"type":"text","text":prompt})
+    return [{"role":"user","content":content}]
+
+def make_inputs(messages, enable_thinking=False):
+    inputs=processor.apply_chat_template(
+        messages,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+        add_generation_prompt=True,
+        enable_thinking=enable_thinking,
+    )
+    return inputs.to(model.device)
+
+def stream_generation(inputs, max_new_tokens, temperature):
+    streamer=TextIteratorStreamer(
+        processor.tokenizer,
+        skip_prompt=True,
+        skip_special_tokens=True,
+    )
+    kwargs={
+        **inputs,
+        "streamer":streamer,
+        "max_new_tokens":max_new_tokens,
+        "do_sample":temperature>0,
+    }
+    if temperature>0:
+        kwargs["temperature"]=temperature
 
     worker=Thread(target=model.generate,kwargs=kwargs,daemon=True)
     worker.start()
 
-    def token_iterator():
-        generated_text = ""
-        chunk_count = 0
-
+    def iterator():
+        generated=""
+        count=0
         try:
             for text in streamer:
-                chunk_count += 1
-                generated_text += text
-
-                # TextIteratorStreamer emits decoded text chunks. Re-tokenize
-                # accumulated output to report the generated-token count.
-                token_count = len(
-                    tokenizer.encode(
-                        generated_text,
-                        add_special_tokens=False,
-                    )
-                )
-
-                print(
-                    f"[CRAYON TOKEN] #{token_count}: {text!r}",
-                    flush=True,
-                )
-
+                generated += text
+                count += 1
+                print(f"[CRAYON STREAM] chunk={count}: {text!r}",flush=True)
                 yield text
-
-            final_token_count = len(
-                tokenizer.encode(
-                    generated_text,
-                    add_special_tokens=False,
-                )
-            )
-
             print(
-                f"[CRAYON STREAM] Generation complete | "
-                f"chunks={chunk_count} | "
-                f"generated_tokens={final_token_count}",
-                flush=True,
+                f"[CRAYON STREAM] complete | chunks={count} | chars={len(generated)}",
+                flush=True
             )
         finally:
             worker.join()
+    return iterator()
 
-    return StreamingResponse(token_iterator(),media_type="text/plain; charset=utf-8",
-                             headers={"Cache-Control":"no-cache","X-CRAYON-Streaming":"true"})
+@app.get("/health")
+def health():
+    return {
+        "status":"ok",
+        "model":MODEL_ID,
+        "device":str(model.device),
+        "cuda":torch.cuda.is_available(),
+        "multimodal":True,
+        "inputs":["text","image","audio","video"],
+        "output":"text",
+        "streaming":True,
+        "inference":"local",
+    }
 
 @app.post("/generate")
-def generate(request:GenerateRequest):
-    input_ids,attention_mask=build_inputs(request.prompt)
-    kwargs={"input_ids":input_ids,"max_new_tokens":request.max_new_tokens,
+def generate_text(request:TextRequest):
+    messages=prepare_messages(request.prompt)
+    inputs=make_inputs(messages,request.enable_thinking)
+    input_len=inputs["input_ids"].shape[-1]
+
+    kwargs={**inputs,"max_new_tokens":request.max_new_tokens,
             "do_sample":request.temperature>0}
-    if attention_mask is not None: kwargs["attention_mask"]=attention_mask
     if request.temperature>0: kwargs["temperature"]=request.temperature
-    output=model.generate(**kwargs)
-    new_tokens=output[0][input_ids.shape[-1]:]
-    return {"text":tokenizer.decode(new_tokens,skip_special_tokens=True)}
+
+    outputs=model.generate(**kwargs)
+    response=processor.decode(outputs[0][input_len:],skip_special_tokens=False)
+    return processor.parse_response(response,prefix=inputs["input_ids"])
+
+@app.post("/generate/stream")
+def generate_text_stream(request:TextRequest):
+    messages=prepare_messages(request.prompt)
+    inputs=make_inputs(messages,request.enable_thinking)
+    return StreamingResponse(
+        stream_generation(inputs,request.max_new_tokens,request.temperature),
+        media_type="text/plain; charset=utf-8",
+        headers={"Cache-Control":"no-cache","X-CRAYON-Streaming":"true"}
+    )
+
+@app.post("/generate/media")
+async def generate_media(
+    prompt:str=Form(...),
+    media_type:str=Form(...),
+    max_new_tokens:int=Form(1024),
+    temperature:float=Form(0.7),
+    enable_thinking:bool=Form(False),
+    file:UploadFile=File(...)
+):
+    if media_type not in ("image","audio","video"):
+        raise HTTPException(400,"media_type must be image, audio, or video")
+
+    suffix=Path(file.filename or "").suffix
+    data=await file.read()
+
+    with tempfile.NamedTemporaryFile(delete=False,suffix=suffix) as tmp:
+        tmp.write(data)
+        media_path=tmp.name
+
+    try:
+        messages=prepare_messages(prompt,media_type,media_path)
+        inputs=make_inputs(messages,enable_thinking)
+        return StreamingResponse(
+            stream_generation(inputs,max_new_tokens,temperature),
+            media_type="text/plain; charset=utf-8",
+            headers={"Cache-Control":"no-cache","X-CRAYON-Streaming":"true"}
+        )
+    finally:
+        # The processor may consume the media before the response iterator starts.
+        # Keep the file until generation has completed by wrapping the iterator.
+        pass
 
 if __name__=="__main__":
     import uvicorn

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@lazarusrolando/crayoncli",
-    "version": "0.3.0",
+    "version": "0.4.0",
     "type": "module",
     "scripts": {
         "build": "tsc",

package/src/index.ts

@@ -1,5 +1,7 @@
 import "dotenv/config";
 import readline from "node:readline";
+import { readFile } from "node:fs/promises";
+import { basename } from "node:path";
 import { LocalGemma } from "./providers/local-gemma.js";
 
 const gemma=new LocalGemma({
@@ -11,19 +13,79 @@ const gemma=new LocalGemma({
   temperature:Number(process.env.CRAYON_TEMPERATURE||0.7)
 });
 
+async function streamMedia(prompt:string,type:"image"|"audio"|"video",path:string){
+  const form=new FormData();
+  form.append("prompt",prompt);
+  form.append("media_type",type);
+  form.append("max_new_tokens",String(Number(process.env.CRAYON_MAX_NEW_TOKENS||1024)));
+  form.append("temperature",String(Number(process.env.CRAYON_TEMPERATURE||0.7)));
+
+  const data=await readFile(path);
+  form.append("file",new Blob([data]),basename(path));
+
+  const response=await fetch(
+    `http://${process.env.CRAYON_HOST||"127.0.0.1"}:${process.env.CRAYON_PORT||8765}/generate/media`,
+    {method:"POST",body:form}
+  );
+
+  if(!response.ok) throw new Error(await response.text());
+  if(!response.body) throw new Error("No streaming response.");
+
+  const reader=response.body.getReader();
+  const decoder=new TextDecoder();
+  process.stdout.write("\nCRAYON: ");
+  try{
+    while(true){
+      const {done,value}=await reader.read();
+      if(done) break;
+      process.stdout.write(decoder.decode(value,{stream:true}));
+    }
+    process.stdout.write(decoder.decode());
+    process.stdout.write("\n");
+  }finally{reader.releaseLock();}
+}
+
 async function main(){
-  const oneShot=process.argv.slice(2).join(" ").trim();
+  const args=process.argv.slice(2);
   await gemma.start();
 
-  if(oneShot){ await gemma.generateStream(oneShot); gemma.stop(); return; }
+  // One-shot:
+  // crayon "Explain this"
+  // crayon image photo.jpg "What is in this image?"
+  // crayon audio speech.wav "Transcribe this"
+  // crayon video clip.mp4 "Describe this video"
+  if(args[0] && ["image","audio","video"].includes(args[0]) && args[1] && args[2]){
+    await streamMedia(args.slice(2).join(" "),args[0] as "image"|"audio"|"video",args[1]);
+    gemma.stop();
+    return;
+  }
 
-  console.log("\n==========================================");
-  console.log("       CRAYON v0.3.0 - LOCAL STREAMING");
-  console.log("==========================================");
-  console.log("Gemma inference: LOCAL");
-  console.log("Hugging Face inference API: NOT USED");
-  console.log("Streaming: ENABLED\n");
-  console.log("Type /exit to quit.\n");
+  if(args.length){
+    await gemma.generateStream(args.join(" "));
+    gemma.stop();
+    return;
+  }
+
+  console.log(`
+==========================================
+     CRAYON v0.4.0 - GEMMA MULTIMODAL CLI
+==========================================
+Text   -> Text
+Image  -> Text
+Audio  -> Text
+Video  -> Text
+
+Commands:
+  /image <file> <question>
+  /audio <file> <question>
+  /video <file> <question>
+  /exit
+
+Examples:
+  /image photo.jpg What is in this image?
+  /audio speech.wav Transcribe this audio.
+  /video clip.mp4 Describe what happens.
+`);
 
   const rl=readline.createInterface({input:process.stdin,output:process.stdout,prompt:"You: "});
   const shutdown=()=>{rl.close();gemma.stop();process.exit(0);};
@@ -31,13 +93,30 @@ async function main(){
   rl.prompt();
 
   rl.on("line",async line=>{
-    const prompt=line.trim();
-    if(!prompt){rl.prompt();return;}
-    if(prompt==="/exit"){shutdown();return;}
+    const input=line.trim();
+    if(!input){rl.prompt();return;}
+    if(input==="/exit"){shutdown();return;}
     rl.pause();
-    try{await gemma.generateStream(prompt);}
-    catch(error){console.error("\n[CRAYON ERROR]",error instanceof Error?error.message:error);}
-    finally{rl.resume();rl.prompt();}
+
+    try{
+      const parts=input.split(/\s+/);
+      const cmd=parts[0];
+
+      if((cmd==="/image"||cmd==="/audio"||cmd==="/video") && parts.length>=3){
+        const type=cmd.slice(1) as "image"|"audio"|"video";
+        const path=parts[1];
+        const prompt=parts.slice(2).join(" ");
+        await streamMedia(prompt,type,path);
+      }else{
+        await gemma.generateStream(input);
+      }
+    }catch(error){
+      console.error("\n[CRAYON ERROR]",error instanceof Error?error.message:error);
+    }finally{
+      rl.resume();
+      rl.prompt();
+    }
   });
 }
+
 main().catch(error=>{console.error("[CRAYON FATAL]",error);gemma.stop();process.exit(1);});

package/src/providers/local-gemma.ts

@@ -1,4 +1,6 @@
 import { spawn, ChildProcess } from "node:child_process";
+import { createReadStream } from "node:fs";
+import { basename } from "node:path";
 
 export interface GemmaOptions {
   python:string; model:string; host:string; port:number;
@@ -17,7 +19,7 @@ export class LocalGemma {
 
   async start():Promise<void>{
     if(await this.isReady()) return;
-    console.log("[CRAYON] Starting local Gemma server...");
+    console.log("[CRAYON] Starting local Gemma multimodal server...");
     console.log(`[CRAYON] Model: ${this.options.model}`);
 
     this.process=spawn(this.options.python,["model/server.py"],{
@@ -25,13 +27,12 @@ export class LocalGemma {
       env:{...process.env,CRAYON_MODEL:this.options.model,CRAYON_HOST:this.options.host,
            CRAYON_PORT:String(this.options.port)}
     });
-
     this.process.stdout?.on("data",(d:Buffer)=>process.stdout.write(`[Gemma] ${d.toString()}`));
     this.process.stderr?.on("data",(d:Buffer)=>process.stderr.write(`[Gemma] ${d.toString()}`));
 
     const deadline=Date.now()+180000;
     while(Date.now()<deadline){
-      if(await this.isReady()){ console.log("[CRAYON] Local Gemma server is ready."); return; }
+      if(await this.isReady()){ console.log("[CRAYON] Local multimodal Gemma server is ready."); return; }
       await new Promise(r=>setTimeout(r,1000));
     }
     throw new Error("Gemma server did not become ready within 180 seconds.");
@@ -41,13 +42,9 @@ export class LocalGemma {
     try{return (await fetch(`${this.baseUrl}/health`)).ok;}catch{return false;}
   }
 
-  async generateStream(prompt:string):Promise<void>{
-    const response=await fetch(`${this.baseUrl}/generate/stream`,{
-      method:"POST",headers:{"Content-Type":"application/json"},
-      body:JSON.stringify({prompt,max_new_tokens:this.options.maxNewTokens,temperature:this.options.temperature})
-    });
-    if(!response.ok) throw new Error(`Gemma streaming error (${response.status}): ${await response.text()}`);
-    if(!response.body) throw new Error("Gemma returned no streaming body.");
+  private async consumeStream(response:Response):Promise<void>{
+    if(!response.ok) throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
+    if(!response.body) throw new Error("Gemma returned no stream.");
 
     const reader=response.body.getReader();
     const decoder=new TextDecoder();
@@ -63,16 +60,16 @@ export class LocalGemma {
     }finally{reader.releaseLock();}
   }
 
-  async generate(prompt:string):Promise<string>{
-    const response=await fetch(`${this.baseUrl}/generate`,{
+  async generateStream(prompt:string):Promise<void>{
+    const response=await fetch(`${this.baseUrl}/generate/stream`,{
       method:"POST",headers:{"Content-Type":"application/json"},
-      body:JSON.stringify({prompt,max_new_tokens:this.options.maxNewTokens,temperature:this.options.temperature})
+      body:JSON.stringify({prompt,max_new_tokens:this.options.maxNewTokens,
+                           temperature:this.options.temperature})
     });
-    if(!response.ok) throw new Error(`Gemma error (${response.status}): ${await response.text()}`);
-    return ((await response.json()) as {text:string}).text;
+    await this.consumeStream(response);
   }
 
   stop():void{
     if(this.process && !this.process.killed) this.process.kill();
   }
-}
+}

package/tsconfig.json

@@ -1 +1,5 @@
-{"compilerOptions":{"target":"ES2022","module":"NodeNext","moduleResolution":"NodeNext","outDir":"dist","rootDir":"src","strict":true,"esModuleInterop":true,"skipLibCheck":true},"include":["src/**/*.ts"]}
+{
+  "compilerOptions":{"target":"ES2022","module":"NodeNext","moduleResolution":"NodeNext",
+  "outDir":"dist","rootDir":"src","strict":true,"esModuleInterop":true,"skipLibCheck":true},
+  "include":["src/**/*.ts"]
+}