@lzhzzzzwill/cofos 1.1.15 → 1.2.0

package/README.md

@@ -14,9 +14,11 @@ The standalone npm package includes a minimal Python runtime under `runtime/`:
 fwdemo/
 ├── bin/cofos.js
 ├── scripts/chat.py
+├── scripts/sync_runtime.py
 ├── runtime/
 │   ├── config/
 │   └── src/
+├── RUNTIME.md
 └── requirements.txt
 ```
 
@@ -139,8 +141,9 @@ under `runtime/` exists only so the published package is self-contained.
 - Fix bugs in `src/` (or `config/`), never in `runtime/`.
 - `npm pack` / `npm publish` rebuild `runtime/` automatically via the package-local
   `prepack` wrapper when the full source checkout is present.
-- Rebuild by hand with `npm run build`, or
-  `python scripts/sync_fwdemo_runtime.py`.
+- Rebuild by hand from `fwdemo/` with `npm run build`.
+- From the repository root, rebuild with `python scripts/sync_fwdemo_runtime.py`.
+- The package-local wrapper `fwdemo/scripts/sync_runtime.py` delegates to the root sync script in a source checkout, and only validates bundled files in an installed package.
 - Check for drift with `python scripts/sync_fwdemo_runtime.py --check`.
 
 The published package requires its bundled `runtime/`; the backend does not fall

package/RUNTIME.md

@@ -21,6 +21,20 @@ cofos --model /path/to/local/merged_model
 
 PDFs passed with `--pdf-dir` are parsed into local chunks and merged into BM25 retrieval data under the COFOS cache root, defaulting to `./.cofos/`. They are not uploaded anywhere.
 
+## Packaged file scope
+
+The npm package intentionally ships only the runtime subset needed for QA:
+
+- `runtime/config/config.yaml`
+- `runtime/config/prompt_templates.yaml`
+- PDF parsing helpers
+- student inference, query routing, KG/BM25 retrieval, BM25 building, and text search
+
+Training, extraction, SFT generation, FAISS building, and full KG construction
+remain repository-only tools and are not included in the package. The package
+uses `fwdemo/scripts/sync_runtime.py` during `prepack`; in a source checkout that
+wrapper delegates to `scripts/sync_fwdemo_runtime.py`.
+
 ## Runtime config scope
 
 The packaged `runtime/config/config.yaml` is generated from the full project config with a strict whitelist. It keeps only the fields needed by the standalone QA path: paths, retrieval, generation, prompt location, ROS normalization, and the student model pointer.

package/bin/cofos.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { spawn, spawnSync } from "node:child_process";
-import { clearLine, createInterface, cursorTo } from "node:readline";
+import blessed from "blessed";
 import { existsSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -16,6 +16,7 @@ const DEFAULT_CONFIG = process.env.COFOS_CONFIG || "config/config.yaml";
 /* ─── ANSI ─── */
 const R = "\x1b[0m", B = "\x1b[1m", D = "\x1b[2m";
 const C = "\x1b[96m", W = "\x1b[97m", G = "\x1b[90m", Y = "\x1b[93m";
+const M = "\x1b[95m", BL = "\x1b[94m", GR = "\x1b[92m", A = "\x1b[38;5;208m";
 
 /* ─── Protocol markers ─── */
 const READY = "===READY===", RESPONSE = "===RESPONSE===", DONE = "===DONE===";
@@ -30,9 +31,16 @@ const ART_W = ART[0].length;
 
 function vis(t) { return t.replace(/\x1b\[[0-9;]*m/g, "").length; }
 function fw() { return Math.min(Math.max((process.stdout.columns || 110) - 2, 82), 150); }
-function top(w)  { return "╭" + "─".repeat(w) + "╮"; }
-function bot(w)  { return "╰" + "─".repeat(w) + "╯"; }
-function sep(w)  { return "├" + "─".repeat(w) + "┤"; }
+function plain(t) { return t.replace(/\x1b\[[0-9;]*m/g, ""); }
+function topTitle(w, left, right = "") {
+  const l = ` ${C}${B}${left}${R} `;
+  const r = right ? ` ${G}${right}${R} ` : "";
+  const room = Math.max(0, w - vis(l) - vis(r));
+  return C + "╭" + R + l + C + "─".repeat(room) + R + r + C + "╮" + R;
+}
+function top(w)  { return C + "╭" + "─".repeat(w) + "╮" + R; }
+function bot(w)  { return C + "╰" + "─".repeat(w) + "╯" + R; }
+function sep(w)  { return C + "├" + "─".repeat(w) + "┤" + R; }
 function vpad(t, w) { return t + " ".repeat(Math.max(0, w - vis(t))); }
 function clip(t, w) {
   const plain = t.replace(/\x1b\[[0-9;]*m/g, "");
@@ -41,17 +49,16 @@ function clip(t, w) {
   return plain.slice(0, w - 1) + "…";
 }
 function cell(t, w) { return vpad(clip(t, w), w); }
-function row(left, right, lw, rw) { return "│ " + cell(left, lw) + " │ " + cell(right, rw) + " │"; }
-function full(t, w) { return "│ " + cell(t, w - 2) + " │"; }
+function row(left, right, lw, rw) { return C + "│ " + R + cell(left, lw) + C + " │ " + R + cell(right, rw) + C + " │" + R; }
+function full(t, w) { return C + "│ " + R + cell(t, w - 2) + C + " │" + R; }
 function empty(w) { return full("", w); }
 
 function logoLines(width) {
-  if (width >= ART_W) return ART.map((line) => C + B + line + R);
+  if (width >= ART_W) return ART.map((line, i) => (i < 2 ? C : i < 4 ? BL : M) + B + line + R);
   return [C + B + "COFOS" + R];
 }
 
 function render({ modelPath, cacheDir, pdfDir }) {
-  console.clear();
   const w = fw();
   const minSplit = 96;
   const cwd = process.cwd();
@@ -60,25 +67,26 @@ function render({ modelPath, cacheDir, pdfDir }) {
     ...logoLines(Math.floor((w - 5) * 0.48)),
     "",
     G + "Scientific RAG CLI" + R,
-    W + "Model" + R + "   " + modelPath,
-    W + "Params" + R + "  9B",
-    W + "Path" + R + "    " + cwd,
-    W + "Cache" + R + "   " + cacheDir,
+    Y + "Model" + R + "   " + W + modelPath + R,
+    Y + "Params" + R + "  " + W + "9B" + R,
+    Y + "Path" + R + "    " + W + cwd + R,
+    Y + "Cache" + R + "   " + W + cacheDir + R,
   ];
   const right = [
-    W + B + "What COFOS Does" + R,
+    GR + B + "What COFOS Does" + R,
     "COFOS answers scientific questions about covalent organic frameworks,",
     "photocatalysis, reactive oxygen species, and oxygen-reduction products.",
     rag,
     "",
-    W + B + "Suggested Commands" + R,
-    "/help      show commands and PDF-RAG usage",
-    "/topk 5    change retrieval depth",
-    "/rag off   answer without retrieval",
-    "cofos --pdf-dir ./new_pdfs   add local papers",
+    A + B + "Suggested Commands" + R,
+    C + "/help" + R + "      show commands and PDF-RAG usage",
+    C + "/topk 5" + R + "    change retrieval depth",
+    C + "/rag off" + R + "   answer without retrieval",
+    M + "cofos --pdf-dir ./new_pdfs" + R + "   add local papers",
   ];
 
-  const lines = [top(w)];
+  const titleLeft = `COFOS CLI v${pkg.version}`;
+  const lines = [topTitle(w, titleLeft)];
   if (w >= minSplit) {
     const inner = w - 5;
     const lw = Math.max(46, Math.floor(inner * 0.48));
@@ -91,7 +99,7 @@ function render({ modelPath, cacheDir, pdfDir }) {
     for (const item of right) lines.push(full(item, w));
   }
   lines.push(bot(w));
-  console.log(lines.join("\n"));
+  return lines.join("\n");
 }
 
 /* ─── Slash commands ─── */
@@ -133,12 +141,149 @@ function findPy() { return cmdExist("python3") ? "python3" : cmdExist("python")
 // Only the modules the chat backend actually imports for the standalone merged-model path.
 const REQUIRED_MODULES = "import transformers, torch, yaml, huggingface_hub, pdfplumber, rank_bm25";
 
-function ensureDeps(py) {
-  console.log("[cofos] Checking Python dependencies …");
+function ensureDeps(py, log = console.log) {
+  log("[cofos] Checking Python dependencies …");
   if (spawnSync(py, ["-c", `${REQUIRED_MODULES}; print('ok')`], { encoding: "utf-8", stdio: "pipe" }).status === 0) return;
-  console.log("[cofos] Installing Python dependencies (first run, may take a while) …");
-  const r = spawnSync(py, ["-m", "pip", "install", "-r", reqFile], { encoding: "utf-8", stdio: "inherit" });
-  if (r.status !== 0) { console.error("[cofos] Failed to install Python dependencies."); process.exit(1); }
+  log("[cofos] Installing Python dependencies (first run, may take a while) …");
+  const r = spawnSync(py, ["-m", "pip", "install", "-r", reqFile], { encoding: "utf-8", stdio: "pipe" });
+  if (r.stdout) log(r.stdout.trimEnd());
+  if (r.stderr) log(r.stderr.trimEnd());
+  if (r.status !== 0) { log("[cofos] Failed to install Python dependencies."); process.exit(1); }
+}
+
+/* ─── TUI ─── */
+
+function createTui() {
+  const screen = blessed.screen({
+    smartCSR: true,
+    fullUnicode: true,
+    title: "COFOS",
+  });
+
+  const history = blessed.box({
+    top: 0,
+    left: 0,
+    width: "100%",
+    height: "100%-3",
+    scrollable: true,
+    alwaysScroll: true,
+    keys: true,
+    vi: true,
+    mouse: true,
+    tags: false,
+    border: { type: "line" },
+    style: {
+      border: { fg: "cyan" },
+      scrollbar: { bg: "cyan" },
+    },
+    scrollbar: { ch: " ", track: { bg: "black" }, style: { bg: "cyan" } },
+  });
+
+  const suggestions = blessed.box({
+    bottom: 3,
+    left: 2,
+    width: "70%",
+    height: 6,
+    hidden: true,
+    border: { type: "line" },
+    tags: true,
+    style: { border: { fg: "yellow" }, fg: "white", bg: "black" },
+  });
+
+  const input = blessed.textbox({
+    bottom: 0,
+    left: 0,
+    width: "100%",
+    height: 3,
+    inputOnFocus: true,
+    keys: true,
+    mouse: true,
+    border: { type: "line" },
+    label: " cofos ",
+    style: {
+      border: { fg: "cyan" },
+      focus: { border: { fg: "green" } },
+    },
+  });
+
+  screen.append(history);
+  screen.append(suggestions);
+  screen.append(input);
+
+  let buffer = "";
+  let onSubmit = () => {};
+
+  function append(text = "") {
+    buffer += String(text).replace(/\r/g, "");
+    if (buffer.length > 250000) buffer = buffer.slice(-220000);
+    history.setContent(buffer);
+    history.setScrollPerc(100);
+    screen.render();
+  }
+
+  function hideSuggestions() {
+    if (!suggestions.hidden) {
+      suggestions.hide();
+      screen.render();
+    }
+  }
+
+  function updateSuggestions() {
+    const value = input.getValue() || "";
+    if (!value.startsWith("/")) {
+      hideSuggestions();
+      return;
+    }
+    const matches = slashMatches(value);
+    if (!matches.length) {
+      hideSuggestions();
+      return;
+    }
+    suggestions.setContent(matches.map(({ cmd, desc }) => `{cyan-fg}${cmd}{/cyan-fg}  ${desc}`).join("\n"));
+    suggestions.height = Math.min(8, matches.length + 2);
+    suggestions.show();
+    screen.render();
+  }
+
+  input.on("submit", (value) => {
+    hideSuggestions();
+    input.clearValue();
+    screen.render();
+    onSubmit(String(value || ""));
+  });
+
+  input.on("keypress", () => setTimeout(updateSuggestions, 0));
+
+  screen.key(["tab"], () => {
+    const value = input.getValue() || "";
+    if (!value.startsWith("/")) return;
+    const matches = slashMatches(value);
+    if (!matches.length) return;
+    if (matches.length === 1) {
+      input.setValue(matches[0].cmd);
+      hideSuggestions();
+      input.focus();
+      screen.render();
+      return;
+    }
+    updateSuggestions();
+  });
+
+  screen.key(["escape"], hideSuggestions);
+  screen.key(["C-c"], () => process.emit("SIGINT"));
+
+  input.focus();
+  screen.render();
+
+  return {
+    screen,
+    input,
+    append,
+    print: (text = "") => append(String(text) + "\n"),
+    setSubmitHandler: (fn) => { onSubmit = fn; },
+    setPrompt: (label) => { input.setLabel(` ${label} `); screen.render(); },
+    destroy: () => screen.destroy(),
+  };
 }
 
 /* ─── main ─── */
@@ -209,17 +354,17 @@ function main() {
   if (pdfDir) backendArgs.push("--pdf-dir", pdfDir);
   if (rebuildPdfIndex) backendArgs.push("--rebuild-pdf-index");
 
-  render({ modelPath, cacheDir, pdfDir });
-
   const py = findPy();
   if (!py) die("Python not found. Install Python 3.9+ and re-run.");
   if (!existsSync(chatScript)) die(`Missing script: ${chatScript}`);
-  ensureDeps(py);
 
-  console.log("  Loading COFOS 9B + RAG backend from Hugging Face … first run may take a while");
-  if (pdfDir) console.log("  Local PDF RAG: enabled for this session");
-  else console.log("  Tip: add local papers with `cofos --pdf-dir ./new_pdfs`");
-  console.log("");
+  const ui = createTui();
+  ui.print(plain(render({ modelPath, cacheDir, pdfDir })));
+  ensureDeps(py, (msg) => ui.print(msg));
+  ui.print("  Loading COFOS 9B + RAG backend from Hugging Face … first run may take a while");
+  if (pdfDir) ui.print("  Local PDF RAG: enabled for this session");
+  else ui.print("  Tip: add local papers with `cofos --pdf-dir ./new_pdfs`");
+  ui.print("");
 
   const backendEnv = {
     ...process.env,
@@ -237,50 +382,78 @@ function main() {
     env: backendEnv,
   });
 
-  // `encoding` is not a spawn() option — without setEncoding each Buffer chunk
-  // is decoded on its own, so a multi-byte character (·OH, ¹O₂, any Chinese)
-  // straddling a chunk boundary comes out as mojibake.
   proc.stdout.setEncoding("utf-8");
   proc.stderr.setEncoding("utf-8");
 
   let buf = "", ready = false, generating = false, responseOpen = false, closed = false;
   const inputQ = [];
-  const mainPrompt = C + "  cofos " + R;
-  const contPrompt = G + "  ...   " + R;
-  const rl = createInterface({
-    input: process.stdin,
-    output: process.stdout,
-    terminal: true,
-    prompt: mainPrompt,
-    completer: slashCompletions,
-  });
-  rl.on("line", (l) => {
-    if (!ready || generating) inputQ.push(l);
-    else if (!handle(l)) ask();
-  });
+  let mlBuf = [];
 
-  function preserveInputWrite(stream, chunk) {
-    const data = String(chunk);
-    if (!rl.terminal || closed) {
-      stream.write(data);
-      return;
+  function shutdown() {
+    if (closed) return;
+    closed = true;
+    try { proc.stdin.end(); } catch (_) {}
+    try { proc.kill("SIGTERM"); } catch (_) {}
+    ui.destroy();
+  }
+
+  process.on("SIGTERM", shutdown);
+  process.on("SIGINT", shutdown);
+
+  function drain() {
+    while (inputQ.length) {
+      const line = inputQ.shift();
+      if (handle(line)) return;
+    }
+  }
+
+  function handle(rawInput) {
+    let input = String(rawInput || "");
+    if (input.endsWith("\\")) {
+      mlBuf.push(input.slice(0, -1));
+      ui.setPrompt("...");
+      return false;
+    }
+    if (mlBuf.length > 0) {
+      mlBuf.push(input);
+      input = mlBuf.join("\n");
+      mlBuf = [];
+      ui.setPrompt("cofos");
+    }
+
+    input = input.trim();
+    if (!input) return false;
+
+    if (/^\/(exit|quit)$/.test(input)) { proc.stdin.write("/exit\n"); proc.stdin.end(); return true; }
+    if (/^\/(help)$/.test(input)) {
+      ui.print(slashHint("/") + "\n  Add local papers to RAG:\n    cofos --pdf-dir ./new_pdfs\n    cofos --pdf-dir ./new_pdfs --rebuild-pdf-index\n\n  Multi-line: end a line with \\ to continue.");
+      return false;
     }
-    const activeInput = !generating;
-    if (activeInput) {
-      clearLine(rl.output, 0);
-      cursorTo(rl.output, 0);
+    if (/^\/(clear)$/.test(input)) {
+      mlBuf = [];
+      responseOpen = false;
+      proc.stdin.write(input + "\n");
+      generating = true;
+      return true;
+    }
+    if (/^\/(info)$/.test(input)) {
+      ui.print(`\n  COFOS CLI — v${pkg.version}\n  Model:  ${modelPath} (Hugging Face)\n  RAG:    Willlzh/COFOS_data/runtime (KG + BM25) + optional --pdf-dir BM25 evidence\n  Cache:  ${cacheDir}\n  HF:     ${backendEnv.HF_HOME}\n  Engine: StudentAdapterInference + transformers + PyTorch\n  Hist:   ~/.cofos_history.jsonl\n`);
+      return false;
     }
-    stream.write(data);
-    if (activeInput) rl.prompt(true);
+
+    ui.print(G + "  you   " + R + input);
+    responseOpen = false;
+    generating = true;
+    proc.stdin.write(input + "\n");
+    return true;
   }
 
-  // Let users type while the Python backend is still loading. Submitted lines
-  // are queued until READY, while status logs are written above and the input
-  // line is restored afterward.
-  ask();
+  ui.setSubmitHandler((line) => {
+    if (!ready || generating) inputQ.push(line);
+    else handle(line);
+  });
 
-  // Forward Python stderr (status / progress messages) without destroying the input line.
-  proc.stderr.on("data", (chunk) => { preserveInputWrite(process.stderr, chunk); });
+  proc.stderr.on("data", (chunk) => ui.append(chunk));
 
   proc.stdout.on("data", (chunk) => {
     buf += chunk;
@@ -294,14 +467,11 @@ function main() {
       }
       return;
     }
-    // Anything arriving while no answer is in flight is not part of a response;
-    // dropping it keeps it from being prepended to the next one.
     if (!generating) { buf = ""; return; }
 
     if (!responseOpen) {
       const ri = buf.indexOf(RESPONSE);
       if (ri === -1) {
-        // Keep enough tail to catch a RESPONSE marker split across chunks.
         const keep = RESPONSE.length - 1;
         if (buf.length > keep) buf = buf.slice(-keep);
         return;
@@ -311,116 +481,34 @@ function main() {
       responseOpen = true;
     }
 
-    // Scan for DONE marker — stream everything before it.
     const di = buf.indexOf(DONE);
     if (di !== -1) {
       const pre = buf.slice(0, di);
-      if (pre) preserveInputWrite(process.stdout, pre);
+      if (pre) ui.append(pre);
       buf = buf.slice(di + DONE.length);
       if (buf[0] === "\n") buf = buf.slice(1);
       generating = false;
       responseOpen = false;
-      preserveInputWrite(process.stdout, "\n");
+      ui.append("\n");
       drain();
     } else {
-      // Stream buffered content, keeping enough tail to catch a partial DONE marker.
       const keep = DONE.length - 1;
       if (buf.length > keep) {
         const out = buf.slice(0, -keep);
-        if (out) preserveInputWrite(process.stdout, out);
+        if (out) ui.append(out);
         buf = buf.slice(-keep);
       }
     }
   });
 
-  proc.on("error", (e) => { console.error("[cofos] Python error:", e.message); process.exit(1); });
+  proc.on("error", (e) => { ui.print("[cofos] Python error: " + e.message); shutdown(); });
   proc.on("exit", (c) => {
     closed = true;
-    if (c && c !== 0) console.error("[cofos] Python exited with code", c);
-    try { rl.close(); } catch (_) {}
-    // Let queued stdout writes flush before tearing the process down.
+    if (c && c !== 0) ui.print("[cofos] Python exited with code " + c);
+    ui.destroy();
     process.exitCode = c ?? 0;
   });
 
-  function shutdown() {
-    if (closed) return;
-    closed = true;
-    try { proc.stdin.end(); } catch (_) {}
-    try { rl.close(); } catch (_) {}
-    proc.kill("SIGTERM");
-  }
-
-  // Ctrl+C and Ctrl+D should stop the backend instead of orphaning it.
-  rl.on("SIGINT", () => { console.log("\n[cofos] Interrupted."); shutdown(); });
-  rl.on("close", () => { if (!closed) shutdown(); });
-  process.on("SIGTERM", shutdown);
-
-  function drain() {
-    while (inputQ.length) {
-      const line = inputQ.shift();
-      if (handle(line)) return;
-    }
-    ask();
-  }
-
-  function ask(prompt = mainPrompt) {
-    if (closed) return;
-    try {
-      rl.setPrompt(prompt);
-      rl.prompt();
-    } catch (_) {}
-  }
-
-  /* ─── multi-line accumulator ─── */
-  let mlBuf = [];
-
-  function handle(input) {
-    // Multi-line continuation: lines ending with "\"
-    if (input.endsWith("\\")) {
-      mlBuf.push(input.slice(0, -1));
-      ask(contPrompt);
-      return true;
-    }
-    if (mlBuf.length > 0) {
-      mlBuf.push(input);
-      input = mlBuf.join("\n");
-      mlBuf = [];
-    }
-
-    input = input.trim();
-    if (!input) return false;
-
-    // Local-only commands (don't hit the backend).
-    if (/^\/(exit|quit)$/.test(input)) { proc.stdin.write("/exit\n"); proc.stdin.end(); return true; }
-    if (/^\/(help)$/.test(input)) {
-      console.log(slashHint("/") + "\n  Add local papers to RAG:\n    cofos --pdf-dir ./new_pdfs\n    cofos --pdf-dir ./new_pdfs --rebuild-pdf-index\n\n  Multi-line: end a line with \\ to continue.\n");
-      return false;
-    }
-    if (/^\/(clear)$/.test(input)) {
-      mlBuf = [];
-      responseOpen = false;
-      proc.stdin.write(input + "\n");
-      generating = true;
-      return true;
-    }
-    if (/^\/(info)$/.test(input)) {
-      console.log(`\n  COFOS CLI — v${pkg.version}`);
-      console.log("  Model:  Willlzh/COFOS (Hugging Face)");
-      console.log("  RAG:    Willlzh/COFOS_data/runtime (KG + BM25) + optional --pdf-dir BM25 evidence");
-      console.log(`  Cache:  ${cacheDir}`);
-      console.log(`  HF:     ${backendEnv.HF_HOME}`);
-      console.log("  Engine: StudentAdapterInference + transformers + PyTorch");
-      console.log("  Hist:   ~/.cofos_history.jsonl\n");
-      return false;
-    }
-
-    // Forward to backend. Echo a stable user turn so pasted/submitted questions remain visible in the transcript.
-    console.log(G + "  you   " + R + input);
-    responseOpen = false;
-    generating = true;
-    proc.stdin.write(input + "\n");
-    return true;
-  }
 }
 
 main();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lzhzzzzwill/cofos",
-  "version": "1.1.15",
+  "version": "1.2.0",
   "description": "COFOS 9B + RAG chat CLI with optional local PDF ingestion.",
   "type": "module",
   "bin": {
@@ -38,5 +38,8 @@
   "license": "MIT",
   "engines": {
     "node": ">=18"
+  },
+  "dependencies": {
+    "blessed": "^0.1.81"
   }
 }