@lzhzzzzwill/cofos 1.1.8 → 1.1.10

package/bin/cofos.js

@@ -175,8 +175,8 @@ function main() {
     HF_HUB_ENABLE_HF_TRANSFER: process.env.HF_HUB_ENABLE_HF_TRANSFER || "0",
     HF_HUB_DOWNLOAD_TIMEOUT: process.env.HF_HUB_DOWNLOAD_TIMEOUT || "120",
     HF_HUB_ETAG_TIMEOUT: process.env.HF_HUB_ETAG_TIMEOUT || "30",
-    HF_HUB_VERBOSITY: process.env.HF_HUB_VERBOSITY || "warning",
-    TRANSFORMERS_VERBOSITY: process.env.TRANSFORMERS_VERBOSITY || "warning",
+    HF_HUB_VERBOSITY: process.env.HF_HUB_VERBOSITY || "error",
+    TRANSFORMERS_VERBOSITY: process.env.TRANSFORMERS_VERBOSITY || "error",
   };
   const proc = spawn(py, backendArgs, {
     stdio: ["pipe", "pipe", "pipe"],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lzhzzzzwill/cofos",
-  "version": "1.1.8",
+  "version": "1.1.10",
   "description": "COFOS 9B + RAG chat CLI with optional local PDF ingestion.",
   "type": "module",
   "bin": {

package/runtime/config/config.yaml

@@ -42,7 +42,7 @@ generation:
   temperature: 0.2
   top_p: 0.9
   max_new_tokens: 1024
-  do_sample: true
+  do_sample: false
   repetition_penalty: 1.08
   num_beams: 1
   no_repeat_ngram_size: 0
@@ -51,6 +51,7 @@ generation:
   refine_answers: false
   stream_refinement: false
   refinement_max_new_tokens: 512
+  remove_invalid_values: true
 data_processing:
   chunk_size: 768
   chunk_overlap: 50

package/runtime/src/inference/student_adapter_inference.py

@@ -2,6 +2,7 @@
 """Local student-model inference with optional confidence-gated KG/BM25 RAG."""
 
 import logging
+import os
 import re
 from pathlib import Path
 from threading import Thread
@@ -181,6 +182,7 @@ class StudentAdapterInference:
         self.num_beams = int(generation_config.get("num_beams", 1))
         self.no_repeat_ngram_size = int(generation_config.get("no_repeat_ngram_size", 0))
         self.min_new_tokens = int(generation_config.get("min_new_tokens", 0))
+        self.remove_invalid_values = bool(generation_config.get("remove_invalid_values", True))
         self.history_max_turns = int(generation_config.get("history_max_turns", 4))
         self.refine_answers = bool(generation_config.get("refine_answers", True))
         self.stream_refinement = bool(generation_config.get("stream_refinement", True))
@@ -224,17 +226,31 @@ class StudentAdapterInference:
 
     @staticmethod
     def _select_dtype() -> torch.dtype:
-        """Pick the fastest dtype the current device actually supports.
+        """Pick a stable dtype for the current runtime.
 
-        bfloat16 was hardcoded, which silently falls back to a very slow
-        emulated path on CPU and on GPUs without bf16 support.
+        CUDA can safely use bf16/fp16.  On Mac/CPU, ``device_map=auto`` may place
+        layers on CPU or disk; fp16 there can produce NaN probabilities or
+        garbage tokens.  Prefer float32 outside CUDA unless the user explicitly
+        opts into another dtype with COFOS_DTYPE.
         """
+        requested = os.environ.get("COFOS_DTYPE", "auto").strip().lower()
+        aliases = {
+            "fp32": torch.float32,
+            "float32": torch.float32,
+            "fp16": torch.float16,
+            "float16": torch.float16,
+            "bf16": torch.bfloat16,
+            "bfloat16": torch.bfloat16,
+        }
+        if requested in aliases:
+            return aliases[requested]
+        if requested not in {"", "auto"}:
+            logging.warning("Ignoring unsupported COFOS_DTYPE=%s; using auto", requested)
+
         if torch.cuda.is_available():
             if torch.cuda.is_bf16_supported():
                 return torch.bfloat16
             return torch.float16
-        if torch.backends.mps.is_available():
-            return torch.float16
         return torch.float32
 
     def load_model(self) -> None:
@@ -283,6 +299,7 @@ class StudentAdapterInference:
             "repetition_penalty": self.repetition_penalty,
             "eos_token_id": self.tokenizer.eos_token_id,
             "pad_token_id": self.tokenizer.pad_token_id,
+            "remove_invalid_values": self.remove_invalid_values,
         }
         if self.do_sample:
             kwargs["temperature"] = self.temperature

package/scripts/chat.py

@@ -24,8 +24,8 @@ os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
 os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "0")
 os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "120")
 os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "30")
-os.environ.setdefault("HF_HUB_VERBOSITY", "warning")
-os.environ.setdefault("TRANSFORMERS_VERBOSITY", "warning")
+os.environ.setdefault("HF_HUB_VERBOSITY", "error")
+os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
 
 HISTORY_FILE = os.path.join(os.path.expanduser("~"), ".cofos_history.jsonl")
 HISTORY_MAX_TURNS = 20
@@ -474,7 +474,7 @@ def main() -> None:
     from inference.student_adapter_inference import StudentAdapterInference, StudentRAGContext
 
     debug = os.environ.get("COFOS_DEBUG", "").lower() in {"1", "true", "yes", "on"}
-    log_level = logging.INFO if debug else logging.WARNING
+    log_level = logging.INFO if debug else logging.ERROR
     logging.basicConfig(
         level=log_level,
         format="%(asctime)s - %(levelname)s - %(message)s",
@@ -586,7 +586,8 @@ def main() -> None:
             answer = "".join(answer_parts)
             gen_elapsed = time.time() - t_gen
             print("\n===DONE===", flush=True)
-            print(f"[chat] Generated in {gen_elapsed:.1f}s", file=sys.stderr, flush=True)
+            if os.environ.get("COFOS_DEBUG", "").lower() in {"1", "true", "yes", "on"}:
+                print(f"[chat] Generated in {gen_elapsed:.1f}s", file=sys.stderr, flush=True)
 
             # Persist both turns only once the answer completed, so an
             # interrupted generation cannot leave a dangling user turn behind.
@@ -595,7 +596,8 @@ def main() -> None:
             _save_turn("user", question)
             _save_turn("assistant", answer)
         except Exception as exc:
-            logging.exception("Generation failed")
+            if os.environ.get("COFOS_DEBUG", "").lower() in {"1", "true", "yes", "on"}:
+                logging.exception("Generation failed")
             print(f"\n[COFOS error] {exc}")
             print("===DONE===", flush=True)