@lzhzzzzwill/cofos 1.1.9 → 1.1.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lzhzzzzwill/cofos",
-  "version": "1.1.9",
+  "version": "1.1.10",
   "description": "COFOS 9B + RAG chat CLI with optional local PDF ingestion.",
   "type": "module",
   "bin": {

package/runtime/src/inference/student_adapter_inference.py

@@ -2,6 +2,7 @@
 """Local student-model inference with optional confidence-gated KG/BM25 RAG."""
 
 import logging
+import os
 import re
 from pathlib import Path
 from threading import Thread
@@ -225,17 +226,31 @@ class StudentAdapterInference:
 
     @staticmethod
     def _select_dtype() -> torch.dtype:
-        """Pick the fastest dtype the current device actually supports.
+        """Pick a stable dtype for the current runtime.
 
-        bfloat16 was hardcoded, which silently falls back to a very slow
-        emulated path on CPU and on GPUs without bf16 support.
+        CUDA can safely use bf16/fp16.  On Mac/CPU, ``device_map=auto`` may place
+        layers on CPU or disk; fp16 there can produce NaN probabilities or
+        garbage tokens.  Prefer float32 outside CUDA unless the user explicitly
+        opts into another dtype with COFOS_DTYPE.
         """
+        requested = os.environ.get("COFOS_DTYPE", "auto").strip().lower()
+        aliases = {
+            "fp32": torch.float32,
+            "float32": torch.float32,
+            "fp16": torch.float16,
+            "float16": torch.float16,
+            "bf16": torch.bfloat16,
+            "bfloat16": torch.bfloat16,
+        }
+        if requested in aliases:
+            return aliases[requested]
+        if requested not in {"", "auto"}:
+            logging.warning("Ignoring unsupported COFOS_DTYPE=%s; using auto", requested)
+
         if torch.cuda.is_available():
             if torch.cuda.is_bf16_supported():
                 return torch.bfloat16
             return torch.float16
-        if torch.backends.mps.is_available():
-            return torch.float16
         return torch.float32
 
     def load_model(self) -> None: