dsh-biocore 0.1.2 → 0.1.4

package/python/mindbase/bm25.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import math
+from collections import Counter
+from typing import Dict, List, Sequence
+
+
+def _tokens(text: str) -> List[str]:
+    text = (text or "").lower()
+    chars = [text[i:i + 2] for i in range(max(0, len(text) - 1))]
+    words = [w for w in text.replace("\n", " ").split() if len(w) > 1]
+    return chars + words
+
+
+class BM25:
+    def __init__(self, docs: Sequence[str], k1: float = 1.5, b: float = 0.75):
+        self.docs = list(docs)
+        self.k1 = k1
+        self.b = b
+        self.freqs: List[Counter] = [Counter(_tokens(d)) for d in self.docs]
+        self.lengths = [sum(f.values()) for f in self.freqs]
+        self.avgdl = sum(self.lengths) / max(1, len(self.lengths))
+        df: Dict[str, int] = {}
+        for f in self.freqs:
+            for t in f:
+                df[t] = df.get(t, 0) + 1
+        n = max(1, len(self.docs))
+        self.idf = {t: math.log(1 + (n - v + 0.5) / (v + 0.5)) for t, v in df.items()}
+
+    def score(self, query: str, doc_index: int) -> float:
+        q = Counter(_tokens(query))
+        f = self.freqs[doc_index]
+        dl = self.lengths[doc_index] or 1
+        total = 0.0
+        for term, qf in q.items():
+            tf = f.get(term, 0)
+            if not tf:
+                continue
+            denom = tf + self.k1 * (1 - self.b + self.b * dl / max(1e-6, self.avgdl))
+            total += self.idf.get(term, 0.0) * tf * (self.k1 + 1) / denom * qf
+        return total
+
+    def rank(self, query: str, k: int = 5) -> List[int]:
+        scores = [(self.score(query, i), i) for i in range(len(self.docs))]
+        scores.sort(reverse=True)
+        return [i for _, i in scores[:k]]

package/python/mindbase/bootstrap.py

@@ -0,0 +1,256 @@
+"""Bootstrap local appraisal heads from silver labels, then run without the LLM.
+
+The LLM is optional and offline-only:
+  texts -> DSH/LLM labeler -> StorySample dataset -> local linear AppraisalModel
+Runtime _blend_appraisal() uses the local head; no LLM call.
+
+The default ScriptedLabeler makes the pipeline testable without any API.
+DSHLabeler is cache-friendly: frozen system prompt + appended JSON payload.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import random
+import time
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from .config import MindConfig
+from .engine import MindBase
+from .lexicon import text_affect
+from .teacher import ScriptedTeacher
+from .training.appraisal import AppraisalModel, TARGETS
+from .training.dataset import StorySample
+from .training.features import feature_names
+from .training.persist import load_heads, save_heads
+from .types import Event
+
+DATA = Path(__file__).resolve().parent / "data"
+DEFAULT_HEADS = DATA / "bootstrap_heads.json"
+DEFAULT_DATASET = DATA / "bootstrap_labels.jsonl"
+
+_LABEL_SYSTEM = (
+    "You are an annotation teacher for a digital companion. Return ONLY JSON. "
+    "Rate the user text on five appraisal dimensions."
+)
+
+
+def _clamp(v: float, lo: float, hi: float) -> float:
+    try:
+        x = float(v)
+    except Exception:
+        return lo
+    return lo if x < lo else hi if x > hi else x
+
+
+def _extract_json(text: str) -> Dict[str, object]:
+    raw = str(text or "").strip()
+    fence = chr(96) * 3
+    if raw.startswith(fence):
+        raw = raw.strip(chr(96))
+        if raw.lower().startswith("json"):
+            raw = raw[4:]
+    i = raw.find("{")
+    j = raw.rfind("}")
+    if i >= 0 and j > i:
+        raw = raw[i:j + 1]
+    return json.loads(raw)
+
+
+class ScriptedLabeler:
+    """Deterministic silver labels; no network, no LLM. Used for tests/fallback."""
+
+    SOCIAL_WORDS = ("你", "陪我", "谢谢", "喜欢", "想", "晚安", "在吗", "我们", "一起")
+
+    def label(self, text: str, directed: bool = True, context: Optional[Dict[str, object]] = None) -> Dict[str, float]:
+        v, a = text_affect(str(text or ""))
+        social = 0.65 if directed or any(w in str(text or "") for w in self.SOCIAL_WORDS) else 0.35
+        relevance = 0.70 if directed else 0.45
+        threat = max(0.0, -float(v)) * 0.85
+        return {
+            "valence": round(_clamp(v, -1.0, 1.0), 4),
+            "arousal": round(_clamp(a, 0.0, 1.0), 4),
+            "threat": round(_clamp(threat, 0.0, 1.0), 4),
+            "social": round(_clamp(social, 0.0, 1.0), 4),
+            "relevance": round(_clamp(relevance, 0.0, 1.0), 4),
+        }
+
+
+class DSHLabeler:
+    """LLM/DSH silver labeler with a stable cached prefix and scripted fallback."""
+
+    def __init__(self, client=None, fallback: Optional[ScriptedLabeler] = None):
+        self.client = client
+        self.fallback = fallback or ScriptedLabeler()
+        self.calls = 0
+
+    def _call(self, system: str, user: str) -> str:
+        if self.client is None:
+            raise RuntimeError("no DSH/LLM client configured")
+        from .llm_adapter import build_prompt
+        from .types import LLMRequest
+        req = LLMRequest(mode="semantic", urgency=0.5, max_tokens=300,
+                         reason="bootstrap_labeler", context={"system": system, "user": user})
+        self.calls += 1
+        return str(self.client.generate(req))
+
+    def label(self, text: str, directed: bool = True, context: Optional[Dict[str, object]] = None) -> Dict[str, float]:
+        payload = json.dumps({
+            "text": str(text or ""),
+            "directed": bool(directed),
+            "context": context or {},
+            "schema": {k: "number" for k in TARGETS},
+        }, ensure_ascii=False)
+        try:
+            data = _extract_json(self._call(_LABEL_SYSTEM, payload))
+            out: Dict[str, float] = {}
+            for k in TARGETS:
+                lo, hi = (-1.0, 1.0) if k == "valence" else (0.0, 1.0)
+                out[k] = round(_clamp(data.get(k, 0.0), lo, hi), 4)
+            return out
+        except Exception:
+            return self.fallback.label(text, directed=directed, context=context)
+
+
+def build_bootstrap_dataset(n_scenarios: int = 50, turns_per_scenario: int = 5,
+                            labeler: Optional[object] = None, seed: int = 1,
+                            mind_seed: int = 42) -> List[StorySample]:
+    labeler = labeler or ScriptedLabeler()
+    teacher = ScriptedTeacher(n_scenarios=max(1, int(n_scenarios)), seed=int(seed))
+    mind = MindBase(MindConfig(seed=int(mind_seed)))
+    samples: List[StorySample] = []
+    for i in range(max(1, int(n_scenarios))):
+        scenario = teacher.scenario(i)
+        for turn in scenario.turns[:max(1, int(turns_per_scenario))]:
+            event = Event(text=str(turn.text), speaker="user", directed=bool(turn.directed),
+                          ts=mind.t + 60.0, domain="chat")
+            try:
+                mind.step(event)
+            except Exception:
+                pass
+            state = mind._state_features()
+            labels = labeler.label(str(turn.text), directed=bool(turn.directed),
+                                   context={"persona": scenario.persona, "scenario": scenario.name})
+            samples.append(StorySample(
+                event=str(turn.text),
+                state=dict(state),
+                appraisal=dict(labels),
+                action="speak",
+                outcome={},
+                reward=0.0,
+                group=str(scenario.name),
+                domain="bootstrap",
+            ))
+    return samples
+
+
+def _split_by_group(samples: Sequence[StorySample], test_ratio: float = 0.25, seed: int = 0) -> Tuple[List[StorySample], List[StorySample]]:
+    groups: Dict[str, List[StorySample]] = {}
+    for s in samples:
+        groups.setdefault(s.group or "default", []).append(s)
+    keys = list(groups)
+    random.Random(int(seed)).shuffle(keys)
+    n_test = max(1, int(round(len(keys) * float(test_ratio)))) if keys else 0
+    test_keys = set(keys[:n_test])
+    train: List[StorySample] = []
+    test: List[StorySample] = []
+    for key, rows in groups.items():
+        (test if key in test_keys else train).extend(rows)
+    return train, test
+
+
+def _mae(model: AppraisalModel, samples: Sequence[StorySample]) -> Dict[str, float]:
+    if not samples:
+        return {}
+    err = {k: 0.0 for k in TARGETS}
+    for s in samples:
+        pred = model.predict(s.event, s.state)
+        for k in TARGETS:
+            err[k] += abs(float(pred.get(k, 0.0)) - float(s.appraisal.get(k, 0.0)))
+    return {f"mae_{k}": round(v / len(samples), 4) for k, v in err.items()}
+
+
+def run_bootstrap(n_scenarios: int = 50, turns_per_scenario: int = 5,
+                  epochs: int = 120, seed: int = 1,
+                  labeler: Optional[object] = None,
+                  out_heads: Optional[Path | str] = None,
+                  out_dataset: Optional[Path | str] = None,
+                  verbose: bool = True) -> Dict[str, object]:
+    t0 = time.perf_counter()
+    labeler = labeler or ScriptedLabeler()
+    samples = build_bootstrap_dataset(n_scenarios=n_scenarios, turns_per_scenario=turns_per_scenario,
+                                      labeler=labeler, seed=seed)
+    train, test = _split_by_group(samples, test_ratio=0.25, seed=seed)
+    model = AppraisalModel(len(feature_names())).fit(train, epochs=int(epochs))
+    metrics = {
+        "n_total": len(samples),
+        "n_train": len(train),
+        "n_test": len(test),
+        "epochs": int(epochs),
+        "labeler": type(labeler).__name__,
+        "train": _mae(model, train),
+        "holdout": _mae(model, test),
+        "elapsed_sec": round(time.perf_counter() - t0, 3),
+    }
+    hp = Path(out_heads) if out_heads else DEFAULT_HEADS
+    dp = Path(out_dataset) if out_dataset else DEFAULT_DATASET
+    hp.parent.mkdir(parents=True, exist_ok=True)
+    save_heads({"appraisal": model}, hp)
+    if out_dataset is not False:
+        dp.parent.mkdir(parents=True, exist_ok=True)
+        with dp.open("w", encoding="utf-8") as f:
+            for s in samples:
+                f.write(json.dumps({
+                    "event": s.event, "state": s.state, "appraisal": s.appraisal,
+                    "action": s.action, "group": s.group, "domain": s.domain,
+                }, ensure_ascii=False) + "\n")
+    metrics["heads_path"] = str(hp)
+    metrics["dataset_path"] = str(dp)
+    if verbose:
+        print(json.dumps(metrics, ensure_ascii=False, indent=2))
+    return metrics
+
+
+def _build_dsh_client():
+    try:
+        from .llm_openai import OpenAICompatClient
+    except Exception:
+        return None
+    key = os.getenv("DEEPSEEK_API_KEY") or os.getenv("LLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+    if not key:
+        return None
+    try:
+        return OpenAICompatClient(model=os.getenv("LLM_MODEL") or "deepseek-chat",
+                                  base_url=os.getenv("LLM_BASE_URL") or "https://api.deepseek.com",
+                                  api_key=key, temperature=0.2)
+    except Exception:
+        return None
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Bootstrap appraisal head from silver labels (LLM optional)")
+    parser.add_argument("--scenarios", type=int, default=50)
+    parser.add_argument("--turns", type=int, default=5)
+    parser.add_argument("--epochs", type=int, default=120)
+    parser.add_argument("--seed", type=int, default=1)
+    parser.add_argument("--teacher", choices=["scripted", "dsh"], default="scripted")
+    parser.add_argument("--out-heads", default="")
+    parser.add_argument("--out-dataset", default="")
+    args = parser.parse_args()
+    labeler = ScriptedLabeler()
+    if args.teacher == "dsh":
+        client = _build_dsh_client()
+        labeler = DSHLabeler(client=client)
+        if client is None:
+            print("[bootstrap] DSH client unavailable; using scripted fallback.")
+    run_bootstrap(n_scenarios=args.scenarios, turns_per_scenario=args.turns,
+                  epochs=args.epochs, seed=args.seed, labeler=labeler,
+                  out_heads=Path(args.out_heads) if args.out_heads else None,
+                  out_dataset=Path(args.out_dataset) if args.out_dataset else None)
+
+
+if __name__ == "__main__":
+    main()

package/python/mindbase/bootstrap_real.py

@@ -0,0 +1,231 @@
+"""Train local appraisal + timing heads from REAL datasets.
+
+Data (already local):
+  EmoBank 10k  -> valence/arousal
+  CPED 11k     -> Chinese emotion/DA/Big Five
+  ESConv 1.3k  -> support strategies
+  When2Speak 21.8k -> SPEAK vs SILENT timing
+
+No LLM is called. Output: mindbase/data/bootstrap_heads_real.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import re
+import time
+from collections import Counter
+from pathlib import Path
+from typing import Dict, List, Sequence, Tuple
+
+from .training.appraisal import TARGETS, AppraisalModel, CompositeAppraisalModel
+from .training.dataset import StorySample
+from .training.features import feature_names
+from .training.loaders import load_cped, load_emobank, load_esconv
+from .training.persist import save_heads
+from .training.timing import (TimingModel, load_human_thread_pairs, load_spontaneous_pairs,
+                              load_when2speak_pairs)
+
+DATA = Path(__file__).resolve().parent / "data"
+REAL = DATA / "real"
+DEFAULT_OUT = DATA / "bootstrap_heads_real.json"
+DEFAULT_SUMMARY = DATA / "bootstrap_real_summary.json"
+
+
+def _limit(n: int):
+    return None if int(n) <= 0 else int(n)
+
+
+def load_real_data(max_per_source: int = 4000) -> Dict[str, List[StorySample]]:
+    lim = _limit(max_per_source)
+    return {
+        "emobank": load_emobank(REAL / "EmoBank" / "emobank.csv", limit=lim),
+        "cped": load_cped(REAL / "CPED" / "valid_split.csv", limit=lim),
+        "esconv": load_esconv(REAL / "ESConv" / "ESConv.json", limit=lim),
+    }
+
+
+def _split_by_group(samples: Sequence[StorySample], test_ratio: float = 0.2, seed: int = 0) -> Tuple[List[StorySample], List[StorySample]]:
+    groups: Dict[str, List[StorySample]] = {}
+    for s in samples:
+        key = s.domain + ":" + (s.group or "default")
+        groups.setdefault(key, []).append(s)
+    keys = list(groups)
+    random.Random(int(seed)).shuffle(keys)
+    n_test = max(1, int(round(len(keys) * float(test_ratio)))) if keys else 0
+    test_keys = set(keys[:n_test])
+    train: List[StorySample] = []
+    test: List[StorySample] = []
+    for key, rows in groups.items():
+        (test if key in test_keys else train).extend(rows)
+    return train, test
+
+
+def _split_appraisal(samples: Sequence[StorySample], seed: int = 0):
+    """Honor EmoBank train/test; group-split CPED/ESConv conversations."""
+    train: List[StorySample] = []
+    test: List[StorySample] = []
+    by_group: Dict[str, List[StorySample]] = {}
+    for s in samples:
+        if s.domain == "appraisal":
+            (test if str(s.group) == "test" else train).append(s)
+        else:
+            by_group.setdefault(s.domain + ":" + (s.group or "default"), []).append(s)
+    keys = list(by_group)
+    random.Random(int(seed)).shuffle(keys)
+    n_test = max(1, int(round(len(keys) * 0.2))) if keys else 0
+    test_keys = set(keys[:n_test])
+    for key, rows in by_group.items():
+        (test if key in test_keys else train).extend(rows)
+    return train, test
+
+def _appraisal_mae(model: AppraisalModel, samples: Sequence[StorySample]) -> Dict[str, float]:
+    if not samples:
+        return {}
+    targets = tuple(getattr(model, "targets", TARGETS))
+    err = {k: 0.0 for k in targets}
+    for s in samples:
+        pred = model.predict(s.event, s.state)
+        for k in targets:
+            err[k] += abs(float(pred.get(k, 0.0)) - float(s.appraisal.get(k, 0.0)))
+    return {"mae_" + k: round(v / len(samples), 4) for k, v in err.items()}
+
+
+def _n_speakers(context: str) -> int:
+    return len(set(re.findall(r"Speaker_(\d+):", str(context or ""))))
+
+def _split_pairs(pairs: Sequence[Tuple[str, int]], test_ratio: float = 0.2, seed: int = 0):
+    idx = list(range(len(pairs)))
+    random.Random(int(seed)).shuffle(idx)
+    n_test = max(1, int(round(len(idx) * float(test_ratio)))) if idx else 0
+    test = [pairs[i] for i in idx[:n_test]]
+    train = [pairs[i] for i in idx[n_test:]]
+    return train, test
+
+
+def _balance_pairs(pairs: Sequence[Tuple[str, int]], seed: int = 0, cap: int = 3000):
+    """Oversample the rare SPEAK class and keep all SILENT examples."""
+    speak = [p for p in pairs if int(p[1]) == 1]
+    silent = [p for p in pairs if int(p[1]) == 0]
+    if not speak or not silent:
+        return list(pairs)
+    mult = max(1, min(4, len(silent) // max(1, len(speak))))
+    out = list(silent) + speak * mult
+    random.Random(int(seed)).shuffle(out)
+    return out
+
+
+def _timing_metrics(model: TimingModel, pairs) -> Dict[str, object]:
+    if not pairs:
+        return {"n": 0}
+    y = []
+    pred = []
+    for item in pairs:
+        ctx, label = item[0], int(item[1])
+        state = item[2] if len(item) > 2 else None
+        y.append(label)
+        pred.append(1 if model.predict(ctx, state) == "speak" else 0)
+    acc = sum(1 for a, b in zip(y, pred) if a == b) / len(y)
+    rec, prec, f1 = [], [], []
+    for lab in (0, 1):
+        tp = sum(1 for a, b in zip(y, pred) if a == lab and b == lab)
+        fn = sum(1 for a, b in zip(y, pred) if a == lab and b != lab)
+        fp = sum(1 for a, b in zip(y, pred) if a != lab and b == lab)
+        r = tp / (tp + fn) if (tp + fn) else 0.0
+        p = tp / (tp + fp) if (tp + fp) else 0.0
+        rec.append(r)
+        prec.append(p)
+        f1.append(2 * p * r / (p + r) if (p + r) else 0.0)
+    return {
+        "n": len(y),
+        "accuracy": round(acc, 4),
+        "balanced_accuracy": round(sum(rec) / 2.0, 4),
+        "macro_f1": round(sum(f1) / 2.0, 4),
+        "speak_recall": round(rec[1], 4),
+        "speak_precision": round(prec[1], 4),
+        "true": dict(Counter("speak" if x == 1 else "silence" for x in y)),
+        "pred": dict(Counter("speak" if x == 1 else "silence" for x in pred)),
+    }
+
+
+def train_real_heads(max_per_source: int = 4000, epochs: int = 60,
+                     seed: int = 1, out: Path | str = DEFAULT_OUT) -> Dict[str, object]:
+    t0 = time.perf_counter()
+    data = load_real_data(max_per_source=max_per_source)
+    va_rows = list(data["emobank"])
+    # CPED is the chat-domain source.  ESConv is kept as a support/action
+    # source, not mixed into the chat appraisal head (labels/domain differ).
+    chat_rows = list(data["cped"])
+    es_rows = list(data["esconv"])
+    appraisal_rows = va_rows + chat_rows + es_rows
+    va_train, va_test = _split_appraisal(va_rows, seed=seed)
+    chat_train, chat_test = _split_appraisal(chat_rows, seed=seed)
+    _es_train, es_test = _split_appraisal(es_rows, seed=seed)
+    ap_test = va_test + chat_test
+
+    when2_all = load_when2speak_pairs(REAL / "When2Speak" / "finetune_val_token.jsonl")
+    when2_1v1 = [p for p in when2_all if _n_speakers(p[0]) <= 2]
+    human_all = load_human_thread_pairs(DATA / "human_thread.jsonl")
+    spont_all = load_spontaneous_pairs(DATA / "spontaneous_48.jsonl")
+    w2_train, w2_test = _split_pairs(when2_1v1, test_ratio=0.2, seed=seed)
+    h_train, h_test = _split_pairs(human_all, test_ratio=0.25, seed=seed)
+    s_train, s_test = _split_pairs(spont_all, test_ratio=0.25, seed=seed)
+    timing_train = w2_train + h_train * 5 + s_train * 5
+    timing_bal = _balance_pairs(timing_train, seed=seed, cap=3000)
+
+    n_features = len(feature_names())
+    va_head = AppraisalModel(n_features).fit(va_train, epochs=int(epochs))
+    vad_head = AppraisalModel(n_features, targets=("valence", "arousal", "dominance")).fit(va_train, epochs=int(epochs))
+    chat_head = AppraisalModel(n_features).fit(chat_train, epochs=int(epochs))
+    # The base runtime is chat; use the chat-domain head for all five targets.
+    # EmoBank head is kept for calibration/reporting, not runtime.
+    appraisal = chat_head
+    timing = TimingModel().fit(timing_bal, epochs=100)
+    timing.calibrate(h_train + s_train, metric="f1")
+
+    metrics: Dict[str, object] = {
+        "sources": {k: len(v) for k, v in data.items()},
+        "when2speak_pairs": len(when2_all),
+        "when2speak_1v1_pairs": len(when2_1v1),
+        "human_thread_pairs": len(human_all),
+        "spontaneous_pairs": len(spont_all),
+        "appraisal_rows": len(appraisal_rows),
+        "appraisal_train": len(va_train) + len(chat_train),
+        "appraisal_test": len(ap_test),
+        "appraisal_cped": _appraisal_mae(chat_head, chat_test),
+        "appraisal_esconv_generalization": _appraisal_mae(chat_head, es_test),
+        "appraisal_emobank": _appraisal_mae(va_head, va_test),
+        "appraisal_vad_emobank": _appraisal_mae(vad_head, va_test),
+        "timing_train": len(timing_bal),
+        "timing_threshold": round(float(timing.threshold), 3),
+        "timing_when2_1v1": _timing_metrics(timing, w2_test),
+        "timing_human": _timing_metrics(timing, h_test),
+        "timing_spontaneous": _timing_metrics(timing, s_test),
+        "timing_train_mix": dict(Counter("speak" if int(v) == 1 else "silence" for v in [p[1] for p in timing_bal])),
+        "elapsed_sec": round(time.perf_counter() - t0, 3),
+    }
+    path = Path(out)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    save_heads({"appraisal": appraisal, "vad": vad_head}, path)
+    metrics["heads_path"] = str(path)
+    summary = Path(DEFAULT_SUMMARY)
+    summary.write_text(json.dumps(metrics, ensure_ascii=False, indent=2), encoding="utf-8")
+    metrics["summary_path"] = str(summary)
+    return metrics
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Train real appraisal + timing heads (no LLM)")
+    parser.add_argument("--max-per-source", type=int, default=4000)
+    parser.add_argument("--epochs", type=int, default=60)
+    parser.add_argument("--seed", type=int, default=1)
+    parser.add_argument("--out", default=str(DEFAULT_OUT))
+    args = parser.parse_args()
+    metrics = train_real_heads(max_per_source=args.max_per_source, epochs=args.epochs,
+                               seed=args.seed, out=args.out)
+    print(json.dumps(metrics, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()