dsh-biocore 0.3.3 → 0.5.0

package/lib/index.js

@@ -28,6 +28,7 @@ import { SERIES_ROUTE_PATH, handleSeriesRoute } from './modules/series.js'
 import { readCacheStats } from './modules/cache-stats.js'
 import { startCoreWorker, stopCoreWorker } from './core-worker.js'
 import { runAutonomyTick } from './modules/autonomy.js'
+import { drainOutbox } from './modules/standby.js'
 import { queueSessionTurn } from './modules/turn-queue.js'
 import { makeCoreSessionListener, registerCoreTurn } from './modules/core-turn.js'
 import { migrateLegacyPluginData, PLUGIN_DATA_DIR, resolveDataDir } from './modules/data-dir.js'
@@ -193,6 +194,8 @@ export function readFullConfig(dataDir) {
     rerank: normalizeRerank(source.rerank),
     expression: normalizeExpression(source.expression),
     autonomy: normalizeAutonomy(source.autonomy),
+    standby: source.standby && typeof source.standby === 'object' ? source.standby : {},
+    standbyEnabled: source.standbyEnabled === true,
     sanitize_reply: source.sanitize_reply !== false,
   }
 }
@@ -389,6 +392,10 @@ function coreConfigPayload(dataDir) {
     rerank: config.rerank,
     expression: config.expression,
     autonomy: config.autonomy,
+    standby: {
+      ...(config.standby && typeof config.standby === 'object' ? config.standby : {}),
+      standby_enabled: config.standbyEnabled === true,
+    },
     sanitize_reply: config.sanitize_reply !== false,
     switches,
     recall_mode: 'query',
@@ -762,8 +769,10 @@ export async function handleStats(req, res, options) {
   } catch { reply = null }
   const body = Object.assign({ ok: false, reason: 'core_unavailable' },
     reply && typeof reply === 'object' ? reply : {})
+  // contract migration 1: the core computes cache stats; the legacy file is a
+  // read-only fallback for one week, never a source of new numbers.
   try {
-    if (typeof opts.dataDir === 'string' && opts.dataDir) {
+    if (!body.cache && typeof opts.dataDir === 'string' && opts.dataDir) {
       body.cache = readCacheStats(opts.dataDir)
     }
   } catch { /* display only */ }
@@ -992,6 +1001,26 @@ export function apply(ctx, config) {
     keep(() => { if (timer !== null) { clearInterval(timer); timer = null } })
   }
 
+  // task-215: standby is off by default; when on, the core writes the outbox
+  // and this adapter only drains it into results.
+  if (config && config.standbyEnabled === true) {
+    let standbyTimer = null
+    const standbyTick = async () => {
+      try {
+        await runtimeAction('standby_tick', {
+          store: resolveMemoryDir(dataDir),
+          data_dir: dataDir,
+          session_id: 'standby',
+          now: Date.now() / 1000,
+        })
+        drainOutbox(dataDir)
+      } catch { /* standby never disturbs a turn */ }
+    }
+    standbyTimer = setInterval(() => { void standbyTick() }, 5 * 60 * 1000)
+    if (typeof standbyTimer.unref === 'function') standbyTimer.unref()
+    keep(() => { if (standbyTimer !== null) { clearInterval(standbyTimer); standbyTimer = null } })
+  }
+
   return () => {
     for (const dispose of disposers) {
       try { if (typeof dispose === 'function') dispose() } catch { /* ignore */ }

package/lib/modules/standby.js

@@ -0,0 +1,61 @@
+/** task-215: adapter side of standby -- read outbox, record results only.
+ *
+ * The core writes <dataDir>/proactive/outbox.jsonl. This module never decides
+ * whether to speak and never sends externally; it records one execution result
+ * per outbox row into <dataDir>/proactive/results.jsonl.
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+export function proactiveDir(dataDir) {
+  return join(String(dataDir || '').trim(), 'proactive')
+}
+
+export function outboxPath(dataDir) {
+  return join(proactiveDir(dataDir), 'outbox.jsonl')
+}
+
+export function resultsPath(dataDir) {
+  return join(proactiveDir(dataDir), 'results.jsonl')
+}
+
+function readLines(path) {
+  try {
+    if (!existsSync(path)) return []
+    return readFileSync(path, 'utf8').split('\n').filter(Boolean)
+  } catch {
+    return []
+  }
+}
+
+/** Record never-seen outbox rows; returns how many were recorded. */
+export function drainOutbox(dataDir, options) {
+  const opts = options || {}
+  const rows = readLines(outboxPath(dataDir))
+  const done = readLines(resultsPath(dataDir)).length
+  if (rows.length <= done) return { ok: true, recorded: 0, pending: 0 }
+  const path = resultsPath(dataDir)
+  try {
+    mkdirSync(dirname(path), { recursive: true })
+    for (let index = done; index < rows.length; index += 1) {
+      let row = {}
+      try { row = JSON.parse(rows[index]) } catch { row = {} }
+      appendFileSync(path, JSON.stringify({
+        ts: row.ts || 0,
+        index,
+        delivered: false,
+        reason: 'outbox_only',
+        result: { ok: true, external: false },
+      }) + '\n', 'utf8')
+    }
+    return { ok: true, recorded: rows.length - done, pending: 0 }
+  } catch {
+    return { ok: false, recorded: 0, pending: rows.length - done }
+  }
+}
+
+export function standbySummary(dataDir) {
+  const outbox = readLines(outboxPath(dataDir)).length
+  const results = readLines(resultsPath(dataDir)).length
+  return { pending: Math.max(0, outbox - results), outbox, results }
+}

package/lib/modules/state.js

@@ -8,7 +8,13 @@ import { readFileSync, statSync } from 'node:fs'
 import { join } from 'node:path'
 import { resolveDataDir } from './data-dir.js'
 
-export const STATE_LIMIT = 16 * 1024
+/** /state is forwarded whole. The core state now carries field, batches,
+ *  dimensions, perception and friends, so ~50KB is a normal size; the old 16KB
+ *  budget only made every session look broken. Keep a hard ceiling so a runaway
+ *  payload still degrades instead of flooding the host. */
+export const STATE_LIMIT = 512 * 1024
+/** Old budget: log once past it so slow growth stays visible. */
+export const STATE_WARN = 16 * 1024
 export const SESSION_ID_RE = /^[A-Za-z0-9._-]{1,128}$/
 
 const TONE_LABELS = {
@@ -170,11 +176,24 @@ export function readStateFile(file) {
   return { state: raw, mtimeMs }
 }
 
-/** True when the serialized payload fits the 16KB /state budget. */
+let lastBytesLogAt = 0
+
+/** True when the serialized payload fits the 512KB /state budget. */
 export function payloadWithinLimit(body) {
+  let bytes = 0
   try {
-    return Buffer.byteLength(JSON.stringify(body), 'utf8') <= STATE_LIMIT
+    bytes = Buffer.byteLength(JSON.stringify(body), 'utf8')
   } catch {
     return false
   }
+  if (bytes > STATE_WARN) {
+    const now = Date.now()
+    if (now - lastBytesLogAt >= 60 * 1000) {
+      lastBytesLogAt = now
+      try {
+        console.error('__bioStateBytes ' + bytes + ' limit=' + STATE_LIMIT)
+      } catch { /* logging must never fail the request */ }
+    }
+  }
+  return bytes <= STATE_LIMIT
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-biocore",
-  "version": "0.3.3",
+  "version": "0.5.0",
   "type": "module",
   "description": "BioCore DSH UI: per-session state tab, standalone instrument panel, /api/biocore/state, session/event one_turn updates, /bio sync.",
   "main": "lib/index.js",

package/python/biocore_memory/core/config.py

@@ -8,7 +8,9 @@ TOMBSTONE_LIMIT = 100
 DEFAULT_BUDGET = 3
 MAX_BUDGET = 5
 
-# recall fusion / tiers (M2)
+# recall fusion / tiers (M2). [engineering] no fitting source; .50/.30/.10/.10
+# is the single口径 now (the old "0.6/0.4" comment and the task's .4/.3/.2/.1
+# are retired, code unchanged).
 CUE_WEIGHT = 0.50
 SEM_WEIGHT = 0.30
 IMPORTANCE_WEIGHT = 0.10
@@ -21,6 +23,7 @@ SOURCE_MIN_CONFIDENCE = 0.55
 SOURCE_CONFIDENCE_DEFAULT = 0.70
 
 # memory value formula (deterministic; trust is NOT part of value)
+# [engineering] product definition, no behavioural data behind the weights.
 VALUE_BASE = 0.20
 VALUE_W_INTENSITY = 0.30
 VALUE_W_VALENCE = 0.20
@@ -28,7 +31,7 @@ VALUE_W_SELF = 0.15
 VALUE_W_MAINLINE = 0.15
 VALUE_W_ROUTINE = 0.20
 
-# eviction weights (read-only score)
+# eviction weights (read-only score) [engineering]
 EVICT_W_SALIENCE = 0.30
 EVICT_W_POSITION = 0.20
 EVICT_W_REDUNDANCY = 0.20
@@ -46,7 +49,7 @@ THREAD_LIMIT = 3
 
 # memory-point gate (OR): value >= MIN_VALUE or importance >= MIN_IMPORTANCE or self >= MIN_SELF
 # (event centrality / emotional intensity; self-reference effect; engineering thresholds)
-MEMORY_POINT_MIN_VALUE = 0.60
+MEMORY_POINT_MIN_VALUE = 0.60   # [engineering]
 MEMORY_POINT_MIN_SELF = 0.45
 MEMORY_POINT_MIN_IMPORTANCE = 0.75
 
@@ -60,6 +63,7 @@ CONSOLIDATE_AUTO_TURNS = 10
 
 # recall -> affect feedback (task-218): a recalled memory nudges the live
 # affect a little. Engine owns the state; recall/feedback.py owns the guards.
+# [engineering] gain / step / cooldown / floor are hand-set (see PAPER-MAP.md).
 FEEDBACK_MAX_STEP = 0.02    # per-turn cap on |delta valence| and |delta arousal|
 FEEDBACK_GAIN = 0.50        # share of (recalled - current) that flows back
 FEEDBACK_COOLDOWN = 1800.0  # same entry: no second feedback inside 30 minutes

package/python/biocore_memory/engine.py

@@ -98,6 +98,11 @@ class MemoryEngine:
                  eviction_readonly=None, identity_writeback=None,
                  config=None, slots_enabled=None, due_hints_enabled=None):
         self.affect = affect
+        # task-228: the shared field (single source of truth). Unbound by
+        # default -- every existing caller keeps reading/writing self.affect.
+        self._field_view = None
+        self._field_arbiter = False
+        self._field_proposals = []
         self.config = config if isinstance(config, dict) else {}
         self.flags = dict(FLAGS)
         overrides = (self.config.get("flags")
@@ -258,6 +263,65 @@ class MemoryEngine:
         elif not self.slots_enabled:
             self.slot_registry = None
 
+    # -- shared field (task-228) -----------------------------------------
+    def bind_field(self, view=None, arbitrate=True):
+        """Bind the shared field (a read-only copy) and its write-back arbiter.
+
+        With a field bound the learners stop reading their own affect copy:
+        mood_recall, affect_encoding and memory_feedback all read the field, and
+        memory_feedback stops writing -- it proposes a delta instead, which
+        drain_field_proposals() hands to biocore_runtime.arbiter at turn_end.
+        bind_field(None) restores the old behaviour exactly.
+        """
+        if isinstance(view, dict):
+            self._field_view = {key: value for key, value in view.items()
+                                if not isinstance(value, (dict, list))}
+        else:
+            self._field_view = None
+        self._field_arbiter = bool(arbitrate and self._field_view is not None)
+        self._field_proposals = []
+        return self.field_view()
+
+    def field_view(self):
+        """Copy of the bound field, or None when no field is bound."""
+        return dict(self._field_view) if isinstance(self._field_view, dict) else None
+
+    def _affect_source(self):
+        """What the learners read: the field when bound, else self.affect."""
+        view = getattr(self, "_field_view", None)
+        if isinstance(view, dict):
+            valence = view.get("v", view.get("valence"))
+            arousal = view.get("a", view.get("arousal"))
+            if valence is not None:
+                return {"v": valence,
+                        "a": 0.5 if arousal is None else arousal}
+        return getattr(self, "affect", None)
+
+    def propose_field(self, learner, values, weight=None):
+        """Record one learner delta; returns True when anything was proposed."""
+        data = values if isinstance(values, dict) else {}
+        added = False
+        for name, delta in data.items():
+            try:
+                number = float(delta)
+            except Exception:
+                continue
+            if not (number == number) or abs(number) < 1e-6:
+                continue
+            item = {"learner": str(learner or "unknown")[:32],
+                    "field": str(name), "delta": round(number, 6)}
+            if weight is not None:
+                item["weight"] = weight
+            self._field_proposals.append(item)
+            added = True
+        return added
+
+    def drain_field_proposals(self):
+        """Pop the proposals recorded since the last drain (turn_end input)."""
+        items = list(getattr(self, "_field_proposals", []) or [])
+        self._field_proposals = []
+        return items
+
     def _auto_turn_limit(self):
         """Resolve the auto-consolidate interval (default 10, flags/config)."""
         for raw in (self.flags.get("consolidate_auto_turns"),
@@ -1023,7 +1087,7 @@ class MemoryEngine:
         _base_importance = explicit_importance if explicit_importance is not None else value_of(ev)
         _encode_on = self._flag("affect_encoding") or str(os.environ.get("BIOCORE_AFFECT_ENCODING") or "").strip().lower() in ("1", "true", "on", "yes")
         if _encode_on and explicit_importance is None:
-            _affect_source = getattr(self, "affect", None)
+            _affect_source = self._affect_source()
             if _affect_source is not None:
                 _ev, _ea = _affect_pair(_affect_source)
                 try:
@@ -1059,7 +1123,8 @@ class MemoryEngine:
             "scope": entry_scope,
         }
         entry.update({k: v for k, v in meta.items() if k not in entry})
-        _affect_source = meta.get("affect") if isinstance(meta.get("affect"), dict) else getattr(self, "affect", None)
+        _affect_source = (meta.get("affect") if isinstance(meta.get("affect"), dict)
+                          else self._affect_source())
         _mv, _ma = _affect_pair(_affect_source)
         entry["affect"] = {"valence": round(_mv, 4), "arousal": round(_ma, 4)}
         try:
@@ -1210,7 +1275,7 @@ class MemoryEngine:
                 _lam = float(os.environ.get("BIOCORE_MOOD_LAMBDA") or 0.25)
             except Exception:
                 _lam = 0.25
-            _current = _affect_pair(getattr(self, "affect", None))
+            _current = _affect_pair(self._affect_source())
             for row in rows:
                 entry = row.get("entry") or {}
                 stored = entry.get("affect") if isinstance(entry.get("affect"), dict) else None
@@ -1293,7 +1358,22 @@ class MemoryEngine:
             return False
 
     def _write_affect(self, valence, arousal):
-        """Write (v, a) back to self.affect, dict or object; never raises."""
+        """Write (v, a) back, or propose it when the shared field is bound.
+
+        task-228: a learner never writes the field. With a field bound the
+        change is recorded as a "memory_feedback" proposal and the arbiter
+        decides at turn_end; without one this is the old direct write.
+        """
+        if isinstance(getattr(self, "_field_view", None), dict) and self._field_arbiter:
+            try:
+                base_v, base_a = _affect_pair(self._affect_source())
+                value_v = max(-1.0, min(1.0, float(valence)))
+                value_a = max(0.0, min(1.0, float(arousal)))
+            except Exception:
+                return False
+            return bool(self.propose_field("memory_feedback", {
+                "v": round(value_v - base_v, 6),
+                "a": round(value_a - base_a, 6)}))
         target = getattr(self, "affect", None)
         if target is None:
             return False
@@ -1357,7 +1437,8 @@ class MemoryEngine:
         if not self._memory_feedback_on():
             self._last_memory_feedback = None
             return None
-        if getattr(self, "affect", None) is None:
+        _source = self._affect_source()
+        if _source is None:
             self._last_memory_feedback = {"enabled": True, "applied": [],
                                           "delta": [0.0, 0.0],
                                           "reason": "no_affect_state"}
@@ -1366,7 +1447,7 @@ class MemoryEngine:
             from .recall import feedback as _feedback_mod
             limit = max(1, int(budget))
             report = _feedback_mod.plan(
-                list(rows or [])[:limit], self.affect, self._feedback_last,
+                list(rows or [])[:limit], _source, self._feedback_last,
                 now=float(self.now()))
         except Exception:
             self._last_memory_feedback = None
@@ -1384,6 +1465,10 @@ class MemoryEngine:
                 report["delta"] = [0.0, 0.0]
                 report["after"] = list(report.get("before") or [0.0, 0.5])
                 report["reason"] = "affect_not_writable"
+            if report.get("applied") and self._field_arbiter:
+                # task-228: recorded as a proposal; the arbiter writes at turn_end.
+                report["deferred"] = True
+                report["field_write"] = "proposal"
         self._last_memory_feedback = report
         return report
 

package/python/biocore_memory/recall/embedding.py

@@ -117,50 +117,6 @@ def _validate_vector(value) -> Optional[List[float]]:
 
 
 CACHE_FILE_ENV = "BIOCORE_EMBEDDING_CACHE_FILE"
-STATS_FILE_ENV = "BIOCORE_EMBED_STATS_FILE"
-_STATS_FLUSHED = {}
-
-
-def _stats_file():
-    return os.environ.get(STATS_FILE_ENV) or None
-
-
-def _flush_stats():
-    """Persist counters so /stats keeps counting across spawn-per-call.
-
-    Each bridge call is a fresh process; the file accumulates deltas since the
-    last flush of this process (never raises, never blocks a turn).
-    """
-    path = _stats_file()
-    if not path:
-        return
-    try:
-        previous = {}
-        if os.path.isfile(path):
-            try:
-                with open(path, encoding="utf-8") as handle:
-                    loaded = json.load(handle)
-                if isinstance(loaded, dict):
-                    previous = loaded
-            except Exception:
-                previous = {}
-        merged = {}
-        for key in EMBED_STATS:
-            try:
-                before = int(previous.get(key, 0) or 0)
-            except Exception:
-                before = 0
-            merged[key] = before + EMBED_STATS[key] - _STATS_FLUSHED.get(key, 0)
-        directory = os.path.dirname(path)
-        if directory:
-            os.makedirs(directory, exist_ok=True)
-        tmp = path + ".tmp"
-        with open(tmp, "w", encoding="utf-8") as handle:
-            json.dump(merged, handle)
-        os.replace(tmp, path)
-        _STATS_FLUSHED.update(EMBED_STATS)
-    except Exception:
-        pass
 
 
 def _cache_file():
@@ -828,13 +784,6 @@ def _call_provider(provider, text, timeout):
 
 def embed(text, provider, cache=None, timeout=None):
     """Return a vector for text, or None (never raises); LRU cache optional."""
-    try:
-        return _embed_impl(text, provider, cache, timeout)
-    finally:
-        _flush_stats()
-
-
-def _embed_impl(text, provider, cache=None, timeout=None):
     if provider is None:
         return None
     normalized = normalize_text(text)

package/python/biocore_memory/recall/feedback.py

@@ -94,9 +94,14 @@ def _weight(entry):
 
 
 def plan(rows, current, cooldowns=None, now=0.0, budget=None,
-         gain=None, max_step=None, cooldown=None, neg_floor=None):
+         gain=None, max_step=None, cooldown=None, neg_floor=None, split=True):
     """Decide the per-turn affect nudge for one recall batch.
 
+    split=True (default): every recalled memory keeps its own proposal row
+    (source id + v/a + weight); no averaged target is invented. The turn then
+    takes the most trusted proposal (ties: the larger pull) instead of a mean.
+    split=False restores the old weighted-average target exactly.
+
     rows       what this recall injected (row dicts or raw entries); trimmed
                to budget when given
     current    live affect: dict {v/a | valence/arousal}, (v, a) or object
@@ -147,6 +152,8 @@ def plan(rows, current, cooldowns=None, now=0.0, budget=None,
         "capped": False,
         "blocked_negative": False,
         "regulation": None,
+        "proposals": [],
+        "selected": "",
         "reason": "",
     }
     if not eligible:
@@ -159,12 +166,32 @@ def plan(rows, current, cooldowns=None, now=0.0, budget=None,
         weights = [1.0] * len(eligible)
     else:
         weights = [weight for _key, _value, weight in eligible]
-    target_v = sum(pair[0] * weight
-                   for (_key, pair, _weight_value), weight
-                   in zip(eligible, weights)) / total_w
-    target_a = sum(pair[1] * weight
-                   for (_key, pair, _weight_value), weight
-                   in zip(eligible, weights)) / total_w
+    # Every recalled memory is a proposal with its own source id. Nothing is
+    # averaged: the turn follows the most trusted proposal (ties -> the larger
+    # pull), so a strong memory can never be diluted by a weak one.
+    proposals = []
+    for (key, pair, weight), mass in zip(eligible, weights):
+        proposals.append({"id": str(key),
+                          "v": round(_clamp(pair[0], -1.0, 1.0), 4),
+                          "a": round(_clamp(pair[1], 0.0, 1.0), 4),
+                          "weight": round(_num(mass, 1.0), 4),
+                          "delta_v": round(_clamp(
+                              (pair[0] - before_v) * gain, -max_step, max_step), 6),
+                          "delta_a": round(_clamp(
+                              (pair[1] - before_a) * gain, -max_step, max_step), 6)})
+    report["proposals"] = proposals
+    if not split and proposals:
+        target_v = sum(row["v"] * row["weight"] for row in proposals) / max(
+            1e-9, sum(row["weight"] for row in proposals))
+        target_a = sum(row["a"] * row["weight"] for row in proposals) / max(
+            1e-9, sum(row["weight"] for row in proposals))
+        report["selected"] = ""
+    else:
+        winner = max(proposals,
+                     key=lambda row: (row["weight"],
+                                      abs(row["delta_v"]) + abs(row["delta_a"])))
+        target_v, target_a = winner["v"], winner["a"]
+        report["selected"] = str(winner["id"])
     target_v = _clamp(target_v, -1.0, 1.0)
     target_a = _clamp(target_a, 0.0, 1.0)
 

package/python/biocore_memory/recall/fusion.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
-"""0.6/0.4 fusion + 0.55/0.35 tiers."""
+"""0.50/0.30/0.10/0.10 fusion (cue/semantic/importance/time) + 0.55/0.35 tiers.
+
+The weights live in biocore_memory.core.config; this docstring used to say
+0.6/0.4, which never matched the constants. [engineering]
+"""
 from __future__ import annotations
 from ..core.config import (CUE_WEIGHT, HIGH_TIER, IMPORTANCE_WEIGHT, MID_TIER,
                            SEM_WEIGHT, TIME_BONUS)