dsh-biocore 0.1.5 → 0.1.7

package/lib/index.js

@@ -28,7 +28,7 @@ import { TRACE_ROUTE_PATH, defaultStoreDir, handleTraceRoute } from './modules/t
 import { queueSessionTurn } from './modules/turn-queue.js'
 import { anySwitchOn } from './modules/expression-inject.js'
 import { makeCoreSessionListener, registerCoreTurn } from './modules/core-turn.js'
-import { PLUGIN_DATA_DIR, resolveDataDir } from './modules/data-dir.js'
+import { migrateLegacyPluginData, PLUGIN_DATA_DIR, resolveDataDir } from './modules/data-dir.js'
 import {
   PANEL_MOUNT_PATH,
   PANEL_ROUTE_PREFIX,
@@ -761,6 +761,7 @@ export async function handleState(req, res, stateDir, options) {
 
 export function apply(ctx, config) {
   const dataDir = resolveDataDir(config)
+  try { migrateLegacyPluginData(dataDir) } catch { /* best-effort data migration */ }
   try { migratePersonaToShared(dataDir) } catch { /* best-effort migration */ }
   const disposers = []
   const keep = (dispose) => {

package/lib/modules/core-turn.js

@@ -94,6 +94,17 @@ export function diagnosticsFromState(state, userText) {
   }
 }
 
+/** Final assistant text: core clean_text wins, JS sanitizer is the fallback.
+ * sanitizeEnabled=false keeps the original (user disabled the switch). */
+export function chooseCleanReply(original, coreCleanText, options) {
+  const opts = options || {}
+  const source = String(original == null ? '' : original)
+  if (opts.sanitizeEnabled === false) return source
+  if (typeof coreCleanText === 'string') return coreCleanText
+  const sanitize = typeof opts.sanitize === 'function' ? opts.sanitize : sanitizeReply
+  return String(sanitize(source))
+}
+
 export function turnStartPayload(options) {
   const opts = options || {}
   const expression = opts.expression && typeof opts.expression === 'object'
@@ -218,6 +229,14 @@ export function makeCoreSessionListener(options) {
           })
           const result = await runAction('turn_end', payload)
           if (!result || result.ok !== true) return
+          const reply = result.reply && typeof result.reply === 'object' ? result.reply : {}
+          const coreCleanText = typeof reply.clean_text === 'string'
+            ? reply.clean_text
+            : (typeof result.clean_text === 'string' ? result.clean_text : null)
+          const finalText = chooseCleanReply(original, coreCleanText, {
+            sanitizeEnabled,
+            sanitize: sanitizeReply,
+          })
           const gate = result.gate && typeof result.gate === 'object' ? result.gate : {}
           const action = stringId(gate.action || 'pass')
           if ((action === 'replace' || action === 'truncate')
@@ -228,9 +247,9 @@ export function makeCoreSessionListener(options) {
               + ' blocked=' + (gate.blocked === true ? '1' : '0')
               + ' rule=' + stringId(gate.rule || '')
               + ' chars=' + original.length + '->' + gate.text.length)
-          } else if (cleaned !== original) {
-            // No gate rewrite: still write the sanitized text back.
-            applyGateText(event, cleaned)
+          } else if (finalText !== original) {
+            // No gate rewrite: write the core-cleaned / JS-fallback text back.
+            applyGateText(event, finalText)
           }
           if (result.trace_file) {
             logLine(log, CORE_TRACE_PREFIX + ' session=' + sessionId

package/lib/modules/data-dir.js

@@ -4,21 +4,24 @@
  *   <dataDir>/sessions/<session_id>.json   (per-session BioCore state)
  *   <dataDir>/flags.json                   (four switches)
  *
- * Resolution order (task-201):
+ * Resolution order:
  *   1. --profile <name> / --profile=<name> from process.argv
  *      -> <homedir>/.dsh/profiles/<name>/data, only when that profile dir exists
  *   2. config.dataDir (cordis insert config)
  *   3. BIOCORE_UI_DATA_DIR
  *   4. <DSH_PROFILE_DIR>/data -- only when it is a real DSH profile directory
  *      (contains package.json or dsh.yml)
- *   5. <plugin>/data (repo/deployed-copy fallback; never cwd)
+ *   5. <home>/.dsh/profiles/web/data -- when profiles/web/package.json exists
+ *   6. <home>/.dsh/profiles/<only>/data -- exactly one profile under profiles/
+ *   7. <plugin>/data -- last resort, logs __bioDataDirFallback
+ *      (never cwd; previous node_modules data is migrated, not deleted)
  *
  * Why argv first: DSH passes the serving process its own --profile flag, but
  * DSH_PROFILE_DIR in the environment can point at the *default* profile (web)
  * even when the instance runs --profile web-bio. Trusting the env first wrote
  * session state into the wrong profile's data dir.
  */
-import { existsSync } from 'node:fs'
+import { copyFileSync, existsSync, mkdirSync, readdirSync } from 'node:fs'
 import { homedir } from 'node:os'
 import { fileURLToPath } from 'node:url'
 import { dirname, join, resolve } from 'node:path'
@@ -28,6 +31,10 @@ export const PROFILE_DIR_ENV = 'DSH_PROFILE_DIR'
 export const PROFILE_MARKERS = Object.freeze(['package.json', 'dsh.yml'])
 /** Fallback: the plugin checkout's own data/ directory. */
 export const PLUGIN_DATA_DIR = fileURLToPath(new URL('../../data', import.meta.url))
+export const FALLBACK_WARN_PREFIX = '__bioDataDirFallback'
+export const MIGRATE_LOG_PREFIX = '__bioDataMigrated'
+
+let warnedFallback = false
 
 function text(value) {
   return typeof value === 'string' ? value.trim() : ''
@@ -69,6 +76,71 @@ export function isProfileDir(dir) {
   }
 }
 
+/** <home>/.dsh/profiles */
+export function profilesRootDir(homeDir) {
+  const home = text(homeDir) || homedir()
+  return join(resolve(home), '.dsh', 'profiles')
+}
+
+/** All real profile names directly under <home>/.dsh/profiles. */
+export function listProfiles(homeDir) {
+  const root = profilesRootDir(homeDir)
+  let names = []
+  try {
+    names = readdirSync(root, { withFileTypes: true })
+      .filter((entry) => entry && entry.isDirectory())
+      .map((entry) => entry.name)
+  } catch { return [] }
+  return names.filter((name) => isProfileDir(join(root, name)))
+}
+
+function warnPluginFallback(path) {
+  if (warnedFallback) return
+  warnedFallback = true
+  try {
+    console.error(FALLBACK_WARN_PREFIX + ' using plugin data dir=' + path
+      + ' reason=no-profile-dir (writes will live inside the plugin package)')
+  } catch { /* logging must not throw */ }
+}
+
+/** Copy existing legacy plugin data into the resolved profile dataDir.
+ * Missing files are copied; existing files are never overwritten; the old
+ * plugin copy stays in place for rollback. */
+export function migrateLegacyPluginData(dataDir, legacyDir) {
+  const target = text(dataDir)
+  const source = text(legacyDir) || PLUGIN_DATA_DIR
+  const result = { migrated: false, copied: 0, target, source }
+  if (!target || resolve(target) === resolve(source)) return result
+  // Only migrate into <profile>/data when the parent is a real profile root.
+  const parent = dirname(resolve(target))
+  if (!isProfileDir(parent)) return result
+  const copyIfMissing = (from, to) => {
+    try {
+      if (!existsSync(from) || existsSync(to)) return false
+      mkdirSync(dirname(to), { recursive: true })
+      copyFileSync(from, to)
+      result.copied += 1
+      return true
+    } catch { return false }
+  }
+  const files = ['flags.json', 'persona_profile.json', 'persona.md', 'config.json']
+  for (const name of files) copyIfMissing(join(source, name), join(target, name))
+  try {
+    const sessions = readdirSync(join(source, 'sessions'))
+    for (const name of sessions) {
+      copyIfMissing(join(source, 'sessions', name), join(target, 'sessions', name))
+    }
+  } catch { /* no legacy sessions */ }
+  result.migrated = result.copied > 0
+  if (result.migrated) {
+    try {
+      console.error(MIGRATE_LOG_PREFIX + ' from=' + source + ' to=' + target
+        + ' copied=' + result.copied)
+    } catch { /* logging must not throw */ }
+  }
+  return result
+}
+
 /**
  * Resolve the one directory that owns flags.json and sessions/.
  * @param {object} [config] plugin config (config.dataDir)
@@ -103,5 +175,18 @@ export function resolveDataDir(config, env, argv, homeDir) {
     if (isProfileDir(candidate)) return join(candidate, 'data')
   }
 
-  return resolve(PLUGIN_DATA_DIR)
-}
+  // 5) default web profile, when it is a real profile.
+  const webProfile = join(profilesRootDir(homeDir), 'web')
+  if (isProfileDir(webProfile)) return join(webProfile, 'data')
+
+  // 6) exactly one profile -> deterministic; multiple profiles are ambiguous.
+  const profiles = listProfiles(homeDir)
+  if (profiles.length === 1) {
+    return join(profilesRootDir(homeDir), profiles[0], 'data')
+  }
+
+  // 7) last resort: the plugin package itself.
+  const fallback = resolve(PLUGIN_DATA_DIR)
+  warnPluginFallback(fallback)
+  return fallback
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-biocore",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "type": "module",
   "description": "BioCore DSH UI: per-session state tab, standalone instrument panel, /api/biocore/state, session/event one_turn updates, /bio sync.",
   "main": "lib/index.js",

package/python/biocore_runtime/pipeline.py

@@ -96,6 +96,11 @@ try:
 except Exception:  # pragma: no cover
     _rerank_mod = None
 
+try:
+    from biocore_model import sanitize as _sanitize
+except Exception:  # pragma: no cover
+    _sanitize = None
+
 try:  # self-layer modules (all pure; optional so a stripped checkout still runs)
     from biocore_model import x_directives as _x_directives
     from biocore_model import self_intent as _self_intent
@@ -1794,10 +1799,35 @@ def dispatch(action, payload=None):
                 return bad("missing session_id")
             if "assistant_text" not in data:
                 return bad("missing assistant_text")
-            assistant_text = clean_text(data.get("assistant_text"))
+            raw_assistant = clean_text(data.get("assistant_text"))
+            do_sanitize = _as_bool(data.get("sanitize_reply"), True)
+            cleaned = raw_assistant
+            sanitized = False
+            removed_tags = 0
+            if do_sanitize and _sanitize is not None:
+                stats = getattr(_sanitize, "STATS", {}) or {}
+                before = (int(stats.get("tag_stripped", 0) or 0),
+                          int(stats.get("json_stripped", 0) or 0),
+                          int(stats.get("fenced_stripped", 0) or 0))
+                try:
+                    cleaned = _sanitize.clean(raw_assistant)
+                except Exception:
+                    cleaned = raw_assistant
+                after = (int(stats.get("tag_stripped", 0) or 0),
+                         int(stats.get("json_stripped", 0) or 0),
+                         int(stats.get("fenced_stripped", 0) or 0))
+                removed_tags = max(0, sum(after) - sum(before))
+                sanitized = cleaned != raw_assistant or removed_tags > 0
+            assistant_text = cleaned
             switches = normalize_switches(data.get("switches"))
             gate = gate_reply(assistant_text, {
                 "switches": switches, "user_text": data.get("user_text", "")})
+            if do_sanitize and _sanitize is not None:
+                try:
+                    gate = dict(gate)
+                    gate["text"] = _sanitize.clean(gate.get("text") or "")
+                except Exception:
+                    pass
             captured = capture(session_id, "assistant", assistant_text, store, {
                 "enabled": _as_bool(data.get("memoryEnabled",
                                              switches.get("memoryEnabled", True)),
@@ -1818,6 +1848,9 @@ def dispatch(action, payload=None):
                 "captured": {"captured": bool(decision.get("capture")),
                              "entry_id": str(entry.get("id") or ""),
                              "reason": str(decision.get("reason") or "")},
+                "clean_text": cleaned,
+                "sanitized": bool(sanitized),
+                "removed_tags": int(removed_tags),
                 "trace": trace,
                 "trace_file": trace_file,
             }