dsh-biocore 0.1.4 → 0.1.6

package/lib/index.js

@@ -167,6 +167,7 @@ export function readFullConfig(dataDir) {
     embedding: storedEmbedding(source),
     rerank: normalizeRerank(source.rerank),
     expression: normalizeExpression(source.expression),
+    sanitize_reply: source.sanitize_reply !== false,
   }
 }
 
@@ -176,6 +177,7 @@ function writeFullConfig(file, config) {
     embedding: config.embedding,
     rerank: config.rerank,
     expression: config.expression,
+    sanitize_reply: config.sanitize_reply !== false,
   }, null, 2) + '\n', { encoding: 'utf8', mode: 0o600 })
   try { chmodSync(file, 0o600) } catch { /* Windows may ignore POSIX modes */ }
 }
@@ -191,6 +193,7 @@ export async function handleSettings(req, res, dataDir) {
       embedding: maskedEmbedding(config.embedding),
       rerank: maskedRerank(config.rerank),
       expression: config.expression,
+      sanitize_reply: config.sanitize_reply !== false,
       restart_required: false,
     })
   }
@@ -213,7 +216,8 @@ export async function handleSettings(req, res, dataDir) {
   const hasEmbedding = body.embedding && typeof body.embedding === 'object'
   const hasRerank = body.rerank && typeof body.rerank === 'object'
   const hasExpression = body.expression && typeof body.expression === 'object'
-  if (!hasEmbedding && !hasRerank && !hasExpression) {
+  const hasSanitize = typeof body.sanitize_reply === 'boolean'
+  if (!hasEmbedding && !hasRerank && !hasExpression && !hasSanitize) {
     return sendJson(res, 400, { ok: false, reason: 'no_settings' })
   }
   if (hasEmbedding) {
@@ -236,12 +240,14 @@ export async function handleSettings(req, res, dataDir) {
     config.rerank = rerank
   }
   if (hasExpression) config.expression = normalizeExpression(body.expression)
+  if (hasSanitize) config.sanitize_reply = body.sanitize_reply === true
   writeFullConfig(paths.file, config)
   return sendJson(res, 200, {
     ok: true,
     embedding: maskedEmbedding(config.embedding),
     rerank: maskedRerank(config.rerank),
     expression: config.expression,
+    sanitize_reply: config.sanitize_reply !== false,
     restart_required: true,
     hint: '重启 BioCore 生效',
   })

package/lib/modules/core-turn.js

@@ -26,6 +26,11 @@ import { applyGateText, assistantTextOf } from './safety-gate.js'
 import { defaultStoreDir } from './trace.js'
 import { runtimeAction, runTurn } from '../py-bridge.js'
 import { EXPRESSION_DEFAULTS, readExpressionSettings } from './expression-config.js'
+import {
+  readSanitizeReplySetting,
+  sanitizeReply,
+  sanitizeReplyEnabled,
+} from './sanitize-reply.js'
 
 export const TURN_CONTEXT_NAME = 'biocore-expression-turn'
 export const TURN_CONTEXT_ORDER = -190
@@ -89,6 +94,17 @@ export function diagnosticsFromState(state, userText) {
   }
 }
 
+/** Final assistant text: core clean_text wins, JS sanitizer is the fallback.
+ * sanitizeEnabled=false keeps the original (user disabled the switch). */
+export function chooseCleanReply(original, coreCleanText, options) {
+  const opts = options || {}
+  const source = String(original == null ? '' : original)
+  if (opts.sanitizeEnabled === false) return source
+  if (typeof coreCleanText === 'string') return coreCleanText
+  const sanitize = typeof opts.sanitize === 'function' ? opts.sanitize : sanitizeReply
+  return String(sanitize(source))
+}
+
 export function turnStartPayload(options) {
   const opts = options || {}
   const expression = opts.expression && typeof opts.expression === 'object'
@@ -118,6 +134,7 @@ export function turnEndPayload(options) {
     store: opts.store || defaultStoreDir(),
     capture: false,
     memoryEnabled: false,
+    sanitize_reply: opts.sanitizeReply !== false,
     diagnostics,
   }, diagnostics, {
     modules_trace: diagnostics.modules,
@@ -135,6 +152,9 @@ export function makeCoreSessionListener(options) {
   const getExpression = typeof opts.getExpression === 'function'
     ? opts.getExpression
     : () => readExpressionSettings(stateDir)
+  const getSanitizeReply = typeof opts.getSanitizeReply === 'function'
+    ? opts.getSanitizeReply
+    : () => readSanitizeReplySetting(stateDir)
   const store = opts.store || defaultStoreDir()
   const log = typeof opts.log === 'function' ? opts.log : (line) => {
     try { console.error(line) } catch { /* noop */ }
@@ -186,9 +206,11 @@ export function makeCoreSessionListener(options) {
         }).catch(() => null)
         return
       }
-      // assistant/message: gate + trace via one turn_end call.
+      // assistant/message: sanitize -> gate + trace via one turn_end call.
       const original = assistantTextOf(event)
       if (!original) return
+      const sanitizeEnabled = sanitizeReplyEnabled(getSanitizeReply())
+      const cleaned = sanitizeEnabled ? sanitizeReply(original) : original
       const userText = lastUser.get(sessionId) || ''
       void (async () => {
         try {
@@ -198,14 +220,23 @@ export function makeCoreSessionListener(options) {
           const state = read && !read.error && !read.missing ? read.state : {}
           const payload = turnEndPayload({
             sessionId,
-            assistantText: original,
+            assistantText: cleaned,
             userText,
             switches,
             store,
             state,
+            sanitizeReply: sanitizeEnabled,
           })
           const result = await runAction('turn_end', payload)
           if (!result || result.ok !== true) return
+          const reply = result.reply && typeof result.reply === 'object' ? result.reply : {}
+          const coreCleanText = typeof reply.clean_text === 'string'
+            ? reply.clean_text
+            : (typeof result.clean_text === 'string' ? result.clean_text : null)
+          const finalText = chooseCleanReply(original, coreCleanText, {
+            sanitizeEnabled,
+            sanitize: sanitizeReply,
+          })
           const gate = result.gate && typeof result.gate === 'object' ? result.gate : {}
           const action = stringId(gate.action || 'pass')
           if ((action === 'replace' || action === 'truncate')
@@ -216,6 +247,9 @@ export function makeCoreSessionListener(options) {
               + ' blocked=' + (gate.blocked === true ? '1' : '0')
               + ' rule=' + stringId(gate.rule || '')
               + ' chars=' + original.length + '->' + gate.text.length)
+          } else if (finalText !== original) {
+            // No gate rewrite: write the core-cleaned / JS-fallback text back.
+            applyGateText(event, finalText)
           }
           if (result.trace_file) {
             logLine(log, CORE_TRACE_PREFIX + ' session=' + sessionId

package/lib/modules/sanitize-reply.js

@@ -0,0 +1,60 @@
+/** Strip BioCore output markers from assistant replies (task-223).
+ *
+ * Removes:
+ *   - <bc>...</bc> and <biocore>...</biocore> blocks (also unclosed to EOL/EOS)
+ *   - bare compact JSON objects that only carry e/i[/n] keys
+ *   - extra whitespace; empty result stays empty (never falls back to input)
+ */
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export const DEFAULT_SANITIZE_REPLY = true
+
+const TAG_RE = /<(bc|biocore)\b[^>]*>/gi
+const BARE_JSON_RE = /\{\s*"e"\s*:\s*"[^"\\]*(?:\\.[^"\\]*)*"\s*,\s*"i"\s*:\s*-?\d+(?:\.\d+)?(?:\s*,\s*"n"\s*:\s*-?\d+(?:\.\d+)?)?\s*\}/g
+
+function stripClosedTags(text) {
+  return String(text)
+    .replace(/<bc\b[^>]*>[\s\S]*?<\/bc\s*>/gi, '')
+    .replace(/<biocore\b[^>]*>[\s\S]*?<\/biocore\s*>/gi, '')
+}
+
+function stripUnclosedTags(text) {
+  // Until line end or a Chinese/ASCII sentence terminator; the terminator stays.
+  return String(text)
+    .replace(/<bc\b[^>]*>[\s\S]*?(?:[\n。！？!?]|$)/gi, '')
+    .replace(/<biocore\b[^>]*>[\s\S]*?(?:[\n。！？!?]|$)/gi, '')
+}
+
+function normalizeWhitespace(text) {
+  return String(text)
+    .replace(/[ \t]{2,}/g, ' ')
+    .replace(/[ \t]*\n[ \t]*/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim()
+}
+
+/** Main entry: returns the cleaned reply (may be ''); never reverts to input. */
+export function sanitizeReply(text, options) {
+  const opts = options || {}
+  if (opts.enabled === false) return String(text == null ? '' : text)
+  const source = String(text == null ? '' : text)
+  let out = stripClosedTags(source)
+  out = stripUnclosedTags(out)
+  out = out.replace(BARE_JSON_RE, '')
+  return normalizeWhitespace(out)
+}
+
+/** Config.json top-level flag: only an explicit false disables cleaning. */
+export function readSanitizeReplySetting(dataDir) {
+  try {
+    const doc = JSON.parse(readFileSync(join(String(dataDir || '').trim(), 'config.json'), 'utf8'))
+    if (doc && typeof doc === 'object' && doc.sanitize_reply === false) return false
+  } catch { /* missing/bad config -> default on */ }
+  return DEFAULT_SANITIZE_REPLY
+}
+
+/** Payload helper: sanitize_reply defaults to true unless explicitly false. */
+export function sanitizeReplyEnabled(value) {
+  return value !== false
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-biocore",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "type": "module",
   "description": "BioCore DSH UI: per-session state tab, standalone instrument panel, /api/biocore/state, session/event one_turn updates, /bio sync.",
   "main": "lib/index.js",

package/python/biocore_runtime/pipeline.py

@@ -96,6 +96,11 @@ try:
 except Exception:  # pragma: no cover
     _rerank_mod = None
 
+try:
+    from biocore_model import sanitize as _sanitize
+except Exception:  # pragma: no cover
+    _sanitize = None
+
 try:  # self-layer modules (all pure; optional so a stripped checkout still runs)
     from biocore_model import x_directives as _x_directives
     from biocore_model import self_intent as _self_intent
@@ -1794,10 +1799,35 @@ def dispatch(action, payload=None):
                 return bad("missing session_id")
             if "assistant_text" not in data:
                 return bad("missing assistant_text")
-            assistant_text = clean_text(data.get("assistant_text"))
+            raw_assistant = clean_text(data.get("assistant_text"))
+            do_sanitize = _as_bool(data.get("sanitize_reply"), True)
+            cleaned = raw_assistant
+            sanitized = False
+            removed_tags = 0
+            if do_sanitize and _sanitize is not None:
+                stats = getattr(_sanitize, "STATS", {}) or {}
+                before = (int(stats.get("tag_stripped", 0) or 0),
+                          int(stats.get("json_stripped", 0) or 0),
+                          int(stats.get("fenced_stripped", 0) or 0))
+                try:
+                    cleaned = _sanitize.clean(raw_assistant)
+                except Exception:
+                    cleaned = raw_assistant
+                after = (int(stats.get("tag_stripped", 0) or 0),
+                         int(stats.get("json_stripped", 0) or 0),
+                         int(stats.get("fenced_stripped", 0) or 0))
+                removed_tags = max(0, sum(after) - sum(before))
+                sanitized = cleaned != raw_assistant or removed_tags > 0
+            assistant_text = cleaned
             switches = normalize_switches(data.get("switches"))
             gate = gate_reply(assistant_text, {
                 "switches": switches, "user_text": data.get("user_text", "")})
+            if do_sanitize and _sanitize is not None:
+                try:
+                    gate = dict(gate)
+                    gate["text"] = _sanitize.clean(gate.get("text") or "")
+                except Exception:
+                    pass
             captured = capture(session_id, "assistant", assistant_text, store, {
                 "enabled": _as_bool(data.get("memoryEnabled",
                                              switches.get("memoryEnabled", True)),
@@ -1818,6 +1848,9 @@ def dispatch(action, payload=None):
                 "captured": {"captured": bool(decision.get("capture")),
                              "entry_id": str(entry.get("id") or ""),
                              "reason": str(decision.get("reason") or "")},
+                "clean_text": cleaned,
+                "sanitized": bool(sanitized),
+                "removed_tags": int(removed_tags),
                 "trace": trace,
                 "trace_file": trace_file,
             }

package/ui/panel.html

@@ -247,6 +247,7 @@
       <dl class="advanced-info">
         <div><dt>数据目录</dt><dd id="dataDirText">宿主托管</dd></div>
         <div><dt>插件版本</dt><dd id="pluginVersion">0.1.0</dd></div>
+        <div><dt>出口清洗</dt><dd id="sanitizeReplyText">开</dd></div>
       </dl>
       <div class="advanced-actions">
         <button id="resetLayout" class="advanced-button" type="button">恢复默认布局</button>

package/ui/panel.js

@@ -1050,6 +1050,7 @@
           fillModel(embedding)
           fillRerank(body.rerank)
           fillExpression(body.expression)
+          setText('sanitizeReplyText', body.sanitize_reply === false ? '关' : '开')
           setModelStatus('已载入')
           return
         }