@theseus-whale/dsh-continuity 0.0.0-stage → 4.12.2

package/bin/ner_direct_zh.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Direct token-classification inference, no transformers pipeline."""
+import json
+import sys
+import time
+
+import torch
+from datasets import load_dataset
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+
+MODEL = "uer/roberta-base-finetuned-cluener2020-chinese"
+DATASET = "xusenlin/clue-ner"
+LIMIT = int(sys.argv[1]) if len(sys.argv) > 1 else 20
+OUT = sys.argv[2] if len(sys.argv) > 2 else None
+MAX_LEN = 256
+
+torch.set_num_threads(1)
+print("load tokenizer", flush=True)
+tok = AutoTokenizer.from_pretrained(MODEL)
+print("fast", tok.is_fast, flush=True)
+model = AutoModelForTokenClassification.from_pretrained(MODEL)
+model.eval()
+id2label = model.config.id2label
+print("load data", flush=True)
+ds = load_dataset(DATASET, split="validation")
+
+
+def predict(text):
+    enc = tok(text, return_tensors="pt", truncation=True, max_length=MAX_LEN, return_offsets_mapping=tok.is_fast)
+    offsets = enc.pop("offset_mapping")[0].tolist()
+    with torch.no_grad():
+        logits = model(**enc).logits[0]
+    ids = logits.argmax(-1).tolist()
+    entities = []
+    current = None
+    for idx, label_id in enumerate(ids):
+        label = id2label[int(label_id)]
+        start, end = offsets[idx]
+        if start == end:
+            continue
+        if label == "O":
+            if current is not None:
+                entities.append(current)
+                current = None
+            continue
+        prefix, _, kind = label.partition("-")
+        if prefix == "S":
+            if current is not None:
+                entities.append(current)
+                current = None
+            entities.append({"text": text[start:end], "label": kind, "start": start, "end": end})
+        elif prefix == "B":
+            if current is not None:
+                entities.append(current)
+            current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+        elif prefix in ("I", "E") and current is not None:
+            current["text"] += text[start:end]
+            current["end"] = end
+            if prefix == "E":
+                entities.append(current)
+                current = None
+    if current is not None:
+        entities.append(current)
+    return entities
+
+
+def score(pred, gold):
+    gold_keys = {(g["entity"], g["label"]) for g in gold}
+    pred_keys = {(p["text"].replace(" ", ""), p["label"]) for p in pred}
+    tp = len(gold_keys & pred_keys)
+    fp = len(pred_keys - gold_keys)
+    fn = len(gold_keys - pred_keys)
+    precision = 0 if not pred_keys else tp / len(pred_keys)
+    recall = 0 if not gold_keys else tp / len(gold_keys)
+    f1 = 0 if precision + recall == 0 else 2 * precision * recall / (precision + recall)
+    return {"tp": tp, "fp": fp, "fn": fn, "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4)}
+
+
+rows = []
+for i in range(min(LIMIT, len(ds))):
+    print("sample", i, flush=True)
+    item = ds[i]
+    t0 = time.time()
+    pred = predict(item["text"])
+    gold = [{"entity": e["entity"], "label": e["label"]} for e in item["entities"]]
+    rows.append({"text": item["text"], "pred": pred, "gold": gold})
+    print("done", i, round(time.time() - t0, 2), pred[:3], flush=True)
+
+result = {
+    "model": MODEL,
+    "dataset": DATASET,
+    "n": len(rows),
+    "score": score([p for r in rows for p in r["pred"]], [g for r in rows for g in r["gold"]]),
+    "rows": rows,
+}
+if OUT:
+    with open(OUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+print(json.dumps({k: v for k, v in result.items() if k != "rows"}, ensure_ascii=False, indent=2), flush=True)

package/bin/ner_pipeline.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Bilingual NER pipeline over local JSONL messages. No training, no live IO."""
+import json
+import re
+import sys
+
+import torch
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+
+ZH_MODEL = "uer/roberta-base-finetuned-cluener2020-chinese"
+EN_MODEL = "elastic/distilbert-base-uncased-finetuned-conll03-english"
+MAX_LEN = 256
+
+ZH_SLOT = {
+    "name": "characters",
+    "address": "location",
+    "scene": "location",
+    "company": "objects",
+    "organization": "objects",
+    "government": "objects",
+    "book": "objects",
+    "movie": "objects",
+    "game": "objects",
+}
+EN_SLOT = {"PER": "characters", "LOC": "location", "ORG": "objects", "MISC": "objects", "PROPER": "objects"}
+PROPER_STOP = {
+    "The", "This", "That", "These", "Those", "If", "Then", "There", "Here",
+    "We", "You", "I", "It", "He", "She", "They", "But", "And", "Or", "So",
+    "DSH", "TAG", "CPU", "GPU", "API", "JSON", "HTML", "HTTP", "HTTPS",
+}
+
+torch.set_num_threads(1)
+print("load zh", flush=True)
+zh_tok = AutoTokenizer.from_pretrained(ZH_MODEL)
+zh_model = AutoModelForTokenClassification.from_pretrained(ZH_MODEL)
+zh_model.eval()
+print("load en", flush=True)
+en_tok = AutoTokenizer.from_pretrained(EN_MODEL)
+en_model = AutoModelForTokenClassification.from_pretrained(EN_MODEL)
+en_model.eval()
+print("models ready", flush=True)
+
+
+def route(text):
+    cjk = sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
+    latin = sum(1 for c in text if c.isascii() and c.isalpha())
+    return "en" if cjk == 0 and latin > 0 else "zh"
+
+
+def decode(text, offsets, ids, id2label, mode):
+    entities = []
+    current = None
+    for idx, label_id in enumerate(ids):
+        label = id2label[int(label_id)]
+        start, end = offsets[idx]
+        if start == end:
+            continue
+        if label == "O":
+            if current is not None:
+                entities.append(current)
+                current = None
+            continue
+        prefix, _, kind = label.partition("-")
+        if mode == "zh":
+            if prefix == "S":
+                if current is not None:
+                    entities.append(current)
+                entities.append({"text": text[start:end], "label": kind, "start": start, "end": end})
+                current = None
+            elif prefix == "B":
+                if current is not None:
+                    entities.append(current)
+                current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+            elif prefix in ("I", "E") and current is not None:
+                current["text"] += text[start:end]
+                current["end"] = end
+                if prefix == "E":
+                    entities.append(current)
+                    current = None
+        else:
+            if prefix == "B":
+                if current is not None:
+                    entities.append(current)
+                current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+            elif prefix == "I" and current is not None and current["label"] == kind:
+                current["text"] += text[start:end]
+                current["end"] = end
+            else:
+                if current is not None:
+                    entities.append(current)
+                current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+    if current is not None:
+        entities.append(current)
+    if mode == "en":
+        merged = []
+        for entity in entities:
+            last = merged[-1] if merged else None
+            if last is not None and last["label"] == entity["label"] and entity["start"] - last["end"] <= 1:
+                last["text"] += entity["text"]
+                last["end"] = entity["end"]
+            else:
+                merged.append(entity)
+        entities = merged
+    return entities
+
+
+def infer(tok, model, text, mode):
+    enc = tok(text, return_tensors="pt", truncation=True, max_length=MAX_LEN, return_offsets_mapping=True)
+    offsets = enc.pop("offset_mapping")[0].tolist()
+    with torch.no_grad():
+        logits = model(**enc).logits[0]
+    return decode(text, offsets, logits.argmax(-1).tolist(), model.config.id2label, mode)
+
+
+def extract(text):
+    lang = route(text)
+    zh_entities = infer(zh_tok, zh_model, text, "zh")
+    en_entities = infer(en_tok, en_model, text, "en")
+    combined = []
+    seen = set()
+    for entity in en_entities:
+        entity_text = entity["text"]
+        if any(c.isascii() and c.isalpha() for c in entity_text):
+            key = (entity["start"], entity["end"], entity_text)
+            if key not in seen:
+                seen.add(key)
+                combined.append(entity)
+    for entity in zh_entities:
+        entity_text = entity["text"]
+        if not any(c.isascii() and c.isalpha() for c in entity_text):
+            key = (entity["start"], entity["end"], entity_text)
+            if key not in seen:
+                seen.add(key)
+                combined.append(entity)
+    combined.sort(key=lambda item: (item["start"], item["end"]))
+    merged = []
+    for entity in combined:
+        last = merged[-1] if merged else None
+        if last is not None and entity["start"] <= last["end"] + 1:
+            between = text[last["end"]:entity["start"]]
+            both_latin = all(any(c.isascii() and c.isalpha() for c in item["text"]) for item in (last, entity))
+            if both_latin and between.strip() == "":
+                last["text"] = text[last["start"]:entity["end"]]
+                last["end"] = entity["end"]
+                if last["label"] not in ("PER", "PERSON", "name"):
+                    last["label"] = entity["label"] if entity["label"] in ("PER", "PERSON", "name") else last["label"]
+                continue
+        merged.append(entity)
+    combined = merged
+    existing_spans = [(item["start"], item["end"], item) for item in combined]
+    for match in re.finditer(r"(?<![A-Za-z0-9_])[A-Z][A-Za-z0-9_-]{1,20}(?![A-Za-z0-9_])", text):
+        token = match.group(0)
+        if token in PROPER_STOP:
+            continue
+        span = (match.start(), match.end())
+        overlap = next(((start, end, item) for start, end, item in existing_spans if start < span[1] and span[0] < end), None)
+        if overlap is not None:
+            if overlap[0] == span[0] and len(token) > len(overlap[2]["text"]):
+                combined.remove(overlap[2])
+                combined.append({"text": token, "label": "PROPER", "start": span[0], "end": span[1]})
+                existing_spans = [(item["start"], item["end"], item) for item in combined]
+            continue
+        combined.append({"text": token, "label": "PROPER", "start": span[0], "end": span[1]})
+        existing_spans.append((span[0], span[1], combined[-1]))
+    combined.sort(key=lambda item: (item["start"], item["end"]))
+    ANIMATE = re.compile(r"(?:说|问|答|看|听|想|做|装|退|关|备份|运行|她|他|你|我)")
+    PRODUCT = re.compile(r"(?:插件|版本|系统|模型|工具|平台|接口|文件|世界|规则|框架|协议|功能)")
+    for entity in combined:
+        if entity["label"] != "PROPER":
+            continue
+        context = text[max(0, entity["start"] - 10):min(len(text), entity["end"] + 10)]
+        if ANIMATE.search(context) and not PRODUCT.search(context):
+            entity["label"] = "PER"
+    slot_map = {}
+    slot_map.update(ZH_SLOT)
+    slot_map.update(EN_SLOT)
+    slots = {"characters": [], "location": None, "objects": []}
+    for entity in combined:
+        slot = slot_map.get(entity["label"])
+        if slot == "characters":
+            slots["characters"].append(entity["text"])
+        elif slot == "location" and slots["location"] is None:
+            slots["location"] = entity["text"]
+        elif slot == "objects":
+            slots["objects"].append(entity["text"])
+    slots["characters"] = list(dict.fromkeys(slots["characters"]))
+    slots["objects"] = list(dict.fromkeys(slots["objects"]))
+    slots["objects"] = [item for item in slots["objects"] if item not in slots["characters"]]
+    return {"lang": lang, "entities": combined, "slots": slots}
+
+
+src, out, limit = sys.argv[1], sys.argv[2], int(sys.argv[3]) if len(sys.argv) > 3 else 100
+rows = []
+for line in open(src, encoding="utf-8"):
+    if len(rows) >= limit:
+        break
+    item = json.loads(line)
+    text = item.get("text", "")
+    if not text.strip():
+        continue
+    rows.append({"turn": item.get("turn"), "text": text, **extract(text)})
+    if len(rows) % 10 == 0:
+        print("done", len(rows), flush=True)
+
+with open(out, "w", encoding="utf-8") as f:
+    json.dump({"n": len(rows), "rows": rows}, f, ensure_ascii=False, indent=2)
+print(json.dumps({"n": len(rows), "langs": {k: sum(1 for r in rows if r["lang"] == k) for k in ["zh", "en"]}, "with_entities": sum(1 for r in rows if r["entities"])}, ensure_ascii=False, indent=2), flush=True)

package/bin/noise-selfcheck.mjs

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { needsSkillInjection } from '../src/inject-policy.mjs';
+import { selectTopic } from '../src/topic.mjs';
+
+const rows = [
+  { id: 'a', alias: 'A', label: 'A', keywords: ['普通词'] },
+  { id: 'b', alias: 'B', label: 'B', keywords: ['专名甲', '专名乙'] },
+];
+const generic = new Set(['普通词']);
+
+// 泛词单独出现：静默。
+assert.equal(selectTopic({ rows, text: '普通词', genericWords: generic, activeId: 'b' }).blocked, true);
+// 两个非泛词：可提示。
+assert.equal(selectTopic({ rows, text: '专名甲 专名乙', genericWords: generic, activeId: 'a' }).blocked, false);
+// 一个专属词：可提示。
+assert.equal(selectTopic({
+  rows,
+  text: '专名甲',
+  genericWords: generic,
+  wordStats: new Map([['专名甲', { precision: 0.6, hits: 6 }]]),
+  activeId: 'a',
+}).blocked, false);
+
+// 技能正文只跟集合变化走；载体被折叠不再重灌整份正文。
+assert.equal(needsSkillInjection({ keyChanged: true }), true);
+assert.equal(needsSkillInjection({ keyChanged: false }), false);
+
+// 静态断言：新标签和存提醒的阈值仍在源码里。
+const engine = readFileSync(new URL('../src/engine.mjs', import.meta.url), 'utf8');
+assert.match(engine, /snippet\.length >= 40/);
+assert.match(engine, /cands\.length === 0 && snippet\.length >= 40/);
+assert.match(engine, /currentTurn - Number\(last\?\.turn.*>= 20/);
+assert.match(engine, /contentAfter >= 8 && gap >= 20/);
+assert.match(engine, /this\.saveAsked/);
+assert.match(engine, /TAG: <T 别名/);
+assert.match(engine, /可选短语/);
+assert.match(engine, /tagLearnShown/);
+assert.match(engine, /turn-gate\.jsonl/);
+assert.match(engine, /auditGate-failed/);
+assert.match(engine, /habits\.md/);
+assert.match(engine, /splitNewToken/);
+assert.match(engine, /tagIdOf\(made\)/);
+assert.match(engine, /gate-steer\.jsonl/);
+assert.match(engine, /snapshotEvents/);
+assert.match(engine, /bumpTagCount/);
+assert.match(engine, /闸·线/);
+
+console.log('noise-selfcheck PASS');

package/bin/pack-integrity.mjs

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+/**
+ * Pack integrity check: compare selected files inside a .tgz with the dev tree.
+ *
+ *   node bin/pack-integrity.mjs --tgz path/to/pkg.tgz --root . \
+ *     --files src/memory/relabel-migrate.mjs,src/memory/bc-pre-step.mjs
+ *
+ * Exits 1 on any mismatch. This catches the "packaged an old public copy"
+ * failure mode without trusting the version number.
+ */
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+
+function arg(name, fallback = null) {
+  const index = process.argv.indexOf(`--${name}`);
+  return index >= 0 ? process.argv[index + 1] ?? fallback : fallback;
+}
+
+const tgz = arg('tgz');
+const root = arg('root', '.');
+const files = String(arg('files', '')).split(',').map((file) => file.trim()).filter(Boolean);
+if (!tgz || files.length === 0) {
+  console.error('usage: node bin/pack-integrity.mjs --tgz <file.tgz> --root <dir> --files a,b');
+  process.exit(2);
+}
+
+const sha = (buffer) => createHash('sha256').update(buffer).digest('hex');
+let failed = 0;
+for (const file of files) {
+  const local = `${root.replace(/[\\/]+$/, '')}/${file}`;
+  const expected = sha(readFileSync(local));
+  const result = spawnSync('tar', ['-xOf', tgz, `package/${file}`]);
+  if (result.status !== 0) {
+    console.error(`MISSING ${file}: ${result.stderr?.toString().trim() || 'tar failed'}`);
+    failed += 1;
+    continue;
+  }
+  const actual = sha(result.stdout);
+  if (actual !== expected) {
+    console.error(`MISMATCH ${file}: dev ${expected.slice(0, 12)} vs tgz ${actual.slice(0, 12)}`);
+    failed += 1;
+  } else {
+    console.log(`OK ${file} ${actual.slice(0, 12)}`);
+  }
+}
+process.exit(failed === 0 ? 0 : 1);

package/bin/panel-check.mjs

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+/**
+ * Diagnose the A/B/C/K panel installation.
+ *
+ * Run from the installed package:
+ *   node bin/panel-check.mjs --url http://127.0.0.1:3095
+ *   node bin/panel-check.mjs --url http://127.0.0.1:3095 --session session-...
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+function arg(name, fallback = null) {
+  const index = process.argv.indexOf(`--${name}`);
+  return index >= 0 ? process.argv[index + 1] ?? fallback : fallback;
+}
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
+const client = path.join(root, 'client.js');
+const url = arg('url', process.env.DSH_WEB_URL ?? null);
+const session = arg('session', null);
+
+console.log(`package: ${pkg.name}@${pkg.version}`);
+console.log(`client.js: ${fs.existsSync(client) ? 'present' : 'MISSING'}`);
+console.log(`dsh.client: ${pkg.dsh?.client ? JSON.stringify(pkg.dsh.client) : 'MISSING'}`);
+if (pkg.dsh?.client?.platform !== 'web') {
+  console.log('warning: dsh.client.platform is not "web"; the browser UI will not load');
+}
+if (!url) {
+  console.log('\nNo --url given. Pass the current DSH Web URL, e.g. --url http://127.0.0.1:3095');
+  process.exit(0);
+}
+const base = url.replace(/\/+$/, '');
+async function get(pathname) {
+  const response = await fetch(`${base}${pathname}`, { headers: { accept: 'application/json' } });
+  return { status: response.status, text: await response.text() };
+}
+try {
+  const state = await get('/dsh-continuity/state.json');
+  console.log(`state.json: HTTP ${state.status}`);
+  console.log(state.text.slice(0, 300));
+} catch (error) {
+  console.log(`state.json: FAILED ${error.message} (is this the DSH Web URL and is the plugin loaded?)`);
+}
+if (session) {
+  try {
+    const bc = await get(`/dsh-continuity/bc.json?sessionId=${encodeURIComponent(session)}`);
+    console.log(`bc.json: HTTP ${bc.status}`);
+    console.log(bc.text.slice(0, 500));
+  } catch (error) {
+    console.log(`bc.json: FAILED ${error.message}`);
+  }
+}
+console.log('\nIf state.json works but the sidebar has no Continuity entry, the client bundle is not mounted.');
+console.log('If the left panel works but the right tab is absent, this DSH lacks sidebarRight/sidebarRightTabs.');
+console.log('If the tab opens with empty tables, send one message; A/C/K start empty by design.');

package/bin/plugin-integrity.mjs

@@ -0,0 +1,74 @@
+/**
+ * Plugin integrity manifest.
+ *
+ * Detects modifications to the installed plugin. Detection is not prevention;
+ * the installed copy must also be read-only / ACL-protected.
+ */
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const SKIP_DIRS = new Set(['node_modules', '.git', '.npm-cache', 'test', 'tests', 'backups']);
+
+function walk(root, current = root, out = []) {
+  for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
+    if (entry.isDirectory()) {
+      if (SKIP_DIRS.has(entry.name)) continue;
+      walk(root, path.join(current, entry.name), out);
+    } else if (entry.isFile()) {
+      out.push(path.relative(root, path.join(current, entry.name)).replace(/\\/g, '/'));
+    }
+  }
+  return out;
+}
+
+export function snapshotPlugin(root) {
+  const files = {};
+  for (const relative of walk(root).sort()) {
+    const full = path.join(root, relative);
+    files[relative] = crypto.createHash('sha256').update(fs.readFileSync(full)).digest('hex');
+  }
+  return { version: 1, algorithm: 'sha256', root: path.basename(root), files };
+}
+
+export function verifyPlugin(root, manifest) {
+  const current = snapshotPlugin(root);
+  const changed = [];
+  const missing = [];
+  const extra = [];
+  for (const [file, hash] of Object.entries(manifest.files ?? {})) {
+    if (!(file in current.files)) missing.push(file);
+    else if (current.files[file] !== hash) changed.push(file);
+  }
+  for (const file of Object.keys(current.files)) {
+    if (!(file in (manifest.files ?? {}))) extra.push(file);
+  }
+  return { ok: changed.length === 0 && missing.length === 0, changed, missing, extra };
+}
+
+export function readManifest(file) {
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+export function writeManifest(file, manifest) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(manifest, null, 2), 'utf8');
+  return file;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+  const [command, root, manifestFile] = process.argv.slice(2);
+  if (command === 'snapshot' && root && manifestFile) {
+    const manifest = snapshotPlugin(path.resolve(root));
+    writeManifest(path.resolve(manifestFile), manifest);
+    console.log(JSON.stringify({ ok: true, files: Object.keys(manifest.files).length }, null, 2));
+  } else if (command === 'verify' && root && manifestFile) {
+    const result = verifyPlugin(path.resolve(root), readManifest(path.resolve(manifestFile)));
+    console.log(JSON.stringify(result, null, 2));
+    if (!result.ok) process.exitCode = 1;
+  } else {
+    console.log('usage: plugin-integrity snapshot <root> <manifest> | verify <root> <manifest>');
+    process.exitCode = 2;
+  }
+}

package/bin/preview-live.mjs

@@ -0,0 +1,13 @@
+import fs from 'node:fs';
+import { createExtractionProviderFromEnv } from '../src/memory/extraction-provider.mjs';
+import { createExtractionPreStep } from '../src/memory/extraction-pre-step.mjs';
+
+const config = 'C:/Users/35794/Documents/DeepSeek\'s Sandbox/coopanion-continuity/backups/continuity-tags-20261031/durable-memory/extraction-live.json';
+const provider = createExtractionProviderFromEnv({ DSH_EXTRACTION_MODE: 'model', DSH_EXTRACTION_CONFIG: config });
+const handler = createExtractionPreStep({ provider, mode: 'model' });
+const text = 'Coo在灯室拿起钥匙，准备去找宁。';
+const decision = { kind: 'accept', messages: [{ role: 'user', content: [{ type: 'text', text }] }] };
+const result = await handler({ step: 1 }, async () => decision);
+fs.writeFileSync('C:/Users/35794/Documents/DeepSeek\'s Sandbox/coopanion-continuity/backups/continuity-tags-20261031/durable-memory/build-a/LIVE-PREVIEW.json', JSON.stringify({ text, messages: result.messages, extraction: result.extraction }, null, 2));
+provider.close?.();
+console.log(JSON.stringify({ messages: result.messages.length, ok: result.extraction?.ok, slots: result.extraction?.data?.slots }, null, 2));