@theseus-whale/dsh-continuity 0.0.0-stage → 4.12.2

package/bin/memory-scenario-test.mjs

@@ -0,0 +1,146 @@
+#!/usr/bin/env node
+/**
+ * Scenario replay: new term, new→old, old→new→old, repeated recall,
+ * Coo interjection. Gate is before the turn; alias proposal is after it.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { readEventLog } from '../src/memory/event-log.mjs';
+import { MemoryLoadState, latestCandidate, candidateSegment } from '../src/memory/load-gate.mjs';
+import { AliasProposalState } from '../src/memory/alias-proposal.mjs';
+
+const args = process.argv.slice(2);
+const argOf = (name) => {
+  const i = args.indexOf(name);
+  return i >= 0 ? args[i + 1] : undefined;
+};
+const dir = argOf('--dir');
+if (!dir) {
+  console.error('usage: node bin/memory-scenario-test.mjs --dir .durable-memory');
+  process.exit(2);
+}
+const records = readEventLog(path.join(dir, 'events.jsonl'));
+const dev = { activity: '开发', topic: '技术' };
+const run = { activity: '跑团', topic: '剧情' };
+
+const scenarios = [
+  {
+    id: 'new-term',
+    steps: [
+      { text: '我们开个新团，叫星港', facets: run, term: '星港', gate: 'load', alias: 'none' },
+      { text: '星港的规则是掷骰', facets: run, term: '星港', gate: 'none', alias: 'propose' },
+      { text: '星港继续', facets: run, term: '星港', gate: 'none', alias: 'none' },
+    ],
+  },
+  {
+    id: 'new-old-back',
+    steps: [
+      { text: '我们开个新团，叫星港', facets: run, term: '星港', gate: 'load', alias: 'none' },
+      { text: '星港里有一扇门', facets: run, term: '星港', gate: 'none', alias: 'propose' },
+      { text: '上次跑团怎么样了', facets: run, gate: 'load', alias: 'none' },
+      { text: '回到星港', facets: run, term: '星港', gate: 'none', alias: 'none' },
+    ],
+  },
+  {
+    id: 'old-new-old',
+    steps: [
+      { text: '规则怎么写', facets: dev, gate: 'none', alias: 'none' },
+      { text: '开始跑团', facets: run, gate: 'load', alias: 'none' },
+      { text: '星港的规则是掷骰', facets: run, term: '星港', gate: 'none', alias: 'none' },
+      { text: '星港继续', facets: run, term: '星港', gate: 'none', alias: 'propose' },
+      { text: '上次规则写哪了', facets: dev, gate: 'load', alias: 'none' },
+    ],
+  },
+  {
+    id: 'repeat-recall',
+    steps: [
+      { text: '上次跑团怎么样了', facets: run, gate: 'load', alias: 'none' },
+      { text: '上次跑团后来呢', facets: run, gate: 'load', alias: 'none' },
+      { text: '继续跑团', facets: run, gate: 'none', alias: 'none' },
+    ],
+  },
+  {
+    id: 'coo-interject',
+    steps: [
+      { text: 'Coo：来一局', facets: run, gate: 'load', alias: 'none', role: 'coo' },
+      { text: 'Coo：灯室见', facets: run, gate: 'none', alias: 'none', role: 'coo' },
+      { text: '我推门进去', facets: run, gate: 'none', alias: 'none' },
+    ],
+  },
+  {
+    id: 'known-alias',
+    steps: [
+      { text: 'TRPG 开一把', facets: run, term: 'TRPG', gate: 'load', alias: 'none' },
+    ],
+  },
+];
+
+const output = [];
+for (const scenario of scenarios) {
+  const loadState = new MemoryLoadState({ loadedFacets: dev, loadedAtTurn: 0 });
+  const aliasState = new AliasProposalState();
+  const events = [];
+  for (let index = 0; index < scenario.steps.length; index += 1) {
+    const step = scenario.steps[index];
+    const nowTurn = 100 + index;
+    const candidate = latestCandidate(records, 'activity', step.facets.activity);
+    const segment = candidateSegment(records, candidate);
+    const gate = loadState.decide({
+      text: step.text,
+      currentFacets: step.facets,
+      confidence: 0.9,
+      nowTurn,
+      hasCandidate: candidate !== null,
+      candidate,
+      role: step.role ?? 'user',
+    });
+    if (gate.action === 'load') {
+      loadState.recordLoad({ facets: step.facets, seq: segment?.seqStart ?? null, nowTurn, reason: gate.reason });
+    }
+    const alias = aliasState.observe({ term: step.term, facets: step.facets, nowTurn });
+    events.push({
+      text: step.text,
+      gate: gate.action,
+      gateReason: gate.reason,
+      expectedGate: step.gate,
+      alias: alias.action,
+      aliasReason: alias.reason,
+      expectedAlias: step.alias,
+      aliasText: alias.text ?? null,
+      candidateTurn: candidate?.turn ?? null,
+      segmentTurns: segment?.turns ?? 0,
+      position: { gate: 'before-response', alias: 'after-turn' },
+    });
+  }
+  const gatePass = events.filter((event) => event.gate === event.expectedGate).length;
+  const aliasPass = events.filter((event) => event.alias === event.expectedAlias).length;
+  output.push({
+    id: scenario.id,
+    steps: events.length,
+    gatePass,
+    aliasPass,
+    pass: gatePass === events.length && aliasPass === events.length,
+    events,
+  });
+}
+const summary = {
+  scenarios: output.length,
+  steps: output.reduce((sum, item) => sum + item.steps, 0),
+  passScenarios: output.filter((item) => item.pass).length,
+  failedScenarios: output.filter((item) => !item.pass).map((item) => item.id),
+  missedAliasProposals: output.flatMap((item) => item.events.filter((event) => event.expectedAlias === 'propose' && event.alias !== 'propose')).length,
+  extraAliasProposals: output.flatMap((item) => item.events.filter((event) => event.expectedAlias !== 'propose' && event.alias === 'propose')).length,
+};
+const result = { dir, summary, output };
+fs.writeFileSync(path.join(dir, 'SCENARIO-TEST.json'), JSON.stringify(result, null, 2));
+console.log(JSON.stringify(summary, null, 2));
+for (const scenario of output) {
+  if (!scenario.pass) {
+    console.log('FAIL', scenario.id);
+    for (const event of scenario.events) {
+      if (event.gate !== event.expectedGate || event.alias !== event.expectedAlias) {
+        console.log(' ', event.text, 'gate', event.gate, 'expected', event.expectedGate, 'alias', event.alias, 'expected', event.expectedAlias);
+      }
+    }
+  }
+}

package/bin/memory-user-loop.mjs

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+/**
+ * Real user-message loop: BGE + MLP facets + NER slots + pre-turn gate.
+ * No live session IO.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { readEventLog } from '../src/memory/event-log.mjs';
+import { PreTurnController } from '../src/memory/pre-turn.mjs';
+import { MemoryLoadState } from '../src/memory/load-gate.mjs';
+import { attachExtraction } from '../src/memory/extraction-adapter.mjs';
+import { embedTexts } from '../src/facets/embedding.mjs';
+import { deserializeMlp, predictMlp } from '../src/facets/mlp-probe.mjs';
+import { BM25Index } from '../src/memory/bm25.mjs';
+import { VectorIndex } from '../src/memory/vector-index.mjs';
+import { summaryIndex } from '../src/memory/planner.mjs';
+import { stabilizeFacets } from '../src/memory/facet-stabilizer.mjs';
+
+const args = process.argv.slice(2);
+const argOf = (name) => {
+  const i = args.indexOf(name);
+  return i >= 0 ? args[i + 1] : undefined;
+};
+const dir = argOf('--dir');
+const nerFile = argOf('--ner');
+const modelPath = argOf('--model');
+if (!dir || !nerFile || !modelPath) {
+  console.error('usage: node bin/memory-user-loop.mjs --dir .durable-memory --ner USER-NER.json --model MLP-MODEL.json');
+  process.exit(2);
+}
+const records = readEventLog(path.join(dir, 'events.jsonl'));
+const ner = JSON.parse(fs.readFileSync(nerFile, 'utf8'));
+const model = deserializeMlp(JSON.parse(fs.readFileSync(modelPath, 'utf8')));
+const bm25 = BM25Index.fromJSON(JSON.parse(fs.readFileSync(path.join(dir, 'bm25.json'), 'utf8')));
+const vectorIndex = VectorIndex.fromJSON(JSON.parse(fs.readFileSync(path.join(dir, 'vectors.json'), 'utf8')));
+const tree = JSON.parse(fs.readFileSync(path.join(dir, 'summary-tree.json'), 'utf8'));
+const summaries = summaryIndex(tree);
+const texts = ner.rows.map((row) => row.text);
+const vectors = await embedTexts(texts);
+const controller = new PreTurnController({
+  records,
+  gate: new MemoryLoadState({ loadedFacets: { activity: '开发', topic: '技术' }, loadedAtTurn: 0 }),
+});
+const rows = [];
+for (let index = 0; index < ner.rows.length; index += 1) {
+  const item = ner.rows[index];
+  const prediction = predictMlp(model, vectors[index]);
+  const keyword = bm25.search(item.text, { topK: 5, minCoverage: 0.3 });
+  const semantic = vectorIndex.search(vectors[index], { topK: 5 });
+  const summary = summaries.search(item.text, { topK: 3, minCoverage: 0.3 });
+  const stable = stabilizeFacets({
+    raw: prediction,
+    text: item.text,
+    records,
+    keyword,
+    semantic,
+    summary,
+  });
+  const result = attachExtraction(controller, {
+    text: item.text,
+    entities: item.entities,
+    slots: item.slots,
+    facets: stable.facets,
+    confidence: stable.confidence.activity,
+    nowTurn: 1000 + index,
+  });
+  rows.push({
+    turn: item.turn,
+    text: item.text,
+    facets: stable.facets,
+    rawFacets: prediction.facets,
+    confidence: +stable.confidence.activity.toFixed(2),
+    stabilizeReason: stable.reason,
+    slots: item.slots,
+    entities: item.entities,
+    ok: result.ok,
+    action: result.gate?.action ?? 'invalid',
+    reason: result.gate?.reason ?? result.reason,
+    boundaryScore: result.gate?.boundaryScore ?? null,
+    candidateTurn: result.gate?.candidateTurn ?? null,
+    segmentTurns: result.gate?.segment?.turns ?? 0,
+  });
+}
+const summary = {
+  messages: rows.length,
+  loads: rows.filter((row) => row.action === 'load').length,
+  hints: rows.filter((row) => row.action === 'hint').length,
+  none: rows.filter((row) => row.action === 'none').length,
+  invalid: rows.filter((row) => row.action === 'invalid').length,
+  withEntities: rows.filter((row) => row.entities.length > 0).length,
+  loadsSample: rows.filter((row) => row.action === 'load').slice(0, 8).map((row) => ({ turn: row.turn, text: row.text.slice(0, 80), reason: row.reason })),
+};
+const output = { summary, rows };
+fs.writeFileSync(path.join(dir, 'USER-LOOP.json'), JSON.stringify(output, null, 2));
+console.log(JSON.stringify(summary, null, 2));

package/bin/ner_bench_tiny_zh.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Tiny local Chinese NER smoke benchmark on CLUE-NER validation (boundary-only)."""
+import json
+import os
+import sys
+from datasets import load_dataset
+from tokenizers import BertWordPieceTokenizer
+from transformers import AutoModelForTokenClassification, PreTrainedTokenizerFast, pipeline
+
+MODEL_DIR = os.path.join(os.environ["TEMP"], "ckip-tiny")
+LIMIT = int(sys.argv[1]) if len(sys.argv) > 1 else 20
+OUT = sys.argv[2] if len(sys.argv) > 2 else None
+
+bw = BertWordPieceTokenizer(vocab=os.path.join(MODEL_DIR, "vocab.txt"), lowercase=False)
+tok = PreTrainedTokenizerFast(
+    tokenizer_object=bw,
+    unk_token="[UNK]",
+    sep_token="[SEP]",
+    pad_token="[PAD]",
+    cls_token="[CLS]",
+    mask_token="[MASK]",
+)
+model = AutoModelForTokenClassification.from_pretrained(MODEL_DIR)
+ner = pipeline("ner", model=model, tokenizer=tok, aggregation_strategy="simple", device=-1)
+ds = load_dataset("xusenlin/clue-ner", split="validation")
+
+rows = []
+tp = fp = fn = 0
+for i in range(min(LIMIT, len(ds))):
+    item = ds[i]
+    pred = ner(item["text"])
+    pred_spans = {p["word"].replace(" ", "") for p in pred}
+    gold_spans = {e["entity"] for e in item["entities"]}
+    tp += len(pred_spans & gold_spans)
+    fp += len(pred_spans - gold_spans)
+    fn += len(gold_spans - pred_spans)
+    rows.append({
+        "text": item["text"],
+        "pred": [{"text": p["word"].replace(" ", ""), "label": p["entity_group"]} for p in pred],
+        "gold": [{"text": e["entity"], "label": e["label"]} for e in item["entities"]],
+        "pred_spans": sorted(pred_spans),
+        "gold_spans": sorted(gold_spans),
+    })
+precision = 0 if tp + fp == 0 else tp / (tp + fp)
+recall = 0 if tp + fn == 0 else tp / (tp + fn)
+f1 = 0 if precision + recall == 0 else 2 * precision * recall / (precision + recall)
+result = {
+    "model": "ckiplab/bert-tiny-chinese-ner (local)",
+    "dataset": "xusenlin/clue-ner validation",
+    "n": len(rows),
+    "boundary_only": {"tp": tp, "fp": fp, "fn": fn, "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4)},
+    "rows": rows,
+}
+if OUT:
+    with open(OUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+print(json.dumps({k: v for k, v in result.items() if k != "rows"}, ensure_ascii=False, indent=2))
+

package/bin/ner_bench_zh.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Chinese NER benchmark: UER CLUENER model on clean CLUE-NER validation."""
+import json
+import sys
+import torch
+from datasets import load_dataset
+from transformers import AutoModelForTokenClassification, AutoTokenizer, pipeline
+
+MODEL = "uer/roberta-base-finetuned-cluener2020-chinese"
+DATASET = "xusenlin/clue-ner"
+LIMIT = int(sys.argv[1]) if len(sys.argv) > 1 else 100
+OUT = sys.argv[2] if len(sys.argv) > 2 else None
+
+torch.set_num_threads(1)
+print("loading model", flush=True)
+tok = AutoTokenizer.from_pretrained(MODEL)
+model = AutoModelForTokenClassification.from_pretrained(MODEL)
+print("labels", model.config.id2label, flush=True)
+ner = pipeline("ner", model=model, tokenizer=tok, aggregation_strategy="simple", device=-1, truncation=True, max_length=512)
+print("loading data", flush=True)
+ds = load_dataset(DATASET, split="validation")
+
+
+def spans(pred, gold):
+    gold_keys = {(g["entity"], g["label"]) for g in gold}
+    pred_keys = {(p["word"].replace(" ", ""), p["entity_group"]) for p in pred}
+    tp = len(gold_keys & pred_keys)
+    fp = len(pred_keys - gold_keys)
+    fn = len(gold_keys - pred_keys)
+    precision = 0 if not pred_keys else tp / len(pred_keys)
+    recall = 0 if not gold_keys else tp / len(gold_keys)
+    f1 = 0 if precision + recall == 0 else 2 * precision * recall / (precision + recall)
+    return {"tp": tp, "fp": fp, "fn": fn, "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4)}
+
+
+rows = []
+for i in range(min(LIMIT, len(ds))):
+    item = ds[i]
+    pred = ner(item["text"])
+    gold = [{"entity": e["entity"], "label": e["label"]} for e in item["entities"]]
+    rows.append({
+        "text": item["text"],
+        "pred": [{"text": p["word"].replace(" ", ""), "label": p["entity_group"]} for p in pred],
+        "gold": gold,
+    })
+    if i < 2:
+        print("sample", i, rows[-1]["pred"], rows[-1]["gold"], flush=True)
+
+result = {
+    "model": MODEL,
+    "dataset": DATASET,
+    "n": len(rows),
+    "score": spans([p for r in rows for p in r["pred"]], [g for r in rows for g in r["gold"]]),
+    "rows": rows,
+}
+if OUT:
+    with open(OUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+print(json.dumps({k: v for k, v in result.items() if k != "rows"}, ensure_ascii=False, indent=2), flush=True)

package/bin/ner_conll_en.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""English NER benchmark: DistilBERT CoNLL03 on the official eng.testa file."""
+import json
+import sys
+
+import torch
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+
+MODEL = "elastic/distilbert-base-uncased-finetuned-conll03-english"
+FILE = sys.argv[1]
+LIMIT = int(sys.argv[2]) if len(sys.argv) > 2 else 200
+OUT = sys.argv[3] if len(sys.argv) > 3 else None
+MAX_LEN = 256
+
+torch.set_num_threads(1)
+tok = AutoTokenizer.from_pretrained(MODEL)
+model = AutoModelForTokenClassification.from_pretrained(MODEL)
+model.eval()
+id2label = model.config.id2label
+
+
+def read_sentences(path):
+    sentences = []
+    tokens, tags = [], []
+    for line in open(path, encoding="utf-8"):
+        line = line.rstrip("\n")
+        if not line.strip():
+            if tokens:
+                sentences.append((tokens, tags))
+                tokens, tags = [], []
+            continue
+        parts = line.split()
+        if len(parts) < 2:
+            continue
+        tokens.append(parts[0])
+        tags.append(parts[-1])
+    if tokens:
+        sentences.append((tokens, tags))
+    return sentences
+
+
+def predict(text):
+    enc = tok(text, return_tensors="pt", truncation=True, max_length=MAX_LEN, return_offsets_mapping=True)
+    offsets = enc.pop("offset_mapping")[0].tolist()
+    with torch.no_grad():
+        logits = model(**enc).logits[0]
+    ids = logits.argmax(-1).tolist()
+    entities = []
+    current = None
+    for idx, label_id in enumerate(ids):
+        label = id2label[int(label_id)]
+        start, end = offsets[idx]
+        if start == end:
+            continue
+        if label == "O":
+            if current is not None:
+                entities.append(current)
+                current = None
+            continue
+        prefix, _, kind = label.partition("-")
+        if prefix == "B":
+            if current is not None:
+                entities.append(current)
+            current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+        elif prefix == "I" and current is not None and current["label"] == kind:
+            current["text"] += text[start:end]
+            current["end"] = end
+        else:
+            if current is not None:
+                entities.append(current)
+            current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+    if current is not None:
+        entities.append(current)
+    merged = []
+    for entity in entities:
+        last = merged[-1] if merged else None
+        if last is not None and last["label"] == entity["label"] and entity["start"] - last["end"] <= 1:
+            last["text"] += entity["text"]
+            last["end"] = entity["end"]
+        else:
+            merged.append(entity)
+    return merged
+
+
+def gold_entities(tokens, tags):
+    out = []
+    current = None
+    for token, tag in zip(tokens, tags):
+        if tag == "O":
+            if current is not None:
+                out.append(current)
+                current = None
+            continue
+        prefix, _, kind = tag.partition("-")
+        if prefix == "B":
+            if current is not None:
+                out.append(current)
+            current = {"text": token, "label": kind}
+        elif prefix == "I" and current is not None:
+            current["text"] += " " + token
+        else:
+            if current is not None:
+                out.append(current)
+            current = {"text": token, "label": kind}
+    if current is not None:
+        out.append(current)
+    return out
+
+
+def score(pred, gold):
+    pred_keys = {(p["text"].replace(" ", ""), p["label"]) for p in pred}
+    gold_keys = {(g["text"].replace(" ", ""), g["label"]) for g in gold}
+    tp = len(pred_keys & gold_keys)
+    fp = len(pred_keys - gold_keys)
+    fn = len(gold_keys - pred_keys)
+    precision = 0 if not pred_keys else tp / len(pred_keys)
+    recall = 0 if not gold_keys else tp / len(gold_keys)
+    f1 = 0 if precision + recall == 0 else 2 * precision * recall / (precision + recall)
+    return {"tp": tp, "fp": fp, "fn": fn, "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4)}
+
+
+sentences = read_sentences(FILE)[:LIMIT]
+rows = []
+for i, (tokens, tags) in enumerate(sentences):
+    text = " ".join(tokens)
+    rows.append({"text": text, "pred": predict(text), "gold": gold_entities(tokens, tags)})
+
+result = {
+    "model": MODEL,
+    "dataset": "CoNLL2003 eng.testa",
+    "n": len(rows),
+    "score": score([p for r in rows for p in r["pred"]], [g for r in rows for g in r["gold"]]),
+    "rows": rows,
+}
+if OUT:
+    with open(OUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+print(json.dumps({k: v for k, v in result.items() if k != "rows"}, ensure_ascii=False, indent=2))

package/bin/ner_direct_en.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""English NER benchmark: DistilBERT CoNLL03 on CoNLL2003 validation."""
+import json
+import sys
+
+import torch
+from datasets import load_dataset
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+
+MODEL = "elastic/distilbert-base-uncased-finetuned-conll03-english"
+DATASET = "jon-fernandes/conll2003"
+LIMIT = int(sys.argv[1]) if len(sys.argv) > 1 else 200
+OUT = sys.argv[2] if len(sys.argv) > 2 else None
+MAX_LEN = 256
+
+torch.set_num_threads(1)
+print("tokenizer", flush=True)
+tok = AutoTokenizer.from_pretrained(MODEL)
+print("model", flush=True)
+model = AutoModelForTokenClassification.from_pretrained(MODEL)
+model.eval()
+id2label = model.config.id2label
+print("data", flush=True)
+ds = load_dataset(DATASET, split="validation")
+print("data ok", len(ds), flush=True)
+
+
+def decode(text, offsets, ids):
+    entities = []
+    current = None
+    for idx, label_id in enumerate(ids):
+        label = id2label[int(label_id)]
+        start, end = offsets[idx]
+        if start == end:
+            continue
+        if label == "O":
+            if current is not None:
+                entities.append(current)
+                current = None
+            continue
+        prefix, _, kind = label.partition("-")
+        if prefix == "B":
+            if current is not None:
+                entities.append(current)
+            current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+        elif prefix == "I" and current is not None and current["label"] == kind:
+            current["text"] += text[start:end]
+            current["end"] = end
+        else:
+            if current is not None:
+                entities.append(current)
+            current = {"text": text[start:end], "label": kind, "start": start, "end": end}
+    if current is not None:
+        entities.append(current)
+    return entities
+
+
+def predict(text):
+    enc = tok(text, return_tensors="pt", truncation=True, max_length=MAX_LEN, return_offsets_mapping=True)
+    offsets = enc.pop("offset_mapping")[0].tolist()
+    with torch.no_grad():
+        logits = model(**enc).logits[0]
+    return decode(text, offsets, logits.argmax(-1).tolist())
+
+
+def gold_entities(tokens, tags):
+    out = []
+    current = None
+    for token, tag in zip(tokens, tags):
+        if tag == "O":
+            if current is not None:
+                out.append(current)
+                current = None
+            continue
+        prefix, _, kind = tag.partition("-")
+        if prefix == "B":
+            if current is not None:
+                out.append(current)
+            current = {"text": token, "label": kind}
+        elif prefix == "I" and current is not None:
+            current["text"] += " " + token
+        else:
+            if current is not None:
+                out.append(current)
+            current = {"text": token, "label": kind}
+    if current is not None:
+        out.append(current)
+    return out
+
+
+def score(pred, gold):
+    pred_keys = {(p["text"].replace(" ", ""), p["label"]) for p in pred}
+    gold_keys = {(g["text"].replace(" ", ""), g["label"]) for g in gold}
+    tp = len(pred_keys & gold_keys)
+    fp = len(pred_keys - gold_keys)
+    fn = len(gold_keys - pred_keys)
+    precision = 0 if not pred_keys else tp / len(pred_keys)
+    recall = 0 if not gold_keys else tp / len(gold_keys)
+    f1 = 0 if precision + recall == 0 else 2 * precision * recall / (precision + recall)
+    return {"tp": tp, "fp": fp, "fn": fn, "precision": round(precision, 4), "recall": round(recall, 4), "f1": round(f1, 4)}
+
+
+rows = []
+for i in range(min(LIMIT, len(ds))):
+    print("sample", i, flush=True)
+    item = ds[i]
+    text = " ".join(item["tokens"])
+    pred = predict(text)
+    gold = gold_entities(item["tokens"], item["ner_tags"])
+    rows.append({"text": text, "pred": pred, "gold": gold})
+
+result = {
+    "model": MODEL,
+    "dataset": DATASET,
+    "n": len(rows),
+    "score": score([p for r in rows for p in r["pred"]], [g for r in rows for g in r["gold"]]),
+    "rows": rows,
+}
+if OUT:
+    with open(OUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+print(json.dumps({k: v for k, v in result.items() if k != "rows"}, ensure_ascii=False, indent=2))