dsh-data-domain-knowledge 0.1.0-alpha.9 → 0.1.1-alpha.2

package/cordis.patch.yml

@@ -1,427 +1,14 @@
 # data-domain-knowledge bundle patch: mount the knowledge / intent audit / convert host.
 #
 # 0.1.7 起这条 entry **自己就是设置文档**：插件导出的 `Config`（见
-# `src/config-schema.ts`）把两块设置合成一个 schema —— `knowledge` / `convert`。
+# `src/config-schema.ts`）把各设置组合成一个 schema —— `knowledge` / `convert` / `behaviorRules`。
 # 会话意图使用固定审计契约，不属于 entry config。设置页里的改动由 `configEditor` 写回**本 entry 的
 # `config:`**，客户端按路径读写（`{op:'set', path:['knowledge','recallTopK']}`）。
 #
-# 所以这里**故意不写 `config:`**：两组字段全都有 schema `.default()`，而
-# `knowledge.agentRoot` 的默认值就是 `~/Desktop/titaagent`（`defaultAgentRoot()`，
-# 可被 `DSH_AGENT_ROOT` 覆盖）。往这里写死一个 `agentRoot` 会把「Agent 的家在哪」
-# 这件事从代码挪进某台机器的配置文件里 —— 那正是 0029 想收拢的那一个答案。
+# 所以这里**故意不写 `config:`**：各组字段全都有 schema `.default()`，而
+# `knowledge.agentRoot` 的默认值就是 DSH 下的 `~/.dsh/tita-agent` 专属目录
+#（`defaultAgentRoot()`，跟随 `DSH_HOME`）。往这里写死一个 `agentRoot` 会把「Agent 的家在哪」
+# 这件事从代码挪进某台机器的配置文件里 —— 那正是这里要避免的。
 - insert:
     - id: tita-data-domain-knowledge
       name: 'dsh-data-domain-knowledge'
-
-# >>> agent preset (generated) >>>
-# Tita Data Agent 的 agent preset 声明。
-#
-# 这一段由 `scripts/sync-agent-preset.mjs` 从 `preset/tita-data-agent/` 生成 ——
-# **不要手改**：改真源再跑 `pnpm sync:preset`。`pnpm doctor` 会校验它与真源一致。
-#
-# 为什么 preset 必须长在插件的 patch 里：0.1.7 起 DSH 不再扫描
-# `$DSH_HOME/.agent-presets/`，preset 的唯一形态就是这条 insert。详见脚本头注释。
-- insert:
-    - id: preset-tita-data-agent
-      name: '@deepseek-ai/dsh-agent-preset'
-      config:
-        id: tita-data-agent
-        name: 'Tita Data Agent'
-        description: 'TitaAgent 自己的组合：数据 Agent 的提示词，加上只有这个 preset 才有的意图记录工具。'
-        order: 10
-        plugins:
-          # Tita Data Agent 的 agent 组合（0029）。
-          #
-          # 这是 `standard` 的一份**拷贝**，只改了四件事。preset 是「一份完整组合」，没有
-          # patch 语义（`dsh-agent-presets` 的 authoring 是 copy-only），所以它必须是一整份
-          # 文件、而不是对 standard 的增量。
-          #
-          # 改了什么：
-          #
-          # 1. **身份**（`persona` 行）：把「你是一个编码 Agent」换成 Tita Data Agent 自己的
-          #    提示词。这一行是**这个 preset 存在的理由** —— 它 shadow 掉部署级的 persona，
-          #    所以只有加入这个 preset 的会话会以数据 Agent 的身份说话。提示词正文就在下面，
-          #    直接改它即可；改完**新建一个会话**才生效（preset 在会话创建时固定）。
-          # 2. **意图工具**（`intent-tools` 行）：`intent_record` / `intent_store_view` 两个工具
-          #    搬到 agent 平面，只有这个 preset 的会话能看到。
-          #    DSH 里「只有某个 preset 能用」唯一支持的形态就是注册作用域，所以它们必须在这里
-          #    挂载，而不是在宿主平面全局注册（那会让每个 agent 都看到 intent_record）。
-          # 3. **子代理不许继承意图工具**：子代理会加入父会话的组合，所以要在 delegation 的每一
-          #    行上显式 deny 掉 —— 意图记录描述的是「用户在跟 Agent 要什么」，子代理做的是被拆
-          #    出来的一个片段，它记一条意图只会污染任务树。
-          # 4. 其余每一行都与 `standard` 逐字相同：工具集不变，这个 preset 换的是身份与意图
-          #    能力，不是一个更小的 Agent。
-          #
-          # 其余纪律与 `standard` 一致：这是 AGENT-PLANE 组合，roster 在 standing scope 下挂载
-          # 一次，每个命名它的会话按作用域父子关系加入。需要 provide 服务的行必须落在带
-          # `isolate` realm 的 group 里，否则它会发布到根 realm（进程级）并被挂载审计拒绝 ——
-          # 本文件里唯一这样的行是 delegation 那个 group。
-
-          # ── identity ────────────────────────────────────────────────────────────────
-
-          # The preset's own persona, shadowing the deployment default for this agent.
-          # `{{model}}` and `{{cwd}}` resolve from the agent's own route and workspace.
-          - id: persona
-            name: '@deepseek-ai/dsh-persona'
-            config:
-              # `{{model}}` / `{{cwd}}` 从本会话自己的路由与工作区解析。
-              suffix: Your working directory is {{cwd}}.
-              # ↓↓↓ Tita Data Agent 的提示词。**要改的就是这一段。** ↓↓↓
-              prefix: |
-                You are Tita Data Agent, the data analyst on this deployment. You are powered
-                by the {{model}} model. Your job is to answer questions about this
-                organisation's business with evidence, and to write down what you learn so the
-                next answer is cheaper than this one.
-
-                ## What you work with
-
-                - **The knowledge store.** Each knowledge base is one directory holding three
-                  things. `knowledge.db` is the knowledge itself: SourceRaw (the original
-                  business documents, never rewritten, kept so a conclusion can point back at
-                  where it came from), the cards people read — one per entity, each with
-                  mandatory `source_refs`, each following a type (`concept`, `metric`, `table`,
-                  `event`, `playbook`, `case`) — and every version of both, plus who changed
-                  what and when. `derived.db` is what `recall_knowledge` actually searches:
-                  wikiNotes, a vector index, a knowledge graph and a keyword index; it is
-                  recomputed, never backed up. `assets/` holds images. There are no
-                  per-segment folders inside a base; use the knowledge tools, not the
-                  filesystem, to read or write documents. There can be several bases.
-
-                - **The data surfaces, which are a different thing.** `workbook_*` reads and
-                  writes the workbooks the user has open; `notebook_*` runs SQL in the user's
-                  own browser. That is where a number is *computed*. The knowledge store is
-                  where a number is *defined* — a `metric` card says how it must be calculated,
-                  a `table` card says what one row means. Computing without the definition gets
-                  you a number nobody agreed to.
-
-                - **Conversion.** `convert_source` turns a local file (including a web page
-                  saved to disk as `.html`) or a video into knowledge-ready Markdown and says
-                  where it landed. It never fetches a URL itself. It is not knowledge until
-                  `convert_ingest` has run, so keep saying which of the two steps you are on.
-
-                ## Working rules
-
-                1. **Record the intent before the work.** Call `intent_record` with what the
-                   user wants and what you are about to do, before you do it. The vocabulary,
-                   and the rule for continuing existing work instead of starting a new task,
-                   live in that tool's own description. The record never gates the request:
-                   record, then continue.
-                2. **Recall before you answer about this organisation.** For its own rules,
-                   definitions, metrics, processes or history, call `recall_knowledge` first and
-                   answer from what comes back. An empty result is a finding, not a licence to
-                   improvise — say the store has nothing on it. The wiki layer answers "what does
-                   this organisation mean by X"; when you need "where exactly does it say that",
-                   or a card and the document behind it disagree, go read the source.
-                3. **Never invent a number.** Every figure you report must come from a tool
-                   result in this turn, and you must say which one. "I don't have that data" is
-                   a correct answer; a made-up number is not.
-                4. **Compiling is the normal path, and it writes cards by itself.** You do not
-                   have to create wiki cards: when a source is added or changed, `rebuild_knowledge`
-                   extracts entities from it, follows each wiki type's schema (required sections and
-                   fields), checks whether the wiki already has that entity, and either appends to
-                   the existing card or creates a new one. A card with missing sections is still
-                   written — the store keeps what it knows and fills the rest in when a later
-                   document covers it. So after adding a source, rebuild; then `recall_knowledge`
-                   and answer from what comes back. **More than one document at a time: use
-                   `import_sources`** — it writes the whole batch and compiles once, in one call.
-                   Ten documents through `add_source` means ten approvals and ten compiles, and the
-                   first nine results are thrown away by the tenth.
-                5. **You can add to the wiki, but you cannot rewrite it.** Call `propose_wiki_cards`
-                   with the `source_id` you just wrote and cards whose `source_refs` already exist in
-                   the store when you already have structured cards in hand — it writes them
-                   straight into `02_wiki`, they are recalled from the next compile on. A card whose
-                   refs point nowhere is rejected, and rightly so. **A card body is read-only**: it
-                   belongs to the compiler. When something in the store is wrong, or a card and its
-                   source disagree, do not rewrite it — attach an annotation with
-                   `annotate_wiki_card` (`kind: anomaly` when the numbers or definitions do not add
-                   up), or call `set_knowledge_status` to mark it `problem` and say what is wrong.
-                   Cards marked `problem` drop out of recall by default — that is how a wrong
-                   definition stops propagating. Everything else is reviewed by a human after entry:
-                   review raises a card's score, and an anomaly lowers it, but an
-                   anomaly-flagged card is still recalled.
-                6. **Never invent a knowledge base id.** Call `list_knowledge_bases` before
-                   addressing one you have not seen this session; omit `kb` only when the
-                   default base is genuinely the right one.
-                7. **Be concrete about uncertainty.** When sources disagree, present both and
-                   name them. When a conclusion rests on a single document, say so.
-                8. **Ask only what the user owns.** Which of two readings they meant, what
-                   matters more, what to do next — ask. Anything you can look up, look up
-                   instead of asking.
-                9. **Deliverables belong to the project, not to your home.** Write every file
-                   you produce for the user under `output/<YYYY-MM-DD>/` in your working
-                   directory, never into the Agent root: the Agent root accumulates across
-                   projects, a deliverable belongs to one. Name the file `<中文名>.<ext>` — at
-                   most 25 characters, keeping 汉字, letters, digits, `-`, `_` and `.`, and
-                   turning spaces and punctuation into `-` (so `销售汇总` stays `销售汇总`, and
-                   you write `月度销售汇总.xlsx`, not `output1.xlsx`). **Do not repeat the date
-                   in the file name**: the directory already carries it, and the name alone has
-                   to say what the file is a year later. Append `-2`, `-3` rather than
-                   overwriting, and say the path in your answer.
-
-                ## How to write
-
-                Answer in the user's language. Lead with the answer, then the evidence. Prefer
-                a table when you are comparing things and a short paragraph when you are not.
-                Skip preamble and praise; the user is here for the analysis.
-              # ↑↑↑ Tita Data Agent 的提示词结束 ↑↑↑
-
-          - id: agent-instructions
-            name: '@deepseek-ai/dsh-agent-instructions'
-            config:
-              maxBytes: 65536
-
-          # ── 意图记录（agent 平面）────────────────────────────────────────────────────
-          #
-          # 0029 之前这三个工具由知识插件在**宿主平面**全局注册，于是每个 agent、每个 preset
-          # 的会话都会在提示词里看到 `intent_record`。现在它们只在这里挂载：这个模块只往
-          # `tools` 注册表里塞定义、不 provide 任何服务，所以和随包的 `tool-fs` 一样不需要
-          # realm，而它的注册只覆盖加入本 preset 的会话。
-          #
-          # 引擎本身（设置、轮次记账、`/intent/**` 只读面）仍在宿主平面，通过
-          # `titaDataAgentIntent` 服务把操作面暴露给这一行 —— 引擎只有一份。
-          - id: intent-tools
-            name: 'dsh-data-domain-knowledge/intent-tools'
-
-          # ── shell ───────────────────────────────────────────────────────────────────
-
-          # `shell-env` stays in the HOST composition: `apps/cli/src/web.ts` injects it to
-          # publish `DSH_WEB_URL`/`DSH_WEB_MODE`, and a host row that injects a service is
-          # the criterion for host-plane ownership — injection resolves before any session
-          # exists, so there is no agent to key by. Behind a preset realm those variables
-          # never reached the model's shell at all. Both shell tools consume the host
-          # registry from here; their executors (`bash-sandbox`/`pwsh-sandbox`) are
-          # host-plane too.
-          - id: tool-bash
-            name: '@deepseek-ai/dsh-tool-bash'
-            disabled: !!js process.platform === 'win32'
-
-          - id: tool-pwsh
-            name: '@deepseek-ai/dsh-tool-pwsh'
-            disabled: !!js process.platform !== 'win32'
-
-          # ── filesystem ──────────────────────────────────────────────────────────────
-
-          # Both register into the host `tools` registry and provide nothing, so
-          # they need no realm. The `fs` service and its policy stay in the host.
-          - id: tool-fs
-            name: '@deepseek-ai/dsh-tool-fs'
-
-          - id: tool-fs-search
-            name: '@deepseek-ai/dsh-tool-fs-search'
-            config:
-              sampleOverCapGlobResults: false
-
-          # ── background jobs ────────────────────────────────────────────────────────
-
-          # Only the model-facing controls. The task REGISTRY stays on the host plane:
-          # its producers sit outside any realm this file could put it in — `tool-bash`
-          # above resolves it with `ctx.get`, and an entry-local realm here is invisible
-          # to every sibling row, so `run_in_background` would answer "background jobs
-          # unavailable" while these controls sat in the catalog. The registry is keyed by
-          # owning agent anyway, so one host instance serves every session. What a preset
-          # chooses is whether its agent can collect and stop background work at all.
-          - id: tool-jobs
-            name: '@deepseek-ai/dsh-tool-jobs'
-
-          # ── skills ──────────────────────────────────────────────────────────────────
-
-          # The skill REGISTRY lives in the host composition and is layered per scope:
-          # these rows register into THIS preset's layer of it, so they need no realm.
-          # `skill-filesystem` contributes local-root discovery for agents on this preset, and
-          # `tool-skill` gives them the catalog and loader; the merged catalog also
-          # carries whatever the deployment registered globally (repository plugins).
-          - id: skill-filesystem
-            name: '@deepseek-ai/dsh-skill-filesystem'
-
-          - id: tool-skill
-            name: '@deepseek-ai/dsh-tool-skill'
-
-          # ── goals ───────────────────────────────────────────────────────────────────
-
-          # The goal service and session driver stay on the host plane, where the Gateway
-          # can resolve them. The human command and model-facing tool register into this
-          # preset's scoped layers.
-          - id: command-goal
-            name: '@deepseek-ai/dsh-command-goal'
-
-          - id: tool-goal
-            name: '@deepseek-ai/dsh-tool-goal'
-
-          # ── plan mode ───────────────────────────────────────────────────────────────
-
-          # Plan state is per-agent by nature, so an entry-local realm is not a
-          # workaround here — it is the correct lifetime.
-          - id: planning
-            name: cordis:group
-            group: true
-            isolate:
-              planMode: true
-            config:
-              - id: plan-mode
-                name: '@deepseek-ai/dsh-plan-mode'
-                config:
-                  section: |
-                        You are in plan mode. Stay in plan mode until exit_plan_mode succeeds or the user switches the session mode. Imperative language to implement changes means plan the implementation, not execute it. A user's conversational agreement — including an answer confirming something you asked — approves nothing and does not end plan mode; fold the confirmed decision into the plan and submit it through exit_plan_mode.
-
-                        Explore first. Use non-mutating reads, searches, static analysis, and checks to ground the plan in the actual repository. Do not edit or write files, change configuration, run formatters or code generation that rewrites tracked files, commit, or otherwise carry out the plan. Prefer existing functions and patterns over new machinery.
-
-                        The tool catalog stays the same across modes for request-cache stability. These plan-mode rules override any later tool description or guidance that suggests using mutation tools; those tools remain listed to keep the tool catalog unchanged. Do not use todo_write to track this planning phase: it tracks implementation after an approved plan, while the plan itself belongs in exit_plan_mode.
-
-                        Resolve discoverable facts by inspection. Use ask_user_question only for user-owned choices or material ambiguity that inspection cannot answer. Do not ask the user where code lives or how current behavior works when you can find out.
-
-                        Make the plan decision-complete: state the goal and success criteria; group implementation changes by subsystem; identify public API, schema, and data-flow changes; cover edge cases, failure modes, tests, acceptance criteria, and explicit assumptions. Keep it concise enough to review but detailed enough that another engineer can implement it without making design decisions.
-
-                        When ready, call exit_plan_mode with the complete plan markdown, starting with a # title. Make exit_plan_mode the only and final tool call in that assistant response: it presents the plan for approval, and implementation begins only in a later step after approval. Do not paste the final plan as a plain reply or ask "should I proceed?" through prose or ask_user_question. If review rejects it, incorporate the feedback and present again. If the review channel is unavailable or aborted, stay in plan mode and ask the user to switch modes manually; do not proceed with implementation.
-
-          # ── compaction ──────────────────────────────────────────────────────────────
-
-          # `compaction-basic` reads `toolResultPrune` through `ctx.get`, so the pruner must
-          # share this realm rather than sit outside it.
-          #
-          # `tokenMeter` is deliberately NOT in this realm: the meter stays on the HOST
-          # plane, and the rows here resolve that one instance. It takes no configuration,
-          # keys every fold by Session, and owns the context-meter projection units the
-          # browser reads for every session — behind a realm those units would come and go
-          # with whichever presets happen to be mounted. What a preset chooses is whether
-          # its agent compacts at all, which is `compaction-basic` below.
-          - id: compaction
-            name: cordis:group
-            group: true
-            isolate:
-              compaction: true
-              toolResultPruner: true
-            config:
-              - id: compaction-basic
-                name: '@deepseek-ai/dsh-compaction-basic'
-
-              - id: command-compact
-                name: '@deepseek-ai/dsh-command-compact'
-
-              - id: tool-result-pruner
-                name: '@deepseek-ai/dsh-compaction-tool-result-pruner'
-                config:
-                  thresholdChars: 8192
-                  headChars: 4096
-                  tailChars: 1024
-
-          # ── delegation and workflows ────────────────────────────────────────────────
-
-          # The `subagents` registry and its spawn/fork backends live in the HOST
-          # composition: the registry is a process singleton whose cross-session queries
-          # the api-proxy serves to the browser, and a provider name may only be
-          # registered once. This preset contributes the delegation TOOLS, which resolve
-          # that host registry.
-          #
-          # `workflows` is different — nothing outside an agent reads it — so every row
-          # that reaches it shares one entry-local realm here, and a consumer left
-          # outside would resolve a host registry this preset does not populate.
-          - id: delegation
-            name: cordis:group
-            group: true
-            isolate:
-              workflowEngine: true
-            config:
-              - id: tool-subagent-control
-                name: '@deepseek-ai/dsh-tool-subagent-control'
-
-              - id: tool-subagent-list-agents
-                name: '@deepseek-ai/dsh-tool-subagent-control/list-agents'
-
-              - id: tool-subagent
-                name: '@deepseek-ai/dsh-tool-subagent'
-                config:
-                  provider: spawn
-                  toolName: subagent
-                  modelSelectionSettings: true
-                  backgroundMode: continuable
-                  toolFilter:
-                    deny:
-                      - intent_record
-                      - intent_store_view
-
-              # Fork omits model selection so provider/model stay equal to the parent and
-              # the inherited history remains eligible for KV Cache reuse. This preset
-              # keeps fork continuable; parent and child inherit the same messaging tool,
-              # while the parent id and return guidance follow the inherited history.
-              - id: tool-subagent-fork
-                name: '@deepseek-ai/dsh-tool-subagent'
-                config:
-                  provider: fork
-                  toolName: subagent_fork
-                  backgroundMode: continuable
-                  toolFilter:
-                    deny:
-                      - intent_record
-                      - intent_store_view
-
-              # Production dsh does not install these optional providers. Install the
-              # matching Bundle in this Profile and restart the Host, then copy this
-              # preset and remove `disabled` from the matching tool row. Host availability
-              # alone grants no tool.
-              - id: tool-subagent-codex
-                name: '@deepseek-ai/dsh-tool-subagent'
-                disabled: true
-                config:
-                  provider: codex
-                  toolName: subagent_codex
-                  backgroundMode: one-shot
-                  maxDepth: provider-managed
-                  toolFilter:
-                    deny:
-                      - intent_record
-                      - intent_store_view
-
-              - id: tool-subagent-claude-code
-                name: '@deepseek-ai/dsh-tool-subagent'
-                disabled: true
-                config:
-                  provider: claude-code
-                  toolName: subagent_claude_code
-                  backgroundMode: one-shot
-                  maxDepth: provider-managed
-                  toolFilter:
-                    deny:
-                      - intent_record
-                      - intent_store_view
-
-              # 0.1.7 把这行换掉了：旧包名 `@deepseek-ai/dsh-workflow-worker-thread` 在
-              # 0.1.7 的安装里已经不存在，对应物是 `@deepseek-ai/dsh-workflow-ptc`
-              # （见 `dsh-web-app` 的 `presets/cordis.patch.yml`）。留着旧名不会报「找不到
-              # 包」，而是让**整个 preset 挂载失败**，症状是它出现在 Agent 预设名单里、
-              # 却带一条 `workflow-worker-thread (…): never started` 诊断 —— 这也是
-              # 「自定义 agent 没生效」最像「配置没生效」的一种失败。
-              - id: workflow-ptc
-                name: '@deepseek-ai/dsh-workflow-ptc'
-                config:
-                  provider: spawn
-
-              - id: tool-workflow
-                name: '@deepseek-ai/dsh-tool-workflow'
-
-              - id: tool-ralph
-                name: '@deepseek-ai/dsh-tool-ralph'
-                config:
-                  subagentProvider: spawn
-                  maxRounds: 64
-
-          # ── remaining model-facing rows ─────────────────────────────────────────────
-
-          - id: tool-ask-user
-            name: '@deepseek-ai/dsh-tool-ask-user'
-
-          - id: tool-todo
-            name: '@deepseek-ai/dsh-tool-todo'
-            config:
-              allowParallelInProgress: true
-
-          # The `web` service and its search provider stay in the host composition; only
-          # the model-facing tool is per-session.
-          - id: tool-web
-            name: '@deepseek-ai/dsh-tool-web'
-            config:
-              fetch: true
-              searchTimeoutMs: 60000
-
-          - id: present
-            name: '@deepseek-ai/dsh-tool-present'
-# <<< agent preset (generated) <<<

package/eval/README.md

@@ -1,19 +1,26 @@
 # Tita Knowledge Retrieval Evaluation
 
 `evaluate-retrieval.mjs` 对同一份已经编译好的 Knowledge Store，逐条运行
-`fts`、`vector`、`hybrid`，输出 Recall、MRR、nDCG 和 p50/p95 latency。它不自动
+`fts`、`vector`、`hybrid`，输出 Recall、MRR、nDCG、弃答准确率和 p50/p95 latency。它不自动
 下载公开数据集，也不会改写知识库；公开数据集需要先转换为本插件的 qrels 格式。
 
 ## qrels 格式
 
 每行一个 JSON 对象，`relevant_unit_ids` 使用编译后的稳定 `unitId`，例如
 `metric:sales_amount`。可以附加 `include_problems` 和 `subset`，用于分组评测文本、
-图片 Markdown、表格截图等场景。
+图片 Markdown、表格截图等场景。`subset` 当前保留在输入中；按子集运行时先分别生成
+qrels 文件，runner 不自动输出子集汇总。
 
 ```json
 {"id":"q1","query":"销售额如何计算","relevant_unit_ids":["metric:sales_amount"],"subset":"text"}
+{"id":"q2","query":"库内没有依据的问题","relevant_unit_ids":[],"subset":"abstention"}
 ```
 
+空 `relevant_unit_ids` 明确表示“不应召回”。必须提供这个字段，避免把漏标注误当作
+弃答样本。正例的 Recall、MRR、nDCG 只使用有正例的查询计算；弃答样本单独输出
+`abstention.count`、`falseAnswers`（任何命中均计为误召回）和 `accuracy`。
+只有弃答样本时，正例指标为 `null`。接口失败会使评测失败，不会当作成功弃答。
+
 ## 本地运行
 
 先在 UI 点击一次「编译」，再运行：
@@ -21,7 +28,7 @@
 ```bash
 pnpm --filter dsh-data-domain-knowledge build
 node plugins/dsh-data-domain-knowledge/eval/evaluate-retrieval.mjs \
-  --root "$HOME/.tita-data-agent/knowledge" \
+  --root "$HOME/.dsh/tita-agent/domain_knowledge/default" \
   --qrels plugins/dsh-data-domain-knowledge/eval/queries.example.jsonl
 ```
 
@@ -29,10 +36,10 @@ node plugins/dsh-data-domain-knowledge/eval/evaluate-retrieval.mjs \
 
 ```bash
 TITA_EVAL_PROVIDER=wemm-python \
-TITA_EVAL_PYTHON="$HOME/.tita-data-agent/embedding-venv/bin/python" \
-TITA_EVAL_MODEL="$HOME/.tita-data-agent/models/WeMM-Embedding-2B-Quantized" \
+TITA_EVAL_PYTHON="$HOME/.dsh/tita-agent/resource/python/bin/python" \
+TITA_EVAL_MODEL="$HOME/.dsh/tita-agent/resource/models/WeMM-Embedding-2B-Quantized" \
 node plugins/dsh-data-domain-knowledge/eval/evaluate-retrieval.mjs \
-  --root "$HOME/.tita-data-agent/knowledge" \
+  --root "$HOME/.dsh/tita-agent/domain_knowledge/default" \
   --qrels ./queries.jsonl
 ```
 
@@ -61,12 +68,13 @@ checkpoint 编译后再跑同一份 qrels。每次比较固定知识卡片、qre
    评测，但它仍然是模型级评测，不是 Tita 端到端评测。
 3. Tita 专属 golden set：建议先人工标注 100–200 条查询，至少包含指标口径、表/字段
    语义、同义词、跨语言、图片中的 UI/图表信息、hard negative 和“不应召回”样本。
-   每条 query 标注 1–3 个正例 unitId；每轮发布前固定这份集合，新增样本走版本控制。
+   有答案的 query 标注 1–3 个正例 unitId；不应召回的 query 使用空集合。每轮发布前
+   固定这份集合，新增样本走版本控制。
 
 ## 指标与门槛
 
 主指标使用 `nDCG@10`、`Recall@10`、`MRR@10`，辅助看 `Recall@5/20`；同时记录
-向量编译吞吐（units/min）、查询 p50/p95、模型常驻内存、`03_compiled` 索引大小和
+弃答准确率和误召回数。另行测量向量编译吞吐（units/min）、模型常驻内存、`derived.db` 大小和
 向量复用率。多模态集合要单独报告 text→image、image→text、text+image 三类结果，
 不能只看总平均。
 
@@ -74,3 +82,6 @@ checkpoint 编译后再跑同一份 qrels。每次比较固定知识卡片、qre
 至少应在语义改写和图片子集上带来可观增益；具体阈值在首轮 100–200 条标注跑完后再
 根据置信区间确定，不预先假设量化版一定优于 BF16。最终结果同时保存 qrels、配置、
 模型 revision、硬件和原始 JSON，确保可复现。
+
+runner 关闭 cross-encoder 重排，比较原始召回路线。完整 Rerank 链路的质量要另行
+测量。自动化回归测试使用小型本地语料和模型替身，不构成真实模型质量门槛的验收结果。

package/eval/benchmark-extraction-format.mjs

@@ -0,0 +1,20 @@
+import { encode as tokens } from 'gpt-tokenizer'
+import { encodeExtractionTable, decodeExtractionTable } from '../lib/types/extract-table.js'
+
+// Compare the same semantic payload with the same o200k_base tokenizer.
+// This measures transport size, not model extraction accuracy or the provider's billing.
+function payload(count) {
+  const quote = '经营报表核对前，需要先统一订单状态、统计周期和退款口径。'
+  const sources = [{ quote, heading_path: ['经营报表', '口径核验'] }]
+  return { subject: '经营报表', knowledge: Array.from({ length: count }, (_, index) => ({ id: `k${index+1}`, text: `第 ${index+1} 项核验：统一统计周期，并检查订单状态与退款口径。`, object: '经营报表', action: '核对', scene: '周报', section: '核验步骤', order: index+1, sources })), events: [], objects: [{ mention: '经营报表', definition: '经营指标报表', evidence: quote }], actions: [{ mention: '核对', definition: '核验报表口径', evidence: quote }], scenes: [{ mention: '周报', definition: '每周报表场景', evidence: quote }], dropped: [] }
+}
+const results = [1, 12, 40].map(count => {
+  const value = payload(count)
+  const table = encodeExtractionTable(value)
+  decodeExtractionTable(table)
+  const json = tokens(JSON.stringify(value)).length
+  const toon = tokens(table).length
+  return { knowledge: count, jsonTokens: json, tableTokens: toon, outputReduction: `${((1-toon/json)*100).toFixed(1)}%` }
+})
+console.table(results)
+console.log('固定表头对小样本有额外开销；真实节省取决于行数、字段长度、引文重复率和所选模型的 tokenizer。')

package/eval/evaluate-retrieval.mjs

@@ -56,8 +56,8 @@ if (!limits.length || !modes.length) usage('invalid --k or --modes')
 const lines = (await readFile(args.qrels, 'utf8')).split(/\r?\n/).map(line => line.trim()).filter(Boolean).filter(line => !line.startsWith('#'))
 const qrels = lines.map((line, index) => {
   const item = JSON.parse(line)
-  const relevant = item.relevant_unit_ids ?? item.relevant ?? []
-  if (!item.query || !Array.isArray(relevant) || relevant.length === 0) throw new Error(`qrels line ${index + 1} needs query and non-empty relevant_unit_ids`)
+  const relevant = item.relevant_unit_ids ?? item.relevant
+  if (!item.query || !Array.isArray(relevant)) throw new Error(`qrels line ${index + 1} needs query and relevant_unit_ids (empty means abstain)`)
   return { ...item, relevant_unit_ids: [...new Set(relevant.map(String))] }
 })
 if (!qrels.length) throw new Error('qrels is empty')
@@ -93,26 +93,35 @@ try {
       // is disabled and the raw fused ranking is measured.
       const response = await runtime.recall(qrel.query, maxLimit, qrel.include_problems === true, mode, undefined, { rerank: false })
       latencies.push(performance.now() - started)
+      if (!response.ok) throw new Error(`${qrel.id ?? qrel.query}: ${response.error?.code ?? 'RECALL_FAILED'}`)
       if (response.data.backend === 'fts' && mode !== 'fts') fallbackCount += 1
-      rows.push({ hits: response.data.hits, relevant: new Set(qrel.relevant_unit_ids) })
+      rows.push({ hits: response.data.hits, relevant: new Set(qrel.relevant_unit_ids), subset: qrel.subset ?? 'default' })
     }
     const metrics = {}
     for (const limit of limits) {
       let recall = 0
       let mrr = 0
       let score = 0
-      for (const row of rows) {
+      const positiveRows = rows.filter(row => row.relevant.size > 0)
+      for (const row of positiveRows) {
         const hits = row.hits.slice(0, limit)
         recall += hits.filter(hit => row.relevant.has(hit.unitId)).length / row.relevant.size
         const rank = hits.findIndex(hit => row.relevant.has(hit.unitId))
         mrr += rank < 0 ? 0 : 1 / (rank + 1)
         score += ndcg(row.hits, row.relevant, limit)
       }
-      metrics[`recall@${limit}`] = Number((recall / rows.length).toFixed(4))
-      metrics[`mrr@${limit}`] = Number((mrr / rows.length).toFixed(4))
-      metrics[`ndcg@${limit}`] = Number((score / rows.length).toFixed(4))
+      const divisor = positiveRows.length || 1
+      metrics[`recall@${limit}`] = positiveRows.length ? Number((recall / divisor).toFixed(4)) : null
+      metrics[`mrr@${limit}`] = positiveRows.length ? Number((mrr / divisor).toFixed(4)) : null
+      metrics[`ndcg@${limit}`] = positiveRows.length ? Number((score / divisor).toFixed(4)) : null
+    }
+    const negativeRows = rows.filter(row => row.relevant.size === 0)
+    const falseAnswers = negativeRows.filter(row => row.hits.length > 0).length
+    output.modes[mode] = {
+      metrics, positiveCount: rows.length - negativeRows.length,
+      abstention: { count: negativeRows.length, falseAnswers, accuracy: negativeRows.length ? Number((1 - falseAnswers / negativeRows.length).toFixed(4)) : null },
+      latencyMs: { p50: percentile(latencies, 0.5), p95: percentile(latencies, 0.95) }, fallbackCount,
     }
-    output.modes[mode] = { metrics, latencyMs: { p50: percentile(latencies, 0.5), p95: percentile(latencies, 0.95) }, fallbackCount }
   }
 } finally {
   runtime.close()

package/eval/queries.example.jsonl

@@ -1,2 +1,3 @@
 {"id":"q-agent-architecture","query":"知识编译与召回的 Agent 架构","relevant_unit_ids":["concept:agent_architecture"],"subset":"text"}
 {"id":"q-ui-architecture","query":"Tita 数据 Agent 的界面和架构","relevant_unit_ids":["concept:agent_architecture","concept:ui_reference"],"subset":"text-image"}
+{"id":"q-no-evidence","query":"火星香蕉的年度产量是多少","relevant_unit_ids":[],"subset":"abstention"}