dsh-data-domain-knowledge 0.1.0-alpha.1

package/cordis.patch.yml

@@ -0,0 +1,410 @@
+# data-domain-knowledge bundle patch: mount the knowledge / intent / convert host.
+#
+# 0.1.7 起这条 entry **自己就是设置文档**：插件导出的 `Config`（见
+# `src/config-schema.ts`）把三块设置合成一个 schema —— `knowledge` / `intent` /
+# `convert`，正是 0.1.5 那三个设置命名空间（`tita-data-agent-knowledge` /
+# `-intent` / `-convert`）。设置页里的改动由 `configEditor` 写回**本 entry 的
+# `config:`**，客户端按路径读写（`{op:'set', path:['knowledge','recallTopK']}`）。
+#
+# 所以这里**故意不写 `config:`**：三组字段全都有 schema `.default()`，而
+# `knowledge.agentRoot` 的默认值就是 `~/Desktop/titaagent`（`defaultAgentRoot()`，
+# 可被 `DSH_AGENT_ROOT` 覆盖）。往这里写死一个 `agentRoot` 会把「Agent 的家在哪」
+# 这件事从代码挪进某台机器的配置文件里 —— 那正是 0029 想收拢的那一个答案。
+- insert:
+    - id: tita-data-domain-knowledge
+      name: 'dsh-data-domain-knowledge'
+
+# >>> agent preset (generated) >>>
+# Tita Data Agent 的 agent preset 声明。
+#
+# 这一段由 `scripts/sync-agent-preset.mjs` 从 `preset/tita-data-agent/` 生成 ——
+# **不要手改**：改真源再跑 `pnpm sync:preset`。`pnpm doctor` 会校验它与真源一致。
+#
+# 为什么 preset 必须长在插件的 patch 里：0.1.7 起 DSH 不再扫描
+# `$DSH_HOME/.agent-presets/`，preset 的唯一形态就是这条 insert。详见脚本头注释。
+- insert:
+    - id: preset-tita-data-agent
+      name: '@deepseek-ai/dsh-agent-preset'
+      config:
+        id: tita-data-agent
+        name: 'Tita Data Agent'
+        description: 'TitaAgent 自己的组合：数据 Agent 的提示词，加上只有这个 preset 才有的意图记录工具。'
+        order: 10
+        plugins:
+          # Tita Data Agent 的 agent 组合（0029）。
+          #
+          # 这是 `standard` 的一份**拷贝**，只改了四件事。preset 是「一份完整组合」，没有
+          # patch 语义（`dsh-agent-presets` 的 authoring 是 copy-only），所以它必须是一整份
+          # 文件、而不是对 standard 的增量。
+          #
+          # 改了什么：
+          #
+          # 1. **身份**（`persona` 行）：把「你是一个编码 Agent」换成 Tita Data Agent 自己的
+          #    提示词。这一行是**这个 preset 存在的理由** —— 它 shadow 掉部署级的 persona，
+          #    所以只有加入这个 preset 的会话会以数据 Agent 的身份说话。提示词正文就在下面，
+          #    直接改它即可；改完**新建一个会话**才生效（preset 在会话创建时固定）。
+          # 2. **意图工具**（`intent-tools` 行）：`intent_record` / `intent_store_view` /
+          #    `intent_registry_list` 三个工具搬到 agent 平面，只有这个 preset 的会话能看到。
+          #    DSH 里「只有某个 preset 能用」唯一支持的形态就是注册作用域，所以它们必须在这里
+          #    挂载，而不是在宿主平面全局注册（那会让每个 agent 都看到 intent_record）。
+          # 3. **子代理不许继承意图工具**：子代理会加入父会话的组合，所以要在 delegation 的每一
+          #    行上显式 deny 掉 —— 意图记录描述的是「用户在跟 Agent 要什么」，子代理做的是被拆
+          #    出来的一个片段，它记一条意图只会污染任务树。
+          # 4. 其余每一行都与 `standard` 逐字相同：工具集不变，这个 preset 换的是身份与意图
+          #    能力，不是一个更小的 Agent。
+          #
+          # 其余纪律与 `standard` 一致：这是 AGENT-PLANE 组合，roster 在 standing scope 下挂载
+          # 一次，每个命名它的会话按作用域父子关系加入。需要 provide 服务的行必须落在带
+          # `isolate` realm 的 group 里，否则它会发布到根 realm（进程级）并被挂载审计拒绝 ——
+          # 本文件里唯一这样的行是 delegation 那个 group。
+
+          # ── identity ────────────────────────────────────────────────────────────────
+
+          # The preset's own persona, shadowing the deployment default for this agent.
+          # `{{model}}` and `{{cwd}}` resolve from the agent's own route and workspace.
+          - id: persona
+            name: '@deepseek-ai/dsh-persona'
+            config:
+              # `{{model}}` / `{{cwd}}` 从本会话自己的路由与工作区解析。
+              suffix: Your working directory is {{cwd}}.
+              # ↓↓↓ Tita Data Agent 的提示词。**要改的就是这一段。** ↓↓↓
+              prefix: |
+                You are Tita Data Agent, the data analyst on this deployment. You are powered
+                by the {{model}} model. Your job is to answer questions about this
+                organisation's business with evidence, and to write down what you learn so the
+                next answer is cheaper than this one.
+
+                ## What you work with
+
+                - **The knowledge store.** Each knowledge base on disk is a tree.
+                  `01_source_truth` holds SourceRaw: the original business documents, never
+                  rewritten, kept so that a conclusion can point back at where it came from.
+                  `02_wiki` holds the cards people read — one per entity, each with mandatory
+                  `source_refs`, each following a type (`concept`, `metric`, `table`, `event`,
+                  `playbook`, `case`). `03_compiled` is what `recall_knowledge` actually
+                  searches: wikiNotes, a vector index and a knowledge graph. `05_diagnosis` is
+                  the attribution graph the panel renders. There can be several bases.
+
+                - **The data surfaces, which are a different thing.** `workbook_*` reads and
+                  writes the workbooks the user has open; `notebook_*` runs SQL in the user's
+                  own browser. That is where a number is *computed*. The knowledge store is
+                  where a number is *defined* — a `metric` card says how it must be calculated,
+                  a `table` card says what one row means. Computing without the definition gets
+                  you a number nobody agreed to.
+
+                - **Conversion.** `convert_source` turns a file, a web page or a video into
+                  knowledge-ready Markdown and says where it landed. It is not knowledge until
+                  `convert_ingest` has run, so keep saying which of the two steps you are on.
+
+                ## Working rules
+
+                1. **Record the intent before the work.** Call `intent_record` with what the
+                   user wants and what you are about to do, before you do it. The vocabulary,
+                   and the rule for continuing existing work instead of starting a new task,
+                   live in that tool's own description. The record never gates the request:
+                   record, then continue.
+                2. **Recall before you answer about this organisation.** For its own rules,
+                   definitions, metrics, processes or history, call `recall_knowledge` first and
+                   answer from what comes back. An empty result is a finding, not a licence to
+                   improvise — say the store has nothing on it. The wiki layer answers "what does
+                   this organisation mean by X"; when you need "where exactly does it say that",
+                   or a card and the document behind it disagree, go read the source.
+                3. **Never invent a number.** Every figure you report must come from a tool
+                   result in this turn, and you must say which one. "I don't have that data" is
+                   a correct answer; a made-up number is not.
+                4. **Put it in writing while you still have the evidence.** A definition, a
+                   metric's exact calculation, a process, a diagnosed cause — if it will be
+                   asked again, write it as a card with `write_knowledge_card`, with
+                   `source_refs` that already exist in the store. A card whose refs point
+                   nowhere is rejected, and rightly so.
+                5. **Correct a card, do not quietly overwrite it.** When something in the store
+                   is wrong, set its status to `problem` and say what is wrong with it. Cards
+                   marked `problem` drop out of recall by default — that is how a wrong
+                   definition stops propagating.
+                6. **Never invent a knowledge base id.** Call `list_knowledge_bases` before
+                   addressing one you have not seen this session; omit `kb` only when the
+                   default base is genuinely the right one.
+                7. **Be concrete about uncertainty.** When sources disagree, present both and
+                   name them. When a conclusion rests on a single document, say so.
+                8. **Ask only what the user owns.** Which of two readings they meant, what
+                   matters more, what to do next — ask. Anything you can look up, look up
+                   instead of asking.
+                9. **Deliverables belong to the project, not to your home.** Write every file
+                   you produce for the user under `output/<YYYY-MM-DD>/` in your working
+                   directory, never into the Agent root: the Agent root accumulates across
+                   projects, a deliverable belongs to one. Name it `<topic>-<kind>.<ext>` where
+                   `<kind>` is exactly one of `报告`, `图表`, `数据集`, `清洗脚本`, `SQL` — the
+                   name alone has to say what the file is a year later. Append `-2`, `-3` rather
+                   than overwriting, and say the path in your answer.
+
+                ## How to write
+
+                Answer in the user's language. Lead with the answer, then the evidence. Prefer
+                a table when you are comparing things and a short paragraph when you are not.
+                Skip preamble and praise; the user is here for the analysis.
+              # ↑↑↑ Tita Data Agent 的提示词结束 ↑↑↑
+
+          - id: agent-instructions
+            name: '@deepseek-ai/dsh-agent-instructions'
+            config:
+              maxBytes: 65536
+
+          # ── 意图记录（agent 平面）────────────────────────────────────────────────────
+          #
+          # 0029 之前这三个工具由知识插件在**宿主平面**全局注册，于是每个 agent、每个 preset
+          # 的会话都会在提示词里看到 `intent_record`。现在它们只在这里挂载：这个模块只往
+          # `tools` 注册表里塞定义、不 provide 任何服务，所以和随包的 `tool-fs` 一样不需要
+          # realm，而它的注册只覆盖加入本 preset 的会话。
+          #
+          # 引擎本身（设置、轮次记账、`/intent/**` 只读面）仍在宿主平面，通过
+          # `titaDataAgentIntent` 服务把操作面暴露给这一行 —— 引擎只有一份。
+          - id: intent-tools
+            name: 'dsh-data-domain-knowledge/intent-tools'
+
+          # ── shell ───────────────────────────────────────────────────────────────────
+
+          # `shell-env` stays in the HOST composition: `apps/cli/src/web.ts` injects it to
+          # publish `DSH_WEB_URL`/`DSH_WEB_MODE`, and a host row that injects a service is
+          # the criterion for host-plane ownership — injection resolves before any session
+          # exists, so there is no agent to key by. Behind a preset realm those variables
+          # never reached the model's shell at all. Both shell tools consume the host
+          # registry from here; their executors (`bash-sandbox`/`pwsh-sandbox`) are
+          # host-plane too.
+          - id: tool-bash
+            name: '@deepseek-ai/dsh-tool-bash'
+            disabled: !!js process.platform === 'win32'
+
+          - id: tool-pwsh
+            name: '@deepseek-ai/dsh-tool-pwsh'
+            disabled: !!js process.platform !== 'win32'
+
+          # ── filesystem ──────────────────────────────────────────────────────────────
+
+          # Both register into the host `tools` registry and provide nothing, so
+          # they need no realm. The `fs` service and its policy stay in the host.
+          - id: tool-fs
+            name: '@deepseek-ai/dsh-tool-fs'
+
+          - id: tool-fs-search
+            name: '@deepseek-ai/dsh-tool-fs-search'
+            config:
+              sampleOverCapGlobResults: false
+
+          # ── background jobs ────────────────────────────────────────────────────────
+
+          # Only the model-facing controls. The task REGISTRY stays on the host plane:
+          # its producers sit outside any realm this file could put it in — `tool-bash`
+          # above resolves it with `ctx.get`, and an entry-local realm here is invisible
+          # to every sibling row, so `run_in_background` would answer "background jobs
+          # unavailable" while these controls sat in the catalog. The registry is keyed by
+          # owning agent anyway, so one host instance serves every session. What a preset
+          # chooses is whether its agent can collect and stop background work at all.
+          - id: tool-jobs
+            name: '@deepseek-ai/dsh-tool-jobs'
+
+          # ── skills ──────────────────────────────────────────────────────────────────
+
+          # The skill REGISTRY lives in the host composition and is layered per scope:
+          # these rows register into THIS preset's layer of it, so they need no realm.
+          # `skill-filesystem` contributes local-root discovery for agents on this preset, and
+          # `tool-skill` gives them the catalog and loader; the merged catalog also
+          # carries whatever the deployment registered globally (repository plugins).
+          - id: skill-filesystem
+            name: '@deepseek-ai/dsh-skill-filesystem'
+
+          - id: tool-skill
+            name: '@deepseek-ai/dsh-tool-skill'
+
+          # ── goals ───────────────────────────────────────────────────────────────────
+
+          # The goal service and session driver stay on the host plane, where the Gateway
+          # can resolve them. The human command and model-facing tool register into this
+          # preset's scoped layers.
+          - id: command-goal
+            name: '@deepseek-ai/dsh-command-goal'
+
+          - id: tool-goal
+            name: '@deepseek-ai/dsh-tool-goal'
+
+          # ── plan mode ───────────────────────────────────────────────────────────────
+
+          # Plan state is per-agent by nature, so an entry-local realm is not a
+          # workaround here — it is the correct lifetime.
+          - id: planning
+            name: cordis:group
+            group: true
+            isolate:
+              planMode: true
+            config:
+              - id: plan-mode
+                name: '@deepseek-ai/dsh-plan-mode'
+                config:
+                  section: |
+                        You are in plan mode. Stay in plan mode until exit_plan_mode succeeds or the user switches the session mode. Imperative language to implement changes means plan the implementation, not execute it. A user's conversational agreement — including an answer confirming something you asked — approves nothing and does not end plan mode; fold the confirmed decision into the plan and submit it through exit_plan_mode.
+
+                        Explore first. Use non-mutating reads, searches, static analysis, and checks to ground the plan in the actual repository. Do not edit or write files, change configuration, run formatters or code generation that rewrites tracked files, commit, or otherwise carry out the plan. Prefer existing functions and patterns over new machinery.
+
+                        The tool catalog stays the same across modes for request-cache stability. These plan-mode rules override any later tool description or guidance that suggests using mutation tools; those tools remain listed to keep the tool catalog unchanged. Do not use todo_write to track this planning phase: it tracks implementation after an approved plan, while the plan itself belongs in exit_plan_mode.
+
+                        Resolve discoverable facts by inspection. Use ask_user_question only for user-owned choices or material ambiguity that inspection cannot answer. Do not ask the user where code lives or how current behavior works when you can find out.
+
+                        Make the plan decision-complete: state the goal and success criteria; group implementation changes by subsystem; identify public API, schema, and data-flow changes; cover edge cases, failure modes, tests, acceptance criteria, and explicit assumptions. Keep it concise enough to review but detailed enough that another engineer can implement it without making design decisions.
+
+                        When ready, call exit_plan_mode with the complete plan markdown, starting with a # title. Make exit_plan_mode the only and final tool call in that assistant response: it presents the plan for approval, and implementation begins only in a later step after approval. Do not paste the final plan as a plain reply or ask "should I proceed?" through prose or ask_user_question. If review rejects it, incorporate the feedback and present again. If the review channel is unavailable or aborted, stay in plan mode and ask the user to switch modes manually; do not proceed with implementation.
+
+          # ── compaction ──────────────────────────────────────────────────────────────
+
+          # `compaction-basic` reads `toolResultPrune` through `ctx.get`, so the pruner must
+          # share this realm rather than sit outside it.
+          #
+          # `tokenMeter` is deliberately NOT in this realm: the meter stays on the HOST
+          # plane, and the rows here resolve that one instance. It takes no configuration,
+          # keys every fold by Session, and owns the context-meter projection units the
+          # browser reads for every session — behind a realm those units would come and go
+          # with whichever presets happen to be mounted. What a preset chooses is whether
+          # its agent compacts at all, which is `compaction-basic` below.
+          - id: compaction
+            name: cordis:group
+            group: true
+            isolate:
+              compaction: true
+              toolResultPruner: true
+            config:
+              - id: compaction-basic
+                name: '@deepseek-ai/dsh-compaction-basic'
+
+              - id: command-compact
+                name: '@deepseek-ai/dsh-command-compact'
+
+              - id: tool-result-pruner
+                name: '@deepseek-ai/dsh-compaction-tool-result-pruner'
+                config:
+                  thresholdChars: 8192
+                  headChars: 4096
+                  tailChars: 1024
+
+          # ── delegation and workflows ────────────────────────────────────────────────
+
+          # The `subagents` registry and its spawn/fork backends live in the HOST
+          # composition: the registry is a process singleton whose cross-session queries
+          # the api-proxy serves to the browser, and a provider name may only be
+          # registered once. This preset contributes the delegation TOOLS, which resolve
+          # that host registry.
+          #
+          # `workflows` is different — nothing outside an agent reads it — so every row
+          # that reaches it shares one entry-local realm here, and a consumer left
+          # outside would resolve a host registry this preset does not populate.
+          - id: delegation
+            name: cordis:group
+            group: true
+            isolate:
+              workflowEngine: true
+            config:
+              - id: tool-subagent-control
+                name: '@deepseek-ai/dsh-tool-subagent-control'
+
+              - id: tool-subagent-list-agents
+                name: '@deepseek-ai/dsh-tool-subagent-control/list-agents'
+
+              - id: tool-subagent
+                name: '@deepseek-ai/dsh-tool-subagent'
+                config:
+                  provider: spawn
+                  toolName: subagent
+                  modelSelectionSettings: true
+                  backgroundMode: continuable
+                  toolFilter:
+                    deny:
+                      - intent_record
+                      - intent_store_view
+                      - intent_registry_list
+
+              # Fork omits model selection so provider/model stay equal to the parent and
+              # the inherited history remains eligible for KV Cache reuse. This preset
+              # keeps fork continuable; parent and child inherit the same messaging tool,
+              # while the parent id and return guidance follow the inherited history.
+              - id: tool-subagent-fork
+                name: '@deepseek-ai/dsh-tool-subagent'
+                config:
+                  provider: fork
+                  toolName: subagent_fork
+                  backgroundMode: continuable
+                  toolFilter:
+                    deny:
+                      - intent_record
+                      - intent_store_view
+                      - intent_registry_list
+
+              # Production dsh does not install these optional providers. Install the
+              # matching Bundle in this Profile and restart the Host, then copy this
+              # preset and remove `disabled` from the matching tool row. Host availability
+              # alone grants no tool.
+              - id: tool-subagent-codex
+                name: '@deepseek-ai/dsh-tool-subagent'
+                disabled: true
+                config:
+                  provider: codex
+                  toolName: subagent_codex
+                  backgroundMode: one-shot
+                  maxDepth: provider-managed
+                  toolFilter:
+                    deny:
+                      - intent_record
+                      - intent_store_view
+                      - intent_registry_list
+
+              - id: tool-subagent-claude-code
+                name: '@deepseek-ai/dsh-tool-subagent'
+                disabled: true
+                config:
+                  provider: claude-code
+                  toolName: subagent_claude_code
+                  backgroundMode: one-shot
+                  maxDepth: provider-managed
+                  toolFilter:
+                    deny:
+                      - intent_record
+                      - intent_store_view
+                      - intent_registry_list
+
+              # 0.1.7 把这行换掉了：旧包名 `@deepseek-ai/dsh-workflow-worker-thread` 在
+              # 0.1.7 的安装里已经不存在，对应物是 `@deepseek-ai/dsh-workflow-ptc`
+              # （见 `dsh-web-app` 的 `presets/cordis.patch.yml`）。留着旧名不会报「找不到
+              # 包」，而是让**整个 preset 挂载失败**，症状是它出现在 Agent 预设名单里、
+              # 却带一条 `workflow-worker-thread (…): never started` 诊断 —— 这也是
+              # 「自定义 agent 没生效」最像「配置没生效」的一种失败。
+              - id: workflow-ptc
+                name: '@deepseek-ai/dsh-workflow-ptc'
+                config:
+                  provider: spawn
+
+              - id: tool-workflow
+                name: '@deepseek-ai/dsh-tool-workflow'
+
+              - id: tool-ralph
+                name: '@deepseek-ai/dsh-tool-ralph'
+                config:
+                  subagentProvider: spawn
+                  maxRounds: 64
+
+          # ── remaining model-facing rows ─────────────────────────────────────────────
+
+          - id: tool-ask-user
+            name: '@deepseek-ai/dsh-tool-ask-user'
+
+          - id: tool-todo
+            name: '@deepseek-ai/dsh-tool-todo'
+            config:
+              allowParallelInProgress: true
+
+          # The `web` service and its search provider stay in the host composition; only
+          # the model-facing tool is per-session.
+          - id: tool-web
+            name: '@deepseek-ai/dsh-tool-web'
+            config:
+              fetch: true
+              searchTimeoutMs: 60000
+
+          - id: present
+            name: '@deepseek-ai/dsh-tool-present'
+# <<< agent preset (generated) <<<

package/eval/README.md

@@ -0,0 +1,76 @@
+# Tita Knowledge Retrieval Evaluation
+
+`evaluate-retrieval.mjs` 对同一份已经编译好的 Knowledge Store，逐条运行
+`fts`、`vector`、`hybrid`，输出 Recall、MRR、nDCG 和 p50/p95 latency。它不自动
+下载公开数据集，也不会改写知识库；公开数据集需要先转换为本插件的 qrels 格式。
+
+## qrels 格式
+
+每行一个 JSON 对象，`relevant_unit_ids` 使用编译后的稳定 `unitId`，例如
+`metric:sales_amount`。可以附加 `source_types`、`include_problems` 和 `subset`，用于
+分组评测文本、图片 Markdown、表格截图等场景。
+
+```json
+{"id":"q1","query":"销售额如何计算","relevant_unit_ids":["metric:sales_amount"],"subset":"text"}
+```
+
+## 本地运行
+
+先在 UI 点击一次「编译」，再运行：
+
+```bash
+pnpm --filter dsh-data-domain-knowledge build
+node plugins/dsh-data-domain-knowledge/eval/evaluate-retrieval.mjs \
+  --root "$HOME/.tita-data-agent/knowledge" \
+  --qrels plugins/dsh-data-domain-knowledge/eval/queries.example.jsonl
+```
+
+如果 Knowledge Store 使用 WeMM：
+
+```bash
+TITA_EVAL_PROVIDER=wemm-python \
+TITA_EVAL_PYTHON="$HOME/.tita-data-agent/embedding-venv/bin/python" \
+TITA_EVAL_MODEL="$HOME/.tita-data-agent/models/WeMM-Embedding-2B-Quantized" \
+node plugins/dsh-data-domain-knowledge/eval/evaluate-retrieval.mjs \
+  --root "$HOME/.tita-data-agent/knowledge" \
+  --qrels ./queries.jsonl
+```
+
+## 建议的对比矩阵
+
+| 组别 | 检索方式 | 目的 |
+| --- | --- | --- |
+| A | FTS | 当前精确匹配基线，观察术语、ID 和公式查询 |
+| B | WeMM 量化 1024 维，binary/Hamming + int8 重排 | Tita 默认本地多模态方案 |
+| C | WeMM 同一模型的未量化/BF16 向量 | 分离量化损失与模型能力 |
+| D | hybrid（当前默认） | 观察精确词命中与语义/图片召回的互补收益 |
+
+A、B、D 可直接由本脚本比较；C 需要在另一份 compiled root 中用官方 BF16
+checkpoint 编译后再跑同一份 qrels。每次比较固定知识卡片、qrels、top-k、查询顺序
+和硬件，避免把数据变化误认为模型差异。
+
+## 数据集分层
+
+1. 公共文本 sanity：从 [BEIR](https://github.com/beir-cellar/beir) 选一个小型
+   domain subset，或从 [MTEB 可用 benchmark 列表](https://docs.mteb.org/overview/available_benchmarks/)
+   选择 retrieval task。它们适合比较文本 embedding，不足以代表 Tita 的 Markdown
+   图片和知识卡片工作流。
+2. 公共多模态：使用 [MMEB-V3](https://huggingface.co/datasets/VLM2Vec/MMEB-V3)
+   的 image/text、visual-document 和 agent-centric 子集；它与 [WeMM 官方仓库](https://github.com/Tencent/WeMM-Embedding)
+   的模型使用场景更接近。MTEB 的 benchmark 列表也收录了 MIEB 这类图像 embedding
+   评测，但它仍然是模型级评测，不是 Tita 端到端评测。
+3. Tita 专属 golden set：建议先人工标注 100–200 条查询，至少包含指标口径、表/字段
+   语义、同义词、跨语言、图片中的 UI/图表信息、hard negative 和“不应召回”样本。
+   每条 query 标注 1–3 个正例 unitId；每轮发布前固定这份集合，新增样本走版本控制。
+
+## 指标与门槛
+
+主指标使用 `nDCG@10`、`Recall@10`、`MRR@10`，辅助看 `Recall@5/20`；同时记录
+向量编译吞吐（units/min）、查询 p50/p95、模型常驻内存、`03_compiled` 索引大小和
+向量复用率。多模态集合要单独报告 text→image、image→text、text+image 三类结果，
+不能只看总平均。
+
+建议先以 FTS 为守门基线：hybrid 在整体和每个关键子集都不能低于 FTS，WeMM vector
+至少应在语义改写和图片子集上带来可观增益；具体阈值在首轮 100–200 条标注跑完后再
+根据置信区间确定，不预先假设量化版一定优于 BF16。最终结果同时保存 qrels、配置、
+模型 revision、硬件和原始 JSON，确保可复现。

package/eval/evaluate-retrieval.mjs

@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+
+import { readFile } from 'node:fs/promises'
+import { performance } from 'node:perf_hooks'
+import { DEFAULT_KNOWLEDGE_CONFIG, KnowledgeRuntime } from '../lib/types/index.js'
+
+function usage(message) {
+  if (message) console.error(`Error: ${message}\n`)
+  console.error('Usage: node eval/evaluate-retrieval.mjs --root <knowledge-root> --qrels <queries.jsonl> [--modes fts,vector,hybrid] [--k 5,10,20]')
+  process.exit(message ? 2 : 0)
+}
+
+function options(argv) {
+  const result = {}
+  for (let index = 0; index < argv.length; index += 1) {
+    const item = argv[index]
+    if (item === '--help') usage()
+    if (!item?.startsWith('--')) usage(`unknown argument: ${item ?? ''}`)
+    const key = item.slice(2)
+    const value = argv[index + 1]
+    if (!value || value.startsWith('--')) usage(`missing value for --${key}`)
+    result[key] = value
+    index += 1
+  }
+  return result
+}
+
+function percentile(values, percentileValue) {
+  if (!values.length) return 0
+  const sorted = [...values].sort((left, right) => left - right)
+  const position = (sorted.length - 1) * percentileValue
+  const lower = Math.floor(position)
+  const upper = Math.ceil(position)
+  if (lower === upper) return Number(sorted[lower].toFixed(2))
+  const value = sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower)
+  return Number(value.toFixed(2))
+}
+
+function ndcg(hits, relevant, limit) {
+  let dcg = 0
+  for (let index = 0; index < Math.min(limit, hits.length); index += 1) {
+    if (relevant.has(hits[index].unitId)) dcg += 1 / Math.log2(index + 2)
+  }
+  const idealCount = Math.min(limit, relevant.size)
+  let ideal = 0
+  for (let index = 0; index < idealCount; index += 1) ideal += 1 / Math.log2(index + 2)
+  return ideal ? dcg / ideal : 0
+}
+
+const args = options(process.argv.slice(2))
+if (!args.root || !args.qrels) usage('--root and --qrels are required')
+const limits = String(args.k ?? '5,10,20').split(',').map(Number).filter(value => Number.isInteger(value) && value > 0 && value <= 50)
+const modes = String(args.modes ?? 'fts,vector,hybrid').split(',').filter(value => ['fts', 'vector', 'hybrid'].includes(value))
+if (!limits.length || !modes.length) usage('invalid --k or --modes')
+
+const lines = (await readFile(args.qrels, 'utf8')).split(/\r?\n/).map(line => line.trim()).filter(Boolean).filter(line => !line.startsWith('#'))
+const qrels = lines.map((line, index) => {
+  const item = JSON.parse(line)
+  const relevant = item.relevant_unit_ids ?? item.relevant ?? []
+  if (!item.query || !Array.isArray(relevant) || relevant.length === 0) throw new Error(`qrels line ${index + 1} needs query and non-empty relevant_unit_ids`)
+  return { ...item, relevant_unit_ids: [...new Set(relevant.map(String))] }
+})
+if (!qrels.length) throw new Error('qrels is empty')
+
+const root = String(args.root)
+const config = {
+  ...DEFAULT_KNOWLEDGE_CONFIG,
+  rootDir: root,
+  embeddingProvider: process.env.TITA_EVAL_PROVIDER ?? DEFAULT_KNOWLEDGE_CONFIG.embeddingProvider,
+  embeddingModel: process.env.TITA_EVAL_MODEL ?? DEFAULT_KNOWLEDGE_CONFIG.embeddingModel,
+  python: process.env.TITA_EVAL_PYTHON ?? DEFAULT_KNOWLEDGE_CONFIG.python,
+  embeddingDevice: process.env.TITA_EVAL_DEVICE ?? DEFAULT_KNOWLEDGE_CONFIG.embeddingDevice,
+}
+const runtime = new KnowledgeRuntime(root, config)
+const maxLimit = Math.max(...limits)
+const output = {
+  root,
+  provider: config.embeddingProvider,
+  model: config.embeddingModel,
+  queryCount: qrels.length,
+  limits,
+  modes: {},
+}
+
+try {
+  for (const mode of modes) {
+    const latencies = []
+    const rows = []
+    let fallbackCount = 0
+    for (const qrel of qrels) {
+      const started = performance.now()
+      // The evaluator compares the recall routes themselves, so stage 4 rerank
+      // is disabled and the raw fused ranking is measured.
+      const response = await runtime.recall(qrel.query, maxLimit, qrel.source_types ?? [], qrel.include_problems === true, mode, undefined, { rerank: false })
+      latencies.push(performance.now() - started)
+      if (response.data.backend === 'fts' && mode !== 'fts') fallbackCount += 1
+      rows.push({ hits: response.data.hits, relevant: new Set(qrel.relevant_unit_ids) })
+    }
+    const metrics = {}
+    for (const limit of limits) {
+      let recall = 0
+      let mrr = 0
+      let score = 0
+      for (const row of rows) {
+        const hits = row.hits.slice(0, limit)
+        recall += hits.filter(hit => row.relevant.has(hit.unitId)).length / row.relevant.size
+        const rank = hits.findIndex(hit => row.relevant.has(hit.unitId))
+        mrr += rank < 0 ? 0 : 1 / (rank + 1)
+        score += ndcg(row.hits, row.relevant, limit)
+      }
+      metrics[`recall@${limit}`] = Number((recall / rows.length).toFixed(4))
+      metrics[`mrr@${limit}`] = Number((mrr / rows.length).toFixed(4))
+      metrics[`ndcg@${limit}`] = Number((score / rows.length).toFixed(4))
+    }
+    output.modes[mode] = { metrics, latencyMs: { p50: percentile(latencies, 0.5), p95: percentile(latencies, 0.95) }, fallbackCount }
+  }
+} finally {
+  runtime.close()
+}
+
+console.log(JSON.stringify(output, null, 2))

package/eval/queries.example.jsonl

@@ -0,0 +1,2 @@
+{"id":"q-agent-architecture","query":"知识编译与召回的 Agent 架构","relevant_unit_ids":["concept:agent_architecture"],"subset":"text"}
+{"id":"q-ui-architecture","query":"Tita 数据 Agent 的界面和架构","relevant_unit_ids":["concept:agent_architecture","concept:ui_reference"],"subset":"text-image"}