markindex 0.0.0-stage → 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,54 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to
+[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-10-26
+
+First release.
+
+### Added
+
+- Markdown section parser built on mdast: setext and ATX headings, preface before the first heading, documents
+  without headings, front matter, `#` inside code blocks, skipped heading levels, BOM and CR / CRLF / LF line
+  endings; every source line belongs to exactly one node. Heading titles are capped at 300 code points and text is
+  NFKC-normalized before segmentation.
+- Section tree with a deterministic optimization pass (a port of PageIndex's merge / expand cost model, with pages
+  replaced by tokens) and stable node ids; oversized sections are split at block boundaries into `chunk` nodes.
+- SQLite store (one file per library, WAL) holding the trees, the original lines and a mixed Chinese / English
+  BM25 index (`Intl.Segmenter` words plus a trigram substring fallback). Incremental indexing by content hash, one
+  transaction per document, a single-writer lock, and an index format fingerprint that triggers a full rebuild
+  (reusing cached summaries) when the format or tree parameters change.
+- Five read-only agent tools (`browse_documents`, `get_document`, `get_document_structure`, `get_node_content`,
+  `search_documents`) with JSON envelopes, `next_steps` guidance, a 95 000-character response budget and a
+  1 024-character cap on string arguments; adapters for the Vercel AI SDK and for MCP (stdio server).
+- Optional LLM node summaries and document descriptions through any AI SDK provider (`--summary-model
+  <provider>:<model>`), cached by node content hash, with a cost estimate (`--estimate`) that makes no LLM calls.
+- Built-in `retrieve()` evidence loop: returns original text re-read from the index by line range, usage and a stop
+  reason (`done`, `budget`, `max_steps`), never a model-written answer.
+- Client-side LLM rate limiting: a dependency-free sliding 60 s window for requests and estimated input tokens
+  (FIFO, no deadlock for oversized requests). `rateLimit` option for summaries and `retrieve()`; CLI flags
+  `index --summary-rpm / --summary-tpm` and `eval --rpm / --tpm`.
+- Setext heading switch: `--setext` / `--no-setext` (library: `setext`) for converter output that uses `---` as a
+  page break, with warnings when suspect setext headings are found; the setting is stored in the index.
+- Ignore support: `.gitignore` files at any level, an optional root `.markindexignore`, and repeatable
+  `--ignore <pattern>` options (library: `ignore`).
+- CLI: `index`, `tree`, `search`, `mcp` (with `--reindex-on-start`) and `eval`.
+- Evaluation harness: a 151-question zh / en / mixed question set over a public 329-document corpus, a BM25 baseline,
+  agent mode through `retrieve()`, evidence size and precision metrics (`--max-evidence-lines`), scale benchmarks,
+  and a Claude Code agent-mode runner restricted to the MarkIndex tools (30/30 on the n = 30 run).
+- English and Chinese READMEs, third-party notices, and GitHub Actions CI on Ubuntu and macOS with Node 22 and 24.
+
+### Notes
+
+- Requires Node.js >= 22.12. LLM features need an [AI SDK provider](https://ai-sdk.dev/providers) package such as
+  `@ai-sdk/anthropic` installed alongside.
+- MarkIndex is an independent, unofficial implementation inspired by
+  [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex); see `THIRD_PARTY_NOTICES.md`.
+
+[Unreleased]: https://github.com/jackjin1997/markindex/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/jackjin1997/markindex/releases/tag/v0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zexu Jin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,282 @@
-# Temporary Holding Version
+# MarkIndex
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+[![CI](https://github.com/jackjin1997/markindex/actions/workflows/ci.yml/badge.svg)](https://github.com/jackjin1997/markindex/actions/workflows/ci.yml) [![npm](https://img.shields.io/npm/v/markindex)](https://www.npmjs.com/package/markindex)
+
+**Vectorless, structure-first retrieval over large libraries of long Markdown documents — for agents.**
+
+MarkIndex parses every Markdown file into a tree of its own sections, stores the trees, the original lines and a
+mixed Chinese/English BM25 index in one SQLite file, and gives an agent five read-only tools to walk
+"library → folder → document → section" and read the original text with line numbers. No embeddings, no vector
+database, no chunking guesswork: the document's own structure is the index, and every piece of evidence can be
+cited back as `[doc_path, L<start>-<end>]`.
+
+> Inspired by [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex), MarkIndex is an independent,
+> unofficial TypeScript implementation focused on Markdown. It is not affiliated with or endorsed by VectifyAI.
+
+中文说明：[README.zh-CN.md](README.zh-CN.md)
+
+## Why
+
+- **Structure-first, not chunk-first.** Long professional documents (filings, guidelines, laws, manuals) are
+  already organized by their authors. Navigating that outline keeps context intact and makes answers explainable
+  and citable; arbitrary chunks + vector similarity cut sections apart.
+- **Deterministic and LLM-free indexing.** Parsing (mdast, not regex), tree building and the token-cost tree
+  optimization never call a model; the same input always produces the same tree. LLMs are optional and only used
+  for node summaries, document descriptions and the built-in `retrieve()` loop.
+- **Markdown done properly.** Setext headings, preface text before the first heading, documents without any
+  heading, front matter, `#` inside code fences, skipped heading levels, BOM / CRLF — every source line belongs to
+  exactly one node, and oversized sections are split at block boundaries into `chunk` nodes.
+- **Keyword fallback that works for Chinese.** BM25 over `Intl.Segmenter` word segmentation (Chinese, English and
+  mixed text, identifiers like `C++`, `ISO-27001`, `v2.1`), plus a trigram substring index as a second pass.
+- **Budgeted tool responses.** Tools never throw; they return JSON envelopes with `next_steps` guidance and stay
+  under a 95 000-character budget, paginating large outlines and sections.
+
+## Install
+
+```bash
+npm install markindex          # library + CLI (Node.js >= 22.12)
+npx markindex --help
+```
+
+From source:
+
+```bash
+git clone https://github.com/jackjin1997/markindex.git && cd markindex
+npm install && npm run build
+node dist/cli.js --help          # or `npm link` to get a `markindex` command
+```
+
+Optional, only for LLM features (summaries, `retrieve()`, agent-mode evaluation): install the
+[AI SDK provider](https://ai-sdk.dev/providers) you want, e.g. `npm install @ai-sdk/anthropic`, and set its API key
+(`ANTHROPIC_API_KEY`, …). With `npx` and no local install, put the provider in the same temporary environment:
+
+```bash
+npx -p markindex -p @ai-sdk/anthropic markindex index ./docs --summary-model anthropic:claude-sonnet-5-5
+```
+
+## CLI
+
+```bash
+# Build or incrementally update the index (stored in <root>/.markindex/index.db)
+markindex index ./docs
+markindex index ./docs --summary-model anthropic:claude-sonnet-5-5 --estimate   # cost estimate, no LLM calls
+markindex index ./docs --summary-model anthropic:claude-sonnet-5-5             # with node summaries
+markindex index ./docs --summary-model anthropic:claude-sonnet-5-5 --summary-rpm 50 --summary-tpm 40000   # rate-limited
+markindex index ./docs --json                                                   # machine-readable report
+markindex index ./docs --no-setext                                              # converter output with `---` page breaks
+markindex index ./docs --ignore 'drafts/' --ignore '*.tmp.md'                   # extra ignore patterns (repeatable)
+
+# Inspect one document's section tree
+markindex tree ./docs guides/setup.md --depth 3
+
+# Keyword search (BM25 + trigram fallback)
+markindex search ./docs "离职 补偿"
+markindex search ./docs "ISO-27001 audit" --folder policies --limit 5 --json
+
+# Serve the library to agents over MCP (stdio, read-only)
+markindex mcp ./docs
+markindex mcp ./docs --reindex-on-start
+
+# Measure retrieval quality on your own documents (see "Evaluation")
+markindex eval ./docs --questions questions.jsonl --mode bm25 --out report.json
+markindex eval ./docs --questions questions.jsonl --mode agent --model anthropic:claude-sonnet-5-5 --out report.json
+markindex eval ./docs --questions questions.jsonl --mode agent --model anthropic:claude-sonnet-5-5 --rpm 50 --tpm 40000
+```
+
+**Rate limits.** Calls are always bounded by concurrency (`--concurrency` for eval; 8 for summaries) and the AI SDK's
+retries, which already honor `retry-after` / `retry-after-ms` on 429 / 5xx responses (up to 60 s). To stay under a
+provider quota, add a client-side limit: `--summary-rpm` / `--summary-tpm` on `index`, `--rpm` / `--tpm` on `eval`
+(agent mode). `rpm` caps requests and `tpm` caps *estimated input* tokens in any sliding 60 s window; both are
+shared by all documents (index) or all questions (eval) of the run, and waiting callers are served in order. Input
+tokens are estimated from the prompt text (about 1 token per CJK character, otherwise 4 characters), so leave
+headroom below the real quota; output tokens and tool definitions are not counted. A single request larger than the
+`tpm` still goes through, alone. In the library, pass `rateLimit: { requestsPerMinute, inputTokensPerMinute }` to
+`summarizeDocument` / `indexLibrary({ summary })` or `retrieve()`; reuse the same object to share one budget.
+
+`--summary-model` / `--model` take `<provider>:<model>`; MarkIndex imports `@ai-sdk/<provider>` dynamically, so
+only the provider you use needs to be installed. Every command accepts `--db <file>` to use a different index file.
+Put the index on a local disk: the writer lock uses hard links and SQLite runs in WAL mode, neither of which works
+reliably on exFAT, SMB / NFS shares or other network volumes. If the documents live there, keep them in place and
+point `--db` at a local path (e.g. `--db ~/.cache/markindex/docs.db`).
+
+Indexing is incremental: files are compared by content hash, only new or changed documents are rebuilt (one
+transaction per document), deleted files are removed, and summaries are cached by node content hash. Scanned files
+are `.md` and `.markdown` (not `.mdx`). Ignored are: paths matched by the library's own `.gitignore` files (at any
+level under the root; global git excludes and `.git/info/exclude` are not read), by an optional `.markindexignore`
+file at the root (same syntax, for rules you don't want in `.gitignore`) and by `--ignore <pattern>` options
+(gitignore syntax, relative to the root, repeatable; `ignore: string[]` in the library API; also accepted by
+`mcp --reindex-on-start`; a leading `./` is dropped, a leading `/` anchors at the root as in gitignore, and
+absolute filesystem paths or `../` are rejected). The sources only add exclusions: a `!` rule in `.markindexignore`
+or `--ignore` cannot re-include a path that a `.gitignore` excludes. A document that becomes ignored is removed
+from the index on the next run.
+Dot-directories such as `.git` and `.markindex` are skipped and symlinks are not followed. One writer at a time (a
+second `index` fails fast); readers such as the MCP server are never blocked. When the index was built by an older MarkIndex version or with different
+tree parameters, the next `index` run rebuilds every document (`full rebuild: …` in the output, `rebuilt` in the
+report); cached summaries are reused.
+
+**Page breaks from PDF / HTML converters.** Converters often emit a bare `---` line right after a page's last
+paragraph. CommonMark reads that as a setext heading, so the whole paragraph becomes a level-2 heading. `index`
+warns when it sees setext h2 headings whose text spans several lines or exceeds 120 characters (`warning <path>: …`
+on stdout, `warnings` in the JSON report). For such libraries pass `--no-setext` (also on
+`mcp --reindex-on-start`; `setext: false` in the library API): `---` then is a thematic break and `===` plain text.
+The setting is stored in the index: later runs without the flag (`index`, `mcp --reindex-on-start`,
+`Library.index()`) keep it; pass `--setext` (`setext: true`) to turn it back on. Changing it rebuilds the index.
+
+## Library usage
+
+```ts
+import { openLibrary } from 'markindex'
+
+// Index (or update) a folder, then open it read-only for agents.
+const writer = openLibrary({ root: './docs', readonly: false })
+await writer.index() // { added, updated, removed, unchanged, skipped, summaryErrors, warnings }
+writer.close()
+
+const lib = openLibrary({ root: './docs' })
+const envelope = await lib.callTool('search_documents', { query: 'termination notice period' })
+console.log(JSON.parse(envelope)) // { success: true, data: { results: [...] }, next_steps: {...} }
+```
+
+### With the Vercel AI SDK (your agent drives the tools)
+
+```ts
+import { generateText, stepCountIs } from 'ai'
+import { anthropic } from '@ai-sdk/anthropic'
+import { AGENT_INSTRUCTIONS, createAiSdkTools, openLibrary } from 'markindex'
+
+const lib = openLibrary({ root: './docs' })
+const { text } = await generateText({
+  model: anthropic('claude-sonnet-5-5'),
+  system: AGENT_INSTRUCTIONS, // browse first, read structure, cite [doc_path, L<start>-<end>]
+  tools: createAiSdkTools(lib),
+  stopWhen: stepCountIs(15),
+  prompt: 'What notice period applies when an employee resigns?',
+})
+```
+
+### Built-in evidence retrieval
+
+`retrieve()` runs the same tools in a model loop and returns **evidence only** (original text re-read from the
+index by line range — never model-written text), plus usage and why it stopped:
+
+```ts
+import { anthropic } from '@ai-sdk/anthropic'
+import { openLibrary, retrieve } from 'markindex'
+
+const lib = openLibrary({ root: './docs' })
+const { evidence, usage, stopReason } = await retrieve(lib, 'What notice period applies when an employee resigns?', {
+  model: anthropic('claude-sonnet-5-5'),
+  maxSteps: 12,
+  maxInputTokens: 200_000,
+  rateLimit: { requestsPerMinute: 50, inputTokensPerMinute: 40_000 }, // optional, client-side
+})
+for (const e of evidence) console.log(`${e.docPath} L${e.startLine}-${e.endLine}`, e.titlePath.join(' > '))
+```
+
+Lower-level building blocks are exported too: `parseMarkdown`, `buildTree`, `optimizeTree`, `openStore`,
+`indexLibrary`, `searchNodes`, `summarizeDocument`, `estimateSummaryCost`, `TOOL_CONTRACT`, `createMcpServer`.
+
+## MCP setup
+
+Build the index first (`npx markindex index /path/to/docs`), then register the server.
+
+**Claude Code**
+
+```bash
+claude mcp add markindex -- npx markindex mcp /path/to/docs
+```
+
+**Cursor** (`~/.cursor/mcp.json` or `.cursor/mcp.json` in a project)
+
+```json
+{
+  "mcpServers": {
+    "markindex": {
+      "command": "npx",
+      "args": ["markindex", "mcp", "/path/to/docs"]
+    }
+  }
+}
+```
+
+The server sends `AGENT_INSTRUCTIONS` as its MCP instructions and marks all tools read-only.
+
+## Tools
+
+| Tool | What it does |
+|---|---|
+| `browse_documents` | List a folder: sub-folders and documents (path, title, description, tokens), paginated. The default discovery tool. |
+| `get_document` | One document's metadata, front matter, size and `read_structure_first` (true above 15 000 tokens). |
+| `get_document_structure` | The section outline (node ids, titles, line ranges, tokens, summaries — no body), `depth` levels (default 2), optionally only one node's subtree; paginated with `part`. |
+| `get_node_content` | Original Markdown of 1-20 nodes with line numbers (`123│ …`), paginated for long nodes. |
+| `search_documents` | BM25 keyword search over titles, merged sub-headings, summaries and text, with trigram fallback; scope by `folder` or `doc_path`. The escalation path when browsing stalls. |
+
+All tools return `{ success: true, data, next_steps }` or `{ success: false, error, code, details, next_steps }` as a JSON string
+and never throw — unknown document paths come back with similar-name suggestions, stale node ids with a
+"re-fetch the structure" hint. Arguments that do not match the schema (wrong type, unknown field, a string over
+1 024 characters) get an `INVALID_INPUT` envelope from `lib.callTool`; over MCP and the AI SDK, however, the
+transport validates arguments against the same schema before the tool runs, so there such errors come back as the
+transport's own error (an MCP tool error / an AI SDK invalid-tool-input error), not as the JSON envelope.
+
+## Security and trust
+
+- **Read-only, library-scoped.** The tools only read the index of the library you pointed them at; they never read
+  files outside it, never write, and never fetch URLs. Paths and node ids are looked up in the index, not on disk.
+- **Documents are untrusted input.** Tool output is raw document text, and LLM summaries / descriptions are
+  generated from it. A document can contain text written to steer an agent (prompt injection). Treat everything
+  the tools return as data, not instructions — especially when the library holds third-party or user-supplied
+  files, and when the agent also has tools that act (send, write, execute).
+- Summaries and `retrieve()` send document text to the model provider you configure; indexing alone makes no
+  network calls.
+
+## Design notes
+
+- **Tree.** Top-level mdast headings define sections; text before the first heading is a `preface` node; a
+  document without headings is one root node. Node ids are short hashes of the ancestor title chain plus the
+  sibling ordinal, so they stay stable while the outline does.
+- **Tree optimization.** The merge / expand pass is a port of PageIndex's `tree_optimize.py` cost model with pages
+  replaced by tokens: a subtree is merged into its parent when reading it whole is no more expensive than routing
+  through it (`S(v) <= tree_cost(v)`, routing cost `R = 150` tokens), merged-away headings are kept as `keyItems`,
+  and a node whose own text exceeds `MAX_NODE_TOKENS` (8 000) is split deterministically at Markdown block
+  boundaries into `chunk` children. Credit for the cost model goes to PageIndex.
+- **Search.** Only a node's own text is indexed (not its subtree), so parents and children don't double-count.
+  Titles weigh 10×, merged headings 5×, summaries 2×, body 1×.
+- **Tool contract.** One zod contract (names, descriptions, schemas) feeds the AI SDK adapter, the MCP server and
+  `retrieve()`; descriptions and agent instructions are adapted from PageIndex's `agent_tools.py` (pages → line
+  ranges) and pinned by snapshot tests.
+- **Tokens.** Counts are approximate (CJK character = 1 token, other characters / 4); a real tokenizer can be
+  injected. Defaults live in `src/config.ts`.
+
+## Evaluation
+
+`markindex eval` measures whether retrieval lands on the right section: give it a library folder and a JSONL
+question file with gold locations (document path + line range, or a title path), and it reports hit rate, token
+cost, steps and evidence size / precision overall and per language, for the BM25 baseline and for the LLM agent
+(`--max-evidence-lines <n>` additionally requires hits to be at most `n` lines long). The question-file format,
+the hit rule, the development corpus (329 public documents, ~38 M tokens, English filings and guidelines plus
+Chinese laws and encyclopedia articles), baseline results and scale benchmarks are in
+[eval/README.md](eval/README.md).
+
+**Headline (n = 30, Claude Code sonnet restricted to the five MarkIndex MCP tools):** the gold passage was found for
+**30/30** questions (93.3% at rank 1), citing 15 lines on average with 88.6% of cited lines inside the gold range — vs
+73.3% hit and 209 lines (12.8% precision) for BM25 top-5 on the same questions, at ~3 agent turns per question
+([details and caveats](eval/README.md#7-results-agent-mode-via-claude-code)).
+
+## Development
+
+```bash
+npm install
+npm test            # vitest
+npm run typecheck
+npm run build
+npx tsx eval/fetch-corpus.ts   # download the evaluation corpus (eval/corpus, not committed)
+npx tsx eval/bench.ts          # scale benchmark
+npx tsx eval/claude-code-eval.ts --root eval/corpus --questions eval/questions.jsonl --per-lang 10 --out cc.json
+                               # agent eval with Claude Code over MCP (spends Claude usage; eval/README.md §7)
+```
+
+## License
+
+[MIT](LICENSE). PageIndex is also MIT-licensed; MarkIndex contains no PageIndex code, but ports its tree cost
+model and adapts its agent-tool descriptions, instructions and prompts, with thanks — see
+[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for its copyright and license notice.

package/README.zh-CN.md

@@ -0,0 +1,164 @@
+# MarkIndex
+
+[![CI](https://github.com/jackjin1997/markindex/actions/workflows/ci.yml/badge.svg)](https://github.com/jackjin1997/markindex/actions/workflows/ci.yml) [![npm](https://img.shields.io/npm/v/markindex)](https://www.npmjs.com/package/markindex)
+
+**面向 Agent 的无向量、结构优先的长篇 Markdown 文档库检索。**
+
+MarkIndex 把每篇 Markdown 按其自身章节解析成树，把树、原文行和中英混排 BM25 索引存进一个 SQLite 文件，
+再给 Agent 提供 5 个只读工具，沿"库 → 文件夹 → 文档 → 章节"逐层导航，并带行号读取原文。
+不用 embedding、不用向量库、不靠猜测切块；文档自身的结构就是索引，每条证据都能以 `[doc_path, L起-止]` 的格式引用回原文。
+
+> 受 [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex) 启发，MarkIndex 是一个独立、非官方的
+> TypeScript 实现，专注 Markdown；与 VectifyAI 无隶属或背书关系。
+
+English: [README.md](README.md)
+
+## 为什么
+
+- **结构优先**：长篇专业文档（年报、临床指南、法规、手册）本来就有作者组织好的目录；沿目录导航保留上下文、可解释、可引用。
+- **建索引确定、不调 LLM**：mdast 解析、建树、token 成本模型优化都不调模型，同一输入结果逐字节一致。LLM 只用于可选的节点摘要、文档描述和内置 `retrieve()`。
+- **认真处理 Markdown**：Setext 标题、首个标题前的前言、无标题文档、frontmatter、代码块里的 `#`、标题跳级、BOM/CRLF；每一行原文都恰好属于一个节点，过大的章节按块边界确定性拆成 `chunk` 节点。
+- **中文可用的关键词兜底**：基于 `Intl.Segmenter` 分词的 BM25（中文、英文、中英混排，`C++`、`ISO-27001`、`v2.1` 这类标识符），另有 trigram 子串索引补位。
+- **响应有预算**：工具从不抛异常，返回带 `next_steps` 指引的 JSON 信封，单次响应不超过 95 000 字符，大目录与长章节自动分段。
+
+## 安装
+
+```bash
+npm install markindex          # 库 + CLI（Node.js >= 22.12）
+npx markindex --help
+```
+
+从源码安装：
+
+```bash
+git clone https://github.com/jackjin1997/markindex.git && cd markindex
+npm install && npm run build
+node dist/cli.js --help          # 或 `npm link` 得到 `markindex` 命令
+```
+
+只有用到 LLM 功能（摘要、`retrieve()`、agent 模式评测）时才需要安装对应的 AI SDK provider，例如 `npm install @ai-sdk/anthropic`，并设置 `ANTHROPIC_API_KEY`。
+不在本地安装、直接用 `npx` 时，把 provider 放进同一个临时环境：
+
+```bash
+npx -p markindex -p @ai-sdk/anthropic markindex index ./docs --summary-model anthropic:claude-sonnet-5-5
+```
+
+## CLI
+
+```bash
+markindex index ./docs                                                       # 建立 / 增量更新索引（<root>/.markindex/index.db）
+markindex index ./docs --summary-model anthropic:claude-sonnet-5-5 --estimate # 只估算摘要成本，不调 LLM
+markindex index ./docs --summary-model anthropic:claude-sonnet-5-5 --summary-rpm 50 --summary-tpm 40000 # 摘要限速
+markindex index ./docs --no-setext                                           # 转换器产物里 `---` 是分页符时使用
+markindex index ./docs --ignore 'drafts/' --ignore '*.tmp.md'                # 额外的忽略规则（可重复）
+markindex tree ./docs 指南/安装.md --depth 3                                  # 查看一篇文档的章节树
+markindex search ./docs "离职 补偿" --limit 5 --json                          # 关键词检索
+markindex mcp ./docs                                                         # 以 MCP server（stdio，只读）提供给 Agent
+markindex eval ./docs --questions questions.jsonl --mode bm25 --out report.json
+markindex eval ./docs --questions questions.jsonl --mode agent --model anthropic:claude-sonnet-5-5 --out report.json
+markindex eval ./docs --questions questions.jsonl --mode agent --model anthropic:claude-sonnet-5-5 --rpm 50 --tpm 40000
+```
+
+**限速**：调用默认受并发数（eval 的 `--concurrency`；摘要为 8）和 AI SDK 重试约束，后者已经会遵循 429 / 5xx 响应里的
+`retry-after` / `retry-after-ms`（上限 60 秒）。要压在服务商配额以内，可再加客户端限速：`index` 用 `--summary-rpm` / `--summary-tpm`，
+`eval`（agent 模式）用 `--rpm` / `--tpm`。`rpm` 限制任意滑动 60 秒窗口内的请求数，`tpm` 限制窗口内**估算的输入** token 数；
+同一次运行的所有文档（index）或所有问题（eval）共用一份额度，排队的调用按先后顺序放行。输入 token 按提示词文本估算
+（中日韩字符约 1 个 token 一个字，其余约 4 个字符 1 个 token），请在真实配额下留出余量；输出 token 与工具定义不计入。
+单个请求超过 `tpm` 时仍会放行（独占整个窗口）。库 API 中给 `summarizeDocument` / `indexLibrary({ summary })` 或 `retrieve()`
+传 `rateLimit: { requestsPerMinute, inputTokensPerMinute }`；复用同一个对象即共用额度。
+
+所有命令都接受 `--db <file>` 指定索引文件。索引请放在本地磁盘：写锁依赖硬链接，SQLite 使用 WAL 模式，二者在 exFAT、
+SMB / NFS 共享或其他网络卷上都不可靠；文档放在这类卷上时，文档原地不动，用 `--db` 指向本地路径
+（如 `--db ~/.cache/markindex/docs.db`）。
+
+索引是增量的：按内容哈希比较，只重建新增或改动的文档，删除的文件会被清理，摘要按节点内容哈希缓存。扫描 `.md` 与
+`.markdown`（不含 `.mdx`）；以下路径会被忽略：文档库自身各级目录下 `.gitignore` 文件匹配的路径（不读取全局 git excludes 与
+`.git/info/exclude`）、库根目录下可选的 `.markindexignore` 文件（语法相同，用于不想写进 `.gitignore` 的规则）匹配的路径，
+以及 `--ignore <pattern>` 选项（gitignore 语法，相对库根，可重复；库 API 为 `ignore: string[]`；`mcp --reindex-on-start` 同样支持）
+匹配的路径（开头的 `./` 会被去掉，开头的 `/` 与 gitignore 一样表示锚定到库根，绝对文件系统路径或 `../` 会报错）。
+这几类来源只会增加排除：`.markindexignore` 或 `--ignore` 中的 `!` 规则不能重新包含被 `.gitignore` 排除的路径。
+变为被忽略的文档会在下一次运行时从索引中删除。跳过 `.git`、`.markindex` 等点目录，不跟随符号链接。同一时间只允许一个写进程（第二个 `index`
+立即失败），MCP server 等读取方不受阻塞。索引由旧版本 MarkIndex 或不同的建树参数生成时，下一次 `index` 会重建全部文档
+（输出 `full rebuild: …`，报告中带 `rebuilt`），已缓存的摘要会被复用。
+
+**PDF / HTML 转换产物的分页符**：转换器常在一页最后一段之后紧跟一行 `---` 作为分页，CommonMark 会把它读成 setext 标题，
+整段文字变成二级标题。`index` 发现文字跨多行或超过 120 个字符的 setext 二级标题时会给出警告（标准输出 `warning <path>: …`，
+JSON 报告中的 `warnings`）。这类文档库请加 `--no-setext`（`mcp --reindex-on-start` 同样支持；库 API 为 `setext: false`）：
+`---` 按分隔线、`===` 按普通文字处理。该设置保存在索引中：之后不带该参数的运行（`index`、`mcp --reindex-on-start`、`Library.index()`）会沿用它；
+要重新开启请传 `--setext`（库 API 为 `setext: true`）。改变该设置会重建整个索引。
+
+## 库用法
+
+```ts
+import { generateText, stepCountIs } from 'ai'
+import { anthropic } from '@ai-sdk/anthropic'
+import { AGENT_INSTRUCTIONS, createAiSdkTools, openLibrary, retrieve } from 'markindex'
+
+const writer = openLibrary({ root: './docs', readonly: false })
+await writer.index()
+writer.close()
+
+const lib = openLibrary({ root: './docs' })
+await lib.callTool('search_documents', { query: '劳动合同 解除' }) // JSON 信封字符串
+
+// 方式一：你的 Agent 自己调用工具
+await generateText({ model: anthropic('claude-sonnet-5-5'), system: AGENT_INSTRUCTIONS, tools: createAiSdkTools(lib), stopWhen: stepCountIs(15), prompt: '员工辞职需要提前多久通知？' })
+
+// 方式二：内置检索循环，只返回原文证据（按行号从索引回读），不生成答案
+const { evidence } = await retrieve(lib, '员工辞职需要提前多久通知？', { model: anthropic('claude-sonnet-5-5') })
+```
+
+## 接入 MCP
+
+先 `npx markindex index /path/to/docs`，然后：
+
+- Claude Code：`claude mcp add markindex -- npx markindex mcp /path/to/docs`
+- Cursor（`~/.cursor/mcp.json`）：
+
+```json
+{ "mcpServers": { "markindex": { "command": "npx", "args": ["markindex", "mcp", "/path/to/docs"] } } }
+```
+
+## 工具
+
+`browse_documents`（按文件夹浏览，默认发现入口）、`get_document`（元数据与是否需要先看结构）、
+`get_document_structure`（不含正文的章节树，默认 2 层，可只展开某节点）、`get_node_content`（带行号原文，长节点分段）、
+`search_documents`（BM25 + trigram 兜底，可限定文件夹或文档）。
+
+工具从不抛异常，返回 `{ success, data | error, code, details, next_steps }` JSON 信封。参数不符合 schema（类型错误、未知字段、
+字符串超过 1 024 字符）时，`lib.callTool` 返回 `INVALID_INPUT` 信封；但经 MCP 与 AI SDK 调用时，传输层会先按同一 schema
+校验参数，此类错误以传输层自身的错误返回（MCP 工具错误 / AI SDK 的 invalid tool input 错误），而不是 JSON 信封。
+
+## 安全与信任
+
+- **只读、限定在文档库内**：工具只读取你指定的那个文档库的索引，从不读取库外文件、不写入、不访问网络；路径与节点 id 都在索引中查找，而不是直接访问磁盘。
+- **文档是不可信输入**：工具输出是文档原文，LLM 摘要 / 描述也由原文生成。文档中可能含有试图操纵 Agent 的文字（提示词注入）。请把工具返回的一切当作数据而非指令——尤其当文档库包含第三方或用户提交的文件，且 Agent 同时拥有发送、写入、执行等有副作用的工具时。
+- 摘要与 `retrieve()` 会把文档内容发送给你配置的模型服务商；单纯建索引不产生任何网络请求。
+
+## 设计要点
+
+- 树优化移植 PageIndex `tree_optimize.py` 的成本模型（页 → token）：读整棵子树不比路由更贵就合并（`S(v) <= tree_cost(v)`，路由成本 150 token），被合并的标题保留为 `keyItems`；自身正文超过 8 000 token 的节点按块边界拆成 `chunk`。成本模型归功于 PageIndex。
+- 只索引节点自身正文，父子节点不会因同一段文字重复命中；标题权重 10、合并标题 5、摘要 2、正文 1。
+- 工具契约只有一份（zod），AI SDK 适配、MCP server 与 `retrieve()` 都是它的消费者。
+
+## 评测（供内部验收）
+
+问题文件为 JSONL，每行一个问题：
+
+```json
+{"id": "q1", "question": "员工辞职需要提前多久通知？", "lang": "zh", "gold": [{"doc_path": "人事/员工手册.md", "start_line": 120, "end_line": 134}]}
+```
+
+`gold` 中任一位置与返回证据的行区间有交集即算命中；没有行号时按 `title_path`（从顶层章节到答案节点的标题链）前缀匹配；
+只有 `doc_path` 时命中该文档任意位置即可。报告还给出每题的证据规模（行数、近似 token）与精度（证据行落在 gold 行区间内的比例），
+并按语言汇总；`--max-evidence-lines <n>` 要求命中的证据条目不超过 `n` 行。完整格式、命中规则、开发期语料、BM25 基线结果与规模基准见
+[eval/README.md](eval/README.md)。
+
+**核心结果（n = 30，Claude Code sonnet，只开放 MarkIndex 的 5 个 MCP 工具）**：30/30 题都找到了标准答案段落（首位命中 93.3%），
+平均引用 15 行，其中 88.6% 落在标准答案区间内；同一批问题上 BM25 top-5 命中 73.3%、平均 209 行（精度 12.8%）。Agent 平均每题约 3 轮
+（[详情与局限](eval/README.md#7-results-agent-mode-via-claude-code)）。
+
+## 许可
+
+[MIT](LICENSE)。PageIndex 同为 MIT；MarkIndex 不含其代码，但移植了其树成本模型、改写了其 Agent 工具描述、指令与提示词，特此致谢；
+其版权与许可声明见 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)。

package/THIRD_PARTY_NOTICES.md

@@ -0,0 +1,36 @@
+# Third-party notices
+
+MarkIndex is an independent TypeScript implementation. It contains no PageIndex source code, but parts of it are
+adapted from [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex):
+
+- the agent instructions (`AGENT_INSTRUCTIONS`) and the tool descriptions in `src/tools/contract.ts`, and the
+  summarization prompts in `src/llm/summarize.ts`, are adapted from PageIndex's agent tools and prompts
+  (pages replaced by line ranges);
+- the tree-optimization cost model in `src/tree/optimize.ts` (merge / expand by token cost) is a port of
+  PageIndex's `tree_optimize.py`, with pages replaced by tokens.
+
+PageIndex is distributed under the following license:
+
+```
+MIT License
+
+Copyright (c) 2025 Vectify AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```

package/dist/adapters/ai-sdk.d.ts

@@ -0,0 +1,8 @@
+import { type Tool } from 'ai';
+import type { Library } from '../library.js';
+import { type ToolName } from '../tools/contract.js';
+/**
+ * The five agent tools as Vercel AI SDK tools, backed by `lib.callTool`. Each `execute` resolves to the JSON
+ * envelope string (never rejects), so the model always sees a structured result.
+ */
+export declare function createAiSdkTools(lib: Library): Record<ToolName, Tool>;

package/dist/adapters/ai-sdk.js

@@ -0,0 +1,20 @@
+import { tool } from 'ai';
+import { TOOL_CONTRACT } from '../tools/contract.js';
+/**
+ * The five agent tools as Vercel AI SDK tools, backed by `lib.callTool`. Each `execute` resolves to the JSON
+ * envelope string (never rejects), so the model always sees a structured result.
+ */
+export function createAiSdkTools(lib) {
+    const entries = Object.keys(TOOL_CONTRACT).map((name) => {
+        const contract = TOOL_CONTRACT[name];
+        return [
+            name,
+            tool({
+                description: contract.description,
+                inputSchema: contract.input,
+                execute: (args) => lib.callTool(name, args),
+            }),
+        ];
+    });
+    return Object.fromEntries(entries);
+}

package/dist/adapters/mcp.d.ts

@@ -0,0 +1,6 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { Library } from '../library.js';
+/** Package version, read from package.json (same relative location from src/ and dist/). */
+export declare function packageVersion(): string;
+/** MCP server exposing the five agent tools over `lib`. Connect it to a transport (stdio for `markindex mcp`). */
+export declare function createMcpServer(lib: Library): McpServer;

package/dist/adapters/mcp.js

@@ -0,0 +1,28 @@
+import { readFileSync } from 'node:fs';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { AGENT_INSTRUCTIONS, TOOL_CONTRACT } from '../tools/contract.js';
+/** Package version, read from package.json (same relative location from src/ and dist/). */
+export function packageVersion() {
+    const url = new URL('../../package.json', import.meta.url);
+    return JSON.parse(readFileSync(url, 'utf8')).version;
+}
+/** MCP server exposing the five agent tools over `lib`. Connect it to a transport (stdio for `markindex mcp`). */
+export function createMcpServer(lib) {
+    const server = new McpServer({ name: 'markindex', version: packageVersion() }, { instructions: AGENT_INSTRUCTIONS });
+    for (const name of Object.keys(TOOL_CONTRACT)) {
+        const contract = TOOL_CONTRACT[name];
+        server.registerTool(name, { description: contract.description, inputSchema: contract.input.shape, annotations: { readOnlyHint: true } }, async (args) => {
+            const text = await lib.callTool(name, args);
+            return { content: [{ type: 'text', text }], isError: !isSuccess(text) };
+        });
+    }
+    return server;
+}
+function isSuccess(envelope) {
+    try {
+        return JSON.parse(envelope).success === true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+export declare function buildProgram(): Command;