@fastrag/pageindex 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 FastRAG Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,251 @@
+# PageIndex TS SDK
+
+[中文文档](./README.zh-CN.md)
+
+> TypeScript rewrite of [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex)
+
+Traditional vector-based RAG relies on semantic *similarity* rather than true *relevance* — and similarity ≠ relevance. [PageIndex](https://pageindex.ai/blog/pageindex-intro) takes a different approach: it builds a **hierarchical tree index** (like a "table of contents") from documents and uses LLM **reasoning** to navigate that tree for retrieval. No chunking, no vector DB required — just structured, human-like document navigation.
+
+This SDK is a TypeScript implementation of the PageIndex framework, designed for Node.js/TypeScript RAG pipelines with full runtime decoupling.
+
+## Highlights
+
+- **Reasoning-based retrieval** — builds a hierarchical tree index from documents, enabling LLM-driven tree search instead of vector similarity
+- **LLM-powered TOC detection** — automatically detects table of contents and chooses the optimal processing mode
+- **Three processing modes** — `toc_with_page_numbers`, `toc_no_page_numbers`, `no_toc` with auto-degradation
+- **Markdown to tree** — `mdToTree()` for structured tree conversion from Markdown
+- **Optional vector enhancement** — indexes tree structures into a vector store for fast segment retrieval within a document, and for multi-document collections where per-document LLM tree-search is impractical
+- **Runtime decoupled** — bring your own LLM (`LlmProvider`), PDF parser (`DocumentParser`), and vector DB (`VectorStore` / `Embedder`)
+
+## Package Layout
+
+```text
+@fastrag/pageindex         ← document structure indexing SDK
+  exports:
+    .           → core pipeline
+    ./types     → shared types + defaults
+    ./vector    → vector enhancement
+  runtime deps:
+    - gpt-tokenizer
+```
+
+| Subpath | Description |
+| --- | --- |
+| `@fastrag/pageindex` | Document structure indexing pipeline |
+| `@fastrag/pageindex/types` | Shared type definitions and interface contracts |
+| `@fastrag/pageindex/vector` | Vector enhancement: chunking, indexing, search |
+
+## Requirements
+
+- Node.js >= 20
+- pnpm >= 10
+
+## Install
+
+For application usage:
+
+```bash
+pnpm add @fastrag/pageindex
+```
+
+For local development in this repository:
+
+```bash
+pnpm install
+pnpm build
+```
+
+## Quick Start
+
+### 1) Build Document Tree Index
+
+```ts
+import { pageIndex } from '@fastrag/pageindex';
+import type { LlmProvider, PageContent } from '@fastrag/pageindex/types';
+
+// pageIndex is 1-based
+const pages: PageContent[] = [
+  { pageIndex: 1, text: 'Table of Contents\n1 Introduction ... 1' },
+  { pageIndex: 2, text: '1 Introduction\nThis document ...' },
+  { pageIndex: 3, text: '2 Methods\nWe propose ...' },
+];
+
+const provider: LlmProvider = {
+  async chat(messages) {
+    // Plug in OpenAI / Claude / local model here
+    return { content: '{}', finishReason: 'stop' };
+  },
+};
+
+const result = await pageIndex(
+  pages,
+  {
+    docName: 'example.pdf',
+    addNodeId: true,
+    addNodeSummary: true,
+    addDocDescription: true,
+  },
+  provider,
+);
+
+console.log(result.metadata.processingMode);
+console.log(result.structure);
+```
+
+### 2) Markdown to Tree
+
+```ts
+import { mdToTree } from '@fastrag/pageindex';
+
+const tree = mdToTree('# Intro\nhello\n## Background\nmore context', {
+  thinning: true,
+  minTokenThreshold: 500,
+});
+```
+
+### 3) Vector Enhancement
+
+```ts
+import { VectorEnhancer, InMemoryAdapter, treeChunker } from '@fastrag/pageindex/vector';
+import type { Embedder } from '@fastrag/pageindex/vector';
+
+const store = new InMemoryAdapter();
+
+const embedder: Embedder = {
+  dimension: 3,
+  async embed(texts: string[]) {
+    // Replace with real embedding service
+    return texts.map((t) => [t.length, t.length / 2, 1]);
+  },
+};
+
+const enhancer = new VectorEnhancer(
+  store,
+  embedder,
+  treeChunker, // you can replace this with a custom chunking strategy
+  { chunkMaxTokens: 1000 },
+);
+
+await enhancer.index(result); // result from pageIndex()
+const hits = await enhancer.search('introduction section');
+```
+
+### 4) Hybrid Search
+
+```ts
+import { HybridSearch } from '@fastrag/pageindex/vector';
+
+const hybrid = new HybridSearch(enhancer, {
+  vectorTopK: 20,
+  rerankTopK: 5,
+});
+
+const results = await hybrid.search('what is the main contribution?');
+```
+
+## Core Pipeline
+
+```text
+PageContent[] → TOC Detection → Mode Selection → Verify/Fix/Degrade → TreeNode[] → Post-processing → PageIndexResult
+```
+
+1. Scan first N pages for table of contents
+2. Choose processing mode based on TOC presence and page numbers
+3. Verify TOC accuracy, auto-fix or degrade to a simpler mode if needed
+4. Build `TreeNode[]` tree structure
+5. Optional post-processing: `nodeId`, `nodeText`, `summary`, `docDescription`
+
+## Vector Enhancement
+
+The original PageIndex uses LLM-based tree search for retrieval — the entire tree structure is sent to an LLM as a prompt, and the LLM reasons about which nodes are relevant. This works well but means every query requires an LLM call.
+
+The `@fastrag/pageindex/vector` subpath export offers an alternative: convert the tree into vector embeddings so retrieval becomes a similarity search — no LLM needed at query time, with millisecond-level latency.
+
+| | Indexing | Retrieval |
+| --- | --- | --- |
+| Tree search (original) | LLM builds tree | LLM reasons over tree per query |
+| Vector enhancement (this SDK) | LLM builds tree → embed into vector store | Vector similarity search (no LLM) |
+
+This covers two scenarios:
+
+- **Within a single document** — chunk tree nodes into embeddings for fast, precise segment retrieval without LLM calls
+- **Across many documents** — index multiple documents into a shared vector store, then search across all of them at once; avoids running LLM tree-search on every document
+
+The flow: `PageIndexResult` → `Chunker` (default: `treeChunker`) → `Embedder` (generates vectors) → `VectorStore` (upsert/search). `treeChunker` currently uses paragraph splitting with a char-based heuristic threshold (`1 token ≈ 4 chars`). Each chunk retains its tree metadata (`docName`, `nodeId`, `title`, page range), so results stay traceable to the original document structure.
+
+## Interface Abstractions
+
+### LlmProvider
+
+```ts
+interface LlmProvider {
+  chat(messages: LlmMessage[], options?: LlmOptions): Promise<LlmResponse>;
+  chatWithSchema?(
+    messages: LlmMessage[],
+    schema: JsonSchema,
+    options?: LlmOptions,
+  ): Promise<LlmResponse>;
+}
+```
+
+### DocumentParser
+
+```ts
+interface DocumentParser {
+  parse(input: string | ArrayBuffer | Uint8Array): Promise<PageContent[]>;
+}
+```
+
+### Embedder
+
+```ts
+interface Embedder {
+  embed(texts: string[]): Promise<number[][]>;
+  readonly dimension: number;
+}
+```
+
+### Chunker
+
+```ts
+type Chunker = (result: PageIndexResult, config?: VectorConfig) => Chunk[];
+```
+
+`treeChunker` is the default strategy implementation you can pass in, but `VectorEnhancer` requires explicit chunker injection so you can replace it with a model-specific strategy when needed.
+
+## Configuration
+
+| Field | Default | Description |
+| --- | ---: | --- |
+| `tocCheckPageNum` | `20` | Max pages scanned for TOC detection |
+| `maxPageNumEachNode` | `10` | Page threshold for large-node recursive splitting |
+| `maxTokenNumEachNode` | `20000` | Token threshold for large-node recursive splitting |
+| `addNodeId` | `true` | Generate 4-digit node IDs |
+| `addNodeSummary` | `true` | Generate node summaries via LLM |
+| `addDocDescription` | `false` | Generate one-sentence document description |
+| `addNodeText` | `false` | Keep raw node text in output |
+| `retryConfig` | exp. backoff | LLM retry strategy (`maxRetries: 10`, `initialDelayMs: 1000`, `maxDelayMs: 30000`, `backoffMultiplier: 2`) |
+| `onDegradation` | `undefined` | Callback when processing mode degrades |
+| `logger` | silent | Custom logger injection |
+
+## Output
+
+```ts
+interface PageIndexResult {
+  docName: string;
+  docDescription?: string;
+  structure: TreeNode[];
+  metadata: {
+    processingMode: ProcessingMode;
+    degradations: DegradationEvent[];
+  };
+}
+```
+
+## Further Reading
+
+- [Retrieval Strategies](./docs/retrieval-strategies.md) — comparison of LLM tree search, MCTS, and vector-based retrieval, with a hybrid architecture design for large-scale document collections
+
+## License
+
+[MIT](./LICENSE)

package/README.zh-CN.md

@@ -0,0 +1,251 @@
+# PageIndex TS SDK
+
+[English](./README.md)
+
+> [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex) 的 TypeScript 重写版
+
+传统向量 RAG 依赖语义*相似度*而非真正的*相关性* — 但相似 ≠ 相关。[PageIndex](https://pageindex.ai/blog/pageindex-intro) 采用不同的思路：从文档中构建**层级树索引**（类似"目录"），然后利用 LLM **推理**在树上导航检索。无需分块，无需向量数据库 — 像人类专家一样结构化地阅读文档。
+
+本 SDK 是 PageIndex 框架的 TypeScript 实现，面向 Node.js/TypeScript RAG 场景，完全运行时解耦。
+
+## 特性
+
+- **基于推理的检索** — 从文档构建层级树索引，通过 LLM 驱动的树搜索替代向量相似度匹配
+- **LLM 驱动的 TOC 检测** — 自动检测目录并选择最优处理模式
+- **三种处理模式** — `toc_with_page_numbers`、`toc_no_page_numbers`、`no_toc`，支持自动降级
+- **Markdown 转树** — `mdToTree()` 将 Markdown 转换为结构化树
+- **可选向量增强** — 将树索引结果向量化，支持单文档内的精确片段检索，也适用于多文档场景下快速定位相关节点、避免逐一 LLM 树搜索
+- **运行时解耦** — 自带 LLM（`LlmProvider`）、PDF 解析器（`DocumentParser`）、向量数据库（`VectorStore` / `Embedder`）
+
+## 包结构
+
+```text
+@fastrag/pageindex         ← 文档结构索引 SDK
+  exports:
+    .           → 核心流程
+    ./types     → 共享类型 + 默认配置
+    ./vector    → 向量增强
+  runtime deps:
+    - gpt-tokenizer
+```
+
+| 子路径 | 说明 |
+| --- | --- |
+| `@fastrag/pageindex` | 文档结构索引主流程 |
+| `@fastrag/pageindex/types` | 共享类型定义与接口契约 |
+| `@fastrag/pageindex/vector` | 向量增强：分块、索引、搜索 |
+
+## 环境要求
+
+- Node.js >= 20
+- pnpm >= 10
+
+## 安装
+
+作为依赖安装：
+
+```bash
+pnpm add @fastrag/pageindex
+```
+
+在本仓库本地开发：
+
+```bash
+pnpm install
+pnpm build
+```
+
+## 快速开始
+
+### 1) 构建文档树索引
+
+```ts
+import { pageIndex } from '@fastrag/pageindex';
+import type { LlmProvider, PageContent } from '@fastrag/pageindex/types';
+
+// pageIndex 为 1-based
+const pages: PageContent[] = [
+  { pageIndex: 1, text: 'Table of Contents\n1 Introduction ... 1' },
+  { pageIndex: 2, text: '1 Introduction\nThis document ...' },
+  { pageIndex: 3, text: '2 Methods\nWe propose ...' },
+];
+
+const provider: LlmProvider = {
+  async chat(messages) {
+    // 接入 OpenAI / Claude / 本地模型
+    return { content: '{}', finishReason: 'stop' };
+  },
+};
+
+const result = await pageIndex(
+  pages,
+  {
+    docName: 'example.pdf',
+    addNodeId: true,
+    addNodeSummary: true,
+    addDocDescription: true,
+  },
+  provider,
+);
+
+console.log(result.metadata.processingMode);
+console.log(result.structure);
+```
+
+### 2) Markdown 转树
+
+```ts
+import { mdToTree } from '@fastrag/pageindex';
+
+const tree = mdToTree('# Intro\nhello\n## Background\nmore context', {
+  thinning: true,
+  minTokenThreshold: 500,
+});
+```
+
+### 3) 向量增强
+
+```ts
+import { VectorEnhancer, InMemoryAdapter, treeChunker } from '@fastrag/pageindex/vector';
+import type { Embedder } from '@fastrag/pageindex/vector';
+
+const store = new InMemoryAdapter();
+
+const embedder: Embedder = {
+  dimension: 3,
+  async embed(texts: string[]) {
+    // 替换为真实 embedding 服务
+    return texts.map((t) => [t.length, t.length / 2, 1]);
+  },
+};
+
+const enhancer = new VectorEnhancer(
+  store,
+  embedder,
+  treeChunker, // 也可以替换为自定义分片策略
+  { chunkMaxTokens: 1000 },
+);
+
+await enhancer.index(result); // 来自 pageIndex() 的结果
+const hits = await enhancer.search('introduction section');
+```
+
+### 4) 混合检索
+
+```ts
+import { HybridSearch } from '@fastrag/pageindex/vector';
+
+const hybrid = new HybridSearch(enhancer, {
+  vectorTopK: 20,
+  rerankTopK: 5,
+});
+
+const results = await hybrid.search('what is the main contribution?');
+```
+
+## 核心流程
+
+```text
+PageContent[] → TOC 检测 → 模式选择 → 验证/修复/降级 → TreeNode[] → 后处理 → PageIndexResult
+```
+
+1. 扫描前 N 页检测目录
+2. 根据 TOC 有无和页码信息选择处理模式
+3. 验证 TOC 准确性，自动修复或降级到更简单的模式
+4. 构建 `TreeNode[]` 树结构
+5. 可选后处理：`nodeId`、`nodeText`、`summary`、`docDescription`
+
+## 向量增强
+
+原版 PageIndex 使用 LLM 树搜索进行检索 — 将整个树结构作为 prompt 发送给 LLM，由 LLM 推理判断哪些节点相关。这种方式效果好，但每次查询都需要调用 LLM。
+
+`@fastrag/pageindex/vector` 子路径导出提供了另一种方案：将树结构转换为向量 embedding，检索变为相似度搜索 — 查询时无需 LLM，延迟在毫秒级。
+
+| | 索引阶段 | 检索阶段 |
+| --- | --- | --- |
+| 树搜索（原版） | LLM 构建树 | 每次查询 LLM 推理遍历树 |
+| 向量增强（本 SDK） | LLM 构建树 → embedding 入库 | 向量相似度搜索（无需 LLM） |
+
+覆盖两种场景：
+
+- **单文档内** — 将树节点分块为 embedding，无需 LLM 调用即可快速精确定位相关片段
+- **跨多文档** — 将多个文档索引到同一个向量库，一次搜索覆盖所有文档；避免对每个文档逐一执行 LLM 树搜索
+
+流程：`PageIndexResult` → `Chunker`（默认实现为 `treeChunker`）→ `Embedder`（生成向量）→ `VectorStore`（存储/检索）。`treeChunker` 当前采用“按段落切分 + 字符近似阈值”（`1 token ≈ 4 chars`）。每个 chunk 保留树结构元数据（`docName`、`nodeId`、`title`、页码范围），检索结果可追溯到原始文档结构。
+
+## 接口抽象
+
+### LlmProvider
+
+```ts
+interface LlmProvider {
+  chat(messages: LlmMessage[], options?: LlmOptions): Promise<LlmResponse>;
+  chatWithSchema?(
+    messages: LlmMessage[],
+    schema: JsonSchema,
+    options?: LlmOptions,
+  ): Promise<LlmResponse>;
+}
+```
+
+### DocumentParser
+
+```ts
+interface DocumentParser {
+  parse(input: string | ArrayBuffer | Uint8Array): Promise<PageContent[]>;
+}
+```
+
+### Embedder
+
+```ts
+interface Embedder {
+  embed(texts: string[]): Promise<number[][]>;
+  readonly dimension: number;
+}
+```
+
+### Chunker
+
+```ts
+type Chunker = (result: PageIndexResult, config?: VectorConfig) => Chunk[];
+```
+
+`treeChunker` 是可直接使用的默认策略实现，但 `VectorEnhancer` 需要显式注入 chunker，因此你可以按模型分词规则替换为自定义分片策略。
+
+## 配置项
+
+| 字段 | 默认值 | 说明 |
+| --- | ---: | --- |
+| `tocCheckPageNum` | `20` | TOC 检测最多扫描页数 |
+| `maxPageNumEachNode` | `10` | 大节点递归拆分页数阈值 |
+| `maxTokenNumEachNode` | `20000` | 大节点递归拆分 token 阈值 |
+| `addNodeId` | `true` | 生成 4 位节点 ID |
+| `addNodeSummary` | `true` | 通过 LLM 生成节点摘要 |
+| `addDocDescription` | `false` | 生成文档一句话描述 |
+| `addNodeText` | `false` | 在输出中保留节点原文 |
+| `retryConfig` | 指数退避 | LLM 重试策略（`maxRetries: 10`、`initialDelayMs: 1000`、`maxDelayMs: 30000`、`backoffMultiplier: 2`） |
+| `onDegradation` | `undefined` | 处理模式降级时的回调 |
+| `logger` | 静默 | 自定义日志注入 |
+
+## 输出结构
+
+```ts
+interface PageIndexResult {
+  docName: string;
+  docDescription?: string;
+  structure: TreeNode[];
+  metadata: {
+    processingMode: ProcessingMode;
+    degradations: DegradationEvent[];
+  };
+}
+```
+
+## 延伸阅读
+
+- [检索策略探讨](./docs/retrieval-strategies.md) — LLM 树搜索、MCTS、向量检索三种方案的对比，以及面向大规模文档集合的混合架构设计
+
+## 许可证
+
+[MIT](./LICENSE)

package/dist/errors/index.d.ts

@@ -0,0 +1,10 @@
+export declare class PageIndexError extends Error {
+    constructor(message: string);
+}
+export declare class LlmError extends PageIndexError {
+    constructor(message: string);
+}
+export declare class TocProcessingError extends PageIndexError {
+    constructor(message: string);
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/errors/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/errors/index.ts"],"names":[],"mappings":"AAAA,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,QAAS,SAAQ,cAAc;gBAC9B,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,kBAAmB,SAAQ,cAAc;gBACxC,OAAO,EAAE,MAAM;CAI5B"}

package/dist/errors/index.js

@@ -0,0 +1,19 @@
+export class PageIndexError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'PageIndexError';
+    }
+}
+export class LlmError extends PageIndexError {
+    constructor(message) {
+        super(message);
+        this.name = 'LlmError';
+    }
+}
+export class TocProcessingError extends PageIndexError {
+    constructor(message) {
+        super(message);
+        this.name = 'TocProcessingError';
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/errors/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/errors/index.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,QAAS,SAAQ,cAAc;IAC1C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;IACzB,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,cAAc;IACpD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF"}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+export { PageIndexError, LlmError, TocProcessingError, } from './errors/index.js';
+export { countTokens, extractJson, mergeConfig, pageListToGroupText, convertPhysicalIndexToInt, addPhysicalIndexTags, createSilentLogger, } from './utils/index.js';
+export type { ResolvedConfig } from './utils/index.js';
+export { LlmClient } from './llm/index.js';
+export { listToTree, getNodes, getLeafNodes, structureToList, postProcessing } from './tree/index.js';
+export { mdToTree } from './markdown/index.js';
+export { extractNodesFromMarkdown, buildTreeFromNodes, treeThinningForIndex } from './markdown/index.js';
+export { writeNodeId } from './post-processing/index.js';
+export { addNodeText, removeStructureText } from './post-processing/index.js';
+export { generateSummariesForStructure } from './post-processing/index.js';
+export { createCleanStructureForDescription, generateDocDescription } from './post-processing/index.js';
+export { pageIndex } from './page-index.js';
+export { treeParser } from './tree-parser.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,cAAc,EACd,QAAQ,EACR,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,WAAW,EACX,WAAW,EACX,WAAW,EACX,mBAAmB,EACnB,yBAAyB,EACzB,oBAAoB,EACpB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGvD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG3C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtG,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAGzG,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAC;AAC3E,OAAO,EAAE,kCAAkC,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAGxG,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/index.js

@@ -0,0 +1,20 @@
+// Errors
+export { PageIndexError, LlmError, TocProcessingError, } from './errors/index.js';
+// Utils
+export { countTokens, extractJson, mergeConfig, pageListToGroupText, convertPhysicalIndexToInt, addPhysicalIndexTags, createSilentLogger, } from './utils/index.js';
+// LLM
+export { LlmClient } from './llm/index.js';
+// Tree
+export { listToTree, getNodes, getLeafNodes, structureToList, postProcessing } from './tree/index.js';
+// Markdown
+export { mdToTree } from './markdown/index.js';
+export { extractNodesFromMarkdown, buildTreeFromNodes, treeThinningForIndex } from './markdown/index.js';
+// Post-processing
+export { writeNodeId } from './post-processing/index.js';
+export { addNodeText, removeStructureText } from './post-processing/index.js';
+export { generateSummariesForStructure } from './post-processing/index.js';
+export { createCleanStructureForDescription, generateDocDescription } from './post-processing/index.js';
+// Main entry
+export { pageIndex } from './page-index.js';
+export { treeParser } from './tree-parser.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,SAAS;AACT,OAAO,EACL,cAAc,EACd,QAAQ,EACR,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAE3B,QAAQ;AACR,OAAO,EACL,WAAW,EACX,WAAW,EACX,WAAW,EACX,mBAAmB,EACnB,yBAAyB,EACzB,oBAAoB,EACpB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAG1B,MAAM;AACN,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO;AACP,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtG,WAAW;AACX,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAEzG,kBAAkB;AAClB,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAC;AAC3E,OAAO,EAAE,kCAAkC,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAExG,aAAa;AACb,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/internal-types/config.d.ts

@@ -0,0 +1,35 @@
+import type { DegradationEvent } from './processing.js';
+import type { Logger } from './logger.js';
+export interface RetryConfig {
+    maxRetries: number;
+    initialDelayMs: number;
+    maxDelayMs: number;
+    backoffMultiplier: number;
+}
+export interface PageIndexConfig {
+    tocCheckPageNum?: number;
+    maxPageNumEachNode?: number;
+    maxTokenNumEachNode?: number;
+    addNodeId?: boolean;
+    addNodeSummary?: boolean;
+    addDocDescription?: boolean;
+    addNodeText?: boolean;
+    onDegradation?: (event: DegradationEvent) => void;
+    retryConfig?: RetryConfig;
+    logger?: Logger;
+}
+export declare const DEFAULT_CONFIG: {
+    readonly tocCheckPageNum: 20;
+    readonly maxPageNumEachNode: 10;
+    readonly maxTokenNumEachNode: 20000;
+    readonly addNodeId: true;
+    readonly addNodeSummary: true;
+    readonly addDocDescription: false;
+    readonly addNodeText: false;
+};
+export declare const DEFAULT_RETRY_CONFIG: RetryConfig;
+export interface MdConfig {
+    thinning?: boolean;
+    minTokenThreshold?: number;
+}
+//# sourceMappingURL=config.d.ts.map

package/dist/internal-types/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/internal-types/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAClD,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,eAAO,MAAM,cAAc;;;;;;;;CAmB1B,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,WAKlC,CAAC;AAEF,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B"}

package/dist/internal-types/config.js

@@ -0,0 +1,16 @@
+export const DEFAULT_CONFIG = {
+    tocCheckPageNum: 20,
+    maxPageNumEachNode: 10,
+    maxTokenNumEachNode: 20000,
+    addNodeId: true,
+    addNodeSummary: true,
+    addDocDescription: false,
+    addNodeText: false,
+};
+export const DEFAULT_RETRY_CONFIG = {
+    maxRetries: 10,
+    initialDelayMs: 1000,
+    maxDelayMs: 30000,
+    backoffMultiplier: 2,
+};
+//# sourceMappingURL=config.js.map

package/dist/internal-types/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/internal-types/config.ts"],"names":[],"mappings":"AAuBA,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,eAAe,EAAE,EAAE;IACnB,kBAAkB,EAAE,EAAE;IACtB,mBAAmB,EAAE,KAAK;IAC1B,SAAS,EAAE,IAAI;IACf,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,KAAK;IACxB,WAAW,EAAE,KAAK;CAYnB,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAC/C,UAAU,EAAE,EAAE;IACd,cAAc,EAAE,IAAI;IACpB,UAAU,EAAE,KAAK;IACjB,iBAAiB,EAAE,CAAC;CACrB,CAAC"}

package/dist/internal-types/document-parser.d.ts

@@ -0,0 +1,5 @@
+import type { PageContent } from './page.js';
+export interface DocumentParser {
+    parse(input: string | ArrayBuffer | Uint8Array): Promise<PageContent[]>;
+}
+//# sourceMappingURL=document-parser.d.ts.map

package/dist/internal-types/document-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-parser.d.ts","sourceRoot":"","sources":["../../src/internal-types/document-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CACzE"}

package/dist/internal-types/document-parser.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=document-parser.js.map