pageindex 1.0.0

package/dist/cli.js

@@ -0,0 +1,2178 @@
+#!/usr/bin/env node
+#!/usr/bin/env node
+
+// src/cli.ts
+import { parseArgs } from "util";
+
+// src/pdf.ts
+import { PDFParse } from "pdf-parse";
+
+// src/utils.ts
+function countTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+function getJsonContent(response) {
+  let content = response;
+  content = content.replace(/<think>[\s\S]*?<\/think>/g, "").trim();
+  content = content.replace(/<\/?output>/gi, "").trim();
+  const jsonBlockMatch = content.match(/```json\s*([\s\S]*?)```/);
+  if (jsonBlockMatch && jsonBlockMatch[1]) {
+    return jsonBlockMatch[1].trim();
+  }
+  const codeBlockMatch = content.match(/```\s*([\s\S]*?)```/);
+  if (codeBlockMatch && codeBlockMatch[1]) {
+    return codeBlockMatch[1].trim();
+  }
+  const jsonMatch = content.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
+  if (jsonMatch && jsonMatch[1]) {
+    return jsonMatch[1].trim();
+  }
+  return content.trim();
+}
+function cleanJsonString(jsonContent) {
+  return jsonContent.replace(/None/g, "null").replace(/True/g, "true").replace(/False/g, "false").replace(/,\s*]/g, "]").replace(/,\s*}/g, "}").replace(/'/g, '"');
+}
+function extractFieldsFromMalformedJson(content) {
+  const result = {};
+  const stringFields = [
+    "toc_detected",
+    "tocDetected",
+    "answer",
+    "start_begin",
+    "startBegin",
+    "completed",
+    "page_index_given",
+    "pageIndexGiven",
+    "page_index_given_in_toc",
+    "pageIndexGivenInToc",
+    "reasoning",
+    "thinking",
+    "structure",
+    "appear_start",
+    "appearStart"
+  ];
+  const numberFields = ["confidence", "page", "physical_index", "physicalIndex"];
+  for (const field of stringFields) {
+    const match = content.match(new RegExp(`["']${field}["']\\s*:\\s*["']([^"']*?)["']`, "i"));
+    if (match && match[1]) {
+      const normalizedField = field.replace(/([A-Z])/g, "_$1").toLowerCase();
+      result[normalizedField] = match[1].trim().toLowerCase();
+      result[field] = match[1].trim().toLowerCase();
+    }
+  }
+  for (const field of numberFields) {
+    const match = content.match(new RegExp(`["']${field}["']\\s*:\\s*([\\d.]+)`, "i"));
+    if (match && match[1]) {
+      const normalizedField = field.replace(/([A-Z])/g, "_$1").toLowerCase();
+      result[normalizedField] = parseFloat(match[1]);
+      result[field] = parseFloat(match[1]);
+    }
+  }
+  const titleMatch = content.match(/["']title["']\s*:\s*["']([^"']+)["']/i);
+  if (titleMatch && titleMatch[1]) {
+    result.title = titleMatch[1];
+  }
+  const tocMatch = content.match(/["']table_of_contents["']\s*:\s*\[/i);
+  if (tocMatch) {
+    const arrayContent = content.slice(content.indexOf("["));
+    try {
+      let depth = 0;
+      let endIndex = 0;
+      for (let i = 0; i < arrayContent.length; i++) {
+        if (arrayContent[i] === "[") depth++;
+        else if (arrayContent[i] === "]") {
+          depth--;
+          if (depth === 0) {
+            endIndex = i + 1;
+            break;
+          }
+        }
+      }
+      if (endIndex > 0) {
+        const arrayStr = cleanJsonString(arrayContent.slice(0, endIndex));
+        result.table_of_contents = JSON.parse(arrayStr);
+      }
+    } catch {
+    }
+  }
+  return Object.keys(result).length > 0 ? result : null;
+}
+function extractJson(content) {
+  try {
+    let jsonContent = getJsonContent(content);
+    jsonContent = cleanJsonString(jsonContent);
+    return JSON.parse(jsonContent);
+  } catch (error) {
+    try {
+      let cleanContent = content.replace(/<think>[\s\S]*?<\/think>/g, "").trim();
+      const startBrace = cleanContent.indexOf("{");
+      const startBracket = cleanContent.indexOf("[");
+      let start = -1;
+      let end = -1;
+      if (startBrace !== -1 && (startBracket === -1 || startBrace < startBracket)) {
+        start = startBrace;
+        end = cleanContent.lastIndexOf("}");
+      } else if (startBracket !== -1) {
+        start = startBracket;
+        end = cleanContent.lastIndexOf("]");
+      }
+      if (start !== -1 && end !== -1 && end > start) {
+        let jsonContent = cleanContent.slice(start, end + 1);
+        jsonContent = cleanJsonString(jsonContent);
+        return JSON.parse(jsonContent);
+      }
+    } catch {
+      const extracted = extractFieldsFromMalformedJson(content);
+      if (extracted) {
+        return extracted;
+      }
+    }
+    console.error("Failed to extract JSON:", error);
+    return null;
+  }
+}
+function writeNodeId(data, nodeId = 0) {
+  if (Array.isArray(data)) {
+    for (const item of data) {
+      nodeId = writeNodeId(item, nodeId);
+    }
+  } else if (data && typeof data === "object") {
+    data.nodeId = String(nodeId).padStart(4, "0");
+    nodeId += 1;
+    if (data.nodes) {
+      nodeId = writeNodeId(data.nodes, nodeId);
+    }
+  }
+  return nodeId;
+}
+function structureToList(structure) {
+  if (Array.isArray(structure)) {
+    const nodes2 = [];
+    for (const item of structure) {
+      nodes2.push(...structureToList(item));
+    }
+    return nodes2;
+  }
+  const nodes = [structure];
+  if (structure.nodes) {
+    nodes.push(...structureToList(structure.nodes));
+  }
+  return nodes;
+}
+function listToTree(data) {
+  const getParentStructure = (structure) => {
+    if (!structure) return null;
+    const parts = structure.split(".");
+    return parts.length > 1 ? parts.slice(0, -1).join(".") : null;
+  };
+  const nodes = /* @__PURE__ */ new Map();
+  const rootNodes = [];
+  for (const item of data) {
+    const structure = item.structure;
+    const node = {
+      title: item.title,
+      startIndex: item.physicalIndex,
+      endIndex: void 0,
+      nodes: []
+    };
+    if (structure) {
+      nodes.set(structure, node);
+    }
+    const parentStructure = getParentStructure(structure);
+    if (parentStructure && nodes.has(parentStructure)) {
+      nodes.get(parentStructure).nodes.push(node);
+    } else {
+      rootNodes.push(node);
+    }
+  }
+  const cleanNode = (node) => {
+    if (!node.nodes || node.nodes.length === 0) {
+      delete node.nodes;
+    } else {
+      node.nodes.forEach(cleanNode);
+    }
+    return node;
+  };
+  return rootNodes.map(cleanNode);
+}
+function addPrefaceIfNeeded(data) {
+  if (!Array.isArray(data) || data.length === 0) return data;
+  const firstItem = data[0];
+  if (firstItem && firstItem.physicalIndex && firstItem.physicalIndex > 1) {
+    data.unshift({
+      structure: "0",
+      title: "Preface",
+      physicalIndex: 1
+    });
+  }
+  return data;
+}
+function postProcessing(structure, endPhysicalIndex) {
+  for (let i = 0; i < structure.length; i++) {
+    const item = structure[i];
+    item.startIndex = item.physicalIndex;
+    if (i < structure.length - 1) {
+      const nextItem = structure[i + 1];
+      if (nextItem.appearStart === "yes") {
+        item.endIndex = (nextItem.physicalIndex || 0) - 1;
+      } else {
+        item.endIndex = nextItem.physicalIndex;
+      }
+    } else {
+      item.endIndex = endPhysicalIndex;
+    }
+  }
+  const tree = listToTree(structure);
+  return tree.length > 0 ? tree : structure;
+}
+function removeFields(data, fields = ["text"]) {
+  if (Array.isArray(data)) {
+    return data.map((item) => removeFields(item, fields));
+  }
+  if (data && typeof data === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(data)) {
+      if (!fields.includes(key)) {
+        result[key] = removeFields(value, fields);
+      }
+    }
+    return result;
+  }
+  return data;
+}
+function convertPhysicalIndexToInt(data) {
+  if (typeof data === "string") {
+    if (data.startsWith("<physical_index_")) {
+      return parseInt(data.split("_").pop().replace(">", "").trim(), 10);
+    }
+    if (data.startsWith("physical_index_")) {
+      return parseInt(data.split("_").pop().trim(), 10);
+    }
+    const num = parseInt(data, 10);
+    return isNaN(num) ? null : num;
+  }
+  if (Array.isArray(data)) {
+    for (const item of data) {
+      if (typeof item.physicalIndex === "string") {
+        const parsed = convertPhysicalIndexToInt(item.physicalIndex);
+        if (typeof parsed === "number") {
+          item.physicalIndex = parsed;
+        }
+      }
+    }
+  }
+  return data;
+}
+function convertPageToInt(data) {
+  for (const item of data) {
+    if (typeof item.page === "string") {
+      const parsed = parseInt(item.page, 10);
+      if (!isNaN(parsed)) {
+        item.page = parsed;
+      }
+    }
+  }
+  return data;
+}
+function reorderDict(data, keyOrder) {
+  if (!keyOrder.length) return data;
+  const result = {};
+  for (const key of keyOrder) {
+    if (key in data) {
+      result[key] = data[key];
+    }
+  }
+  return result;
+}
+function formatStructure(structure, order) {
+  if (!order) return structure;
+  if (Array.isArray(structure)) {
+    return structure.map((item) => formatStructure(item, order));
+  }
+  if (structure.nodes) {
+    structure.nodes = formatStructure(structure.nodes, order);
+  }
+  if (!structure.nodes || structure.nodes.length === 0) {
+    delete structure.nodes;
+  }
+  return reorderDict(structure, order);
+}
+function createCleanStructureForDescription(structure) {
+  if (Array.isArray(structure)) {
+    return structure.map(
+      (item) => createCleanStructureForDescription(item)
+    );
+  }
+  const cleanNode = {};
+  const essentialFields = [
+    "title",
+    "nodeId",
+    "summary",
+    "prefixSummary"
+  ];
+  for (const key of essentialFields) {
+    if (key in structure) {
+      cleanNode[key] = structure[key];
+    }
+  }
+  if (structure.nodes && structure.nodes.length > 0) {
+    cleanNode.nodes = createCleanStructureForDescription(
+      structure.nodes
+    );
+  }
+  return cleanNode;
+}
+
+// src/pdf.ts
+import * as fs from "fs/promises";
+async function parsePdf(input) {
+  let data;
+  if (typeof input === "string") {
+    const buffer = await fs.readFile(input);
+    data = new Uint8Array(buffer);
+  } else if (input instanceof ArrayBuffer) {
+    data = new Uint8Array(input);
+  } else {
+    data = new Uint8Array(input);
+  }
+  const parser = new PDFParse({ data });
+  const textResult = await parser.getText();
+  const infoResult = await parser.getInfo();
+  const pages = [];
+  let title = "Untitled";
+  if (infoResult?.info?.Title) {
+    title = infoResult.info.Title;
+  }
+  for (const pageText of textResult.pages) {
+    const text = pageText.text;
+    const tokenCount = countTokens(text);
+    pages.push({ text, tokenCount });
+  }
+  await parser.destroy();
+  return {
+    title,
+    numPages: textResult.pages.length,
+    pages
+  };
+}
+function getPdfName(pdfPath) {
+  const parts = pdfPath.split("/");
+  const basename3 = parts[parts.length - 1] || "Untitled";
+  return basename3.replace(/\.pdf$/i, "");
+}
+
+// src/ocr.ts
+import { exec } from "child_process";
+import { promisify } from "util";
+import * as path from "path";
+import * as os from "os";
+import * as fs2 from "fs/promises";
+import OpenAI from "openai";
+var execAsync = promisify(exec);
+var DEFAULT_OCR_OPTIONS = {
+  ocrModel: "mlx-community/GLM-OCR-bf16",
+  imageFormat: "png",
+  imageDpi: 150,
+  ocrPromptType: "text",
+  concurrency: 3
+};
+var OCR_PROMPTS = {
+  text: "Text Recognition:",
+  formula: "Formula Recognition:",
+  table: "Table Recognition:"
+};
+async function checkPopplerInstalled() {
+  try {
+    await execAsync("which pdftocairo");
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function pdfToImages(pdfPath, options = {}) {
+  const format = options.imageFormat || DEFAULT_OCR_OPTIONS.imageFormat;
+  const dpi = options.imageDpi || DEFAULT_OCR_OPTIONS.imageDpi;
+  const installed = await checkPopplerInstalled();
+  if (!installed) {
+    throw new Error(
+      "Poppler tools not installed. Install with:\n  macOS: brew install poppler\n  Ubuntu: sudo apt-get install poppler-utils"
+    );
+  }
+  const tempDir = await fs2.mkdtemp(path.join(os.tmpdir(), "pageindex-ocr-"));
+  const outputPrefix = path.join(tempDir, "page");
+  const formatFlag = format === "png" ? "-png" : "-jpeg";
+  const cmd = `pdftocairo ${formatFlag} -r ${dpi} "${pdfPath}" "${outputPrefix}"`;
+  try {
+    await execAsync(cmd);
+  } catch (error) {
+    await fs2.rm(tempDir, { recursive: true, force: true }).catch(() => {
+    });
+    throw new Error(`PDF conversion failed: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  const files = await fs2.readdir(tempDir);
+  const imageFiles = files.filter((f) => f.endsWith(`.${format}`)).sort((a, b) => {
+    const numA = parseInt(a.match(/-(\d+)\./)?.[1] || "0");
+    const numB = parseInt(b.match(/-(\d+)\./)?.[1] || "0");
+    return numA - numB;
+  }).map((f) => path.join(tempDir, f));
+  return imageFiles;
+}
+async function pdfBufferToImages(pdfBuffer, options = {}) {
+  const tempDir = await fs2.mkdtemp(path.join(os.tmpdir(), "pageindex-pdf-"));
+  const tempPdfPath = path.join(tempDir, "input.pdf");
+  const buffer = pdfBuffer instanceof ArrayBuffer ? Buffer.from(pdfBuffer) : pdfBuffer;
+  await fs2.writeFile(tempPdfPath, buffer);
+  try {
+    return await pdfToImages(tempPdfPath, options);
+  } finally {
+    await fs2.unlink(tempPdfPath).catch(() => {
+    });
+    await fs2.rmdir(tempDir).catch(() => {
+    });
+  }
+}
+async function ocrImage(imagePath, options = {}) {
+  const model = options.ocrModel || DEFAULT_OCR_OPTIONS.ocrModel;
+  const promptType = options.ocrPromptType || DEFAULT_OCR_OPTIONS.ocrPromptType;
+  const prompt = OCR_PROMPTS[promptType] ?? OCR_PROMPTS.text ?? "Text Recognition:";
+  const apiKey = options.apiKey || process.env.OPENAI_API_KEY || "lm-studio";
+  const baseUrl = options.baseUrl || process.env.OPENAI_BASE_URL || "http://localhost:1234/v1";
+  const client = new OpenAI({
+    apiKey,
+    baseURL: baseUrl
+  });
+  const imageData = await fs2.readFile(imagePath);
+  const base64Image = imageData.toString("base64");
+  const mimeType = imagePath.endsWith(".png") ? "image/png" : "image/jpeg";
+  try {
+    const contentParts = [
+      {
+        type: "image_url",
+        image_url: {
+          url: `data:${mimeType};base64,${base64Image}`
+        }
+      },
+      {
+        type: "text",
+        text: prompt
+      }
+    ];
+    const response = await client.chat.completions.create({
+      model,
+      messages: [
+        {
+          role: "user",
+          content: contentParts
+        }
+      ],
+      max_tokens: 4096
+    });
+    return response.choices[0]?.message?.content || "";
+  } catch (error) {
+    console.error(`[OCR Error] Failed to process ${imagePath}:`, error);
+    return "";
+  }
+}
+async function ocrImages(imagePaths, options = {}) {
+  const concurrency = options.concurrency || DEFAULT_OCR_OPTIONS.concurrency;
+  const results = [];
+  for (let i = 0; i < imagePaths.length; i += concurrency) {
+    const batch = imagePaths.slice(i, i + concurrency);
+    const batchResults = await Promise.all(
+      batch.map((imagePath) => ocrImage(imagePath, options))
+    );
+    results.push(...batchResults);
+    const processed = Math.min(i + concurrency, imagePaths.length);
+    console.log(`[OCR] Processed ${processed}/${imagePaths.length} pages`);
+  }
+  return results;
+}
+async function parsePdfWithOcr(input, options = {}) {
+  console.log("[OCR Mode] Converting PDF to images...");
+  let imagePaths;
+  let tempDir;
+  if (typeof input === "string") {
+    imagePaths = await pdfToImages(input, options);
+  } else {
+    const buffer = input instanceof ArrayBuffer ? Buffer.from(input) : input;
+    imagePaths = await pdfBufferToImages(buffer, options);
+  }
+  if (imagePaths.length > 0) {
+    tempDir = path.dirname(imagePaths[0]);
+  }
+  console.log(`[OCR Mode] Extracted ${imagePaths.length} page images`);
+  console.log("[OCR Mode] Running OCR on pages...");
+  const texts = await ocrImages(imagePaths, options);
+  const pages = texts.map((text) => ({
+    text,
+    tokenCount: countTokens(text)
+  }));
+  await cleanupTempImages(imagePaths);
+  return { pages };
+}
+async function cleanupTempImages(imagePaths) {
+  if (imagePaths.length === 0) return;
+  const firstPath = imagePaths[0];
+  if (!firstPath) return;
+  const tempDir = path.dirname(firstPath);
+  await Promise.all(
+    imagePaths.map((p) => fs2.unlink(p).catch(() => {
+    }))
+  );
+  await fs2.rmdir(tempDir).catch(() => {
+  });
+}
+
+// src/openai.ts
+import OpenAI2 from "openai";
+var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+var clientInstance = null;
+var currentBaseUrl;
+function getClient(config = {}) {
+  const apiKey = config.apiKey || process.env.OPENAI_API_KEY || "lm-studio";
+  const baseUrl = config.baseUrl || process.env.OPENAI_BASE_URL;
+  if (clientInstance && currentBaseUrl === baseUrl) {
+    return clientInstance;
+  }
+  clientInstance = new OpenAI2({
+    apiKey,
+    baseURL: baseUrl
+  });
+  currentBaseUrl = baseUrl;
+  return clientInstance;
+}
+async function chatGPT(options) {
+  if (isLMStudio(options.baseUrl) && isThinkingModel(options.model)) {
+    const result2 = await chatLMStudioNative(options);
+    return result2.content;
+  }
+  const result = await chatGPTWithFinishReason(options);
+  return result.content;
+}
+async function chatGPTWithFinishReason(options) {
+  const {
+    model,
+    prompt,
+    apiKey,
+    baseUrl,
+    chatHistory,
+    temperature = 0,
+    maxRetries = 10
+  } = options;
+  const client = getClient({ apiKey, baseUrl });
+  const messages = chatHistory ? [...chatHistory, { role: "user", content: prompt }] : [{ role: "user", content: prompt }];
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const response = await client.chat.completions.create({
+        model,
+        messages,
+        temperature
+      });
+      const choice = response.choices[0];
+      if (!choice) {
+        throw new Error("No response from model");
+      }
+      const finishReason = choice.finish_reason === "length" ? "max_output_reached" : "finished";
+      return {
+        content: choice.message.content || "",
+        finishReason
+      };
+    } catch (error) {
+      console.error(`[Retry ${attempt + 1}/${maxRetries}]`, error);
+      if (attempt < maxRetries - 1) {
+        await sleep(1e3 * (attempt + 1));
+      } else {
+        console.error("Max retries reached for prompt:", prompt.slice(0, 100));
+        return { content: "Error", finishReason: "error" };
+      }
+    }
+  }
+  return { content: "Error", finishReason: "error" };
+}
+function isLMStudio(baseUrl) {
+  return baseUrl?.includes("localhost:1234") ?? false;
+}
+function isThinkingModel(model) {
+  const thinkingModelPatterns = [
+    /qwen3\.5/i,
+    // qwen3.5 series supports reasoning
+    /qwen3-coder/i,
+    // qwen3-coder supports reasoning
+    /deepseek/i,
+    // deepseek models support reasoning
+    /o1/i,
+    // OpenAI o1 models
+    /o3/i
+    // OpenAI o3 models
+  ];
+  return thinkingModelPatterns.some((pattern) => pattern.test(model));
+}
+var modelsWithoutReasoning = /* @__PURE__ */ new Set();
+async function chatLMStudioNative(options) {
+  const {
+    model,
+    prompt,
+    temperature = 0,
+    maxRetries = 10
+  } = options;
+  const baseUrl = "http://localhost:1234";
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const body = {
+        model,
+        input: prompt,
+        temperature
+      };
+      if (!modelsWithoutReasoning.has(model)) {
+        body.reasoning = "off";
+      }
+      const response = await fetch(`${baseUrl}/api/v1/chat`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify(body)
+      });
+      const data = await response.json();
+      if (!response.ok) {
+        if (data.type === "invalid_request" && data.param === "reasoning") {
+          console.log(`[LM Studio] Model ${model} doesn't support reasoning parameter, retrying without it`);
+          modelsWithoutReasoning.add(model);
+          const retryResponse = await fetch(`${baseUrl}/api/v1/chat`, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+              model,
+              input: prompt,
+              temperature
+            })
+          });
+          if (!retryResponse.ok) {
+            const retryError = await retryResponse.text();
+            throw new Error(`LM Studio API error on retry: ${retryResponse.status} - ${retryError}`);
+          }
+          const retryData = await retryResponse.json();
+          const messageOutput2 = retryData.output?.find((o) => o.type === "message");
+          return {
+            content: messageOutput2?.content || "",
+            finishReason: "finished"
+          };
+        }
+        throw new Error(`LM Studio API error: ${response.status} - ${data.message || JSON.stringify(data)}`);
+      }
+      const messageOutput = data.output?.find((o) => o.type === "message");
+      const content = messageOutput?.content || "";
+      return {
+        content,
+        finishReason: "finished"
+      };
+    } catch (error) {
+      console.error(`[LM Studio Retry ${attempt + 1}/${maxRetries}]`, error);
+      if (attempt < maxRetries - 1) {
+        await sleep(1e3 * (attempt + 1));
+      } else {
+        console.error("Max retries reached for prompt:", prompt.slice(0, 100));
+        return { content: "Error", finishReason: "error" };
+      }
+    }
+  }
+  return { content: "Error", finishReason: "error" };
+}
+
+// src/prompts.ts
+function tocDetectorPrompt(content) {
+  return `Your job is to detect if there is a table of content provided in the given text.
+
+Given text: ${content}
+
+return the following JSON format:
+{
+    "thinking": <why do you think there is a table of content in the given text>
+    "toc_detected": "<yes or no>",
+}
+
+Directly return the final JSON structure. Do not output anything else.
+Please note: abstract, summary, notation list, figure list, table list, etc. are not table of contents.`;
+}
+function checkTitleAppearancePrompt(title, pageText) {
+  return `Your job is to check if the given section appears or starts in the given page_text.
+
+Note: do fuzzy matching, ignore any space inconsistency in the page_text.
+
+The given section title is ${title}.
+The given page_text is ${pageText}.
+
+Reply format:
+{
+    "thinking": <why do you think the section appears or starts in the page_text>
+    "answer": "yes or no" (yes if the section appears or starts in the page_text, no otherwise)
+}
+Directly return the final JSON structure. Do not output anything else.`;
+}
+function checkTitleStartAtBeginningPrompt(title, pageText) {
+  return `You will be given the current section title and the current page_text.
+Your job is to check if the current section starts in the beginning of the given page_text.
+If there are other contents before the current section title, then the current section does not start in the beginning of the given page_text.
+If the current section title is the first content in the given page_text, then the current section starts in the beginning of the given page_text.
+
+Note: do fuzzy matching, ignore any space inconsistency in the page_text.
+
+The given section title is ${title}.
+The given page_text is ${pageText}.
+
+reply format:
+{
+    "thinking": <why do you think the section appears or starts in the page_text>
+    "start_begin": "yes or no" (yes if the section starts in the beginning of the page_text, no otherwise)
+}
+Directly return the final JSON structure. Do not output anything else.`;
+}
+function checkTocTransformationCompletePrompt(rawToc, cleanedToc) {
+  return `You are given a raw table of contents and a cleaned table of contents.
+Your job is to check if the cleaned table of contents is complete.
+
+Reply format:
+{
+    "thinking": <why do you think the cleaned table of contents is complete or not>
+    "completed": "yes" or "no"
+}
+Directly return the final JSON structure. Do not output anything else.
+
+Raw Table of contents:
+${rawToc}
+
+Cleaned Table of contents:
+${cleanedToc}`;
+}
+function detectPageIndexPrompt(tocContent) {
+  return `You will be given a table of contents.
+
+Your job is to detect if there are page numbers/indices given within the table of contents.
+
+Given text: ${tocContent}
+
+Reply format:
+{
+    "thinking": <why do you think there are page numbers/indices given within the table of contents>
+    "page_index_given_in_toc": "<yes or no>"
+}
+Directly return the final JSON structure. Do not output anything else.`;
+}
+function tocTransformerPrompt(tocContent) {
+  return `You are given a table of contents, You job is to transform the whole table of content into a JSON format included table_of_contents.
+
+structure is the numeric system which represents the index of the hierarchy section in the table of contents. For example, the first section has structure index 1, the first subsection has structure index 1.1, the second subsection has structure index 1.2, etc.
+
+The response should be in the following JSON format: 
+{
+table_of_contents: [
+    {
+        "structure": <structure index, "x.x.x" or None> (string),
+        "title": <title of the section>,
+        "page": <page number or None>,
+    },
+    ...
+    ],
+}
+You should transform the full table of contents in one go.
+Directly return the final JSON structure, do not output anything else.
+
+Given table of contents:
+${tocContent}`;
+}
+function tocTransformerContinuePrompt(rawToc, incompleteToc) {
+  return `Your task is to continue the table of contents json structure, directly output the remaining part of the json structure.
+The response should be in the following JSON format: 
+
+The raw table of contents json structure is:
+${rawToc}
+
+The incomplete transformed table of contents json structure is:
+${incompleteToc}
+
+Please continue the json structure, directly output the remaining part of the json structure.`;
+}
+function tocIndexExtractorPrompt(toc, content) {
+  return `You are given a table of contents in a json format and several pages of a document, your job is to add the physical_index to the table of contents in the json format.
+
+The provided pages contains tags like <physical_index_X> and <physical_index_X> to indicate the physical location of the page X.
+
+The structure variable is the numeric system which represents the index of the hierarchy section in the table of contents. For example, the first section has structure index 1, the first subsection has structure index 1.1, the second subsection has structure index 1.2, etc.
+
+The response should be in the following JSON format: 
+[
+    {
+        "structure": <structure index, "x.x.x" or None> (string),
+        "title": <title of the section>,
+        "physical_index": "<physical_index_X>" (keep the format)
+    },
+    ...
+]
+
+Only add the physical_index to the sections that are in the provided pages.
+If the section is not in the provided pages, do not add the physical_index to it.
+Directly return the final JSON structure. Do not output anything else.
+
+Table of contents:
+${toc}
+
+Document pages:
+${content}`;
+}
+function addPageNumberToTocPrompt(part, structure) {
+  return `You are given an JSON structure of a document and a partial part of the document. Your task is to check if the title that is described in the structure is started in the partial given document.
+
+The provided text contains tags like <physical_index_X> and <physical_index_X> to indicate the physical location of the page X. 
+
+If the full target section starts in the partial given document, insert the given JSON structure with the "start": "yes", and "start_index": "<physical_index_X>".
+
+If the full target section does not start in the partial given document, insert "start": "no",  "start_index": None.
+
+The response should be in the following format. 
+    [
+        {
+            "structure": <structure index, "x.x.x" or None> (string),
+            "title": <title of the section>,
+            "start": "<yes or no>",
+            "physical_index": "<physical_index_X> (keep the format)" or None
+        },
+        ...
+    ]    
+The given structure contains the result of the previous part, you need to fill the result of the current part, do not change the previous result.
+Directly return the final JSON structure. Do not output anything else.
+
+Current Partial Document:
+${part}
+
+Given Structure
+${structure}`;
+}
+function generateTocInitPrompt(part) {
+  return `You are an expert in extracting hierarchical tree structure, your task is to generate the tree structure of the document.
+
+The structure variable is the numeric system which represents the index of the hierarchy section in the table of contents. For example, the first section has structure index 1, the first subsection has structure index 1.1, the second subsection has structure index 1.2, etc.
+
+For the title, you need to extract the original title from the text, only fix the space inconsistency.
+
+The provided text contains tags like <physical_index_X> and <physical_index_X> to indicate the start and end of page X. 
+
+For the physical_index, you need to extract the physical index of the start of the section from the text. Keep the <physical_index_X> format.
+
+The response should be in the following format. 
+    [
+        {
+            "structure": <structure index, "x.x.x"> (string),
+            "title": <title of the section, keep the original title>,
+            "physical_index": "<physical_index_X> (keep the format)"
+        },
+        
+    ],
+
+
+Directly return the final JSON structure. Do not output anything else.
+
+Given text:
+${part}`;
+}
+function generateTocContinuePrompt(part, previousStructure) {
+  return `You are an expert in extracting hierarchical tree structure.
+You are given a tree structure of the previous part and the text of the current part.
+Your task is to continue the tree structure from the previous part to include the current part.
+
+The structure variable is the numeric system which represents the index of the hierarchy section in the table of contents. For example, the first section has structure index 1, the first subsection has structure index 1.1, the second subsection has structure index 1.2, etc.
+
+For the title, you need to extract the original title from the text, only fix the space inconsistency.
+
+The provided text contains tags like <physical_index_X> and <physical_index_X> to indicate the start and end of page X.
+
+For the physical_index, you need to extract the physical index of the start of the section from the text. Keep the <physical_index_X> format.
+
+The response should be in the following format. 
+    [
+        {
+            "structure": <structure index, "x.x.x"> (string),
+            "title": <title of the section, keep the original title>,
+            "physical_index": "<physical_index_X> (keep the format)"
+        },
+        ...
+    ]    
+
+Directly return the additional part of the final JSON structure. Do not output anything else.
+
+Given text:
+${part}
+
+Previous tree structure:
+${previousStructure}`;
+}
+function singleTocItemIndexFixerPrompt(sectionTitle, content) {
+  return `You are given a section title and several pages of a document, your job is to find the physical index of the start page of the section in the partial document.
+
+The provided pages contains tags like <physical_index_X> and <physical_index_X> to indicate the physical location of the page X.
+
+Reply in a JSON format:
+{
+    "thinking": <explain which page, started and closed by <physical_index_X>, contains the start of this section>,
+    "physical_index": "<physical_index_X>" (keep the format)
+}
+Directly return the final JSON structure. Do not output anything else.
+
+Section Title:
+${sectionTitle}
+
+Document pages:
+${content}`;
+}
+function generateNodeSummaryPrompt(nodeText) {
+  return `You are given a part of a document, your task is to generate a description of the partial document about what are main points covered in the partial document.
+
+Partial Document Text: ${nodeText}
+
+Directly return the description, do not include any other text.`;
+}
+function generateDocDescriptionPrompt(structure) {
+  return `Your are an expert in generating descriptions for a document.
+You are given a structure of a document. Your task is to generate a one-sentence description for the document, which makes it easy to distinguish the document from other documents.
+    
+Document Structure: ${structure}
+
+Directly return the description, do not include any other text.`;
+}
+
+// src/toc.ts
+async function tocDetectorSinglePage(content, options) {
+  const prompt = tocDetectorPrompt(content);
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  return json?.toc_detected === "yes" ? "yes" : "no";
+}
+async function findTocPages(startPageIndex, pages, options) {
+  let lastPageIsYes = false;
+  const tocPageList = [];
+  let i = startPageIndex;
+  while (i < pages.length) {
+    if (i >= options.tocCheckPageNum && !lastPageIsYes) {
+      break;
+    }
+    const page = pages[i];
+    if (!page) {
+      i++;
+      continue;
+    }
+    const detected = await tocDetectorSinglePage(page.text, options);
+    if (detected === "yes") {
+      tocPageList.push(i);
+      lastPageIsYes = true;
+    } else if (detected === "no" && lastPageIsYes) {
+      break;
+    }
+    i++;
+  }
+  return tocPageList;
+}
+function transformDotsToColon(text) {
+  text = text.replace(/\.{5,}/g, ": ");
+  text = text.replace(/(?:\. ){5,}\.?/g, ": ");
+  return text;
+}
+async function detectPageIndex(tocContent, options) {
+  const prompt = detectPageIndexPrompt(tocContent);
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  return json?.page_index_given_in_toc === "yes" ? "yes" : "no";
+}
+async function tocExtractor(pages, tocPageList, options) {
+  let tocContent = "";
+  for (const pageIndex of tocPageList) {
+    const page = pages[pageIndex];
+    if (page) {
+      tocContent += page.text;
+    }
+  }
+  tocContent = transformDotsToColon(tocContent);
+  const hasPageIndex = await detectPageIndex(tocContent, options);
+  return {
+    tocContent,
+    pageIndexGivenInToc: hasPageIndex
+  };
+}
+async function checkTocTransformationComplete(rawToc, cleanedToc, options) {
+  const prompt = checkTocTransformationCompletePrompt(rawToc, cleanedToc);
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  return json?.completed === "yes";
+}
+async function tocTransformer(tocContent, options) {
+  const prompt = tocTransformerPrompt(tocContent);
+  let { content: lastComplete, finishReason } = await chatGPTWithFinishReason({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  let isComplete = await checkTocTransformationComplete(tocContent, lastComplete, options);
+  if (isComplete && finishReason === "finished") {
+    const json = extractJson(lastComplete);
+    if (json?.table_of_contents) {
+      return convertPageToInt(json.table_of_contents);
+    }
+  }
+  lastComplete = getJsonContent(lastComplete);
+  let attempts = 0;
+  const maxAttempts = 5;
+  while (!(isComplete && finishReason === "finished") && attempts < maxAttempts) {
+    const position = lastComplete.lastIndexOf("}");
+    if (position !== -1) {
+      lastComplete = lastComplete.slice(0, position + 2);
+    }
+    const continuePrompt = tocTransformerContinuePrompt(tocContent, lastComplete);
+    const result = await chatGPTWithFinishReason({
+      model: options.model,
+      prompt: continuePrompt,
+      apiKey: options.apiKey,
+      baseUrl: options.baseUrl
+    });
+    let newContent = result.content;
+    finishReason = result.finishReason;
+    if (newContent.startsWith("```json")) {
+      newContent = getJsonContent(newContent);
+    }
+    lastComplete = lastComplete + newContent;
+    isComplete = await checkTocTransformationComplete(tocContent, lastComplete, options);
+    attempts++;
+  }
+  try {
+    const parsed = JSON.parse(lastComplete);
+    return convertPageToInt(parsed.table_of_contents || parsed);
+  } catch {
+    console.error("Failed to parse TOC JSON");
+    return [];
+  }
+}
+async function tocIndexExtractor(toc, content, options) {
+  const prompt = tocIndexExtractorPrompt(JSON.stringify(toc), content);
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  return json || [];
+}
+async function generateTocInit(part, options) {
+  const prompt = generateTocInitPrompt(part);
+  const { content, finishReason } = await chatGPTWithFinishReason({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  if (finishReason === "finished") {
+    const json = extractJson(content);
+    return json || [];
+  }
+  throw new Error(`Generation incomplete: ${finishReason}`);
+}
+async function generateTocContinue(tocContent, part, options) {
+  const prompt = generateTocContinuePrompt(part, JSON.stringify(tocContent, null, 2));
+  const { content, finishReason } = await chatGPTWithFinishReason({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  if (finishReason === "finished") {
+    const json = extractJson(content);
+    return json || [];
+  }
+  throw new Error(`Generation incomplete: ${finishReason}`);
+}
+async function addPageNumberToToc(part, structure, options) {
+  const prompt = addPageNumberToTocPrompt(part, JSON.stringify(structure, null, 2));
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  if (!json) return structure;
+  for (const item of json) {
+    delete item.start;
+  }
+  return json;
+}
+async function checkTitleAppearance(item, pages, startIndex, options) {
+  const title = item.title;
+  if (!item.physicalIndex) {
+    return { listIndex: item.listIndex, answer: "no", title, pageNumber: void 0 };
+  }
+  const pageNumber = item.physicalIndex;
+  const pageText = pages[pageNumber - startIndex]?.text || "";
+  const prompt = checkTitleAppearancePrompt(title, pageText);
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  const answer = json?.answer === "yes" ? "yes" : "no";
+  return { listIndex: item.listIndex, answer, title, pageNumber };
+}
+async function checkTitleAppearanceInStart(title, pageText, options) {
+  const prompt = checkTitleStartAtBeginningPrompt(title, pageText);
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  return json?.start_begin === "yes" ? "yes" : "no";
+}
+async function checkTitleAppearanceInStartConcurrent(structure, pages, options) {
+  const results = [];
+  for (const item of structure) {
+    if (!item.physicalIndex) {
+      results.push({ ...item, appearStart: "no" });
+      continue;
+    }
+    const pageText = pages[item.physicalIndex - 1]?.text || "";
+    const appearStart = await checkTitleAppearanceInStart(item.title, pageText, options);
+    results.push({ ...item, appearStart });
+  }
+  return results;
+}
+async function checkToc(pages, options) {
+  const tocPageList = await findTocPages(0, pages, options);
+  if (tocPageList.length === 0) {
+    return {
+      tocContent: null,
+      tocPageList: [],
+      pageIndexGivenInToc: "no"
+    };
+  }
+  const tocResult = await tocExtractor(pages, tocPageList, options);
+  if (tocResult.pageIndexGivenInToc === "yes") {
+    return {
+      tocContent: tocResult.tocContent,
+      tocPageList,
+      pageIndexGivenInToc: "yes"
+    };
+  }
+  const lastTocPage = tocPageList[tocPageList.length - 1];
+  let currentStartIndex = lastTocPage !== void 0 ? lastTocPage + 1 : 0;
+  while (currentStartIndex < pages.length && currentStartIndex < options.tocCheckPageNum) {
+    const additionalTocPages = await findTocPages(currentStartIndex, pages, options);
+    if (additionalTocPages.length === 0) {
+      break;
+    }
+    const additionalTocResult = await tocExtractor(pages, additionalTocPages, options);
+    if (additionalTocResult.pageIndexGivenInToc === "yes") {
+      return {
+        tocContent: additionalTocResult.tocContent,
+        tocPageList: additionalTocPages,
+        pageIndexGivenInToc: "yes"
+      };
+    }
+    const lastAdditionalPage = additionalTocPages[additionalTocPages.length - 1];
+    currentStartIndex = lastAdditionalPage !== void 0 ? lastAdditionalPage + 1 : pages.length;
+  }
+  return {
+    tocContent: tocResult.tocContent,
+    tocPageList,
+    pageIndexGivenInToc: "no"
+  };
+}
+async function singleTocItemIndexFixer(sectionTitle, content, options) {
+  const prompt = singleTocItemIndexFixerPrompt(sectionTitle, content);
+  const response = await chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+  const json = extractJson(response);
+  if (!json?.physical_index) return null;
+  const result = convertPhysicalIndexToInt(json.physical_index);
+  return typeof result === "number" ? result : null;
+}
+
+// src/tree.ts
+function pageListToGroupText(pageContents, tokenLengths, maxTokens = 2e4, overlapPage = 1) {
+  const numTokens = tokenLengths.reduce((a, b) => a + b, 0);
+  if (numTokens <= maxTokens) {
+    return [pageContents.join("")];
+  }
+  const subsets = [];
+  let currentSubset = [];
+  let currentTokenCount = 0;
+  const expectedPartsNum = Math.ceil(numTokens / maxTokens);
+  const averageTokensPerPart = Math.ceil(
+    (numTokens / expectedPartsNum + maxTokens) / 2
+  );
+  for (let i = 0; i < pageContents.length; i++) {
+    const pageContent = pageContents[i] || "";
+    const pageTokens = tokenLengths[i] || 0;
+    if (currentTokenCount + pageTokens > averageTokensPerPart) {
+      subsets.push(currentSubset.join(""));
+      const overlapStart = Math.max(i - overlapPage, 0);
+      currentSubset = pageContents.slice(overlapStart, i);
+      currentTokenCount = tokenLengths.slice(overlapStart, i).reduce((a, b) => a + b, 0);
+    }
+    currentSubset.push(pageContent);
+    currentTokenCount += pageTokens;
+  }
+  if (currentSubset.length > 0) {
+    subsets.push(currentSubset.join(""));
+  }
+  return subsets;
+}
+function extractMatchingPagePairs(tocPage, tocPhysicalIndex, startPageIndex) {
+  const pairs = [];
+  for (const phyItem of tocPhysicalIndex) {
+    for (const pageItem of tocPage) {
+      if (phyItem.title === pageItem.title) {
+        const physicalIndex = phyItem.physicalIndex;
+        if (physicalIndex !== void 0 && physicalIndex >= startPageIndex) {
+          pairs.push({
+            title: phyItem.title,
+            page: pageItem.page || 0,
+            physicalIndex
+          });
+        }
+      }
+    }
+  }
+  return pairs;
+}
+function calculatePageOffset(pairs) {
+  const differences = [];
+  for (const pair of pairs) {
+    const difference = pair.physicalIndex - pair.page;
+    differences.push(difference);
+  }
+  if (differences.length === 0) return null;
+  const counts = /* @__PURE__ */ new Map();
+  for (const diff of differences) {
+    counts.set(diff, (counts.get(diff) || 0) + 1);
+  }
+  let maxCount = 0;
+  let mostCommon = 0;
+  for (const [diff, count] of counts) {
+    if (count > maxCount) {
+      maxCount = count;
+      mostCommon = diff;
+    }
+  }
+  return mostCommon;
+}
+function addPageOffsetToTocJson(data, offset) {
+  for (const item of data) {
+    if (item.page !== void 0 && typeof item.page === "number") {
+      item.physicalIndex = item.page + offset;
+      delete item.page;
+    }
+  }
+  return data;
+}
+async function processNoToc(pages, startIndex, options) {
+  const pageContents = [];
+  const tokenLengths = [];
+  for (let pageIndex = startIndex; pageIndex < startIndex + pages.length; pageIndex++) {
+    const pageText = `<physical_index_${pageIndex}>
+${pages[pageIndex - startIndex]?.text || ""}
+<physical_index_${pageIndex}>
+
+`;
+    pageContents.push(pageText);
+    tokenLengths.push(countTokens(pageText));
+  }
+  const groupTexts = pageListToGroupText(pageContents, tokenLengths);
+  let tocWithPageNumber = await generateTocInit(groupTexts[0] || "", options);
+  for (let i = 1; i < groupTexts.length; i++) {
+    const additional = await generateTocContinue(tocWithPageNumber, groupTexts[i] || "", options);
+    tocWithPageNumber.push(...additional);
+  }
+  return tocWithPageNumber;
+}
+async function processTocNoPageNumbers(tocContent, pages, startIndex, options) {
+  const pageContents = [];
+  const tokenLengths = [];
+  const tocItems = await tocTransformer(tocContent, options);
+  for (let pageIndex = startIndex; pageIndex < startIndex + pages.length; pageIndex++) {
+    const pageText = `<physical_index_${pageIndex}>
+${pages[pageIndex - startIndex]?.text || ""}
+<physical_index_${pageIndex}>
+
+`;
+    pageContents.push(pageText);
+    tokenLengths.push(countTokens(pageText));
+  }
+  const groupTexts = pageListToGroupText(pageContents, tokenLengths);
+  let tocWithPageNumber = [...tocItems];
+  for (const groupText of groupTexts) {
+    tocWithPageNumber = await addPageNumberToToc(groupText, tocWithPageNumber, options);
+  }
+  return tocWithPageNumber;
+}
+async function processTocWithPageNumbers(tocContent, tocPageList, pages, options) {
+  const tocWithPageNumber = await tocTransformer(tocContent, options);
+  const tocNoPageNumber = tocWithPageNumber.map((item) => {
+    const newItem = { ...item };
+    delete newItem.page;
+    return newItem;
+  });
+  const startPageIndex = (tocPageList[tocPageList.length - 1] || 0) + 1;
+  let mainContent = "";
+  for (let pageIndex = startPageIndex; pageIndex < Math.min(startPageIndex + options.tocCheckPageNum, pages.length); pageIndex++) {
+    mainContent += `<physical_index_${pageIndex + 1}>
+${pages[pageIndex]?.text || ""}
+<physical_index_${pageIndex + 1}>
+
+`;
+  }
+  const tocWithPhysicalIndex = await tocIndexExtractor(tocNoPageNumber, mainContent, options);
+  const matchingPairs = extractMatchingPagePairs(
+    tocWithPageNumber,
+    tocWithPhysicalIndex,
+    startPageIndex
+  );
+  const offset = calculatePageOffset(matchingPairs);
+  if (offset !== null) {
+    addPageOffsetToTocJson(tocWithPageNumber, offset);
+  }
+  return tocWithPageNumber;
+}
+function addNodeText(node, pages) {
+  if (Array.isArray(node)) {
+    for (const item of node) {
+      addNodeText(item, pages);
+    }
+    return;
+  }
+  const startPage = node.startIndex;
+  const endPage = node.endIndex;
+  if (startPage !== void 0 && endPage !== void 0) {
+    let text = "";
+    for (let i = startPage - 1; i < endPage && i < pages.length; i++) {
+      text += pages[i]?.text || "";
+    }
+    node.text = text;
+  }
+  if (node.nodes) {
+    addNodeText(node.nodes, pages);
+  }
+}
+async function generateNodeSummary(node, options) {
+  if (!node.text) return "";
+  const prompt = generateNodeSummaryPrompt(node.text);
+  return chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+}
+async function generateSummariesForStructure(structure, options) {
+  const nodes = structureToList(structure);
+  const batchSize = 5;
+  for (let i = 0; i < nodes.length; i += batchSize) {
+    const batch = nodes.slice(i, i + batchSize);
+    const summaries = await Promise.all(
+      batch.map((node) => generateNodeSummary(node, options))
+    );
+    for (let j = 0; j < batch.length; j++) {
+      batch[j].summary = summaries[j];
+    }
+  }
+}
+async function generateDocDescription(structure, options) {
+  const cleanStructure = createCleanStructureForDescription(structure);
+  const prompt = generateDocDescriptionPrompt(JSON.stringify(cleanStructure));
+  return chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+}
+async function verifyToc(pages, listResult, startIndex, options) {
+  const correct = [];
+  const incorrect = [];
+  for (let i = 0; i < listResult.length; i++) {
+    const item = listResult[i];
+    if (!item) continue;
+    const itemWithIndex = { ...item, listIndex: i };
+    const result = await checkTitleAppearance(itemWithIndex, pages, startIndex, options);
+    if (result.answer === "yes") {
+      correct.push(item);
+    } else {
+      incorrect.push({
+        listIndex: i,
+        title: item.title,
+        physicalIndex: item.physicalIndex
+      });
+    }
+  }
+  return { correct, incorrect };
+}
+async function fixIncorrectToc(tocWithPageNumber, pages, incorrectResults, startIndex, options) {
+  const fixed = [...tocWithPageNumber];
+  const stillIncorrect = [];
+  const incorrectIndices = new Set(incorrectResults.map((r) => r.listIndex));
+  const endIndex = pages.length + startIndex - 1;
+  for (const incorrectItem of incorrectResults) {
+    const { listIndex } = incorrectItem;
+    let prevCorrect = startIndex - 1;
+    for (let i = listIndex - 1; i >= 0; i--) {
+      if (!incorrectIndices.has(i)) {
+        const item = tocWithPageNumber[i];
+        if (item?.physicalIndex !== void 0) {
+          prevCorrect = item.physicalIndex;
+          break;
+        }
+      }
+    }
+    let nextCorrect = endIndex;
+    for (let i = listIndex + 1; i < tocWithPageNumber.length; i++) {
+      if (!incorrectIndices.has(i)) {
+        const item = tocWithPageNumber[i];
+        if (item?.physicalIndex !== void 0) {
+          nextCorrect = item.physicalIndex;
+          break;
+        }
+      }
+    }
+    const pageContents = [];
+    for (let pageIndex = prevCorrect; pageIndex <= nextCorrect; pageIndex++) {
+      const idx = pageIndex - startIndex;
+      if (idx >= 0 && idx < pages.length) {
+        pageContents.push(
+          `<physical_index_${pageIndex}>
+${pages[idx]?.text || ""}
+<physical_index_${pageIndex}>
+
+`
+        );
+      }
+    }
+    const contentRange = pageContents.join("");
+    const physicalIndexInt = await singleTocItemIndexFixer(
+      incorrectItem.title,
+      contentRange,
+      options
+    );
+    if (physicalIndexInt !== null && fixed[listIndex]) {
+      fixed[listIndex].physicalIndex = physicalIndexInt;
+      const checkItem = { ...fixed[listIndex], listIndex };
+      const checkResult = await checkTitleAppearance(checkItem, pages, startIndex, options);
+      if (checkResult.answer !== "yes") {
+        stillIncorrect.push({
+          listIndex,
+          title: incorrectItem.title,
+          physicalIndex: physicalIndexInt
+        });
+      }
+    } else {
+      stillIncorrect.push(incorrectItem);
+    }
+  }
+  return { fixed, stillIncorrect };
+}
+function buildTree(tocItems, endPhysicalIndex, options) {
+  const withPreface = addPrefaceIfNeeded(tocItems);
+  const tree = postProcessing(withPreface, endPhysicalIndex);
+  if (options.addNodeId) {
+    writeNodeId(tree);
+  }
+  const keyOrder = ["title", "nodeId", "startIndex", "endIndex", "summary", "text", "nodes"];
+  return formatStructure(tree, keyOrder);
+}
+
+// src/pageindex.ts
+var DEFAULT_OPTIONS = {
+  model: "gpt-4o-2024-11-20",
+  tocCheckPageNum: 20,
+  maxPageNumEachNode: 10,
+  maxTokenNumEachNode: 2e4,
+  addNodeId: true,
+  addNodeSummary: true,
+  addDocDescription: false,
+  addNodeText: false,
+  // OCR defaults
+  extractionMode: "text",
+  ocrModel: "mlx-community/GLM-OCR-bf16",
+  ocrPromptType: "text",
+  imageDpi: 150,
+  imageFormat: "png",
+  ocrConcurrency: 3
+};
+var PageIndex = class {
+  options;
+  constructor(options = {}) {
+    this.options = {
+      model: options.model || DEFAULT_OPTIONS.model,
+      tocCheckPageNum: options.tocCheckPageNum || DEFAULT_OPTIONS.tocCheckPageNum,
+      maxPageNumEachNode: options.maxPageNumEachNode || DEFAULT_OPTIONS.maxPageNumEachNode,
+      maxTokenNumEachNode: options.maxTokenNumEachNode || DEFAULT_OPTIONS.maxTokenNumEachNode,
+      addNodeId: options.addNodeId ?? DEFAULT_OPTIONS.addNodeId,
+      addNodeSummary: options.addNodeSummary ?? DEFAULT_OPTIONS.addNodeSummary,
+      addDocDescription: options.addDocDescription ?? DEFAULT_OPTIONS.addDocDescription,
+      addNodeText: options.addNodeText ?? DEFAULT_OPTIONS.addNodeText,
+      apiKey: options.apiKey,
+      baseUrl: options.baseUrl,
+      // OCR options
+      extractionMode: options.extractionMode || DEFAULT_OPTIONS.extractionMode,
+      ocrModel: options.ocrModel || DEFAULT_OPTIONS.ocrModel,
+      ocrPromptType: options.ocrPromptType || DEFAULT_OPTIONS.ocrPromptType,
+      imageDpi: options.imageDpi || DEFAULT_OPTIONS.imageDpi,
+      imageFormat: options.imageFormat || DEFAULT_OPTIONS.imageFormat,
+      ocrConcurrency: options.ocrConcurrency || DEFAULT_OPTIONS.ocrConcurrency
+    };
+  }
+  /**
+   * Set base URL for OpenAI-compatible API (e.g., LM Studio)
+   */
+  setBaseUrl(baseUrl) {
+    this.options.baseUrl = baseUrl;
+    return this;
+  }
+  /**
+   * Use LM Studio configuration
+   */
+  useLMStudio() {
+    this.options.baseUrl = "http://localhost:1234/v1";
+    this.options.apiKey = "lm-studio";
+    return this;
+  }
+  /**
+   * Use Ollama configuration
+   */
+  useOllama() {
+    this.options.baseUrl = "http://localhost:11434/v1";
+    this.options.apiKey = "ollama";
+    return this;
+  }
+  /**
+   * Enable OCR mode for scanned PDFs
+   */
+  useOcrMode(ocrModel) {
+    this.options.extractionMode = "ocr";
+    if (ocrModel) {
+      this.options.ocrModel = ocrModel;
+    }
+    return this;
+  }
+  /**
+   * Process a PDF file and build tree index
+   */
+  async fromPdf(input) {
+    let pages;
+    let pdfName;
+    if (this.options.extractionMode === "ocr") {
+      console.log("[OCR Mode] Processing PDF with OCR...");
+      const ocrOptions = {
+        ocrModel: this.options.ocrModel,
+        apiKey: this.options.apiKey,
+        baseUrl: this.options.baseUrl,
+        imageFormat: this.options.imageFormat,
+        imageDpi: this.options.imageDpi,
+        ocrPromptType: this.options.ocrPromptType,
+        concurrency: this.options.ocrConcurrency
+      };
+      const result = await parsePdfWithOcr(input, ocrOptions);
+      pages = result.pages;
+      pdfName = typeof input === "string" ? getPdfName(input) : "Untitled";
+    } else {
+      const pdfInfo = await parsePdf(input);
+      pages = pdfInfo.pages;
+      pdfName = typeof input === "string" ? getPdfName(input) : pdfInfo.title;
+    }
+    return this.processPdfPages(pages, pdfName);
+  }
+  /**
+   * Process PDF pages directly
+   */
+  async processPdfPages(pages, docName) {
+    const startIndex = 1;
+    const endPhysicalIndex = pages.length;
+    const tocResult = await checkToc(pages, this.options);
+    console.log(
+      `TOC found: ${tocResult.tocContent !== null}, Pages: ${tocResult.tocPageList.length}, Has page numbers: ${tocResult.pageIndexGivenInToc}`
+    );
+    let tocItems;
+    if (tocResult.tocContent === null) {
+      console.log("Generating structure from document content...");
+      tocItems = await processNoToc(pages, startIndex, this.options);
+    } else if (tocResult.pageIndexGivenInToc === "no") {
+      console.log("Processing TOC without page numbers...");
+      tocItems = await processTocNoPageNumbers(
+        tocResult.tocContent,
+        pages,
+        startIndex,
+        this.options
+      );
+    } else {
+      console.log("Processing TOC with page numbers...");
+      tocItems = await processTocWithPageNumbers(
+        tocResult.tocContent,
+        tocResult.tocPageList,
+        pages,
+        this.options
+      );
+    }
+    tocItems = convertPhysicalIndexToInt(tocItems);
+    tocItems = await checkTitleAppearanceInStartConcurrent(tocItems, pages, this.options);
+    console.log("Verifying TOC...");
+    const { incorrect } = await verifyToc(pages, tocItems, startIndex, this.options);
+    if (incorrect.length > 0) {
+      console.log(`Fixing ${incorrect.length} incorrect TOC items...`);
+      const { fixed } = await fixIncorrectToc(
+        tocItems,
+        pages,
+        incorrect,
+        startIndex,
+        this.options
+      );
+      tocItems = fixed;
+    }
+    const tree = buildTree(tocItems, endPhysicalIndex, this.options);
+    if (this.options.addNodeText || this.options.addNodeSummary) {
+      addNodeText(tree, pages);
+    }
+    if (this.options.addNodeSummary) {
+      console.log("Generating summaries...");
+      await generateSummariesForStructure(tree, this.options);
+    }
+    let docDescription;
+    if (this.options.addDocDescription) {
+      console.log("Generating document description...");
+      docDescription = await generateDocDescription(tree, this.options);
+    }
+    let finalStructure = tree;
+    if (!this.options.addNodeText) {
+      finalStructure = removeFields(tree, ["text"]);
+    }
+    return {
+      docName,
+      docDescription,
+      structure: finalStructure
+    };
+  }
+};
+
+// src/markdown.ts
+import * as path2 from "path";
+import * as fs3 from "fs/promises";
+var DEFAULT_MARKDOWN_OPTIONS = {
+  model: "gpt-4o-2024-11-20",
+  tocCheckPageNum: 20,
+  maxPageNumEachNode: 10,
+  maxTokenNumEachNode: 2e4,
+  addNodeId: true,
+  addNodeSummary: true,
+  addDocDescription: false,
+  addNodeText: false,
+  thinning: false,
+  thinningThreshold: 5e3,
+  summaryTokenThreshold: 200
+};
+function extractNodesFromMarkdown(markdownContent) {
+  const headerPattern = /^(#{1,6})\s+(.+)$/;
+  const codeBlockPattern = /^```/;
+  const nodeList = [];
+  const lines = markdownContent.split("\n");
+  let inCodeBlock = false;
+  for (let lineNum = 0; lineNum < lines.length; lineNum++) {
+    const line = lines[lineNum];
+    const strippedLine = line.trim();
+    if (codeBlockPattern.test(strippedLine)) {
+      inCodeBlock = !inCodeBlock;
+      continue;
+    }
+    if (!strippedLine) {
+      continue;
+    }
+    if (!inCodeBlock) {
+      const match = strippedLine.match(headerPattern);
+      if (match) {
+        const title = match[2].trim();
+        nodeList.push({ nodeTitle: title, lineNum: lineNum + 1 });
+      }
+    }
+  }
+  return { nodeList, lines };
+}
+function extractNodeTextContent(nodeList, markdownLines) {
+  const allNodes = [];
+  for (const node of nodeList) {
+    const lineContent = markdownLines[node.lineNum - 1];
+    if (!lineContent) continue;
+    const headerMatch = lineContent.match(/^(#{1,6})/);
+    if (!headerMatch) {
+      console.warn(
+        `Warning: Line ${node.lineNum} does not contain a valid header: '${lineContent}'`
+      );
+      continue;
+    }
+    const processedNode = {
+      title: node.nodeTitle,
+      lineNum: node.lineNum,
+      level: headerMatch[1].length
+    };
+    allNodes.push(processedNode);
+  }
+  for (let i = 0; i < allNodes.length; i++) {
+    const node = allNodes[i];
+    const startLine = node.lineNum - 1;
+    let endLine;
+    if (i + 1 < allNodes.length) {
+      endLine = allNodes[i + 1].lineNum - 1;
+    } else {
+      endLine = markdownLines.length;
+    }
+    node.text = markdownLines.slice(startLine, endLine).join("\n").trim();
+  }
+  return allNodes;
+}
+function findAllChildren(parentIndex, parentLevel, nodeList) {
+  const childrenIndices = [];
+  for (let i = parentIndex + 1; i < nodeList.length; i++) {
+    const currentLevel = nodeList[i].level || 0;
+    if (currentLevel <= parentLevel) {
+      break;
+    }
+    childrenIndices.push(i);
+  }
+  return childrenIndices;
+}
+function updateNodeListWithTextTokenCount(nodeList) {
+  const resultList = nodeList.map((n) => ({ ...n }));
+  for (let i = resultList.length - 1; i >= 0; i--) {
+    const currentNode = resultList[i];
+    const currentLevel = currentNode.level || 0;
+    const childrenIndices = findAllChildren(i, currentLevel, resultList);
+    const nodeText = currentNode.text || "";
+    let totalText = nodeText;
+    for (const childIndex of childrenIndices) {
+      const childText = resultList[childIndex]?.text || "";
+      if (childText) {
+        totalText += "\n" + childText;
+      }
+    }
+    resultList[i].textTokenCount = countTokens(totalText);
+  }
+  return resultList;
+}
+function treeThinningForIndex(nodeList, minNodeToken) {
+  const resultList = nodeList.map((n) => ({ ...n }));
+  const nodesToRemove = /* @__PURE__ */ new Set();
+  for (let i = resultList.length - 1; i >= 0; i--) {
+    if (nodesToRemove.has(i)) {
+      continue;
+    }
+    const currentNode = resultList[i];
+    const currentLevel = currentNode.level || 0;
+    const totalTokens = currentNode.textTokenCount || 0;
+    if (totalTokens < minNodeToken) {
+      const childrenIndices = findAllChildren(i, currentLevel, resultList);
+      const childrenTexts = [];
+      for (const childIndex of childrenIndices.sort((a, b) => a - b)) {
+        if (!nodesToRemove.has(childIndex)) {
+          const childText = resultList[childIndex]?.text || "";
+          if (childText.trim()) {
+            childrenTexts.push(childText);
+          }
+          nodesToRemove.add(childIndex);
+        }
+      }
+      if (childrenTexts.length > 0) {
+        let parentText = currentNode.text || "";
+        let mergedText = parentText;
+        for (const childText of childrenTexts) {
+          if (mergedText && !mergedText.endsWith("\n")) {
+            mergedText += "\n\n";
+          }
+          mergedText += childText;
+        }
+        resultList[i].text = mergedText;
+        resultList[i].textTokenCount = countTokens(mergedText);
+      }
+    }
+  }
+  const indicesToRemove = Array.from(nodesToRemove).sort((a, b) => b - a);
+  for (const index of indicesToRemove) {
+    resultList.splice(index, 1);
+  }
+  return resultList;
+}
+function buildTreeFromNodes(nodeList) {
+  if (nodeList.length === 0) {
+    return [];
+  }
+  const stack = [];
+  const rootNodes = [];
+  let nodeCounter = 1;
+  for (const node of nodeList) {
+    const currentLevel = node.level || 1;
+    const treeNode = {
+      title: node.title,
+      nodeId: String(nodeCounter).padStart(4, "0"),
+      text: node.text,
+      lineNum: node.lineNum,
+      nodes: []
+    };
+    nodeCounter++;
+    while (stack.length > 0 && stack[stack.length - 1][1] >= currentLevel) {
+      stack.pop();
+    }
+    if (stack.length === 0) {
+      rootNodes.push(treeNode);
+    } else {
+      const [parentNode] = stack[stack.length - 1];
+      parentNode.nodes.push(treeNode);
+    }
+    stack.push([treeNode, currentLevel]);
+  }
+  return rootNodes;
+}
+async function getNodeSummary(node, summaryTokenThreshold, options) {
+  const nodeText = node.text || "";
+  const numTokens = countTokens(nodeText);
+  if (numTokens < summaryTokenThreshold) {
+    return nodeText;
+  }
+  const prompt = generateNodeSummaryPrompt(nodeText);
+  return chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+}
+async function generateSummariesForStructureMd(structure, summaryTokenThreshold, options) {
+  const nodes = structureToList(structure);
+  const batchSize = 5;
+  for (let i = 0; i < nodes.length; i += batchSize) {
+    const batch = nodes.slice(i, i + batchSize);
+    const summaries = await Promise.all(
+      batch.map(
+        (node) => getNodeSummary(node, summaryTokenThreshold, options)
+      )
+    );
+    for (let j = 0; j < batch.length; j++) {
+      const node = batch[j];
+      if (!node.nodes || node.nodes.length === 0) {
+        node.summary = summaries[j];
+      } else {
+        node.prefixSummary = summaries[j];
+      }
+    }
+  }
+}
+async function generateDocDescriptionMd(structure, options) {
+  const cleanStructure = createCleanStructureForDescription(structure);
+  const prompt = generateDocDescriptionPrompt(
+    JSON.stringify(cleanStructure)
+  );
+  return chatGPT({
+    model: options.model,
+    prompt,
+    apiKey: options.apiKey,
+    baseUrl: options.baseUrl
+  });
+}
+async function mdToTree(mdPath, options = {}) {
+  const opts = {
+    ...DEFAULT_MARKDOWN_OPTIONS,
+    ...options
+  };
+  const markdownContent = await fs3.readFile(mdPath, "utf-8");
+  console.log("Extracting nodes from markdown...");
+  const { nodeList, lines: markdownLines } = extractNodesFromMarkdown(markdownContent);
+  console.log("Extracting text content from nodes...");
+  let nodesWithContent = extractNodeTextContent(nodeList, markdownLines);
+  if (opts.thinning) {
+    nodesWithContent = updateNodeListWithTextTokenCount(nodesWithContent);
+    console.log("Thinning nodes...");
+    nodesWithContent = treeThinningForIndex(nodesWithContent, opts.thinningThreshold);
+  }
+  console.log("Building tree from nodes...");
+  let treeStructure = buildTreeFromNodes(nodesWithContent);
+  if (opts.addNodeId) {
+    writeNodeId(treeStructure);
+  }
+  console.log("Formatting tree structure...");
+  const keyOrder = [
+    "title",
+    "nodeId",
+    "summary",
+    "prefixSummary",
+    "text",
+    "lineNum",
+    "nodes"
+  ];
+  if (opts.addNodeSummary) {
+    treeStructure = formatStructure(treeStructure, keyOrder);
+    console.log("Generating summaries for each node...");
+    await generateSummariesForStructureMd(
+      treeStructure,
+      opts.summaryTokenThreshold,
+      {
+        model: opts.model,
+        apiKey: opts.apiKey,
+        baseUrl: void 0
+      }
+    );
+    if (!opts.addNodeText) {
+      const orderWithoutText = keyOrder.filter((k) => k !== "text");
+      treeStructure = formatStructure(treeStructure, orderWithoutText);
+    }
+    if (opts.addDocDescription) {
+      console.log("Generating document description...");
+      const docDescription = await generateDocDescriptionMd(treeStructure, {
+        model: opts.model,
+        apiKey: opts.apiKey,
+        baseUrl: void 0
+      });
+      return {
+        docName: path2.basename(mdPath, path2.extname(mdPath)),
+        docDescription,
+        structure: treeStructure
+      };
+    }
+  } else {
+    const orderToUse = opts.addNodeText ? keyOrder : keyOrder.filter((k) => k !== "text");
+    treeStructure = formatStructure(treeStructure, orderToUse);
+  }
+  return {
+    docName: path2.basename(mdPath, path2.extname(mdPath)),
+    structure: treeStructure
+  };
+}
+
+// src/cli.ts
+import * as path3 from "path";
+import * as fs4 from "fs";
+import * as fsp from "fs/promises";
+function printHelp() {
+  console.log(`
+bun-pageindex - Vectorless, reasoning-based RAG for document understanding
+
+USAGE:
+  bun-pageindex --pdf <path>     Process a PDF file
+  bun-pageindex --md <path>      Process a Markdown file
+
+OPTIONS:
+  --pdf <path>                 Path to PDF file
+  --md <path>                  Path to Markdown file
+  --output, -o <path>          Output file path (default: ./results/<name>_structure.json)
+  
+  MODEL OPTIONS:
+  --model <name>               Model to use (default: gpt-4o-2024-11-20)
+  --lmstudio                   Use LM Studio (localhost:1234)
+  --ollama                     Use Ollama (localhost:11434)
+  --base-url <url>             Custom OpenAI-compatible API URL
+  
+  PDF OPTIONS:
+  --toc-check-pages <n>        Pages to check for TOC (default: 20)
+  --max-pages-per-node <n>     Max pages per node (default: 10)
+  --max-tokens-per-node <n>    Max tokens per node (default: 20000)
+  
+  OCR OPTIONS (for scanned PDFs):
+  --ocr                        Enable OCR mode for scanned PDFs
+  --ocr-model <name>           OCR model (default: mlx-community/GLM-OCR-bf16)
+  --ocr-prompt-type <type>     OCR prompt: text, formula, table (default: text)
+  --image-dpi <n>              Image DPI for OCR (default: 150)
+  
+  MARKDOWN OPTIONS:
+  --thinning                   Apply tree thinning
+  --thinning-threshold <n>     Min tokens for thinning (default: 5000)
+  --summary-token-threshold <n> Token threshold for summaries (default: 200)
+  
+  OUTPUT OPTIONS:
+  --add-node-id                Add node IDs (default: true)
+  --no-node-id                 Don't add node IDs
+  --add-node-summary           Add node summaries (default: true)
+  --no-node-summary            Don't add node summaries
+  --add-doc-description        Add document description
+  --add-node-text              Include raw text in output
+  
+  --help, -h                   Show this help message
+
+EXAMPLES:
+  bun-pageindex --pdf document.pdf
+  bun-pageindex --md README.md --add-doc-description
+  bun-pageindex --pdf paper.pdf --lmstudio --model llama3
+  bun-pageindex --pdf report.pdf --base-url http://localhost:8080/v1
+  bun-pageindex --pdf scanned.pdf --ocr --lmstudio --model qwen/qwen3-vl-30b
+`);
+}
+function parseCliArgs() {
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      pdf: { type: "string" },
+      md: { type: "string" },
+      model: { type: "string", default: "gpt-4o-2024-11-20" },
+      "toc-check-pages": { type: "string", default: "20" },
+      "max-pages-per-node": { type: "string", default: "10" },
+      "max-tokens-per-node": { type: "string", default: "20000" },
+      "add-node-id": { type: "boolean", default: true },
+      "no-node-id": { type: "boolean", default: false },
+      "add-node-summary": { type: "boolean", default: true },
+      "no-node-summary": { type: "boolean", default: false },
+      "add-doc-description": { type: "boolean", default: false },
+      "add-node-text": { type: "boolean", default: false },
+      thinning: { type: "boolean", default: false },
+      "thinning-threshold": { type: "string", default: "5000" },
+      "summary-token-threshold": { type: "string", default: "200" },
+      output: { type: "string", short: "o" },
+      lmstudio: { type: "boolean", default: false },
+      ollama: { type: "boolean", default: false },
+      "base-url": { type: "string" },
+      // OCR options
+      ocr: { type: "boolean", default: false },
+      "ocr-model": { type: "string", default: "mlx-community/GLM-OCR-bf16" },
+      "ocr-prompt-type": { type: "string", default: "text" },
+      "image-dpi": { type: "string", default: "150" },
+      help: { type: "boolean", short: "h", default: false }
+    },
+    allowPositionals: true
+  });
+  return {
+    pdf: values.pdf,
+    md: values.md,
+    model: values.model || "gpt-4o-2024-11-20",
+    tocCheckPages: parseInt(values["toc-check-pages"] || "20", 10),
+    maxPagesPerNode: parseInt(values["max-pages-per-node"] || "10", 10),
+    maxTokensPerNode: parseInt(values["max-tokens-per-node"] || "20000", 10),
+    addNodeId: values["no-node-id"] ? false : values["add-node-id"] ?? true,
+    addNodeSummary: values["no-node-summary"] ? false : values["add-node-summary"] ?? true,
+    addDocDescription: values["add-doc-description"] ?? false,
+    addNodeText: values["add-node-text"] ?? false,
+    thinning: values.thinning ?? false,
+    thinningThreshold: parseInt(values["thinning-threshold"] || "5000", 10),
+    summaryTokenThreshold: parseInt(values["summary-token-threshold"] || "200", 10),
+    output: values.output,
+    lmstudio: values.lmstudio ?? false,
+    ollama: values.ollama ?? false,
+    baseUrl: values["base-url"],
+    // OCR options
+    ocr: values.ocr ?? false,
+    ocrModel: values["ocr-model"] || "mlx-community/GLM-OCR-bf16",
+    ocrPromptType: values["ocr-prompt-type"] || "text",
+    imageDpi: parseInt(values["image-dpi"] || "150", 10),
+    help: values.help ?? false
+  };
+}
+async function main() {
+  const args = parseCliArgs();
+  if (args.help) {
+    printHelp();
+    process.exit(0);
+  }
+  if (!args.pdf && !args.md) {
+    console.error("Error: Either --pdf or --md must be specified");
+    console.error("Use --help for usage information");
+    process.exit(1);
+  }
+  if (args.pdf && args.md) {
+    console.error("Error: Only one of --pdf or --md can be specified");
+    process.exit(1);
+  }
+  const inputPath = args.pdf || args.md;
+  const inputName = path3.basename(inputPath, path3.extname(inputPath));
+  const outputDir = "./results";
+  const outputPath = args.output || path3.join(outputDir, `${inputName}_structure.json`);
+  if (!fs4.existsSync(outputDir)) {
+    fs4.mkdirSync(outputDir, { recursive: true });
+  }
+  let result;
+  if (args.pdf) {
+    if (!args.pdf.toLowerCase().endsWith(".pdf")) {
+      console.error("Error: PDF file must have .pdf extension");
+      process.exit(1);
+    }
+    if (!fs4.existsSync(args.pdf)) {
+      console.error(`Error: PDF file not found: ${args.pdf}`);
+      process.exit(1);
+    }
+    console.log(`Processing PDF: ${args.pdf}`);
+    if (args.ocr) {
+      console.log(`[OCR Mode] Using OCR model: ${args.ocrModel}`);
+    }
+    const pageIndex = new PageIndex({
+      model: args.model,
+      tocCheckPageNum: args.tocCheckPages,
+      maxPageNumEachNode: args.maxPagesPerNode,
+      maxTokenNumEachNode: args.maxTokensPerNode,
+      addNodeId: args.addNodeId,
+      addNodeSummary: args.addNodeSummary,
+      addDocDescription: args.addDocDescription,
+      addNodeText: args.addNodeText,
+      // OCR options
+      extractionMode: args.ocr ? "ocr" : "text",
+      ocrModel: args.ocrModel,
+      ocrPromptType: args.ocrPromptType,
+      imageDpi: args.imageDpi
+    });
+    if (args.lmstudio) {
+      pageIndex.useLMStudio();
+    } else if (args.ollama) {
+      pageIndex.useOllama();
+    } else if (args.baseUrl) {
+      pageIndex.setBaseUrl(args.baseUrl);
+    }
+    result = await pageIndex.fromPdf(args.pdf);
+  } else {
+    const mdPath = args.md;
+    if (!mdPath.toLowerCase().endsWith(".md") && !mdPath.toLowerCase().endsWith(".markdown")) {
+      console.error("Error: Markdown file must have .md or .markdown extension");
+      process.exit(1);
+    }
+    if (!fs4.existsSync(mdPath)) {
+      console.error(`Error: Markdown file not found: ${mdPath}`);
+      process.exit(1);
+    }
+    console.log(`Processing Markdown: ${mdPath}`);
+    result = await mdToTree(mdPath, {
+      model: args.model,
+      addNodeId: args.addNodeId,
+      addNodeSummary: args.addNodeSummary,
+      addDocDescription: args.addDocDescription,
+      addNodeText: args.addNodeText,
+      thinning: args.thinning,
+      thinningThreshold: args.thinningThreshold,
+      summaryTokenThreshold: args.summaryTokenThreshold
+    });
+  }
+  console.log("Parsing done, saving to file...");
+  await fsp.writeFile(outputPath, JSON.stringify(result, null, 2));
+  console.log(`Tree structure saved to: ${outputPath}`);
+}
+main().catch((error) => {
+  console.error("Error:", error.message);
+  process.exit(1);
+});