nexum-code 0.15.3 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -13
- package/dist/XMLParser-DFAnGqkc.mjs +3956 -0
- package/dist/{app-Dzqtno-_.mjs → app-CYLTD0b3.mjs} +21151 -12642
- package/dist/{dispatch-RwKBVuHW.mjs → dispatch-DnZRjRNm.mjs} +55743 -34590
- package/dist/{dispatch-ClUS04uO.mjs → dispatch-cUxbpzTE.mjs} +1 -1
- package/dist/{markit-DG5XsAGz.mjs → esm-C0toZ4e0.mjs} +19 -6205
- package/dist/main.mjs +1 -1
- package/dist/markdown-BwvhOvCA.mjs +25943 -0
- package/dist/markit-LwaDGgYt.mjs +2334 -0
- package/dist/public/assets/astro-DAp21e1Q.js +2 -0
- package/dist/public/assets/astro-DAp21e1Q.js.map +1 -0
- package/dist/public/assets/bash-Yzrsuije.js +2 -0
- package/dist/public/assets/bash-Yzrsuije.js.map +1 -0
- package/dist/public/assets/c-BIGW1oBm.js +2 -0
- package/dist/public/assets/c-BIGW1oBm.js.map +1 -0
- package/dist/public/assets/cpp-BRuaLJcg.js +2 -0
- package/dist/public/assets/cpp-BRuaLJcg.js.map +1 -0
- package/dist/public/assets/csharp-COcwbKMJ.js +2 -0
- package/dist/public/assets/csharp-COcwbKMJ.js.map +1 -0
- package/dist/public/assets/css-DEUVOdPa.js +2 -0
- package/dist/public/assets/css-DEUVOdPa.js.map +1 -0
- package/dist/public/assets/dart-CF10PKvl.js +2 -0
- package/dist/public/assets/dart-CF10PKvl.js.map +1 -0
- package/dist/public/assets/docker-BcOcwvcX.js +2 -0
- package/dist/public/assets/docker-BcOcwvcX.js.map +1 -0
- package/dist/public/assets/go-CxLEBnE3.js +2 -0
- package/dist/public/assets/go-CxLEBnE3.js.map +1 -0
- package/dist/public/assets/graphql-ChdNCCLP.js +2 -0
- package/dist/public/assets/graphql-ChdNCCLP.js.map +1 -0
- package/dist/public/assets/html-CNxCFRDw.js +2 -0
- package/dist/public/assets/html-CNxCFRDw.js.map +1 -0
- package/dist/public/assets/index-BfTRKbmy.css +1 -0
- package/dist/public/assets/index-C28kWE_L.js +240 -0
- package/dist/public/assets/index-C28kWE_L.js.map +1 -0
- package/dist/public/assets/ini-BEwlwnbL.js +2 -0
- package/dist/public/assets/ini-BEwlwnbL.js.map +1 -0
- package/dist/public/assets/inter-cyrillic-ext-wght-normal-BOeWTOD4.woff2 +0 -0
- package/dist/public/assets/inter-cyrillic-wght-normal-DqGufNeO.woff2 +0 -0
- package/dist/public/assets/inter-greek-ext-wght-normal-DlzME5K_.woff2 +0 -0
- package/dist/public/assets/inter-greek-wght-normal-CkhJZR-_.woff2 +0 -0
- package/dist/public/assets/inter-latin-ext-wght-normal-DO1Apj_S.woff2 +0 -0
- package/dist/public/assets/inter-latin-wght-normal-Dx4kXJAl.woff2 +0 -0
- package/dist/public/assets/inter-vietnamese-wght-normal-CBcvBZtf.woff2 +0 -0
- package/dist/public/assets/java-CylS5w8V.js +2 -0
- package/dist/public/assets/java-CylS5w8V.js.map +1 -0
- package/dist/public/assets/javascript-wDzz0qaB.js +2 -0
- package/dist/public/assets/javascript-wDzz0qaB.js.map +1 -0
- package/dist/public/assets/jetbrains-mono-cyrillic-wght-normal-D73BlboJ.woff2 +0 -0
- package/dist/public/assets/jetbrains-mono-greek-wght-normal-Bw9x6K1M.woff2 +0 -0
- package/dist/public/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
- package/dist/public/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
- package/dist/public/assets/jetbrains-mono-vietnamese-wght-normal-Bt-aOZkq.woff2 +0 -0
- package/dist/public/assets/json-Cp-IABpG.js +2 -0
- package/dist/public/assets/json-Cp-IABpG.js.map +1 -0
- package/dist/public/assets/jsx-g9-lgVsj.js +2 -0
- package/dist/public/assets/jsx-g9-lgVsj.js.map +1 -0
- package/dist/public/assets/kotlin-BdnUsdx6.js +2 -0
- package/dist/public/assets/kotlin-BdnUsdx6.js.map +1 -0
- package/dist/public/assets/lua-BaeVxFsk.js +2 -0
- package/dist/public/assets/lua-BaeVxFsk.js.map +1 -0
- package/dist/public/assets/make-CHLpvVh8.js +2 -0
- package/dist/public/assets/make-CHLpvVh8.js.map +1 -0
- package/dist/public/assets/markdown-Cvjx9yec.js +2 -0
- package/dist/public/assets/markdown-Cvjx9yec.js.map +1 -0
- package/dist/public/assets/perl-CptaX86O.js +2 -0
- package/dist/public/assets/perl-CptaX86O.js.map +1 -0
- package/dist/public/assets/postcss-DnzrXTVv.js +2 -0
- package/dist/public/assets/postcss-DnzrXTVv.js.map +1 -0
- package/dist/public/assets/protobuf-C7zT0LnQ.js +2 -0
- package/dist/public/assets/protobuf-C7zT0LnQ.js.map +1 -0
- package/dist/public/assets/python-B6aJPvgy.js +2 -0
- package/dist/public/assets/python-B6aJPvgy.js.map +1 -0
- package/dist/public/assets/r-Dspwwk_N.js +2 -0
- package/dist/public/assets/r-Dspwwk_N.js.map +1 -0
- package/dist/public/assets/ruby-CcrC3V9I.js +2 -0
- package/dist/public/assets/ruby-CcrC3V9I.js.map +1 -0
- package/dist/public/assets/rust-B1yitclQ.js +2 -0
- package/dist/public/assets/rust-B1yitclQ.js.map +1 -0
- package/dist/public/assets/scala-C151Ov-r.js +2 -0
- package/dist/public/assets/scala-C151Ov-r.js.map +1 -0
- package/dist/public/assets/sql-BLtJtn59.js +2 -0
- package/dist/public/assets/sql-BLtJtn59.js.map +1 -0
- package/dist/public/assets/svelte-U4fHntn6.js +2 -0
- package/dist/public/assets/svelte-U4fHntn6.js.map +1 -0
- package/dist/public/assets/swift-D82vCrfD.js +2 -0
- package/dist/public/assets/swift-D82vCrfD.js.map +1 -0
- package/dist/public/assets/toml-vGWfd6FD.js +2 -0
- package/dist/public/assets/toml-vGWfd6FD.js.map +1 -0
- package/dist/public/assets/tsx-COt5Ahok.js +2 -0
- package/dist/public/assets/tsx-COt5Ahok.js.map +1 -0
- package/dist/public/assets/typescript-BPQ3VLAy.js +2 -0
- package/dist/public/assets/typescript-BPQ3VLAy.js.map +1 -0
- package/dist/public/assets/xml-sdJ4AIDG.js +2 -0
- package/dist/public/assets/xml-sdJ4AIDG.js.map +1 -0
- package/dist/public/assets/yaml-Buea-lGh.js +2 -0
- package/dist/public/assets/yaml-Buea-lGh.js.map +1 -0
- package/dist/public/index.html +2 -2
- package/dist/{src-CtMkYz0K.mjs → src-C6YAs3fd.mjs} +1 -1
- package/dist/src-C8mW2wSe.mjs +7765 -0
- package/dist/{text-input-dialog-WSd537y1.mjs → text-input-dialog-BbyJmVbH.mjs} +1 -1
- package/dist/{text-input-dialog-Dvux6kIR.mjs → text-input-dialog-CAxwn6Ze.mjs} +1437 -111
- package/docs/architecture.md +80 -0
- package/docs/configuration.md +204 -0
- package/docs/line.md +159 -0
- package/docs/mcp.md +141 -0
- package/docs/plugins.md +198 -0
- package/docs/sessions.md +78 -0
- package/docs/skills.md +76 -0
- package/package.json +18 -7
- package/dist/base64-js-DzVmk6Nb.mjs +0 -95
- package/dist/public/assets/index-CGwN8W_j.js +0 -93
- package/dist/public/assets/index-CGwN8W_j.js.map +0 -1
- package/dist/public/assets/index-DMyG0a38.css +0 -1
- package/dist/public/assets/wasm-CG6Dc4jp.js +0 -2
- package/dist/public/assets/wasm-CG6Dc4jp.js.map +0 -1
- package/dist/src-DE9LHL58.mjs +0 -2249
|
@@ -0,0 +1,2334 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
3
|
+
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
|
+
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
|
+
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
+
import { o as __toESM } from "./chunk-D90kvbyJ.mjs";
|
|
7
|
+
import { t as XMLParser } from "./XMLParser-DFAnGqkc.mjs";
|
|
8
|
+
import { i as require_lib, n as require_turndown_plugin_gfm_cjs, r as require_turndown_cjs, t as unzipSync } from "./esm-C0toZ4e0.mjs";
|
|
9
|
+
import * as path$1 from "node:path";
|
|
10
|
+
import * as fs$1 from "node:fs/promises";
|
|
11
|
+
//#region ../../packages/agent-core/src/utils/turndown.ts
|
|
12
|
+
var import_lib = /* @__PURE__ */ __toESM(require_lib(), 1);
|
|
13
|
+
var import_turndown_cjs = /* @__PURE__ */ __toESM(require_turndown_cjs(), 1);
|
|
14
|
+
var import_turndown_plugin_gfm_cjs = require_turndown_plugin_gfm_cjs();
|
|
15
|
+
/**
|
|
16
|
+
* Build a Turndown instance configured for GFM with the fixes this codebase
|
|
17
|
+
* needs: `~~strikethrough~~`, unescaped heading periods, and single-space list
|
|
18
|
+
* markers.
|
|
19
|
+
*
|
|
20
|
+
* Shared by the web scrapers (HTML → markdown) and the markit document engine
|
|
21
|
+
* (`src/markit`). The rule set must stay identical across both call sites.
|
|
22
|
+
*/
|
|
23
|
+
function createTurndown() {
|
|
24
|
+
const turndown = new import_turndown_cjs.default({
|
|
25
|
+
headingStyle: "atx",
|
|
26
|
+
codeBlockStyle: "fenced",
|
|
27
|
+
bulletListMarker: "-"
|
|
28
|
+
});
|
|
29
|
+
turndown.use(import_turndown_plugin_gfm_cjs.gfm);
|
|
30
|
+
turndown.addRule("strikethrough", {
|
|
31
|
+
filter: [
|
|
32
|
+
"del",
|
|
33
|
+
"s",
|
|
34
|
+
"strike"
|
|
35
|
+
],
|
|
36
|
+
replacement(content) {
|
|
37
|
+
return `~~${content}~~`;
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
turndown.addRule("heading", {
|
|
41
|
+
filter: [
|
|
42
|
+
"h1",
|
|
43
|
+
"h2",
|
|
44
|
+
"h3",
|
|
45
|
+
"h4",
|
|
46
|
+
"h5",
|
|
47
|
+
"h6"
|
|
48
|
+
],
|
|
49
|
+
replacement(content, node) {
|
|
50
|
+
const level = Number(node.nodeName.charAt(1));
|
|
51
|
+
return `\n\n${"#".repeat(level)} ${content.replaceAll(/\\([.])/g, "$1").trim()}\n\n`;
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
turndown.addRule("listItem", {
|
|
55
|
+
filter: "li",
|
|
56
|
+
replacement(content, node, options) {
|
|
57
|
+
const body = content.replace(/^\n+/, "").replace(/\n+$/, "\n").replaceAll("\n", "\n ");
|
|
58
|
+
const parent = node.parentNode;
|
|
59
|
+
let prefix = `${options.bulletListMarker} `;
|
|
60
|
+
if (parent?.nodeName === "OL") {
|
|
61
|
+
const start = parent.getAttribute("start");
|
|
62
|
+
const index = Array.prototype.indexOf.call(parent.children, node);
|
|
63
|
+
prefix = `${(start ? Number(start) : 1) + index}. `;
|
|
64
|
+
}
|
|
65
|
+
return prefix + body + (node.nextSibling ? "\n" : "");
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
return turndown;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Normalize HTML tables so turndown-plugin-gfm can render them:
|
|
72
|
+
* - strip `<p>` tags inside `<td>`/`<th>` cells (joining paragraphs with a space)
|
|
73
|
+
* - wrap the first row in `<thead>` when missing
|
|
74
|
+
*/
|
|
75
|
+
function normalizeTablesHtml(html) {
|
|
76
|
+
let result = html.replaceAll(/<(td|th)([^>]*)>([\s\S]*?)<\/(td|th)>/gi, (_match, tag, attrs, inner, closeTag) => {
|
|
77
|
+
return `<${tag}${attrs}>${inner.replace(/^\s*<p>/i, "").replace(/<\/p>\s*$/i, "").replaceAll(/<\/p>\s*<p>/gi, " ")}</${closeTag}>`;
|
|
78
|
+
});
|
|
79
|
+
result = result.replaceAll(/<table([^>]*)>\s*(?:<tbody>\s*)?(<tr[\s\S]*?<\/tr>)([\s\S]*?)<\/(?:tbody>\s*<\/)?table>/gi, (_match, attrs, firstRow, rest) => {
|
|
80
|
+
return `<table${attrs}><thead>${firstRow.replaceAll("<td", "<th").replaceAll("</td>", "</th>")}</thead><tbody>${rest}</tbody></table>`;
|
|
81
|
+
});
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
//#endregion
|
|
85
|
+
//#region ../../packages/agent-core/src/markit/converters/docx.ts
|
|
86
|
+
const EXTENSIONS$4 = new Set([".docx"]);
|
|
87
|
+
const MIMETYPES$4 = ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"];
|
|
88
|
+
var DocxConverter = class {
|
|
89
|
+
name = "docx";
|
|
90
|
+
accepts(streamInfo) {
|
|
91
|
+
if (streamInfo.extension && EXTENSIONS$4.has(streamInfo.extension)) return true;
|
|
92
|
+
if (streamInfo.mimetype && MIMETYPES$4.some((m) => streamInfo.mimetype?.startsWith(m))) return true;
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
async convert(input, streamInfo) {
|
|
96
|
+
const imageDir = streamInfo.imageDir;
|
|
97
|
+
let imageCount = 0;
|
|
98
|
+
const convertImage = imageDir ? import_lib.images.imgElement((image) => {
|
|
99
|
+
imageCount++;
|
|
100
|
+
const ext = (image.contentType?.split("/")[1] || "png").replace("jpeg", "jpg");
|
|
101
|
+
const filename = `image_${imageCount}.${ext}`;
|
|
102
|
+
const filepath = path$1.join(imageDir, filename);
|
|
103
|
+
return image.read("base64").then(async (base64) => {
|
|
104
|
+
await fs$1.writeFile(filepath, Buffer.from(base64, "base64"));
|
|
105
|
+
return {
|
|
106
|
+
src: filepath,
|
|
107
|
+
alt: `image_${imageCount}`
|
|
108
|
+
};
|
|
109
|
+
});
|
|
110
|
+
}) : import_lib.images.imgElement((image) => {
|
|
111
|
+
imageCount++;
|
|
112
|
+
const contentType = image.contentType || "image/png";
|
|
113
|
+
return image.read("base64").then((base64) => {
|
|
114
|
+
return {
|
|
115
|
+
src: `data:${contentType};base64,${base64.slice(0, 0)}`,
|
|
116
|
+
alt: `image_${imageCount}`
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
const { value: html } = await import_lib.convertToHtml({ buffer: input }, { convertImage });
|
|
121
|
+
let markdown = createTurndown().turndown(normalizeTablesHtml(html));
|
|
122
|
+
if (!imageDir) markdown = markdown.replaceAll(/!\[([^\]]*)\]\(data:[^)]*\)/g, "<!-- image: $1 -->");
|
|
123
|
+
return { markdown: markdown.trim() };
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
//#endregion
|
|
127
|
+
//#region ../../packages/agent-core/src/utils/zip.ts
|
|
128
|
+
/**
|
|
129
|
+
* Minimal ZIP reading for the markit document converters (docx/pptx/xlsx/
|
|
130
|
+
* epub are all ZIP containers). Read-only: stored + deflate entries via
|
|
131
|
+
* fflate, which is pure JS and works on every platform.
|
|
132
|
+
*
|
|
133
|
+
* The converters only ever need `unzip`/`unzipText`, so a small fflate-backed
|
|
134
|
+
* reader is enough; no tar support is required here.
|
|
135
|
+
*/
|
|
136
|
+
const UTF8_DECODER = new TextDecoder();
|
|
137
|
+
/** Read a single ZIP entry as UTF-8 text, or `undefined` when the entry is absent. */
|
|
138
|
+
function unzipText(entries, entryPath) {
|
|
139
|
+
const data = entries[entryPath];
|
|
140
|
+
return data !== void 0 ? UTF8_DECODER.decode(data) : void 0;
|
|
141
|
+
}
|
|
142
|
+
/** Decode a ZIP archive to a `path → bytes` map. Throws on malformed input. */
|
|
143
|
+
function unzip(bytes) {
|
|
144
|
+
return unzipSync(bytes);
|
|
145
|
+
}
|
|
146
|
+
//#endregion
|
|
147
|
+
//#region ../../packages/agent-core/src/markit/converters/epub.ts
|
|
148
|
+
const EXTENSIONS$3 = new Set([".epub"]);
|
|
149
|
+
const MIMETYPES$3 = [
|
|
150
|
+
"application/epub",
|
|
151
|
+
"application/epub+zip",
|
|
152
|
+
"application/x-epub+zip"
|
|
153
|
+
];
|
|
154
|
+
var EpubConverter = class {
|
|
155
|
+
name = "epub";
|
|
156
|
+
accepts(streamInfo) {
|
|
157
|
+
if (streamInfo.extension && EXTENSIONS$3.has(streamInfo.extension)) return true;
|
|
158
|
+
if (streamInfo.mimetype && MIMETYPES$3.some((m) => streamInfo.mimetype?.startsWith(m))) return true;
|
|
159
|
+
return false;
|
|
160
|
+
}
|
|
161
|
+
async convert(input, _streamInfo) {
|
|
162
|
+
const entries = unzip(input);
|
|
163
|
+
const parser = new XMLParser({
|
|
164
|
+
ignoreAttributes: false,
|
|
165
|
+
attributeNamePrefix: "@_",
|
|
166
|
+
textNodeName: "#text",
|
|
167
|
+
processEntities: { maxTotalExpansions: 1e6 }
|
|
168
|
+
});
|
|
169
|
+
const containerXml = unzipText(entries, "META-INF/container.xml");
|
|
170
|
+
if (!containerXml) throw new Error("Invalid EPUB: missing container.xml");
|
|
171
|
+
const rootfile = parser.parse(containerXml).container?.rootfiles?.rootfile;
|
|
172
|
+
const opfPath = Array.isArray(rootfile) ? rootfile[0]["@_full-path"] : rootfile?.["@_full-path"];
|
|
173
|
+
if (!opfPath) throw new Error("Invalid EPUB: missing rootfile path");
|
|
174
|
+
const opfXml = unzipText(entries, opfPath);
|
|
175
|
+
if (!opfXml) throw new Error("Invalid EPUB: missing content.opf");
|
|
176
|
+
const opf = parser.parse(opfXml);
|
|
177
|
+
const meta = opf.package?.metadata ?? {};
|
|
178
|
+
const metadata = {
|
|
179
|
+
title: this.getText(meta["dc:title"]),
|
|
180
|
+
authors: this.getTextArray(meta["dc:creator"]).join(", ") || void 0,
|
|
181
|
+
language: this.getText(meta["dc:language"]),
|
|
182
|
+
publisher: this.getText(meta["dc:publisher"]),
|
|
183
|
+
date: this.getText(meta["dc:date"]),
|
|
184
|
+
description: this.getText(meta["dc:description"])
|
|
185
|
+
};
|
|
186
|
+
const manifestItems = opf.package?.manifest?.item;
|
|
187
|
+
const itemList = Array.isArray(manifestItems) ? manifestItems : manifestItems ? [manifestItems] : [];
|
|
188
|
+
const manifest = /* @__PURE__ */ new Map();
|
|
189
|
+
for (const item of itemList) manifest.set(item["@_id"], item["@_href"]);
|
|
190
|
+
const spineItems = opf.package?.spine?.itemref;
|
|
191
|
+
const spineOrder = (Array.isArray(spineItems) ? spineItems : spineItems ? [spineItems] : []).map((s) => s["@_idref"]);
|
|
192
|
+
const basePath = opfPath.includes("/") ? opfPath.substring(0, opfPath.lastIndexOf("/")) : "";
|
|
193
|
+
const turndown = createTurndown();
|
|
194
|
+
const sections = [];
|
|
195
|
+
const metaLines = [];
|
|
196
|
+
for (const key in metadata) {
|
|
197
|
+
const value = metadata[key];
|
|
198
|
+
if (value) metaLines.push(`**${key.charAt(0).toUpperCase() + key.slice(1)}:** ${value}`);
|
|
199
|
+
}
|
|
200
|
+
if (metaLines.length > 0) sections.push(metaLines.join("\n"));
|
|
201
|
+
for (const idref of spineOrder) {
|
|
202
|
+
const href = manifest.get(idref);
|
|
203
|
+
if (!href) continue;
|
|
204
|
+
const html = unzipText(entries, basePath ? `${basePath}/${href}` : href);
|
|
205
|
+
if (!html) continue;
|
|
206
|
+
const cleaned = html.replaceAll(/<script[\s\S]*?<\/script>/gi, "").replaceAll(/<style[\s\S]*?<\/style>/gi, "");
|
|
207
|
+
const md = turndown.turndown(normalizeTablesHtml(cleaned)).trim();
|
|
208
|
+
if (md) sections.push(md);
|
|
209
|
+
}
|
|
210
|
+
return {
|
|
211
|
+
markdown: sections.join("\n\n").trim(),
|
|
212
|
+
title: metadata.title
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
getText(node) {
|
|
216
|
+
if (!node) return void 0;
|
|
217
|
+
if (typeof node === "string") return node;
|
|
218
|
+
if (node["#text"]) return String(node["#text"]);
|
|
219
|
+
if (Array.isArray(node)) return this.getText(node[0]);
|
|
220
|
+
}
|
|
221
|
+
getTextArray(node) {
|
|
222
|
+
if (!node) return [];
|
|
223
|
+
return (Array.isArray(node) ? node : [node]).map((n) => this.getText(n)).filter(Boolean);
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
//#endregion
|
|
227
|
+
//#region ../../packages/agent-core/src/markit/converters/pdf/columns.ts
|
|
228
|
+
/**
|
|
229
|
+
* Minimum gap as a fraction of the total text width to consider a column
|
|
230
|
+
* boundary. A two-column layout typically has ~50% gap; we use a lower
|
|
231
|
+
* threshold to catch asymmetric columns.
|
|
232
|
+
*/
|
|
233
|
+
const MIN_GAP_RATIO = .15;
|
|
234
|
+
/** Minimum number of text boxes on each side of the gap. */
|
|
235
|
+
const MIN_BOXES_PER_COLUMN = 4;
|
|
236
|
+
/** Minimum gap in absolute points to avoid splitting on small whitespace. */
|
|
237
|
+
const MIN_GAP_PTS = 40;
|
|
238
|
+
/**
|
|
239
|
+
* Detect column layout and return text boxes grouped by column.
|
|
240
|
+
*
|
|
241
|
+
* For single-column pages, returns all boxes in one group.
|
|
242
|
+
* For multi-column pages, returns boxes split by column in reading order.
|
|
243
|
+
*/
|
|
244
|
+
function detectColumns(textBoxes) {
|
|
245
|
+
if (textBoxes.length < MIN_BOXES_PER_COLUMN * 2) return {
|
|
246
|
+
columnCount: 1,
|
|
247
|
+
columns: [textBoxes],
|
|
248
|
+
boundaries: []
|
|
249
|
+
};
|
|
250
|
+
const lefts = [...new Set(textBoxes.map((tb) => Math.round(tb.bounds.left)))].toSorted((a, b) => a - b);
|
|
251
|
+
if (lefts.length < 2) return {
|
|
252
|
+
columnCount: 1,
|
|
253
|
+
columns: [textBoxes],
|
|
254
|
+
boundaries: []
|
|
255
|
+
};
|
|
256
|
+
const textXMin = lefts[0];
|
|
257
|
+
const textWidth = Math.max(...textBoxes.map((tb) => Math.round(tb.bounds.right))) - textXMin;
|
|
258
|
+
if (textWidth <= 0) return {
|
|
259
|
+
columnCount: 1,
|
|
260
|
+
columns: [textBoxes],
|
|
261
|
+
boundaries: []
|
|
262
|
+
};
|
|
263
|
+
let maxGap = 0;
|
|
264
|
+
let gapLeft = 0;
|
|
265
|
+
let gapRight = 0;
|
|
266
|
+
for (let i = 1; i < lefts.length; i++) {
|
|
267
|
+
const gap = lefts[i] - lefts[i - 1];
|
|
268
|
+
if (gap > maxGap) {
|
|
269
|
+
maxGap = gap;
|
|
270
|
+
gapLeft = lefts[i - 1];
|
|
271
|
+
gapRight = lefts[i];
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (maxGap / textWidth < MIN_GAP_RATIO || maxGap < MIN_GAP_PTS) return {
|
|
275
|
+
columnCount: 1,
|
|
276
|
+
columns: [textBoxes],
|
|
277
|
+
boundaries: []
|
|
278
|
+
};
|
|
279
|
+
const splitX = (gapLeft + gapRight) / 2;
|
|
280
|
+
const leftCol = [];
|
|
281
|
+
const rightCol = [];
|
|
282
|
+
for (const tb of textBoxes) if ((tb.bounds.left + tb.bounds.right) / 2 < splitX) leftCol.push(tb);
|
|
283
|
+
else rightCol.push(tb);
|
|
284
|
+
if (leftCol.length < MIN_BOXES_PER_COLUMN || rightCol.length < MIN_BOXES_PER_COLUMN) return {
|
|
285
|
+
columnCount: 1,
|
|
286
|
+
columns: [textBoxes],
|
|
287
|
+
boundaries: []
|
|
288
|
+
};
|
|
289
|
+
return {
|
|
290
|
+
columnCount: 2,
|
|
291
|
+
columns: [leftCol, rightCol],
|
|
292
|
+
boundaries: [splitX]
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
//#endregion
|
|
296
|
+
//#region ../../packages/agent-core/src/markit/converters/pdf/extract.ts
|
|
297
|
+
let mupdfModule;
|
|
298
|
+
async function loadMupdf() {
|
|
299
|
+
if (!mupdfModule) mupdfModule = await import("mupdf");
|
|
300
|
+
return mupdfModule;
|
|
301
|
+
}
|
|
302
|
+
/** Y tolerance for merging text fragments on the same visual line. */
|
|
303
|
+
const SAME_LINE_Y_TOLERANCE = 2;
|
|
304
|
+
/** Max horizontal gap (pts) to merge adjacent fragments into one text box. */
|
|
305
|
+
const MAX_MERGE_GAP = 14;
|
|
306
|
+
/**
|
|
307
|
+
* Merge horizontally adjacent raw text items on the same visual line into
|
|
308
|
+
* word/phrase-level text boxes.
|
|
309
|
+
*/
|
|
310
|
+
function mergeIntoWords(raws) {
|
|
311
|
+
if (raws.length === 0) return [];
|
|
312
|
+
const sorted = [...raws].toSorted((a, b) => {
|
|
313
|
+
const dy = b.y - a.y;
|
|
314
|
+
return Math.abs(dy) > SAME_LINE_Y_TOLERANCE ? dy : a.x - b.x;
|
|
315
|
+
});
|
|
316
|
+
const merged = [];
|
|
317
|
+
let cur = { ...sorted[0] };
|
|
318
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
319
|
+
const next = sorted[i];
|
|
320
|
+
const sameY = Math.abs(next.y - cur.y) <= SAME_LINE_Y_TOLERANCE;
|
|
321
|
+
const close = next.x <= cur.x + cur.width + MAX_MERGE_GAP;
|
|
322
|
+
if (sameY && close) {
|
|
323
|
+
const sep = next.x - (cur.x + cur.width) > 1 ? " " : "";
|
|
324
|
+
cur.text += sep + next.text;
|
|
325
|
+
cur.width = next.x + next.width - cur.x;
|
|
326
|
+
cur.height = Math.max(cur.height, next.height);
|
|
327
|
+
cur.fontSize = Math.max(cur.fontSize, next.fontSize);
|
|
328
|
+
cur.isBold = cur.isBold || next.isBold;
|
|
329
|
+
} else {
|
|
330
|
+
merged.push(cur);
|
|
331
|
+
cur = { ...next };
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
merged.push(cur);
|
|
335
|
+
return merged;
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* Extract text boxes from a mupdf page using structured text output.
|
|
339
|
+
*
|
|
340
|
+
* mupdf's structured text JSON uses top-left origin; we convert to
|
|
341
|
+
* bottom-left (standard PDF coordinates) using the page height.
|
|
342
|
+
*/
|
|
343
|
+
function extractTextBoxes(page, pageNumber, pageHeight, stext) {
|
|
344
|
+
if (!stext) stext = JSON.parse(page.toStructuredText("preserve-whitespace").asJSON());
|
|
345
|
+
const raws = [];
|
|
346
|
+
for (const block of stext.blocks) {
|
|
347
|
+
if (block.type !== "text") continue;
|
|
348
|
+
for (const line of block.lines) {
|
|
349
|
+
const text = line.text?.trim();
|
|
350
|
+
if (!text) continue;
|
|
351
|
+
const fontSize = line.font?.size ?? 0;
|
|
352
|
+
const weight = line.font?.weight ?? "normal";
|
|
353
|
+
const fontName = line.font?.name ?? "";
|
|
354
|
+
const isBold = weight === "bold" || /bold/i.test(fontName) || /Black|Heavy/i.test(fontName);
|
|
355
|
+
const bboxY = line.bbox.y;
|
|
356
|
+
const bboxH = line.bbox.h;
|
|
357
|
+
const pdfY = pageHeight - (bboxY + bboxH);
|
|
358
|
+
raws.push({
|
|
359
|
+
text,
|
|
360
|
+
x: line.bbox.x,
|
|
361
|
+
y: pdfY,
|
|
362
|
+
width: line.bbox.w,
|
|
363
|
+
height: bboxH,
|
|
364
|
+
fontSize,
|
|
365
|
+
isBold
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
return mergeIntoWords(raws).map((w, i) => ({
|
|
370
|
+
id: `p${pageNumber}-t${i}`,
|
|
371
|
+
text: w.text.trim(),
|
|
372
|
+
pageNumber,
|
|
373
|
+
fontSize: w.fontSize,
|
|
374
|
+
isBold: w.isBold,
|
|
375
|
+
bounds: {
|
|
376
|
+
left: w.x,
|
|
377
|
+
right: w.x + w.width,
|
|
378
|
+
bottom: w.y,
|
|
379
|
+
top: w.y + w.height
|
|
380
|
+
}
|
|
381
|
+
})).filter((b) => b.text.length > 0);
|
|
382
|
+
}
|
|
383
|
+
/** Minimum aspect ratio for a filled rect to be considered a line. */
|
|
384
|
+
const LINE_ASPECT_THRESHOLD = 6;
|
|
385
|
+
/** Minimum length (pts) for a segment to count. */
|
|
386
|
+
const MIN_LENGTH = 2;
|
|
387
|
+
/** Maximum thickness (pts) for a border line (filters out filled areas). */
|
|
388
|
+
const MAX_THICKNESS = 3;
|
|
389
|
+
/**
|
|
390
|
+
* Convert a thin filled rectangle to a horizontal or vertical segment.
|
|
391
|
+
* Returns null if the rect doesn't look like a border line.
|
|
392
|
+
*/
|
|
393
|
+
function thinRectToSegment(id, x, y, w, h) {
|
|
394
|
+
const aw = Math.abs(w);
|
|
395
|
+
const ah = Math.abs(h);
|
|
396
|
+
if (aw > ah * LINE_ASPECT_THRESHOLD && aw >= MIN_LENGTH && ah <= MAX_THICKNESS) {
|
|
397
|
+
const cy = y + ah / 2;
|
|
398
|
+
return {
|
|
399
|
+
id,
|
|
400
|
+
x1: x,
|
|
401
|
+
y1: cy,
|
|
402
|
+
x2: x + aw,
|
|
403
|
+
y2: cy
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
if (ah > aw * LINE_ASPECT_THRESHOLD && ah >= MIN_LENGTH && aw <= MAX_THICKNESS) {
|
|
407
|
+
const cx = x + aw / 2;
|
|
408
|
+
return {
|
|
409
|
+
id,
|
|
410
|
+
x1: cx,
|
|
411
|
+
y1: y,
|
|
412
|
+
x2: cx,
|
|
413
|
+
y2: y + ah
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
return null;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Emit 4 edge segments from a stroked rectangle.
|
|
420
|
+
*/
|
|
421
|
+
function pushStrokedRectEdges(segments, id, x, y, w, h) {
|
|
422
|
+
const aw = Math.abs(w);
|
|
423
|
+
const ah = Math.abs(h);
|
|
424
|
+
const base = id;
|
|
425
|
+
if (aw >= MIN_LENGTH) {
|
|
426
|
+
segments.push({
|
|
427
|
+
id: `${base}-b`,
|
|
428
|
+
x1: x,
|
|
429
|
+
y1: y,
|
|
430
|
+
x2: x + aw,
|
|
431
|
+
y2: y
|
|
432
|
+
});
|
|
433
|
+
segments.push({
|
|
434
|
+
id: `${base}-t`,
|
|
435
|
+
x1: x,
|
|
436
|
+
y1: y + ah,
|
|
437
|
+
x2: x + aw,
|
|
438
|
+
y2: y + ah
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
if (ah >= MIN_LENGTH) {
|
|
442
|
+
segments.push({
|
|
443
|
+
id: `${base}-l`,
|
|
444
|
+
x1: x,
|
|
445
|
+
y1: y,
|
|
446
|
+
x2: x,
|
|
447
|
+
y2: y + ah
|
|
448
|
+
});
|
|
449
|
+
segments.push({
|
|
450
|
+
id: `${base}-r`,
|
|
451
|
+
x1: x + aw,
|
|
452
|
+
y1: y,
|
|
453
|
+
x2: x + aw,
|
|
454
|
+
y2: y + ah
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
const CTM_IDENTITY = [
|
|
459
|
+
1,
|
|
460
|
+
0,
|
|
461
|
+
0,
|
|
462
|
+
1,
|
|
463
|
+
0,
|
|
464
|
+
0
|
|
465
|
+
];
|
|
466
|
+
/** Concatenate two affine matrices: result = parent × child. */
|
|
467
|
+
function ctmConcat(p, c) {
|
|
468
|
+
return [
|
|
469
|
+
p[0] * c[0] + p[2] * c[1],
|
|
470
|
+
p[1] * c[0] + p[3] * c[1],
|
|
471
|
+
p[0] * c[2] + p[2] * c[3],
|
|
472
|
+
p[1] * c[2] + p[3] * c[3],
|
|
473
|
+
p[0] * c[4] + p[2] * c[5] + p[4],
|
|
474
|
+
p[1] * c[4] + p[3] * c[5] + p[5]
|
|
475
|
+
];
|
|
476
|
+
}
|
|
477
|
+
function ctmApply(m, x, y) {
|
|
478
|
+
return [m[0] * x + m[2] * y + m[4], m[1] * x + m[3] * y + m[5]];
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Parse a PDF content stream and extract line segments from thin filled
|
|
482
|
+
* rectangles (re+f), stroked rectangles (re+S), and explicit lines (m/l+S).
|
|
483
|
+
* Tracks the CTM via q/Q/cm operators so coordinates are in page space.
|
|
484
|
+
*/
|
|
485
|
+
function extractSegmentsFromContentStream(raw, pageNumber) {
|
|
486
|
+
const segments = [];
|
|
487
|
+
const tokens = tokenizeContentStream(raw);
|
|
488
|
+
let idx = 0;
|
|
489
|
+
let strokeWidth = 1;
|
|
490
|
+
let ctm = [...CTM_IDENTITY];
|
|
491
|
+
const stateStack = [];
|
|
492
|
+
let curX = 0;
|
|
493
|
+
let curY = 0;
|
|
494
|
+
let pathStartX = 0;
|
|
495
|
+
let pathStartY = 0;
|
|
496
|
+
const pendingRects = [];
|
|
497
|
+
const pendingLines = [];
|
|
498
|
+
function flushPath(mode) {
|
|
499
|
+
const sid = () => `p${pageNumber}-s${segments.length}`;
|
|
500
|
+
if (mode === "fill") for (const r of pendingRects) {
|
|
501
|
+
const [x0, y0] = ctmApply(ctm, r.x, r.y);
|
|
502
|
+
const [x1, y1] = ctmApply(ctm, r.x + r.w, r.y + r.h);
|
|
503
|
+
const seg = thinRectToSegment(sid(), Math.min(x0, x1), Math.min(y0, y1), Math.abs(x1 - x0), Math.abs(y1 - y0));
|
|
504
|
+
if (seg) segments.push(seg);
|
|
505
|
+
}
|
|
506
|
+
else if (mode === "stroke" && strokeWidth <= MAX_THICKNESS) {
|
|
507
|
+
for (const r of pendingRects) {
|
|
508
|
+
const [x0, y0] = ctmApply(ctm, r.x, r.y);
|
|
509
|
+
const [x1, y1] = ctmApply(ctm, r.x + r.w, r.y + r.h);
|
|
510
|
+
pushStrokedRectEdges(segments, sid(), Math.min(x0, x1), Math.min(y0, y1), Math.abs(x1 - x0), Math.abs(y1 - y0));
|
|
511
|
+
}
|
|
512
|
+
for (const l of pendingLines) {
|
|
513
|
+
const [lx1, ly1] = ctmApply(ctm, l.x1, l.y1);
|
|
514
|
+
const [lx2, ly2] = ctmApply(ctm, l.x2, l.y2);
|
|
515
|
+
const dx = Math.abs(lx2 - lx1);
|
|
516
|
+
const dy = Math.abs(ly2 - ly1);
|
|
517
|
+
if (dx >= MIN_LENGTH && dy < 1 || dy >= MIN_LENGTH && dx < 1) segments.push({
|
|
518
|
+
id: sid(),
|
|
519
|
+
x1: lx1,
|
|
520
|
+
y1: ly1,
|
|
521
|
+
x2: lx2,
|
|
522
|
+
y2: ly2
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
pendingRects.length = 0;
|
|
527
|
+
pendingLines.length = 0;
|
|
528
|
+
}
|
|
529
|
+
while (idx < tokens.length) {
|
|
530
|
+
const t = tokens[idx];
|
|
531
|
+
if (t === "q") stateStack.push({
|
|
532
|
+
ctm: [...ctm],
|
|
533
|
+
strokeWidth
|
|
534
|
+
});
|
|
535
|
+
else if (t === "Q") {
|
|
536
|
+
const saved = stateStack.pop();
|
|
537
|
+
if (saved) {
|
|
538
|
+
ctm = saved.ctm;
|
|
539
|
+
strokeWidth = saved.strokeWidth;
|
|
540
|
+
}
|
|
541
|
+
} else if (t === "cm" && idx >= 6) {
|
|
542
|
+
const a = Number(tokens[idx - 6]);
|
|
543
|
+
const b = Number(tokens[idx - 5]);
|
|
544
|
+
const c = Number(tokens[idx - 4]);
|
|
545
|
+
const d = Number(tokens[idx - 3]);
|
|
546
|
+
const e = Number(tokens[idx - 2]);
|
|
547
|
+
const f = Number(tokens[idx - 1]);
|
|
548
|
+
ctm = ctmConcat(ctm, [
|
|
549
|
+
a,
|
|
550
|
+
b,
|
|
551
|
+
c,
|
|
552
|
+
d,
|
|
553
|
+
e,
|
|
554
|
+
f
|
|
555
|
+
]);
|
|
556
|
+
} else if (t === "w" && idx >= 1) strokeWidth = Number(tokens[idx - 1]) || strokeWidth;
|
|
557
|
+
else if (t === "re" && idx >= 4) {
|
|
558
|
+
const x = Number(tokens[idx - 4]);
|
|
559
|
+
const y = Number(tokens[idx - 3]);
|
|
560
|
+
const w = Number(tokens[idx - 2]);
|
|
561
|
+
const h = Number(tokens[idx - 1]);
|
|
562
|
+
if (Number.isFinite(x + y + w + h)) pendingRects.push({
|
|
563
|
+
x,
|
|
564
|
+
y,
|
|
565
|
+
w,
|
|
566
|
+
h
|
|
567
|
+
});
|
|
568
|
+
} else if (t === "m" && idx >= 2) {
|
|
569
|
+
curX = Number(tokens[idx - 2]);
|
|
570
|
+
curY = Number(tokens[idx - 1]);
|
|
571
|
+
pathStartX = curX;
|
|
572
|
+
pathStartY = curY;
|
|
573
|
+
} else if (t === "l" && idx >= 2) {
|
|
574
|
+
const x2 = Number(tokens[idx - 2]);
|
|
575
|
+
const y2 = Number(tokens[idx - 1]);
|
|
576
|
+
pendingLines.push({
|
|
577
|
+
x1: curX,
|
|
578
|
+
y1: curY,
|
|
579
|
+
x2,
|
|
580
|
+
y2
|
|
581
|
+
});
|
|
582
|
+
curX = x2;
|
|
583
|
+
curY = y2;
|
|
584
|
+
} else if (t === "h") {
|
|
585
|
+
if (curX !== pathStartX || curY !== pathStartY) pendingLines.push({
|
|
586
|
+
x1: curX,
|
|
587
|
+
y1: curY,
|
|
588
|
+
x2: pathStartX,
|
|
589
|
+
y2: pathStartY
|
|
590
|
+
});
|
|
591
|
+
curX = pathStartX;
|
|
592
|
+
curY = pathStartY;
|
|
593
|
+
} else if (t === "f" || t === "F" || t === "f*") flushPath("fill");
|
|
594
|
+
else if (t === "S" || t === "s") {
|
|
595
|
+
if (t === "s") {
|
|
596
|
+
if (curX !== pathStartX || curY !== pathStartY) pendingLines.push({
|
|
597
|
+
x1: curX,
|
|
598
|
+
y1: curY,
|
|
599
|
+
x2: pathStartX,
|
|
600
|
+
y2: pathStartY
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
flushPath("stroke");
|
|
604
|
+
} else if (t === "B" || t === "B*" || t === "b" || t === "b*") {
|
|
605
|
+
flushPath("fill");
|
|
606
|
+
flushPath("stroke");
|
|
607
|
+
} else if (t === "n") {
|
|
608
|
+
pendingRects.length = 0;
|
|
609
|
+
pendingLines.length = 0;
|
|
610
|
+
}
|
|
611
|
+
idx++;
|
|
612
|
+
}
|
|
613
|
+
return segments;
|
|
614
|
+
}
|
|
615
|
+
/**
|
|
616
|
+
* Fast tokenizer for PDF content streams.
|
|
617
|
+
* Splits on whitespace, skipping comments, string literals, and inline image payloads.
|
|
618
|
+
*/
|
|
619
|
+
function tokenizeContentStream(raw) {
|
|
620
|
+
const tokens = [];
|
|
621
|
+
const len = raw.length;
|
|
622
|
+
let i = 0;
|
|
623
|
+
let inInlineImage = false;
|
|
624
|
+
while (i < len) {
|
|
625
|
+
const ch = raw.codePointAt(i);
|
|
626
|
+
if (ch <= 32) {
|
|
627
|
+
i++;
|
|
628
|
+
continue;
|
|
629
|
+
}
|
|
630
|
+
if (ch === 37) {
|
|
631
|
+
while (i < len && raw.codePointAt(i) !== 10) i++;
|
|
632
|
+
continue;
|
|
633
|
+
}
|
|
634
|
+
if (ch === 40) {
|
|
635
|
+
let depth = 1;
|
|
636
|
+
i++;
|
|
637
|
+
while (i < len && depth > 0) {
|
|
638
|
+
const c = raw.codePointAt(i);
|
|
639
|
+
if (c === 92) i++;
|
|
640
|
+
else if (c === 40) depth++;
|
|
641
|
+
else if (c === 41) depth--;
|
|
642
|
+
i++;
|
|
643
|
+
}
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
if (ch === 60 && i + 1 < len && raw.codePointAt(i + 1) !== 60) {
|
|
647
|
+
i++;
|
|
648
|
+
while (i < len && raw.codePointAt(i) !== 62) i++;
|
|
649
|
+
i++;
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
if (ch === 60 && i + 1 < len && raw.codePointAt(i + 1) === 60) {
|
|
653
|
+
i += 2;
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
if (ch === 62 && i + 1 < len && raw.codePointAt(i + 1) === 62) {
|
|
657
|
+
i += 2;
|
|
658
|
+
continue;
|
|
659
|
+
}
|
|
660
|
+
if (ch === 41 || ch === 62) {
|
|
661
|
+
i++;
|
|
662
|
+
continue;
|
|
663
|
+
}
|
|
664
|
+
const start = i;
|
|
665
|
+
while (i < len) {
|
|
666
|
+
const c = raw.codePointAt(i);
|
|
667
|
+
if (c <= 32 || c === 40 || c === 41 || c === 60 || c === 62 || c === 37) break;
|
|
668
|
+
i++;
|
|
669
|
+
}
|
|
670
|
+
if (i > start) {
|
|
671
|
+
const token = raw.substring(start, i);
|
|
672
|
+
tokens.push(token);
|
|
673
|
+
if (token === "BI") inInlineImage = true;
|
|
674
|
+
else if (token === "ID" && inInlineImage) {
|
|
675
|
+
while (i < len && raw.codePointAt(i) <= 32) i++;
|
|
676
|
+
while (i < len) {
|
|
677
|
+
const c = raw.codePointAt(i);
|
|
678
|
+
const prev = i === 0 ? 32 : raw.codePointAt(i - 1);
|
|
679
|
+
const next = i + 2 >= len ? 32 : raw.codePointAt(i + 2);
|
|
680
|
+
if (c === 69 && raw.codePointAt(i + 1) === 73 && prev <= 32 && next <= 32) {
|
|
681
|
+
i += 2;
|
|
682
|
+
break;
|
|
683
|
+
}
|
|
684
|
+
i++;
|
|
685
|
+
}
|
|
686
|
+
inInlineImage = false;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
return tokens;
|
|
691
|
+
}
|
|
692
|
+
/** Minimum area (pts²) for an image to be considered a diagram, not an icon. */
|
|
693
|
+
const MIN_IMAGE_AREA = 5e3;
|
|
694
|
+
function extractImageRegions(stext, pageNumber, pageHeight) {
|
|
695
|
+
const regions = [];
|
|
696
|
+
for (const block of stext.blocks) {
|
|
697
|
+
if (block.type !== "image") continue;
|
|
698
|
+
const { x, y, w, h } = block.bbox;
|
|
699
|
+
if (w * h < MIN_IMAGE_AREA) continue;
|
|
700
|
+
const pdfTopY = pageHeight - y;
|
|
701
|
+
regions.push({
|
|
702
|
+
id: `p${pageNumber}-img${regions.length}`,
|
|
703
|
+
pageNumber,
|
|
704
|
+
bbox: {
|
|
705
|
+
x,
|
|
706
|
+
y,
|
|
707
|
+
w,
|
|
708
|
+
h
|
|
709
|
+
},
|
|
710
|
+
topY: pdfTopY
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
return regions;
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* Render an image region from a PDF page as a PNG buffer.
|
|
717
|
+
* Uses mupdf's DrawDevice to render just the cropped area at 2x resolution.
|
|
718
|
+
*/
|
|
719
|
+
async function renderImageRegion(input, region) {
|
|
720
|
+
const m = await loadMupdf();
|
|
721
|
+
const page = m.Document.openDocument(input, "application/pdf").loadPage(region.pageNumber - 1);
|
|
722
|
+
const pad = 10;
|
|
723
|
+
const bx = region.bbox.x - pad;
|
|
724
|
+
const by = region.bbox.y - pad;
|
|
725
|
+
const bw = region.bbox.w + 2 * pad;
|
|
726
|
+
const bh = region.bbox.h + 2 * pad;
|
|
727
|
+
const scale = 2;
|
|
728
|
+
const pw = Math.round(bw * scale);
|
|
729
|
+
const ph = Math.round(bh * scale);
|
|
730
|
+
const pix = new m.Pixmap(m.ColorSpace.DeviceRGB, [
|
|
731
|
+
0,
|
|
732
|
+
0,
|
|
733
|
+
pw,
|
|
734
|
+
ph
|
|
735
|
+
], false);
|
|
736
|
+
pix.clear(255);
|
|
737
|
+
const matrix = [
|
|
738
|
+
scale,
|
|
739
|
+
0,
|
|
740
|
+
0,
|
|
741
|
+
scale,
|
|
742
|
+
-bx * scale,
|
|
743
|
+
-by * scale
|
|
744
|
+
];
|
|
745
|
+
const dl = page.toDisplayList();
|
|
746
|
+
const dev = new m.DrawDevice(matrix, pix);
|
|
747
|
+
dl.run(dev, m.Matrix.identity);
|
|
748
|
+
dev.close();
|
|
749
|
+
return pix.asPNG();
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* Extract text boxes and vector segments from all pages of a PDF buffer.
|
|
753
|
+
*/
|
|
754
|
+
async function extractPages(input) {
|
|
755
|
+
const doc = (await loadMupdf()).Document.openDocument(input, "application/pdf");
|
|
756
|
+
const pages = [];
|
|
757
|
+
for (let i = 0; i < doc.countPages(); i++) {
|
|
758
|
+
const pageNumber = i + 1;
|
|
759
|
+
const page = doc.loadPage(i);
|
|
760
|
+
const bounds = page.getBounds();
|
|
761
|
+
const pageHeight = bounds[3] - bounds[1];
|
|
762
|
+
const stext = JSON.parse(page.toStructuredText("preserve-whitespace,preserve-images").asJSON());
|
|
763
|
+
const textBoxes = extractTextBoxes(page, pageNumber, pageHeight, stext);
|
|
764
|
+
const images = extractImageRegions(stext, pageNumber, pageHeight);
|
|
765
|
+
let segments = [];
|
|
766
|
+
try {
|
|
767
|
+
const contents = page.getObject().get("Contents");
|
|
768
|
+
if (contents) {
|
|
769
|
+
let rawBytes;
|
|
770
|
+
if (contents.isArray()) {
|
|
771
|
+
const parts = [];
|
|
772
|
+
const len = contents.length ?? 0;
|
|
773
|
+
for (let j = 0; j < len; j++) {
|
|
774
|
+
const stream = contents.get(j);
|
|
775
|
+
if (stream?.readStream) parts.push(stream.readStream().asUint8Array());
|
|
776
|
+
}
|
|
777
|
+
const totalLen = parts.reduce((s, p) => s + p.length, 0);
|
|
778
|
+
rawBytes = new Uint8Array(totalLen);
|
|
779
|
+
let offset = 0;
|
|
780
|
+
for (const part of parts) {
|
|
781
|
+
rawBytes.set(part, offset);
|
|
782
|
+
offset += part.length;
|
|
783
|
+
}
|
|
784
|
+
} else rawBytes = contents.readStream().asUint8Array();
|
|
785
|
+
segments = extractSegmentsFromContentStream(new TextDecoder().decode(rawBytes), pageNumber);
|
|
786
|
+
}
|
|
787
|
+
} catch {}
|
|
788
|
+
pages.push({
|
|
789
|
+
pageNumber,
|
|
790
|
+
textBoxes,
|
|
791
|
+
segments,
|
|
792
|
+
images
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
return pages;
|
|
796
|
+
}
|
|
797
|
+
//#endregion
|
|
798
|
+
//#region ../../packages/agent-core/src/markit/converters/pdf/grid.ts
|
|
799
|
+
function castRaysForTextBox(textBox, segments) {
|
|
800
|
+
const cx = (textBox.bounds.left + textBox.bounds.right) / 2;
|
|
801
|
+
const cy = (textBox.bounds.top + textBox.bounds.bottom) / 2;
|
|
802
|
+
let up = {
|
|
803
|
+
direction: "up",
|
|
804
|
+
segmentId: null,
|
|
805
|
+
distance: Infinity
|
|
806
|
+
};
|
|
807
|
+
let down = {
|
|
808
|
+
direction: "down",
|
|
809
|
+
segmentId: null,
|
|
810
|
+
distance: Infinity
|
|
811
|
+
};
|
|
812
|
+
let left = {
|
|
813
|
+
direction: "left",
|
|
814
|
+
segmentId: null,
|
|
815
|
+
distance: Infinity
|
|
816
|
+
};
|
|
817
|
+
let right = {
|
|
818
|
+
direction: "right",
|
|
819
|
+
segmentId: null,
|
|
820
|
+
distance: Infinity
|
|
821
|
+
};
|
|
822
|
+
for (const seg of segments) {
|
|
823
|
+
const isH = Math.abs(seg.y1 - seg.y2) < .5;
|
|
824
|
+
const isV = Math.abs(seg.x1 - seg.x2) < .5;
|
|
825
|
+
if (isH) {
|
|
826
|
+
const minX = Math.min(seg.x1, seg.x2);
|
|
827
|
+
const maxX = Math.max(seg.x1, seg.x2);
|
|
828
|
+
if (cx >= minX && cx <= maxX) {
|
|
829
|
+
const d = seg.y1 - cy;
|
|
830
|
+
if (d >= 0 && d < up.distance) up = {
|
|
831
|
+
direction: "up",
|
|
832
|
+
segmentId: seg.id,
|
|
833
|
+
distance: d
|
|
834
|
+
};
|
|
835
|
+
const dd = cy - seg.y1;
|
|
836
|
+
if (dd >= 0 && dd < down.distance) down = {
|
|
837
|
+
direction: "down",
|
|
838
|
+
segmentId: seg.id,
|
|
839
|
+
distance: dd
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
if (isV) {
|
|
844
|
+
const minY = Math.min(seg.y1, seg.y2);
|
|
845
|
+
const maxY = Math.max(seg.y1, seg.y2);
|
|
846
|
+
if (cy >= minY && cy <= maxY) {
|
|
847
|
+
const d = cx - seg.x1;
|
|
848
|
+
if (d >= 0 && d < left.distance) left = {
|
|
849
|
+
direction: "left",
|
|
850
|
+
segmentId: seg.id,
|
|
851
|
+
distance: d
|
|
852
|
+
};
|
|
853
|
+
const rd = seg.x1 - cx;
|
|
854
|
+
if (rd >= 0 && rd < right.distance) right = {
|
|
855
|
+
direction: "right",
|
|
856
|
+
segmentId: seg.id,
|
|
857
|
+
distance: rd
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
return [
|
|
863
|
+
up,
|
|
864
|
+
down,
|
|
865
|
+
left,
|
|
866
|
+
right
|
|
867
|
+
];
|
|
868
|
+
}
|
|
869
|
+
const AXIS_EPSILON = .8;
|
|
870
|
+
const PAGE_MARGIN = 20;
|
|
871
|
+
function uniqueSorted(values) {
|
|
872
|
+
const sorted = [...values].toSorted((a, b) => a - b);
|
|
873
|
+
const result = [];
|
|
874
|
+
for (const v of sorted) if (result.length === 0 || Math.abs(result.at(-1) - v) > 1) result.push(v);
|
|
875
|
+
return result;
|
|
876
|
+
}
|
|
877
|
+
function chainCoversRange(intervals, lowerY, upperY, eps) {
|
|
878
|
+
const sorted = [...intervals].toSorted((a, b) => a.min - b.min);
|
|
879
|
+
let covered = lowerY;
|
|
880
|
+
for (const iv of sorted) {
|
|
881
|
+
if (iv.min > covered + eps) break;
|
|
882
|
+
if (iv.max > covered) covered = iv.max;
|
|
883
|
+
if (covered >= upperY - eps) return true;
|
|
884
|
+
}
|
|
885
|
+
return false;
|
|
886
|
+
}
|
|
887
|
+
function countBridgingVLineCols(upperY, lowerY, verticals) {
|
|
888
|
+
const eps = 1.5;
|
|
889
|
+
const byX = /* @__PURE__ */ new Map();
|
|
890
|
+
for (const seg of verticals) {
|
|
891
|
+
const rx = Math.round(seg.x1);
|
|
892
|
+
if (!byX.has(rx)) byX.set(rx, []);
|
|
893
|
+
byX.get(rx)?.push({
|
|
894
|
+
min: Math.min(seg.y1, seg.y2),
|
|
895
|
+
max: Math.max(seg.y1, seg.y2)
|
|
896
|
+
});
|
|
897
|
+
}
|
|
898
|
+
let count = 0;
|
|
899
|
+
for (const intervals of byX.values()) if (chainCoversRange(intervals, lowerY, upperY, eps)) count++;
|
|
900
|
+
return count;
|
|
901
|
+
}
|
|
902
|
+
function bridgingXSet(upperY, lowerY, verticals) {
|
|
903
|
+
const eps = 1.5;
|
|
904
|
+
const xs = /* @__PURE__ */ new Set();
|
|
905
|
+
const byX = /* @__PURE__ */ new Map();
|
|
906
|
+
for (const seg of verticals) {
|
|
907
|
+
const rx = Math.round(seg.x1);
|
|
908
|
+
if (!byX.has(rx)) byX.set(rx, []);
|
|
909
|
+
byX.get(rx)?.push({
|
|
910
|
+
min: Math.min(seg.y1, seg.y2),
|
|
911
|
+
max: Math.max(seg.y1, seg.y2)
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
for (const [rx, intervals] of byX) if (chainCoversRange(intervals, lowerY, upperY, eps)) xs.add(rx);
|
|
915
|
+
return xs;
|
|
916
|
+
}
|
|
917
|
+
const MIN_RICH_BRIDGING_COLS = 3;
|
|
918
|
+
function splitYLinesIntoGroups(yLines, verticals) {
|
|
919
|
+
if (yLines.length === 0) return [];
|
|
920
|
+
const eps = 1.5;
|
|
921
|
+
const allX = verticals.map((s) => Math.round(s.x1));
|
|
922
|
+
const globalXMin = allX.length > 0 ? Math.min(...allX) : 0;
|
|
923
|
+
const globalXMax = allX.length > 0 ? Math.max(...allX) : 0;
|
|
924
|
+
const groups = [];
|
|
925
|
+
let currentGroup = [yLines[0]];
|
|
926
|
+
let prevBridgingCols = -1;
|
|
927
|
+
for (let i = 1; i < yLines.length; i++) {
|
|
928
|
+
const upperY = yLines[i - 1];
|
|
929
|
+
const lowerY = yLines[i];
|
|
930
|
+
const cols = countBridgingVLineCols(upperY, lowerY, verticals);
|
|
931
|
+
if (cols === 0) {
|
|
932
|
+
groups.push(currentGroup);
|
|
933
|
+
currentGroup = [yLines[i]];
|
|
934
|
+
prevBridgingCols = -1;
|
|
935
|
+
continue;
|
|
936
|
+
}
|
|
937
|
+
if (prevBridgingCols >= MIN_RICH_BRIDGING_COLS && cols < MIN_RICH_BRIDGING_COLS) {
|
|
938
|
+
if (![...bridgingXSet(upperY, lowerY, verticals)].every((x) => Math.abs(x - globalXMin) <= eps || Math.abs(x - globalXMax) <= eps)) {
|
|
939
|
+
groups.push(currentGroup);
|
|
940
|
+
currentGroup = [yLines[i - 1], yLines[i]];
|
|
941
|
+
prevBridgingCols = cols;
|
|
942
|
+
continue;
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
currentGroup.push(yLines[i]);
|
|
946
|
+
prevBridgingCols = cols;
|
|
947
|
+
}
|
|
948
|
+
groups.push(currentGroup);
|
|
949
|
+
return groups;
|
|
950
|
+
}
|
|
951
|
+
const Y_CLUSTER_GAP = 10;
|
|
952
|
+
const MIN_COLS_IN_TOP_CLUSTER = 2;
|
|
953
|
+
function assignToYCluster(y, clusters) {
|
|
954
|
+
let closest = 0;
|
|
955
|
+
let closestDist = Math.abs(y - clusters[0]);
|
|
956
|
+
for (let k = 1; k < clusters.length; k++) {
|
|
957
|
+
const d = Math.abs(y - clusters[k]);
|
|
958
|
+
if (d < closestDist) {
|
|
959
|
+
closestDist = d;
|
|
960
|
+
closest = k;
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
return closest;
|
|
964
|
+
}
|
|
965
|
+
function expandSubRowsByYClusters(originalRows, cols, cells, cellBoxes) {
|
|
966
|
+
let addedRows = 0;
|
|
967
|
+
for (let origRow = 0; origRow < originalRows; origRow++) {
|
|
968
|
+
const currentRow = origRow + addedRows;
|
|
969
|
+
const rowCellInfos = [];
|
|
970
|
+
for (let col = 0; col < cols; col++) {
|
|
971
|
+
const cell = cells.find((c) => c.row === currentRow && c.col === col);
|
|
972
|
+
if (!cell) continue;
|
|
973
|
+
const boxes = cellBoxes.get(cell);
|
|
974
|
+
if (boxes && boxes.length > 0) rowCellInfos.push({
|
|
975
|
+
cell,
|
|
976
|
+
col,
|
|
977
|
+
boxes
|
|
978
|
+
});
|
|
979
|
+
}
|
|
980
|
+
if (rowCellInfos.length === 0) continue;
|
|
981
|
+
const allMidYs = rowCellInfos.flatMap(({ boxes }) => boxes.map((b) => (b.bounds.top + b.bounds.bottom) / 2));
|
|
982
|
+
const sortedY = [...new Set(allMidYs.map((y) => Math.round(y * 10) / 10))].toSorted((a, b) => b - a);
|
|
983
|
+
const clusters = [sortedY[0]];
|
|
984
|
+
for (let i = 1; i < sortedY.length; i++) if (clusters.at(-1) - sortedY[i] > Y_CLUSTER_GAP) clusters.push(sortedY[i]);
|
|
985
|
+
if (clusters.length < 2) continue;
|
|
986
|
+
const colsInTopCluster = /* @__PURE__ */ new Set();
|
|
987
|
+
const totalNonEmptyCols = /* @__PURE__ */ new Set();
|
|
988
|
+
for (const { col, boxes } of rowCellInfos) {
|
|
989
|
+
totalNonEmptyCols.add(col);
|
|
990
|
+
if (boxes.some((b) => assignToYCluster((b.bounds.top + b.bounds.bottom) / 2, clusters) === 0)) colsInTopCluster.add(col);
|
|
991
|
+
}
|
|
992
|
+
if (colsInTopCluster.size < MIN_COLS_IN_TOP_CLUSTER) continue;
|
|
993
|
+
if (colsInTopCluster.size >= totalNonEmptyCols.size) continue;
|
|
994
|
+
if (!rowCellInfos.some(({ col, boxes }) => !colsInTopCluster.has(col) && boxes.length > 1)) continue;
|
|
995
|
+
const numSubRows = clusters.length;
|
|
996
|
+
const numNewRows = numSubRows - 1;
|
|
997
|
+
for (const cell of cells) if (cell.row > currentRow) cell.row += numNewRows;
|
|
998
|
+
for (let subRow = 1; subRow < numSubRows; subRow++) for (let col = 0; col < cols; col++) cells.push({
|
|
999
|
+
row: currentRow + subRow,
|
|
1000
|
+
col,
|
|
1001
|
+
text: "",
|
|
1002
|
+
rowSpan: 1,
|
|
1003
|
+
colSpan: 1
|
|
1004
|
+
});
|
|
1005
|
+
for (const { cell: origCell, col, boxes } of rowCellInfos) {
|
|
1006
|
+
const subRowBoxGroups = Array.from({ length: numSubRows }, () => []);
|
|
1007
|
+
for (const box of boxes) subRowBoxGroups[assignToYCluster((box.bounds.top + box.bounds.bottom) / 2, clusters)].push(box);
|
|
1008
|
+
cellBoxes.set(origCell, subRowBoxGroups[0]);
|
|
1009
|
+
if (subRowBoxGroups[0].length === 0) cellBoxes.delete(origCell);
|
|
1010
|
+
for (let subRow = 1; subRow < numSubRows; subRow++) if (subRowBoxGroups[subRow].length > 0) {
|
|
1011
|
+
const newCell = cells.find((c) => c.row === currentRow + subRow && c.col === col);
|
|
1012
|
+
if (newCell) cellBoxes.set(newCell, subRowBoxGroups[subRow]);
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
addedRows += numNewRows;
|
|
1016
|
+
}
|
|
1017
|
+
return originalRows + addedRows;
|
|
1018
|
+
}
|
|
1019
|
+
/**
|
|
1020
|
+
* Find which column a horizontal position falls into.
|
|
1021
|
+
* Returns -1 if outside the grid.
|
|
1022
|
+
*/
|
|
1023
|
+
function findCol(x, xLines) {
|
|
1024
|
+
for (let i = 0; i < xLines.length - 1; i++) if (x >= xLines[i] && x <= xLines[i + 1]) return i;
|
|
1025
|
+
return -1;
|
|
1026
|
+
}
|
|
1027
|
+
/**
|
|
1028
|
+
* When a text box spans across one or more vertical column boundaries,
|
|
1029
|
+
* split it into multiple virtual text boxes — one per column — with the
|
|
1030
|
+
* text divided proportionally by width.
|
|
1031
|
+
*
|
|
1032
|
+
* We split at word boundaries closest to the proportional split point
|
|
1033
|
+
* so we don't chop words in half.
|
|
1034
|
+
*/
|
|
1035
|
+
function splitCrossColumnBoxes(textBoxes, xLines) {
|
|
1036
|
+
const result = [];
|
|
1037
|
+
const MARGIN = 5;
|
|
1038
|
+
for (const tb of textBoxes) {
|
|
1039
|
+
const leftCol = findCol(tb.bounds.left + MARGIN, xLines);
|
|
1040
|
+
const rightCol = findCol(tb.bounds.right - MARGIN, xLines);
|
|
1041
|
+
if (leftCol < 0 || rightCol < 0 || leftCol === rightCol) {
|
|
1042
|
+
result.push(tb);
|
|
1043
|
+
continue;
|
|
1044
|
+
}
|
|
1045
|
+
const totalWidth = tb.bounds.right - tb.bounds.left;
|
|
1046
|
+
if (totalWidth <= 0) {
|
|
1047
|
+
result.push(tb);
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
const words = tb.text.split(/\s+/);
|
|
1051
|
+
if (words.length <= 1) {
|
|
1052
|
+
result.push(tb);
|
|
1053
|
+
continue;
|
|
1054
|
+
}
|
|
1055
|
+
let remainingWords = [...words];
|
|
1056
|
+
let currentLeft = tb.bounds.left;
|
|
1057
|
+
for (let col = leftCol; col <= rightCol && remainingWords.length > 0; col++) {
|
|
1058
|
+
const colRight = col < xLines.length - 1 ? xLines[col + 1] : tb.bounds.right;
|
|
1059
|
+
const segmentRight = Math.min(colRight, tb.bounds.right);
|
|
1060
|
+
if (col === rightCol) {
|
|
1061
|
+
result.push({
|
|
1062
|
+
...tb,
|
|
1063
|
+
id: `${tb.id}-split${col}`,
|
|
1064
|
+
text: remainingWords.join(" "),
|
|
1065
|
+
bounds: {
|
|
1066
|
+
...tb.bounds,
|
|
1067
|
+
left: currentLeft,
|
|
1068
|
+
right: tb.bounds.right
|
|
1069
|
+
}
|
|
1070
|
+
});
|
|
1071
|
+
remainingWords = [];
|
|
1072
|
+
} else {
|
|
1073
|
+
const fractionOfTotal = (segmentRight - currentLeft) / totalWidth;
|
|
1074
|
+
const approxChars = Math.round(fractionOfTotal * tb.text.length);
|
|
1075
|
+
let charCount = 0;
|
|
1076
|
+
let splitIdx = 0;
|
|
1077
|
+
for (let w = 0; w < remainingWords.length; w++) {
|
|
1078
|
+
const nextCount = charCount + remainingWords[w].length + (w > 0 ? 1 : 0);
|
|
1079
|
+
if (nextCount > approxChars && splitIdx > 0) break;
|
|
1080
|
+
charCount = nextCount;
|
|
1081
|
+
splitIdx = w + 1;
|
|
1082
|
+
}
|
|
1083
|
+
if (splitIdx === 0) splitIdx = 1;
|
|
1084
|
+
if (splitIdx >= remainingWords.length) {
|
|
1085
|
+
result.push({
|
|
1086
|
+
...tb,
|
|
1087
|
+
id: `${tb.id}-split${col}`,
|
|
1088
|
+
text: remainingWords.join(" "),
|
|
1089
|
+
bounds: {
|
|
1090
|
+
...tb.bounds,
|
|
1091
|
+
left: currentLeft,
|
|
1092
|
+
right: segmentRight
|
|
1093
|
+
}
|
|
1094
|
+
});
|
|
1095
|
+
remainingWords = [];
|
|
1096
|
+
} else {
|
|
1097
|
+
const partWords = remainingWords.slice(0, splitIdx);
|
|
1098
|
+
result.push({
|
|
1099
|
+
...tb,
|
|
1100
|
+
id: `${tb.id}-split${col}`,
|
|
1101
|
+
text: partWords.join(" "),
|
|
1102
|
+
bounds: {
|
|
1103
|
+
...tb.bounds,
|
|
1104
|
+
left: currentLeft,
|
|
1105
|
+
right: segmentRight
|
|
1106
|
+
}
|
|
1107
|
+
});
|
|
1108
|
+
remainingWords = remainingWords.slice(splitIdx);
|
|
1109
|
+
currentLeft = segmentRight;
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
return result;
|
|
1115
|
+
}
|
|
1116
|
+
function buildCells(rows, cols) {
|
|
1117
|
+
const cells = [];
|
|
1118
|
+
for (let row = 0; row < rows; row++) for (let col = 0; col < cols; col++) cells.push({
|
|
1119
|
+
row,
|
|
1120
|
+
col,
|
|
1121
|
+
text: "",
|
|
1122
|
+
rowSpan: 1,
|
|
1123
|
+
colSpan: 1
|
|
1124
|
+
});
|
|
1125
|
+
return cells;
|
|
1126
|
+
}
|
|
1127
|
+
function buildTableGrid(pageNumber, yLines, xLines, filteredSegments, textBoxes) {
|
|
1128
|
+
let rows = yLines.length - 1;
|
|
1129
|
+
const cols = xLines.length - 1;
|
|
1130
|
+
const cells = buildCells(rows, cols);
|
|
1131
|
+
const consumedIds = [];
|
|
1132
|
+
const yMin = yLines.at(-1);
|
|
1133
|
+
const yMax = yLines[0];
|
|
1134
|
+
const xMin = xLines[0];
|
|
1135
|
+
const xMax = xLines.at(-1);
|
|
1136
|
+
const splitBoxes = splitCrossColumnBoxes(textBoxes, xLines);
|
|
1137
|
+
const placedSplitIds = /* @__PURE__ */ new Set();
|
|
1138
|
+
const maxHeaderBoxWidth = (xMax - xMin) / cols * 1.5;
|
|
1139
|
+
const headerBoxes = textBoxes.filter((tb) => {
|
|
1140
|
+
const cy = (tb.bounds.top + tb.bounds.bottom) / 2;
|
|
1141
|
+
const cx = (tb.bounds.left + tb.bounds.right) / 2;
|
|
1142
|
+
const boxWidth = tb.bounds.right - tb.bounds.left;
|
|
1143
|
+
return cy > yMax && cy <= yMax + 20 && cx >= xMin && cx <= xMax && boxWidth <= maxHeaderBoxWidth;
|
|
1144
|
+
});
|
|
1145
|
+
if (headerBoxes.length > 0) {
|
|
1146
|
+
rows += 1;
|
|
1147
|
+
for (const cell of cells) cell.row += 1;
|
|
1148
|
+
for (let col = 0; col < cols; col++) cells.push({
|
|
1149
|
+
row: 0,
|
|
1150
|
+
col,
|
|
1151
|
+
text: "",
|
|
1152
|
+
rowSpan: 1,
|
|
1153
|
+
colSpan: 1
|
|
1154
|
+
});
|
|
1155
|
+
for (const tb of headerBoxes) {
|
|
1156
|
+
const cx = (tb.bounds.left + tb.bounds.right) / 2;
|
|
1157
|
+
const col = xLines.findIndex((lineX, idx) => {
|
|
1158
|
+
const next = xLines[idx + 1];
|
|
1159
|
+
return next !== void 0 && cx >= lineX && cx <= next;
|
|
1160
|
+
});
|
|
1161
|
+
if (col >= 0 && col < cols) {
|
|
1162
|
+
const cell = cells.find((c) => c.row === 0 && c.col === col);
|
|
1163
|
+
if (cell) {
|
|
1164
|
+
cell.text = cell.text.length === 0 ? tb.text : `${cell.text} ${tb.text}`;
|
|
1165
|
+
consumedIds.push(tb.id);
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
const cellBoxes = /* @__PURE__ */ new Map();
|
|
1171
|
+
for (const tb of splitBoxes) {
|
|
1172
|
+
const cx = (tb.bounds.left + tb.bounds.right) / 2;
|
|
1173
|
+
const cy = (tb.bounds.top + tb.bounds.bottom) / 2;
|
|
1174
|
+
if (cy < yMin || cy > yMax || cx < xMin || cx > xMax) continue;
|
|
1175
|
+
const rayConfidence = castRaysForTextBox(tb, filteredSegments).filter((r) => r.segmentId !== null).length;
|
|
1176
|
+
let row = yLines.findIndex((lineY, idx) => {
|
|
1177
|
+
const next = yLines[idx + 1];
|
|
1178
|
+
return next !== void 0 && cy <= lineY && cy >= next;
|
|
1179
|
+
});
|
|
1180
|
+
if (row < 0 || row >= (headerBoxes.length > 0 ? rows - 1 : rows)) continue;
|
|
1181
|
+
if (headerBoxes.length > 0) row += 1;
|
|
1182
|
+
const col = xLines.findIndex((lineX, idx) => {
|
|
1183
|
+
const next = xLines[idx + 1];
|
|
1184
|
+
return next !== void 0 && cx >= lineX && cx <= next;
|
|
1185
|
+
});
|
|
1186
|
+
if (col < 0 || col >= cols) continue;
|
|
1187
|
+
if (rayConfidence === 0) continue;
|
|
1188
|
+
const cell = cells.find((c) => c.row === row && c.col === col);
|
|
1189
|
+
if (!cell) continue;
|
|
1190
|
+
if (!cellBoxes.has(cell)) cellBoxes.set(cell, []);
|
|
1191
|
+
cellBoxes.get(cell)?.push(tb);
|
|
1192
|
+
consumedIds.push(tb.id);
|
|
1193
|
+
if (tb.id.includes("-split")) placedSplitIds.add(tb.id);
|
|
1194
|
+
}
|
|
1195
|
+
rows = expandSubRowsByYClusters(rows, cols, cells, cellBoxes);
|
|
1196
|
+
for (const [cell, boxes] of cellBoxes.entries()) {
|
|
1197
|
+
boxes.sort((a, b) => b.bounds.top - a.bounds.top);
|
|
1198
|
+
const lines = [];
|
|
1199
|
+
let currentLine = [];
|
|
1200
|
+
let currentY = boxes[0].bounds.top;
|
|
1201
|
+
for (const box of boxes) if (Math.abs(box.bounds.top - currentY) > 5) {
|
|
1202
|
+
lines.push(currentLine.join(" "));
|
|
1203
|
+
currentLine = [box.text];
|
|
1204
|
+
currentY = box.bounds.top;
|
|
1205
|
+
} else currentLine.push(box.text);
|
|
1206
|
+
if (currentLine.length > 0) lines.push(currentLine.join(" "));
|
|
1207
|
+
cell.text = lines.join("<br>");
|
|
1208
|
+
}
|
|
1209
|
+
const grid = pruneEmptyRowsAndCols({
|
|
1210
|
+
pageNumber,
|
|
1211
|
+
rows,
|
|
1212
|
+
cols,
|
|
1213
|
+
cells,
|
|
1214
|
+
warnings: [],
|
|
1215
|
+
topY: yLines[0],
|
|
1216
|
+
isBorderless: false
|
|
1217
|
+
});
|
|
1218
|
+
for (const splitId of placedSplitIds) {
|
|
1219
|
+
const origId = splitId.replace(/-split\d+$/, "");
|
|
1220
|
+
if (!consumedIds.includes(origId)) consumedIds.push(origId);
|
|
1221
|
+
}
|
|
1222
|
+
return {
|
|
1223
|
+
grid,
|
|
1224
|
+
consumedIds
|
|
1225
|
+
};
|
|
1226
|
+
}
|
|
1227
|
+
const COL_GAP_THRESHOLD = 20;
|
|
1228
|
+
const HONLY_ROW_GAP = 30;
|
|
1229
|
+
const HONLY_ROW_TOLERANCE = 8;
|
|
1230
|
+
const MIN_TABLE_HEIGHT = 24;
|
|
1231
|
+
const MIN_LEFT_SPREAD = 50;
|
|
1232
|
+
function inferXLinesFromBoxes(textBoxes, xMin, xMax) {
|
|
1233
|
+
const centers = textBoxes.map((tb) => (tb.bounds.left + tb.bounds.right) / 2).toSorted((a, b) => a - b);
|
|
1234
|
+
if (centers.length === 0) return [xMin, xMax];
|
|
1235
|
+
const boundaries = [xMin];
|
|
1236
|
+
for (let i = 1; i < centers.length; i++) if (centers[i] - centers[i - 1] >= COL_GAP_THRESHOLD) boundaries.push((centers[i - 1] + centers[i]) / 2);
|
|
1237
|
+
boundaries.push(xMax);
|
|
1238
|
+
return boundaries;
|
|
1239
|
+
}
|
|
1240
|
+
function buildHLineOnlyTable(pageNumber, yLines, xMin, xMax, textBoxes, alreadyConsumed) {
|
|
1241
|
+
const yMax = yLines[0];
|
|
1242
|
+
const yMin = yLines.at(-1);
|
|
1243
|
+
const candidates = textBoxes.filter((tb) => !alreadyConsumed.has(tb.id));
|
|
1244
|
+
const BOX_LEFT_TOLERANCE = 30;
|
|
1245
|
+
const inRange = candidates.filter((tb) => {
|
|
1246
|
+
const cy = (tb.bounds.top + tb.bounds.bottom) / 2;
|
|
1247
|
+
return tb.bounds.left >= xMin - BOX_LEFT_TOLERANCE && tb.bounds.right <= xMax + BOX_LEFT_TOLERANCE && cy >= yMin && cy <= yMax;
|
|
1248
|
+
});
|
|
1249
|
+
const belowYMin = candidates.filter((tb) => {
|
|
1250
|
+
const cx = (tb.bounds.left + tb.bounds.right) / 2;
|
|
1251
|
+
const cy = (tb.bounds.top + tb.bounds.bottom) / 2;
|
|
1252
|
+
return cx >= xMin && cx <= xMax && cy < yMin;
|
|
1253
|
+
}).toSorted((a, b) => (b.bounds.top + b.bounds.bottom) / 2 - (a.bounds.top + a.bounds.bottom) / 2);
|
|
1254
|
+
const extensionBoxes = [];
|
|
1255
|
+
let lastY = yMin;
|
|
1256
|
+
for (const tb of belowYMin) {
|
|
1257
|
+
const cy = (tb.bounds.top + tb.bounds.bottom) / 2;
|
|
1258
|
+
if (lastY - cy > HONLY_ROW_GAP) break;
|
|
1259
|
+
extensionBoxes.push(tb);
|
|
1260
|
+
lastY = cy;
|
|
1261
|
+
}
|
|
1262
|
+
const allBoxes = [...inRange, ...extensionBoxes];
|
|
1263
|
+
if (allBoxes.length === 0) return null;
|
|
1264
|
+
const leftEdges = allBoxes.map((tb) => tb.bounds.left);
|
|
1265
|
+
if (Math.max(...leftEdges) - Math.min(...leftEdges) < MIN_LEFT_SPREAD) return null;
|
|
1266
|
+
const xLines = inferXLinesFromBoxes(allBoxes, xMin, xMax);
|
|
1267
|
+
if (xLines.length < 2) return null;
|
|
1268
|
+
const cols = xLines.length - 1;
|
|
1269
|
+
const visualRows = [];
|
|
1270
|
+
const sortedBoxes = [...allBoxes].toSorted((a, b) => {
|
|
1271
|
+
const ya = (a.bounds.top + a.bounds.bottom) / 2;
|
|
1272
|
+
const yb = (b.bounds.top + b.bounds.bottom) / 2;
|
|
1273
|
+
if (Math.abs(ya - yb) > .5) return yb - ya;
|
|
1274
|
+
return a.bounds.left - b.bounds.left;
|
|
1275
|
+
});
|
|
1276
|
+
for (const box of sortedBoxes) {
|
|
1277
|
+
const cy = (box.bounds.top + box.bounds.bottom) / 2;
|
|
1278
|
+
const last = visualRows.at(-1);
|
|
1279
|
+
if (last && Math.abs(last.midY - cy) <= HONLY_ROW_TOLERANCE) last.boxes.push(box);
|
|
1280
|
+
else visualRows.push({
|
|
1281
|
+
midY: cy,
|
|
1282
|
+
boxes: [box]
|
|
1283
|
+
});
|
|
1284
|
+
}
|
|
1285
|
+
if (visualRows.length === 0) return null;
|
|
1286
|
+
const cells = [];
|
|
1287
|
+
const consumedIds = [];
|
|
1288
|
+
for (let rowIdx = 0; rowIdx < visualRows.length; rowIdx++) {
|
|
1289
|
+
const vrow = visualRows[rowIdx];
|
|
1290
|
+
const colBoxes = /* @__PURE__ */ new Map();
|
|
1291
|
+
for (const box of vrow.boxes) {
|
|
1292
|
+
const cx = (box.bounds.left + box.bounds.right) / 2;
|
|
1293
|
+
const col = xLines.findIndex((lineX, idx) => {
|
|
1294
|
+
const next = xLines[idx + 1];
|
|
1295
|
+
return next !== void 0 && cx >= lineX && cx <= next;
|
|
1296
|
+
});
|
|
1297
|
+
if (col >= 0 && col < cols) {
|
|
1298
|
+
if (!colBoxes.has(col)) colBoxes.set(col, []);
|
|
1299
|
+
colBoxes.get(col)?.push(box);
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
for (let c = 0; c < cols; c++) {
|
|
1303
|
+
const cbs = (colBoxes.get(c) ?? []).toSorted((a, b) => a.bounds.left - b.bounds.left);
|
|
1304
|
+
cells.push({
|
|
1305
|
+
row: rowIdx,
|
|
1306
|
+
col: c,
|
|
1307
|
+
text: cbs.map((b) => b.text).join(" "),
|
|
1308
|
+
rowSpan: 1,
|
|
1309
|
+
colSpan: 1
|
|
1310
|
+
});
|
|
1311
|
+
consumedIds.push(...cbs.map((b) => b.id));
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
const contentTopY = visualRows.length > 0 ? visualRows[0].midY : yMax;
|
|
1315
|
+
return {
|
|
1316
|
+
grid: pruneEmptyRowsAndCols({
|
|
1317
|
+
pageNumber,
|
|
1318
|
+
rows: visualRows.length,
|
|
1319
|
+
cols,
|
|
1320
|
+
cells,
|
|
1321
|
+
warnings: [],
|
|
1322
|
+
topY: contentTopY,
|
|
1323
|
+
isBorderless: false
|
|
1324
|
+
}),
|
|
1325
|
+
consumedIds
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
function pruneEmptyRowsAndCols(table) {
|
|
1329
|
+
const occupiedRows = new Set(table.cells.filter((c) => c.text.trim().length > 0).map((c) => c.row));
|
|
1330
|
+
const occupiedCols = new Set(table.cells.filter((c) => c.text.trim().length > 0).map((c) => c.col));
|
|
1331
|
+
if (occupiedRows.size === 0) return table;
|
|
1332
|
+
const rowMap = /* @__PURE__ */ new Map();
|
|
1333
|
+
let newRow = 0;
|
|
1334
|
+
for (let r = 0; r < table.rows; r++) if (occupiedRows.has(r)) rowMap.set(r, newRow++);
|
|
1335
|
+
const colMap = /* @__PURE__ */ new Map();
|
|
1336
|
+
let newCol = 0;
|
|
1337
|
+
for (let c = 0; c < table.cols; c++) if (occupiedCols.has(c)) colMap.set(c, newCol++);
|
|
1338
|
+
const prunedCells = table.cells.filter((c) => occupiedRows.has(c.row) && occupiedCols.has(c.col)).map((c) => ({
|
|
1339
|
+
...c,
|
|
1340
|
+
row: rowMap.get(c.row) ?? c.row,
|
|
1341
|
+
col: colMap.get(c.col) ?? c.col
|
|
1342
|
+
}));
|
|
1343
|
+
return {
|
|
1344
|
+
...table,
|
|
1345
|
+
rows: newRow,
|
|
1346
|
+
cols: newCol,
|
|
1347
|
+
cells: prunedCells
|
|
1348
|
+
};
|
|
1349
|
+
}
|
|
1350
|
+
/** Maximum column count for a plausible data table. */
|
|
1351
|
+
const MAX_TABLE_COLS = 25;
|
|
1352
|
+
/**
|
|
1353
|
+
* Returns true if a grid looks like a vector diagram rather than a data table.
|
|
1354
|
+
*
|
|
1355
|
+
* Heuristics (any match → diagram):
|
|
1356
|
+
* 1. Column count > 25 (diagrams create many X-lines from box edges)
|
|
1357
|
+
* 2. Fill ratio < 25% (most cells empty — scattered boxes)
|
|
1358
|
+
* 3. Fill < 50% AND duplicate text ratio > 30% (repeating labels in a
|
|
1359
|
+
* diagram layout, e.g. "Hash", "Transaction" appearing in each column)
|
|
1360
|
+
* 4. Fill < 50% AND cols >= 6 (moderate sparseness with wide grid)
|
|
1361
|
+
*/
|
|
1362
|
+
function isDiagram(grid) {
|
|
1363
|
+
const totalCells = grid.rows * grid.cols;
|
|
1364
|
+
if (totalCells === 0) return true;
|
|
1365
|
+
const filled = grid.cells.filter((c) => c.text.trim().length > 0);
|
|
1366
|
+
const fillRatio = filled.length / totalCells;
|
|
1367
|
+
if (grid.cols > MAX_TABLE_COLS) return true;
|
|
1368
|
+
if (fillRatio < .25) return true;
|
|
1369
|
+
const substantive = filled.filter((c) => c.text.trim().length > 3);
|
|
1370
|
+
const uniqueTexts = new Set(substantive.map((c) => c.text.trim())).size;
|
|
1371
|
+
const dupRatio = substantive.length > 2 ? 1 - uniqueTexts / substantive.length : 0;
|
|
1372
|
+
if (fillRatio < .5 && dupRatio > .3) return true;
|
|
1373
|
+
if (dupRatio > .4 && grid.cols >= 6) return true;
|
|
1374
|
+
if (fillRatio < .4 && grid.cols >= 6) return true;
|
|
1375
|
+
return false;
|
|
1376
|
+
}
|
|
1377
|
+
/**
|
|
1378
|
+
* Detect all table grids on a single page from its text boxes and segments.
|
|
1379
|
+
*/
|
|
1380
|
+
function resolveTableGrids(pageNumber, textBoxes, segments) {
|
|
1381
|
+
const vertical = segments.filter((s) => Math.abs(s.x1 - s.x2) <= AXIS_EPSILON);
|
|
1382
|
+
const horizontal = segments.filter((s) => Math.abs(s.y1 - s.y2) <= AXIS_EPSILON);
|
|
1383
|
+
const textYValues = textBoxes.flatMap((t) => [t.bounds.bottom, t.bounds.top]);
|
|
1384
|
+
const textYMin = textYValues.length > 0 ? Math.min(...textYValues) - PAGE_MARGIN : -Infinity;
|
|
1385
|
+
const textYMax = textYValues.length > 0 ? Math.max(...textYValues) + PAGE_MARGIN : Infinity;
|
|
1386
|
+
const textXValues = textBoxes.flatMap((t) => [t.bounds.left, t.bounds.right]);
|
|
1387
|
+
const textXMin = textXValues.length > 0 ? Math.min(...textXValues) - 100 : -Infinity;
|
|
1388
|
+
const textXMax = textXValues.length > 0 ? Math.max(...textXValues) + 100 : Infinity;
|
|
1389
|
+
const filteredH = horizontal.filter((s) => s.y1 >= textYMin && s.y1 <= textYMax && s.x1 <= textXMax && s.x2 >= textXMin);
|
|
1390
|
+
const hMaxX2 = filteredH.length > 0 ? Math.max(...filteredH.map((s) => s.x2)) : textXMax;
|
|
1391
|
+
const vSegXMax = Math.max(textXMax, hMaxX2 + PAGE_MARGIN);
|
|
1392
|
+
const filteredV = vertical.filter((s) => {
|
|
1393
|
+
const segMin = Math.min(s.y1, s.y2);
|
|
1394
|
+
return Math.max(s.y1, s.y2) >= textYMin && segMin <= textYMax && s.x1 >= textXMin && s.x1 <= vSegXMax;
|
|
1395
|
+
});
|
|
1396
|
+
const allYLines = uniqueSorted(filteredH.flatMap((s) => [s.y1, s.y2])).toSorted((a, b) => b - a);
|
|
1397
|
+
if (allYLines.length < 2) return {
|
|
1398
|
+
grids: [],
|
|
1399
|
+
consumedIds: []
|
|
1400
|
+
};
|
|
1401
|
+
const filteredSegments = [...filteredH, ...filteredV];
|
|
1402
|
+
const yGroups = splitYLinesIntoGroups(allYLines, filteredV);
|
|
1403
|
+
const grids = [];
|
|
1404
|
+
const gridConsumedIds = [];
|
|
1405
|
+
const allConsumedIds = [];
|
|
1406
|
+
for (const yLines of yGroups) {
|
|
1407
|
+
if (yLines.length < 2) continue;
|
|
1408
|
+
const yMin = yLines.at(-1);
|
|
1409
|
+
const yMax = yLines[0];
|
|
1410
|
+
const groupXLines = uniqueSorted(filteredV.filter((s) => {
|
|
1411
|
+
const segMin = Math.min(s.y1, s.y2);
|
|
1412
|
+
const segMax = Math.max(s.y1, s.y2);
|
|
1413
|
+
return segMin < yMax - 1.5 && segMax > yMin + 1.5;
|
|
1414
|
+
}).flatMap((s) => [s.x1, s.x2]));
|
|
1415
|
+
if (groupXLines.length < 2) {
|
|
1416
|
+
if (yMax - yMin < MIN_TABLE_HEIGHT) continue;
|
|
1417
|
+
const groupHoriz = filteredH.filter((s) => s.y1 >= yMin - 1.5 && s.y1 <= yMax + 1.5);
|
|
1418
|
+
if (groupHoriz.length === 0) continue;
|
|
1419
|
+
const result = buildHLineOnlyTable(pageNumber, yLines, Math.min(...groupHoriz.map((s) => s.x1)), Math.max(...groupHoriz.map((s) => s.x2)), textBoxes, new Set(allConsumedIds));
|
|
1420
|
+
if (result) {
|
|
1421
|
+
grids.push(result.grid);
|
|
1422
|
+
gridConsumedIds.push(result.consumedIds);
|
|
1423
|
+
allConsumedIds.push(...result.consumedIds);
|
|
1424
|
+
}
|
|
1425
|
+
continue;
|
|
1426
|
+
}
|
|
1427
|
+
if (yMax - yMin < MIN_TABLE_HEIGHT) continue;
|
|
1428
|
+
const result = buildTableGrid(pageNumber, yLines, groupXLines, filteredSegments, textBoxes);
|
|
1429
|
+
grids.push(result.grid);
|
|
1430
|
+
gridConsumedIds.push(result.consumedIds);
|
|
1431
|
+
allConsumedIds.push(...result.consumedIds);
|
|
1432
|
+
}
|
|
1433
|
+
const filteredGrids = [];
|
|
1434
|
+
const filteredConsumedIds = [];
|
|
1435
|
+
for (let i = 0; i < grids.length; i++) {
|
|
1436
|
+
if (isDiagram(grids[i])) continue;
|
|
1437
|
+
filteredGrids.push(grids[i]);
|
|
1438
|
+
filteredConsumedIds.push(...gridConsumedIds[i]);
|
|
1439
|
+
}
|
|
1440
|
+
return {
|
|
1441
|
+
grids: filteredGrids,
|
|
1442
|
+
consumedIds: filteredConsumedIds
|
|
1443
|
+
};
|
|
1444
|
+
}
|
|
1445
|
+
//#endregion
|
|
1446
|
+
//#region ../../packages/agent-core/src/markit/converters/pdf/headers.ts
|
|
1447
|
+
/** Minimum number of pages to enable header/footer detection. */
|
|
1448
|
+
const MIN_PAGES = 5;
|
|
1449
|
+
/** Minimum Y position for top zone (from bottom of page in PDF coords). */
|
|
1450
|
+
const TOP_ZONE_MIN_Y = 700;
|
|
1451
|
+
/** Maximum Y position for bottom zone. */
|
|
1452
|
+
const BOTTOM_ZONE_MAX_Y = 80;
|
|
1453
|
+
/**
|
|
1454
|
+
* Minimum consecutive pages a text must appear on to be considered a
|
|
1455
|
+
* running header/footer. Catches both document-wide headers (appearing
|
|
1456
|
+
* on every page) and chapter-specific headers (appearing on 4+ consecutive
|
|
1457
|
+
* pages within a chapter).
|
|
1458
|
+
*/
|
|
1459
|
+
const MIN_CONSECUTIVE_PAGES = 8;
|
|
1460
|
+
/**
|
|
1461
|
+
* Detect and remove running headers and footers from all pages.
|
|
1462
|
+
* Mutates the pages array in place, removing header/footer text boxes.
|
|
1463
|
+
*
|
|
1464
|
+
* Uses two strategies:
|
|
1465
|
+
* 1. Global frequency: text appearing on > 20% of all pages
|
|
1466
|
+
* 2. Consecutive runs: text appearing on 8+ consecutive pages
|
|
1467
|
+
*/
|
|
1468
|
+
function stripHeadersFooters(pages) {
|
|
1469
|
+
if (pages.length < MIN_PAGES) return;
|
|
1470
|
+
const pageZoneTexts = [];
|
|
1471
|
+
for (const page of pages) {
|
|
1472
|
+
const zoneTexts = /* @__PURE__ */ new Set();
|
|
1473
|
+
for (const tb of page.textBoxes) {
|
|
1474
|
+
const midY = (tb.bounds.top + tb.bounds.bottom) / 2;
|
|
1475
|
+
if (midY >= TOP_ZONE_MIN_Y || midY <= BOTTOM_ZONE_MAX_Y) {
|
|
1476
|
+
const key = tb.text.trim().replaceAll(/\s+/g, " ");
|
|
1477
|
+
if (key.length > 0) zoneTexts.add(key);
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
pageZoneTexts.push(zoneTexts);
|
|
1481
|
+
}
|
|
1482
|
+
const globalCount = /* @__PURE__ */ new Map();
|
|
1483
|
+
const maxConsecutive = /* @__PURE__ */ new Map();
|
|
1484
|
+
const allTexts = /* @__PURE__ */ new Set();
|
|
1485
|
+
for (const zts of pageZoneTexts) for (const t of zts) allTexts.add(t);
|
|
1486
|
+
for (const text of allTexts) {
|
|
1487
|
+
let total = 0;
|
|
1488
|
+
let consecutive = 0;
|
|
1489
|
+
let maxRun = 0;
|
|
1490
|
+
for (const zts of pageZoneTexts) if (zts.has(text)) {
|
|
1491
|
+
total++;
|
|
1492
|
+
consecutive++;
|
|
1493
|
+
if (consecutive > maxRun) maxRun = consecutive;
|
|
1494
|
+
} else consecutive = 0;
|
|
1495
|
+
globalCount.set(text, total);
|
|
1496
|
+
maxConsecutive.set(text, maxRun);
|
|
1497
|
+
}
|
|
1498
|
+
const globalThreshold = Math.max(3, Math.floor(pages.length * .2));
|
|
1499
|
+
const repeatedTexts = /* @__PURE__ */ new Set();
|
|
1500
|
+
for (const text of allTexts) {
|
|
1501
|
+
const gc = globalCount.get(text) ?? 0;
|
|
1502
|
+
const mc = maxConsecutive.get(text) ?? 0;
|
|
1503
|
+
if (gc >= globalThreshold) {
|
|
1504
|
+
repeatedTexts.add(text);
|
|
1505
|
+
continue;
|
|
1506
|
+
}
|
|
1507
|
+
if (mc >= MIN_CONSECUTIVE_PAGES) repeatedTexts.add(text);
|
|
1508
|
+
}
|
|
1509
|
+
if (repeatedTexts.size === 0) return;
|
|
1510
|
+
for (const page of pages) page.textBoxes = page.textBoxes.filter((tb) => {
|
|
1511
|
+
const midY = (tb.bounds.top + tb.bounds.bottom) / 2;
|
|
1512
|
+
if (midY < TOP_ZONE_MIN_Y && midY > BOTTOM_ZONE_MAX_Y) return true;
|
|
1513
|
+
const normalized = tb.text.trim().replaceAll(/\s+/g, " ");
|
|
1514
|
+
return !repeatedTexts.has(normalized);
|
|
1515
|
+
});
|
|
1516
|
+
}
|
|
1517
|
+
//#endregion
|
|
1518
|
+
//#region ../../packages/agent-core/src/markit/converters/pdf/render.ts
|
|
1519
|
+
/** Convert full-width ASCII characters (A→A, !→! etc.) to normal ASCII. */
|
|
1520
|
+
function normalizeFullWidthAscii(text) {
|
|
1521
|
+
return text.replaceAll(/[!-~]/g, (ch) => String.fromCodePoint(ch.codePointAt(0) - 65248));
|
|
1522
|
+
}
|
|
1523
|
+
function escapePipes(text) {
|
|
1524
|
+
return normalizeFullWidthAscii(text).replaceAll("|", "\\|").replaceAll("\n", "<br>");
|
|
1525
|
+
}
|
|
1526
|
+
/** Parse a markdown pipe-delimited row into cell strings. */
|
|
1527
|
+
function parsePipeRow(line) {
|
|
1528
|
+
const trimmed = line.trim();
|
|
1529
|
+
if (!trimmed.startsWith("|") || !trimmed.endsWith("|")) return [];
|
|
1530
|
+
return trimmed.slice(1, -1).split("|").map((cell) => cell.trim());
|
|
1531
|
+
}
|
|
1532
|
+
/**
|
|
1533
|
+
* Render a TableGrid as a markdown table.
|
|
1534
|
+
*/
|
|
1535
|
+
function renderTableToMarkdown(table) {
|
|
1536
|
+
if (table.rows === 0 || table.cols === 0) return "";
|
|
1537
|
+
const matrix = Array.from({ length: table.rows }, () => Array.from({ length: table.cols }, () => ""));
|
|
1538
|
+
for (const cell of table.cells) if (cell.row < table.rows && cell.col < table.cols) matrix[cell.row][cell.col] = escapePipes(cell.text.trim());
|
|
1539
|
+
const promoted = promoteSubHeaderPrefixes(normalizeShiftedSparseColumns(matrix));
|
|
1540
|
+
return [
|
|
1541
|
+
`| ${promoted[0].join(" | ")} |`,
|
|
1542
|
+
`| ${Array.from({ length: promoted[0].length }, () => "---").join(" | ")} |`,
|
|
1543
|
+
promoted.slice(1).map((row) => `| ${row.join(" | ")} |`).join("\n")
|
|
1544
|
+
].filter((l) => l.length > 0).join("\n");
|
|
1545
|
+
}
|
|
1546
|
+
/**
|
|
1547
|
+
* Fix tables with ≥5 columns where sparse single-value columns are
|
|
1548
|
+
* misaligned. Shifts those values to the adjacent dense column and
|
|
1549
|
+
* removes the now-empty sparse columns.
|
|
1550
|
+
*/
|
|
1551
|
+
function normalizeShiftedSparseColumns(matrix) {
|
|
1552
|
+
if (matrix.length === 0 || matrix[0].length < 5) return matrix;
|
|
1553
|
+
matrix.length;
|
|
1554
|
+
const cols = matrix[0].length;
|
|
1555
|
+
const counts = Array.from({ length: cols }, (_, c) => matrix.reduce((n, row) => n + (row[c].trim().length > 0 ? 1 : 0), 0));
|
|
1556
|
+
const denseCols = new Set(counts.map((count, col) => ({
|
|
1557
|
+
count,
|
|
1558
|
+
col
|
|
1559
|
+
})).filter(({ col, count }) => col === 0 || count >= 2).map(({ col }) => col));
|
|
1560
|
+
const sparseCols = counts.map((count, col) => ({
|
|
1561
|
+
count,
|
|
1562
|
+
col
|
|
1563
|
+
})).filter(({ col, count }) => col > 0 && col < cols - 1 && count === 1).map(({ col }) => col);
|
|
1564
|
+
if (sparseCols.length < 2 || denseCols.size < 4) return matrix;
|
|
1565
|
+
const moves = [];
|
|
1566
|
+
for (const from of sparseCols) {
|
|
1567
|
+
const row = matrix.findIndex((r) => r[from].trim().length > 0);
|
|
1568
|
+
const to = from + 1;
|
|
1569
|
+
if (row < 0) return matrix;
|
|
1570
|
+
if (!denseCols.has(to)) return matrix;
|
|
1571
|
+
if (matrix[row][to].trim().length > 0) return matrix;
|
|
1572
|
+
moves.push({
|
|
1573
|
+
from,
|
|
1574
|
+
to,
|
|
1575
|
+
row
|
|
1576
|
+
});
|
|
1577
|
+
}
|
|
1578
|
+
const copy = matrix.map((row) => [...row]);
|
|
1579
|
+
for (const { from, to, row } of moves) {
|
|
1580
|
+
copy[row][to] = copy[row][to].trim().length > 0 ? `${copy[row][to]} ${copy[row][from]}` : copy[row][from];
|
|
1581
|
+
copy[row][from] = "";
|
|
1582
|
+
}
|
|
1583
|
+
const keepCols = Array.from({ length: cols }, (_, c) => c).filter((c) => copy.some((row) => row[c].trim().length > 0));
|
|
1584
|
+
if (keepCols.length === cols) return copy;
|
|
1585
|
+
return copy.map((row) => keepCols.map((c) => row[c]));
|
|
1586
|
+
}
|
|
1587
|
+
/**
|
|
1588
|
+
* When a data row has ≥2 parenthesized qualifiers in non-first columns
|
|
1589
|
+
* (and the first column is empty), promote them into the header row.
|
|
1590
|
+
*/
|
|
1591
|
+
function promoteSubHeaderPrefixes(matrix) {
|
|
1592
|
+
if (matrix.length < 2) return matrix;
|
|
1593
|
+
const PAREN_RE = /^\([^)]{1,40}\)$/;
|
|
1594
|
+
const result = matrix.map((row) => [...row]);
|
|
1595
|
+
const cols = matrix[0].length;
|
|
1596
|
+
const rowsToRemove = /* @__PURE__ */ new Set();
|
|
1597
|
+
for (let r = 1; r < result.length; r++) {
|
|
1598
|
+
if (rowsToRemove.has(r)) continue;
|
|
1599
|
+
const promotable = [];
|
|
1600
|
+
for (let col = 1; col < cols; col++) {
|
|
1601
|
+
const cell = (result[r][col] ?? "").trim();
|
|
1602
|
+
if (!cell) continue;
|
|
1603
|
+
const parts = cell.split("<br>");
|
|
1604
|
+
if (parts.length === 1 && PAREN_RE.test(cell)) promotable.push({
|
|
1605
|
+
col,
|
|
1606
|
+
prefix: cell,
|
|
1607
|
+
isFullCell: true
|
|
1608
|
+
});
|
|
1609
|
+
else if (parts.length >= 2 && PAREN_RE.test(parts[0].trim())) promotable.push({
|
|
1610
|
+
col,
|
|
1611
|
+
prefix: parts[0].trim(),
|
|
1612
|
+
isFullCell: false
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1615
|
+
if (promotable.length < 2) continue;
|
|
1616
|
+
if (promotable.some((p) => p.isFullCell) && result[r][0].trim().length > 0) continue;
|
|
1617
|
+
for (const { col, prefix, isFullCell } of promotable) {
|
|
1618
|
+
result[0][col] = result[0][col].trim() ? `${result[0][col]} ${prefix}` : prefix;
|
|
1619
|
+
if (isFullCell) result[r][col] = "";
|
|
1620
|
+
else {
|
|
1621
|
+
const parts = result[r][col].split("<br>");
|
|
1622
|
+
result[r][col] = parts.slice(1).join("<br>");
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
if (result[r].every((cell) => cell.trim().length === 0)) rowsToRemove.add(r);
|
|
1626
|
+
}
|
|
1627
|
+
return result.filter((_, r) => !rowsToRemove.has(r));
|
|
1628
|
+
}
|
|
1629
|
+
/** Y tolerance for grouping text boxes onto the same visual line. */
|
|
1630
|
+
const TEXT_LINE_Y_TOLERANCE = 3;
|
|
1631
|
+
/** Minimum X gap between adjacent boxes to mark line as tabular. */
|
|
1632
|
+
const TABULAR_X_GAP = 30;
|
|
1633
|
+
/**
|
|
1634
|
+
* Minimum font size (pts) to consider when computing the modal body font.
|
|
1635
|
+
* Tiny labels from diagrams, footnote markers, and superscripts are excluded
|
|
1636
|
+
* so they don't skew the modal toward small sizes.
|
|
1637
|
+
*/
|
|
1638
|
+
const MIN_BODY_FONT_SIZE = 7;
|
|
1639
|
+
/**
|
|
1640
|
+
* Compute the most frequent font size among text boxes, ignoring very small
|
|
1641
|
+
* text that likely comes from diagrams, footnotes, or superscripts.
|
|
1642
|
+
*/
|
|
1643
|
+
function modalFontSize(textBoxes) {
|
|
1644
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1645
|
+
for (const tb of textBoxes) {
|
|
1646
|
+
const size = Math.round((tb.fontSize ?? 0) * 10) / 10;
|
|
1647
|
+
if (size < MIN_BODY_FONT_SIZE) continue;
|
|
1648
|
+
counts.set(size, (counts.get(size) ?? 0) + 1);
|
|
1649
|
+
}
|
|
1650
|
+
let modal = 0;
|
|
1651
|
+
let maxCount = 0;
|
|
1652
|
+
for (const [size, count] of counts) if (count > maxCount) {
|
|
1653
|
+
maxCount = count;
|
|
1654
|
+
modal = size;
|
|
1655
|
+
}
|
|
1656
|
+
return modal;
|
|
1657
|
+
}
|
|
1658
|
+
/** Group free text boxes into horizontal lines, sorted top-to-bottom. */
|
|
1659
|
+
function groupFreeTextIntoLines(textBoxes) {
|
|
1660
|
+
if (textBoxes.length === 0) return [];
|
|
1661
|
+
const sorted = [...textBoxes].toSorted((a, b) => {
|
|
1662
|
+
const ya = (a.bounds.top + a.bounds.bottom) / 2;
|
|
1663
|
+
const dy = (b.bounds.top + b.bounds.bottom) / 2 - ya;
|
|
1664
|
+
if (Math.abs(dy) > TEXT_LINE_Y_TOLERANCE) return dy;
|
|
1665
|
+
return a.bounds.left - b.bounds.left;
|
|
1666
|
+
});
|
|
1667
|
+
const lines = [];
|
|
1668
|
+
let curParts = [sorted[0].text];
|
|
1669
|
+
let curBoxes = [sorted[0]];
|
|
1670
|
+
let curY = (sorted[0].bounds.top + sorted[0].bounds.bottom) / 2;
|
|
1671
|
+
let curTopY = curY;
|
|
1672
|
+
let curFontSize = sorted[0].fontSize;
|
|
1673
|
+
let curIsBold = sorted[0].isBold;
|
|
1674
|
+
const finishLine = () => {
|
|
1675
|
+
let isTabular = false;
|
|
1676
|
+
for (let j = 1; j < curBoxes.length; j++) if (curBoxes[j].bounds.left - curBoxes[j - 1].bounds.right > TABULAR_X_GAP) {
|
|
1677
|
+
isTabular = true;
|
|
1678
|
+
break;
|
|
1679
|
+
}
|
|
1680
|
+
lines.push({
|
|
1681
|
+
text: curParts.join(" "),
|
|
1682
|
+
topY: curTopY,
|
|
1683
|
+
fontSize: curFontSize,
|
|
1684
|
+
isBold: curIsBold,
|
|
1685
|
+
isTabular
|
|
1686
|
+
});
|
|
1687
|
+
};
|
|
1688
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1689
|
+
const box = sorted[i];
|
|
1690
|
+
const cy = (box.bounds.top + box.bounds.bottom) / 2;
|
|
1691
|
+
if (Math.abs(cy - curY) <= TEXT_LINE_Y_TOLERANCE) {
|
|
1692
|
+
curParts.push(box.text);
|
|
1693
|
+
curBoxes.push(box);
|
|
1694
|
+
curFontSize = Math.max(curFontSize, box.fontSize);
|
|
1695
|
+
curIsBold = curIsBold || box.isBold;
|
|
1696
|
+
} else {
|
|
1697
|
+
finishLine();
|
|
1698
|
+
curParts = [box.text];
|
|
1699
|
+
curBoxes = [box];
|
|
1700
|
+
curY = cy;
|
|
1701
|
+
curTopY = cy;
|
|
1702
|
+
curFontSize = box.fontSize;
|
|
1703
|
+
curIsBold = box.isBold;
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
finishLine();
|
|
1707
|
+
return lines;
|
|
1708
|
+
}
|
|
1709
|
+
/** Determine markdown heading prefix based on font size relative to body. */
|
|
1710
|
+
function headingPrefix(fontSize, bodyFontSize, isBold) {
|
|
1711
|
+
if (bodyFontSize <= 0) return "";
|
|
1712
|
+
const ratio = fontSize / bodyFontSize;
|
|
1713
|
+
if (ratio >= 2) return "# ";
|
|
1714
|
+
if (ratio >= 1.4) return "## ";
|
|
1715
|
+
if (ratio >= 1.1 && isBold) return "### ";
|
|
1716
|
+
return "";
|
|
1717
|
+
}
|
|
1718
|
+
/** Merge consecutive blocks with the same heading prefix (wrapped headings). */
|
|
1719
|
+
function mergeConsecutiveHeadings(blocks, bodyFS) {
|
|
1720
|
+
if (blocks.length === 0) return [];
|
|
1721
|
+
const HEADING_RE = /^(#{1,6} )/;
|
|
1722
|
+
const maxGap = Math.max(bodyFS * 3, 30);
|
|
1723
|
+
const merged = [];
|
|
1724
|
+
let cur = { ...blocks[0] };
|
|
1725
|
+
for (let i = 1; i < blocks.length; i++) {
|
|
1726
|
+
const next = blocks[i];
|
|
1727
|
+
const curMatch = cur.content.match(HEADING_RE);
|
|
1728
|
+
const nextMatch = next.content.match(HEADING_RE);
|
|
1729
|
+
const gap = cur.topY - next.topY;
|
|
1730
|
+
if (curMatch && nextMatch && curMatch[1] === nextMatch[1] && gap <= maxGap) cur = {
|
|
1731
|
+
topY: cur.topY,
|
|
1732
|
+
content: `${cur.content} ${next.content.slice(nextMatch[1].length)}`,
|
|
1733
|
+
isTabular: cur.isTabular || next.isTabular
|
|
1734
|
+
};
|
|
1735
|
+
else {
|
|
1736
|
+
merged.push(cur);
|
|
1737
|
+
cur = { ...next };
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
merged.push(cur);
|
|
1741
|
+
return merged;
|
|
1742
|
+
}
|
|
1743
|
+
/**
|
|
1744
|
+
* Merge consecutive plain-text blocks that are wrapped lines of the same paragraph.
|
|
1745
|
+
*/
|
|
1746
|
+
function mergeParagraphWraps(blocks, bodyFS) {
|
|
1747
|
+
if (blocks.length === 0 || bodyFS <= 0) return blocks;
|
|
1748
|
+
const HEADING_RE = /^#{1,6} /;
|
|
1749
|
+
const SENTENCE_END_RE = /[.!?…)\]]\s*$/;
|
|
1750
|
+
const maxGap = bodyFS * 2;
|
|
1751
|
+
const MIN_WRAP_LENGTH = 25;
|
|
1752
|
+
const merged = [];
|
|
1753
|
+
let cur = {
|
|
1754
|
+
...blocks[0],
|
|
1755
|
+
lastTopY: blocks[0].topY
|
|
1756
|
+
};
|
|
1757
|
+
for (let i = 1; i < blocks.length; i++) {
|
|
1758
|
+
const next = blocks[i];
|
|
1759
|
+
const curIsBody = !HEADING_RE.test(cur.content) && !cur.content.startsWith("|");
|
|
1760
|
+
const nextIsBody = !HEADING_RE.test(next.content) && !next.content.startsWith("|");
|
|
1761
|
+
const gap = cur.lastTopY - next.topY;
|
|
1762
|
+
if (curIsBody && nextIsBody && !cur.isTabular && !next.isTabular && gap > 0 && gap <= maxGap && cur.content.length > MIN_WRAP_LENGTH && !SENTENCE_END_RE.test(cur.content)) cur = {
|
|
1763
|
+
topY: cur.topY,
|
|
1764
|
+
lastTopY: next.topY,
|
|
1765
|
+
content: `${cur.content.trimEnd()} ${next.content.trimStart()}`,
|
|
1766
|
+
isTabular: false
|
|
1767
|
+
};
|
|
1768
|
+
else {
|
|
1769
|
+
merged.push({
|
|
1770
|
+
topY: cur.topY,
|
|
1771
|
+
content: cur.content
|
|
1772
|
+
});
|
|
1773
|
+
cur = {
|
|
1774
|
+
...next,
|
|
1775
|
+
lastTopY: next.topY
|
|
1776
|
+
};
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
merged.push({
|
|
1780
|
+
topY: cur.topY,
|
|
1781
|
+
content: cur.content
|
|
1782
|
+
});
|
|
1783
|
+
return merged;
|
|
1784
|
+
}
|
|
1785
|
+
/** Remove page number blocks near the bottom of the page. */
|
|
1786
|
+
function removePageNumbers(blocks) {
|
|
1787
|
+
const PAGE_NUM_RE = /^(?:#{1,6}\s*)?\d+\s*$/;
|
|
1788
|
+
const BOTTOM_Y = 120;
|
|
1789
|
+
return blocks.filter((block, idx) => {
|
|
1790
|
+
const isBottom = idx >= blocks.length - 3;
|
|
1791
|
+
const isLowY = block.topY <= BOTTOM_Y;
|
|
1792
|
+
const isPageNum = PAGE_NUM_RE.test(block.content.trim());
|
|
1793
|
+
return !(isBottom && isLowY && isPageNum);
|
|
1794
|
+
});
|
|
1795
|
+
}
|
|
1796
|
+
/**
|
|
1797
|
+
* Fix tables where the first column was emitted as free text blocks
|
|
1798
|
+
* around a markdown table containing only the right-side columns.
|
|
1799
|
+
*
|
|
1800
|
+
* Detects: a plain-text header line with (N+1) tokens above an N-column
|
|
1801
|
+
* markdown table, plus short label lines whose count matches the table's
|
|
1802
|
+
* logical row count. Reconstructs into a proper (N+1)-column table.
|
|
1803
|
+
*/
|
|
1804
|
+
function normalizeDetachedFirstColumnTables(blocks) {
|
|
1805
|
+
const HEADING_RE = /^#{1,6}\s/;
|
|
1806
|
+
const isTableBlock = (text) => text.trimStart().startsWith("|");
|
|
1807
|
+
const isPlainBlock = (text) => !HEADING_RE.test(text) && !isTableBlock(text);
|
|
1808
|
+
const isShortLabel = (text) => {
|
|
1809
|
+
const t = text.trim();
|
|
1810
|
+
return t.length > 0 && t.length <= 40;
|
|
1811
|
+
};
|
|
1812
|
+
const splitTokens = (text) => text.trim().split(/[ \t]+/).filter(Boolean);
|
|
1813
|
+
const replacements = /* @__PURE__ */ new Map();
|
|
1814
|
+
const remove = /* @__PURE__ */ new Set();
|
|
1815
|
+
for (let tableIdx = 0; tableIdx < blocks.length; tableIdx++) {
|
|
1816
|
+
if (remove.has(tableIdx)) continue;
|
|
1817
|
+
const tableBlock = blocks[tableIdx];
|
|
1818
|
+
if (!isTableBlock(tableBlock.content)) continue;
|
|
1819
|
+
const dataRows = tableBlock.content.split("\n").map((line) => line.trim()).filter((line) => line.startsWith("|")).filter((line) => !/^\|\s*[-: ]+\|/.test(line)).map(parsePipeRow).filter((row) => row.length > 0);
|
|
1820
|
+
if (dataRows.length === 0) continue;
|
|
1821
|
+
const cols = dataRows[0].length;
|
|
1822
|
+
if (cols < 2 || dataRows.some((row) => row.length !== cols)) continue;
|
|
1823
|
+
const logicalRows = [];
|
|
1824
|
+
for (const row of dataRows) {
|
|
1825
|
+
const splitCells = row.map((cell) => cell.split("<br>").map((p) => p.trim()));
|
|
1826
|
+
const rowSpan = Math.max(...splitCells.map((parts) => parts.length));
|
|
1827
|
+
for (let k = 0; k < rowSpan; k++) logicalRows.push(splitCells.map((parts) => parts[k] ?? ""));
|
|
1828
|
+
}
|
|
1829
|
+
if (logicalRows.length < 2) continue;
|
|
1830
|
+
let headerIdx = -1;
|
|
1831
|
+
let headerTokens = [];
|
|
1832
|
+
for (let i = Math.max(0, tableIdx - 4); i <= tableIdx - 1; i++) {
|
|
1833
|
+
const text = normalizeFullWidthAscii(blocks[i].content).trim();
|
|
1834
|
+
if (!isPlainBlock(text)) continue;
|
|
1835
|
+
const tokens = splitTokens(text);
|
|
1836
|
+
if (tokens.length === cols + 1 && tokens.every((tok) => !/[0-9]/.test(tok))) {
|
|
1837
|
+
headerIdx = i;
|
|
1838
|
+
headerTokens = tokens;
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
if (headerIdx < 0) continue;
|
|
1842
|
+
const aboveLabels = [];
|
|
1843
|
+
for (let i = tableIdx - 1; i > headerIdx; i--) {
|
|
1844
|
+
const text = normalizeFullWidthAscii(blocks[i].content).trim();
|
|
1845
|
+
if (!isPlainBlock(text) || !isShortLabel(text)) break;
|
|
1846
|
+
aboveLabels.push({
|
|
1847
|
+
idx: i,
|
|
1848
|
+
text
|
|
1849
|
+
});
|
|
1850
|
+
}
|
|
1851
|
+
aboveLabels.reverse();
|
|
1852
|
+
const belowLabels = [];
|
|
1853
|
+
for (let i = tableIdx + 1; i < blocks.length; i++) {
|
|
1854
|
+
const text = normalizeFullWidthAscii(blocks[i].content).trim();
|
|
1855
|
+
if (!isPlainBlock(text) || !isShortLabel(text)) break;
|
|
1856
|
+
belowLabels.push({
|
|
1857
|
+
idx: i,
|
|
1858
|
+
text
|
|
1859
|
+
});
|
|
1860
|
+
}
|
|
1861
|
+
const labels = [...aboveLabels, ...belowLabels];
|
|
1862
|
+
if (labels.length !== logicalRows.length) continue;
|
|
1863
|
+
const normalizedLines = [];
|
|
1864
|
+
normalizedLines.push(`| ${headerTokens.join(" | ")} |`);
|
|
1865
|
+
normalizedLines.push(`| ${Array.from({ length: cols + 1 }, () => "---").join(" | ")} |`);
|
|
1866
|
+
for (let r = 0; r < logicalRows.length; r++) normalizedLines.push(`| ${labels[r].text} | ${logicalRows[r].join(" | ")} |`);
|
|
1867
|
+
replacements.set(tableIdx, normalizedLines.join("\n"));
|
|
1868
|
+
remove.add(headerIdx);
|
|
1869
|
+
for (const label of labels) remove.add(label.idx);
|
|
1870
|
+
}
|
|
1871
|
+
if (replacements.size === 0 && remove.size === 0) return blocks;
|
|
1872
|
+
const out = [];
|
|
1873
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
1874
|
+
if (remove.has(i)) continue;
|
|
1875
|
+
const replaced = replacements.get(i);
|
|
1876
|
+
if (replaced) out.push({
|
|
1877
|
+
topY: blocks[i].topY,
|
|
1878
|
+
content: replaced
|
|
1879
|
+
});
|
|
1880
|
+
else out.push(blocks[i]);
|
|
1881
|
+
}
|
|
1882
|
+
return out;
|
|
1883
|
+
}
|
|
1884
|
+
/**
|
|
1885
|
+
* Render one page's content: free text and tables interleaved top-to-bottom.
|
|
1886
|
+
*/
|
|
1887
|
+
function renderPageContent(freeTextBoxes, tables, imageBlocks = [], allTextBoxes) {
|
|
1888
|
+
const blocks = [];
|
|
1889
|
+
const bodyFS = modalFontSize(allTextBoxes ?? freeTextBoxes);
|
|
1890
|
+
for (const line of groupFreeTextIntoLines(freeTextBoxes)) {
|
|
1891
|
+
const prefix = headingPrefix(line.fontSize, bodyFS, line.isBold);
|
|
1892
|
+
blocks.push({
|
|
1893
|
+
topY: line.topY,
|
|
1894
|
+
content: prefix + line.text,
|
|
1895
|
+
isTabular: prefix === "" && line.isTabular
|
|
1896
|
+
});
|
|
1897
|
+
}
|
|
1898
|
+
for (const table of tables) {
|
|
1899
|
+
const md = renderTableToMarkdown(table);
|
|
1900
|
+
if (md.length > 0) blocks.push({
|
|
1901
|
+
topY: table.topY,
|
|
1902
|
+
content: md
|
|
1903
|
+
});
|
|
1904
|
+
}
|
|
1905
|
+
for (const img of imageBlocks) blocks.push({
|
|
1906
|
+
topY: img.topY,
|
|
1907
|
+
content: img.markdown
|
|
1908
|
+
});
|
|
1909
|
+
blocks.sort((a, b) => b.topY - a.topY);
|
|
1910
|
+
return normalizeDetachedFirstColumnTables(mergeParagraphWraps(mergeConsecutiveHeadings(removePageNumbers(blocks), bodyFS), bodyFS)).map((b) => b.content).join("\n\n").trim();
|
|
1911
|
+
}
|
|
1912
|
+
//#endregion
|
|
1913
|
+
//#region ../../packages/agent-core/src/markit/converters/pdf/index.ts
|
|
1914
|
+
/**
|
|
1915
|
+
* PDF to Markdown converter.
|
|
1916
|
+
*
|
|
1917
|
+
* Uses mupdf (native WASM) for fast PDF parsing and a custom pipeline for
|
|
1918
|
+
* table detection via vector line extraction + raycasting.
|
|
1919
|
+
*
|
|
1920
|
+
* Pipeline:
|
|
1921
|
+
* 1. Extract text boxes + vector segments + image regions per page (mupdf)
|
|
1922
|
+
* 2. Detect column layout (single vs multi-column)
|
|
1923
|
+
* 3. Per column: detect table grids from segments (grid detection + raycasting)
|
|
1924
|
+
* 4. Render diagrams as PNG files (if output directory provided)
|
|
1925
|
+
* 5. Render tables as markdown tables, free text as paragraphs/headings
|
|
1926
|
+
*/
|
|
1927
|
+
const EXTENSIONS$2 = new Set([".pdf"]);
|
|
1928
|
+
const MIMETYPES$2 = ["application/pdf", "application/x-pdf"];
|
|
1929
|
+
/**
|
|
1930
|
+
* Process a set of text boxes (one column or full page): run table detection,
|
|
1931
|
+
* separate free text, and render to markdown.
|
|
1932
|
+
*/
|
|
1933
|
+
function processColumn(pageNumber, textBoxes, segments, imageBlocks) {
|
|
1934
|
+
const { grids, consumedIds } = resolveTableGrids(pageNumber, textBoxes, segments);
|
|
1935
|
+
const consumedSet = new Set(consumedIds);
|
|
1936
|
+
return renderPageContent(textBoxes.filter((tb) => !consumedSet.has(tb.id)), grids, imageBlocks, textBoxes);
|
|
1937
|
+
}
|
|
1938
|
+
var PdfConverter = class {
|
|
1939
|
+
name = "pdf";
|
|
1940
|
+
accepts(streamInfo) {
|
|
1941
|
+
if (streamInfo.extension && EXTENSIONS$2.has(streamInfo.extension)) return true;
|
|
1942
|
+
if (streamInfo.mimetype && MIMETYPES$2.some((m) => streamInfo.mimetype?.startsWith(m))) return true;
|
|
1943
|
+
return false;
|
|
1944
|
+
}
|
|
1945
|
+
async convert(input, streamInfo) {
|
|
1946
|
+
const pdfBytes = new Uint8Array(input);
|
|
1947
|
+
const pages = await extractPages(pdfBytes);
|
|
1948
|
+
stripHeadersFooters(pages);
|
|
1949
|
+
const imageDir = streamInfo.imageDir;
|
|
1950
|
+
const pageMarkdowns = [];
|
|
1951
|
+
for (const page of pages) {
|
|
1952
|
+
const imageBlocks = [];
|
|
1953
|
+
if (imageDir && page.images.length > 0) for (const img of page.images) {
|
|
1954
|
+
const filename = `${img.id}.png`;
|
|
1955
|
+
const filepath = path$1.join(imageDir, filename);
|
|
1956
|
+
try {
|
|
1957
|
+
const png = await renderImageRegion(pdfBytes, img);
|
|
1958
|
+
await fs$1.writeFile(filepath, png);
|
|
1959
|
+
imageBlocks.push({
|
|
1960
|
+
topY: img.topY,
|
|
1961
|
+
markdown: ``
|
|
1962
|
+
});
|
|
1963
|
+
} catch {}
|
|
1964
|
+
}
|
|
1965
|
+
else if (page.images.length > 0) for (const img of page.images) imageBlocks.push({
|
|
1966
|
+
topY: img.topY,
|
|
1967
|
+
markdown: `<!-- image: ${img.id} (page ${img.pageNumber}, ${img.bbox.w}x${img.bbox.h}pt) -->`
|
|
1968
|
+
});
|
|
1969
|
+
const layout = detectColumns(page.textBoxes);
|
|
1970
|
+
if (layout.columnCount > 1 && page.segments.some((s) => Math.abs(s.x1 - s.x2) <= .8)) {
|
|
1971
|
+
const pageXMin = Math.min(...page.textBoxes.map((tb) => tb.bounds.left));
|
|
1972
|
+
const pageWidth = Math.max(...page.textBoxes.map((tb) => tb.bounds.right)) - pageXMin;
|
|
1973
|
+
const minColFraction = .3;
|
|
1974
|
+
if (layout.columns.some((col) => {
|
|
1975
|
+
const colXMin = Math.min(...col.map((tb) => tb.bounds.left));
|
|
1976
|
+
return (Math.max(...col.map((tb) => tb.bounds.right)) - colXMin) / pageWidth < minColFraction;
|
|
1977
|
+
})) {
|
|
1978
|
+
layout.columnCount = 1;
|
|
1979
|
+
layout.columns = [page.textBoxes];
|
|
1980
|
+
layout.boundaries = [];
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
if (layout.columnCount === 1) {
|
|
1984
|
+
const md = processColumn(page.pageNumber, page.textBoxes, page.segments, imageBlocks);
|
|
1985
|
+
if (md.length > 0) pageMarkdowns.push(md);
|
|
1986
|
+
} else {
|
|
1987
|
+
const columnMarkdowns = [];
|
|
1988
|
+
for (const colBoxes of layout.columns) {
|
|
1989
|
+
const colXMin = Math.min(...colBoxes.map((tb) => tb.bounds.left));
|
|
1990
|
+
const colXMax = Math.max(...colBoxes.map((tb) => tb.bounds.right));
|
|
1991
|
+
const margin = 10;
|
|
1992
|
+
const colSegments = page.segments.filter((seg) => {
|
|
1993
|
+
const segXMin = Math.min(seg.x1, seg.x2);
|
|
1994
|
+
return Math.max(seg.x1, seg.x2) >= colXMin - margin && segXMin <= colXMax + margin;
|
|
1995
|
+
});
|
|
1996
|
+
const md = processColumn(page.pageNumber, colBoxes, colSegments, columnMarkdowns.length === 0 ? imageBlocks : []);
|
|
1997
|
+
if (md.length > 0) columnMarkdowns.push(md);
|
|
1998
|
+
}
|
|
1999
|
+
const joined = columnMarkdowns.join("\n\n");
|
|
2000
|
+
if (joined.length > 0) pageMarkdowns.push(joined);
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
return { markdown: pageMarkdowns.join("\n\n") };
|
|
2004
|
+
}
|
|
2005
|
+
};
|
|
2006
|
+
//#endregion
|
|
2007
|
+
//#region ../../packages/agent-core/src/markit/converters/pptx.ts
|
|
2008
|
+
const EXTENSIONS$1 = new Set([".pptx"]);
|
|
2009
|
+
const MIMETYPES$1 = ["application/vnd.openxmlformats-officedocument.presentationml.presentation"];
|
|
2010
|
+
var PptxConverter = class {
|
|
2011
|
+
name = "pptx";
|
|
2012
|
+
accepts(streamInfo) {
|
|
2013
|
+
if (streamInfo.extension && EXTENSIONS$1.has(streamInfo.extension)) return true;
|
|
2014
|
+
if (streamInfo.mimetype && MIMETYPES$1.some((m) => streamInfo.mimetype?.startsWith(m))) return true;
|
|
2015
|
+
return false;
|
|
2016
|
+
}
|
|
2017
|
+
async convert(input, streamInfo) {
|
|
2018
|
+
const entries = unzip(input);
|
|
2019
|
+
const parser = new XMLParser({
|
|
2020
|
+
ignoreAttributes: false,
|
|
2021
|
+
attributeNamePrefix: "@_",
|
|
2022
|
+
textNodeName: "#text",
|
|
2023
|
+
processEntities: { maxTotalExpansions: 1e6 }
|
|
2024
|
+
});
|
|
2025
|
+
const presXml = unzipText(entries, "ppt/presentation.xml");
|
|
2026
|
+
if (!presXml) throw new Error("Invalid PPTX: missing presentation.xml");
|
|
2027
|
+
const sldIdList = parser.parse(presXml)["p:presentation"]?.["p:sldIdLst"]?.["p:sldId"];
|
|
2028
|
+
const sldIds = Array.isArray(sldIdList) ? sldIdList : sldIdList ? [sldIdList] : [];
|
|
2029
|
+
const relsXml = unzipText(entries, "ppt/_rels/presentation.xml.rels");
|
|
2030
|
+
const relList = (relsXml ? parser.parse(relsXml) : null)?.Relationships?.Relationship;
|
|
2031
|
+
const relArray = Array.isArray(relList) ? relList : relList ? [relList] : [];
|
|
2032
|
+
const relMap = /* @__PURE__ */ new Map();
|
|
2033
|
+
for (const r of relArray) relMap.set(r["@_Id"], r["@_Target"]);
|
|
2034
|
+
const slidePaths = [];
|
|
2035
|
+
for (const sld of sldIds) {
|
|
2036
|
+
const rId = sld["@_r:id"];
|
|
2037
|
+
const target = relMap.get(rId);
|
|
2038
|
+
if (target) slidePaths.push(`ppt/${target}`);
|
|
2039
|
+
}
|
|
2040
|
+
if (slidePaths.length === 0) {
|
|
2041
|
+
const slideFiles = Object.keys(entries).filter((f) => /^ppt\/slides\/slide\d+\.xml$/.test(f)).toSorted((a, b) => {
|
|
2042
|
+
return parseInt(a.match(/slide(\d+)/)?.[1] || "0", 10) - parseInt(b.match(/slide(\d+)/)?.[1] || "0", 10);
|
|
2043
|
+
});
|
|
2044
|
+
slidePaths.push(...slideFiles);
|
|
2045
|
+
}
|
|
2046
|
+
const imageDir = streamInfo.imageDir;
|
|
2047
|
+
const sections = [];
|
|
2048
|
+
let imageCount = 0;
|
|
2049
|
+
for (let i = 0; i < slidePaths.length; i++) {
|
|
2050
|
+
const slideXml = unzipText(entries, slidePaths[i]);
|
|
2051
|
+
if (!slideXml) continue;
|
|
2052
|
+
const spTree = parser.parse(slideXml)["p:sld"]?.["p:cSld"]?.["p:spTree"];
|
|
2053
|
+
if (!spTree) continue;
|
|
2054
|
+
const slideRelsXml = unzipText(entries, `${slidePaths[i].replace("slides/slide", "slides/_rels/slide")}.rels`);
|
|
2055
|
+
const slideRelMap = /* @__PURE__ */ new Map();
|
|
2056
|
+
if (slideRelsXml) {
|
|
2057
|
+
const relItems = toList(parser.parse(slideRelsXml)?.Relationships?.Relationship);
|
|
2058
|
+
for (const r of relItems) slideRelMap.set(r["@_Id"], r["@_Target"]);
|
|
2059
|
+
}
|
|
2060
|
+
const slideLines = [`<!-- Slide ${i + 1} -->`];
|
|
2061
|
+
const shapes = spTree["p:sp"];
|
|
2062
|
+
const shapeList = Array.isArray(shapes) ? shapes : shapes ? [shapes] : [];
|
|
2063
|
+
let isTitle = true;
|
|
2064
|
+
for (const shape of shapeList) {
|
|
2065
|
+
const text = this.extractText(shape);
|
|
2066
|
+
if (!text) continue;
|
|
2067
|
+
if (isTitle) {
|
|
2068
|
+
slideLines.push(`# ${text}`);
|
|
2069
|
+
isTitle = false;
|
|
2070
|
+
} else slideLines.push(text);
|
|
2071
|
+
}
|
|
2072
|
+
const pics = toList(spTree["p:pic"]);
|
|
2073
|
+
for (const pic of pics) {
|
|
2074
|
+
const rEmbed = pic["p:blipFill"]?.["a:blip"]?.["@_r:embed"];
|
|
2075
|
+
if (!rEmbed) continue;
|
|
2076
|
+
const target = slideRelMap.get(rEmbed);
|
|
2077
|
+
if (!target) continue;
|
|
2078
|
+
const normalizedPath = (target.startsWith("/") ? target.slice(1) : `ppt/slides/${target}`).split("/").reduce((parts, seg) => {
|
|
2079
|
+
if (seg === "..") parts.pop();
|
|
2080
|
+
else parts.push(seg);
|
|
2081
|
+
return parts;
|
|
2082
|
+
}, []).join("/");
|
|
2083
|
+
const buf = entries[normalizedPath];
|
|
2084
|
+
if (!buf) continue;
|
|
2085
|
+
imageCount++;
|
|
2086
|
+
const name = pic["p:nvSpPr"]?.["p:cNvPr"]?.["@_name"] || pic["p:nvPicPr"]?.["p:cNvPr"]?.["@_name"] || `image_${imageCount}`;
|
|
2087
|
+
if (imageDir) try {
|
|
2088
|
+
const ext = normalizedPath.split(".").pop() || "png";
|
|
2089
|
+
const filename = `slide${i + 1}_${imageCount}.${ext}`;
|
|
2090
|
+
const filepath = path$1.join(imageDir, filename);
|
|
2091
|
+
await fs.writeFile(filepath, buf);
|
|
2092
|
+
slideLines.push(``);
|
|
2093
|
+
} catch {
|
|
2094
|
+
slideLines.push(`<!-- image: ${name} (slide ${i + 1}) -->`);
|
|
2095
|
+
}
|
|
2096
|
+
else slideLines.push(`<!-- image: ${name} (slide ${i + 1}) -->`);
|
|
2097
|
+
}
|
|
2098
|
+
const graphicFrames = spTree["p:graphicFrame"];
|
|
2099
|
+
const gfList = Array.isArray(graphicFrames) ? graphicFrames : graphicFrames ? [graphicFrames] : [];
|
|
2100
|
+
for (const gf of gfList) {
|
|
2101
|
+
const table = this.extractTable(gf);
|
|
2102
|
+
if (table) slideLines.push(table);
|
|
2103
|
+
}
|
|
2104
|
+
const noteXml = unzipText(entries, slidePaths[i].replace("slides/slide", "notesSlides/notesSlide"));
|
|
2105
|
+
if (noteXml) {
|
|
2106
|
+
const noteSpTree = parser.parse(noteXml)["p:notes"]?.["p:cSld"]?.["p:spTree"];
|
|
2107
|
+
if (noteSpTree) {
|
|
2108
|
+
const noteShapes = noteSpTree["p:sp"];
|
|
2109
|
+
const noteList = Array.isArray(noteShapes) ? noteShapes : noteShapes ? [noteShapes] : [];
|
|
2110
|
+
const noteTexts = [];
|
|
2111
|
+
for (const ns of noteList) {
|
|
2112
|
+
if (ns["p:nvSpPr"]?.["p:nvPr"]?.["p:ph"]?.["@_type"] === "sldImg") continue;
|
|
2113
|
+
const t = this.extractText(ns);
|
|
2114
|
+
if (t) noteTexts.push(t);
|
|
2115
|
+
}
|
|
2116
|
+
if (noteTexts.length > 0) {
|
|
2117
|
+
slideLines.push("\n### Notes:");
|
|
2118
|
+
slideLines.push(noteTexts.join("\n"));
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
sections.push(slideLines.join("\n"));
|
|
2123
|
+
}
|
|
2124
|
+
return { markdown: sections.join("\n\n").trim() };
|
|
2125
|
+
}
|
|
2126
|
+
extractText(shape) {
|
|
2127
|
+
const txBody = shape["p:txBody"];
|
|
2128
|
+
if (!txBody) return "";
|
|
2129
|
+
const paragraphs = txBody["a:p"];
|
|
2130
|
+
const pList = Array.isArray(paragraphs) ? paragraphs : paragraphs ? [paragraphs] : [];
|
|
2131
|
+
const lines = [];
|
|
2132
|
+
for (const p of pList) {
|
|
2133
|
+
const runs = p["a:r"];
|
|
2134
|
+
const rList = Array.isArray(runs) ? runs : runs ? [runs] : [];
|
|
2135
|
+
const parts = [];
|
|
2136
|
+
for (const r of rList) {
|
|
2137
|
+
const t = r["a:t"];
|
|
2138
|
+
if (t !== null && t !== void 0) parts.push(typeof t === "object" ? t["#text"] || "" : String(t));
|
|
2139
|
+
}
|
|
2140
|
+
if (parts.length > 0) lines.push(parts.join(""));
|
|
2141
|
+
}
|
|
2142
|
+
return lines.join("\n").trim();
|
|
2143
|
+
}
|
|
2144
|
+
extractTable(gf) {
|
|
2145
|
+
const tbl = gf?.["a:graphic"]?.["a:graphicData"]?.["a:tbl"];
|
|
2146
|
+
if (!tbl) return null;
|
|
2147
|
+
const rows = tbl["a:tr"];
|
|
2148
|
+
const rowList = Array.isArray(rows) ? rows : rows ? [rows] : [];
|
|
2149
|
+
if (rowList.length === 0) return null;
|
|
2150
|
+
const mdRows = [];
|
|
2151
|
+
for (const row of rowList) {
|
|
2152
|
+
const cells = row["a:tc"];
|
|
2153
|
+
const cellList = Array.isArray(cells) ? cells : cells ? [cells] : [];
|
|
2154
|
+
const cellTexts = [];
|
|
2155
|
+
for (const cell of cellList) {
|
|
2156
|
+
const txBody = cell["a:txBody"];
|
|
2157
|
+
if (!txBody) {
|
|
2158
|
+
cellTexts.push("");
|
|
2159
|
+
continue;
|
|
2160
|
+
}
|
|
2161
|
+
const paragraphs = txBody["a:p"];
|
|
2162
|
+
const pList = Array.isArray(paragraphs) ? paragraphs : paragraphs ? [paragraphs] : [];
|
|
2163
|
+
const parts = [];
|
|
2164
|
+
for (const p of pList) {
|
|
2165
|
+
const runs = p["a:r"];
|
|
2166
|
+
const rList = Array.isArray(runs) ? runs : runs ? [runs] : [];
|
|
2167
|
+
for (const r of rList) {
|
|
2168
|
+
const t = r["a:t"];
|
|
2169
|
+
if (t !== null && t !== void 0) parts.push(typeof t === "object" ? t["#text"] || "" : String(t));
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
cellTexts.push(parts.join(" "));
|
|
2173
|
+
}
|
|
2174
|
+
mdRows.push(cellTexts);
|
|
2175
|
+
}
|
|
2176
|
+
if (mdRows.length === 0) return null;
|
|
2177
|
+
const [header, ...body] = mdRows;
|
|
2178
|
+
const lines = [];
|
|
2179
|
+
lines.push(`| ${header.join(" | ")} |`);
|
|
2180
|
+
lines.push(`| ${header.map(() => "---").join(" | ")} |`);
|
|
2181
|
+
for (const row of body) {
|
|
2182
|
+
while (row.length < header.length) row.push("");
|
|
2183
|
+
lines.push(`| ${row.join(" | ")} |`);
|
|
2184
|
+
}
|
|
2185
|
+
return lines.join("\n");
|
|
2186
|
+
}
|
|
2187
|
+
};
|
|
2188
|
+
function toList(val) {
|
|
2189
|
+
if (!val) return [];
|
|
2190
|
+
return Array.isArray(val) ? val : [val];
|
|
2191
|
+
}
|
|
2192
|
+
//#endregion
|
|
2193
|
+
//#region ../../packages/agent-core/src/markit/converters/xlsx.ts
|
|
2194
|
+
const EXTENSIONS = new Set([".xlsx"]);
|
|
2195
|
+
const MIMETYPES = ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"];
|
|
2196
|
+
var XlsxConverter = class {
|
|
2197
|
+
name = "xlsx";
|
|
2198
|
+
accepts(streamInfo) {
|
|
2199
|
+
if (streamInfo.extension && EXTENSIONS.has(streamInfo.extension)) return true;
|
|
2200
|
+
if (streamInfo.mimetype && MIMETYPES.some((m) => streamInfo.mimetype?.startsWith(m))) return true;
|
|
2201
|
+
return false;
|
|
2202
|
+
}
|
|
2203
|
+
async convert(input, _streamInfo) {
|
|
2204
|
+
const entries = unzip(input);
|
|
2205
|
+
const parser = new XMLParser({
|
|
2206
|
+
ignoreAttributes: false,
|
|
2207
|
+
attributeNamePrefix: "@_",
|
|
2208
|
+
textNodeName: "#text",
|
|
2209
|
+
processEntities: { maxTotalExpansions: 1e6 }
|
|
2210
|
+
});
|
|
2211
|
+
const ssXml = unzipText(entries, "xl/sharedStrings.xml");
|
|
2212
|
+
const siList = (ssXml ? parser.parse(ssXml) : null)?.sst?.si;
|
|
2213
|
+
const shared = toArray(siList);
|
|
2214
|
+
const wbXml = unzipText(entries, "xl/workbook.xml");
|
|
2215
|
+
if (!wbXml) throw new Error("Invalid XLSX: missing workbook.xml");
|
|
2216
|
+
const sheets = toArray(parser.parse(wbXml).workbook?.sheets?.sheet);
|
|
2217
|
+
const relsXml = unzipText(entries, "xl/_rels/workbook.xml.rels");
|
|
2218
|
+
const relList = toArray((relsXml ? parser.parse(relsXml) : null)?.Relationships?.Relationship);
|
|
2219
|
+
const relMap = /* @__PURE__ */ new Map();
|
|
2220
|
+
for (const r of relList) relMap.set(r["@_Id"], r["@_Target"]);
|
|
2221
|
+
const sections = [];
|
|
2222
|
+
for (const sheet of sheets) {
|
|
2223
|
+
const sheetName = sheet["@_name"];
|
|
2224
|
+
const rId = sheet["@_r:id"];
|
|
2225
|
+
const target = relMap.get(rId);
|
|
2226
|
+
if (!target) continue;
|
|
2227
|
+
const sheetXml = unzipText(entries, target.startsWith("/") ? target.slice(1) : `xl/${target}`);
|
|
2228
|
+
if (!sheetXml) continue;
|
|
2229
|
+
const rows = toArray(parser.parse(sheetXml).worksheet?.sheetData?.row);
|
|
2230
|
+
if (rows.length === 0) continue;
|
|
2231
|
+
const tableRows = [];
|
|
2232
|
+
for (const row of rows) {
|
|
2233
|
+
const cells = toArray(row.c);
|
|
2234
|
+
const values = [];
|
|
2235
|
+
for (const cell of cells) values.push(this.getCellValue(cell, shared));
|
|
2236
|
+
tableRows.push(values);
|
|
2237
|
+
}
|
|
2238
|
+
if (tableRows.length === 0) continue;
|
|
2239
|
+
const maxCols = Math.max(...tableRows.map((r) => r.length));
|
|
2240
|
+
for (const row of tableRows) while (row.length < maxCols) row.push("");
|
|
2241
|
+
sections.push(`## ${sheetName}`);
|
|
2242
|
+
const [header, ...body] = tableRows;
|
|
2243
|
+
const lines = [];
|
|
2244
|
+
lines.push(`| ${header.join(" | ")} |`);
|
|
2245
|
+
lines.push(`| ${header.map(() => "---").join(" | ")} |`);
|
|
2246
|
+
for (const row of body) lines.push(`| ${row.join(" | ")} |`);
|
|
2247
|
+
sections.push(lines.join("\n"));
|
|
2248
|
+
}
|
|
2249
|
+
return { markdown: sections.join("\n\n") };
|
|
2250
|
+
}
|
|
2251
|
+
getCellValue(cell, shared) {
|
|
2252
|
+
if (cell["@_t"] === "s") return this.getSharedString(shared, Number(cell.v));
|
|
2253
|
+
if (cell["@_t"] === "inlineStr") {
|
|
2254
|
+
const is = cell.is;
|
|
2255
|
+
if (!is) return "";
|
|
2256
|
+
if (is.t !== null && is.t !== void 0) return textValue(is.t);
|
|
2257
|
+
if (is.r) return toArray(is.r).map((r) => textValue(r.t)).join("");
|
|
2258
|
+
return "";
|
|
2259
|
+
}
|
|
2260
|
+
if (cell["@_t"] === "b") return cell.v === 1 || cell.v === "1" ? "TRUE" : "FALSE";
|
|
2261
|
+
if (cell.v !== null && cell.v !== void 0) return String(cell.v);
|
|
2262
|
+
return "";
|
|
2263
|
+
}
|
|
2264
|
+
getSharedString(shared, idx) {
|
|
2265
|
+
const si = shared[idx];
|
|
2266
|
+
if (!si) return "";
|
|
2267
|
+
if (si.t !== null && si.t !== void 0) return textValue(si.t);
|
|
2268
|
+
if (si.r) return toArray(si.r).map((r) => textValue(r.t)).join("");
|
|
2269
|
+
return "";
|
|
2270
|
+
}
|
|
2271
|
+
};
|
|
2272
|
+
function textValue(t) {
|
|
2273
|
+
if (t === null || t === void 0) return "";
|
|
2274
|
+
if (typeof t === "object") return t["#text"] || "";
|
|
2275
|
+
return String(t);
|
|
2276
|
+
}
|
|
2277
|
+
function toArray(val) {
|
|
2278
|
+
if (!val) return [];
|
|
2279
|
+
return Array.isArray(val) ? val : [val];
|
|
2280
|
+
}
|
|
2281
|
+
//#endregion
|
|
2282
|
+
//#region ../../packages/agent-core/src/markit/registry.ts
|
|
2283
|
+
/**
|
|
2284
|
+
* In-house document → markdown engine.
|
|
2285
|
+
*
|
|
2286
|
+
* Only the document converters this codebase needs are registered (pdf, docx,
|
|
2287
|
+
* pptx, xlsx, epub). The first converter whose `accepts()` returns true and
|
|
2288
|
+
* whose `convert()` succeeds wins.
|
|
2289
|
+
*/
|
|
2290
|
+
var Markit = class {
|
|
2291
|
+
#converters;
|
|
2292
|
+
#options;
|
|
2293
|
+
constructor(options = {}) {
|
|
2294
|
+
this.#options = options;
|
|
2295
|
+
this.#converters = [
|
|
2296
|
+
new PdfConverter(),
|
|
2297
|
+
new DocxConverter(),
|
|
2298
|
+
new PptxConverter(),
|
|
2299
|
+
new XlsxConverter(),
|
|
2300
|
+
new EpubConverter()
|
|
2301
|
+
];
|
|
2302
|
+
}
|
|
2303
|
+
async convertFile(filePath, extra) {
|
|
2304
|
+
const buffer = await fs$1.readFile(filePath);
|
|
2305
|
+
const streamInfo = {
|
|
2306
|
+
localPath: filePath,
|
|
2307
|
+
extension: path$1.extname(filePath).toLowerCase(),
|
|
2308
|
+
filename: path$1.basename(filePath),
|
|
2309
|
+
...extra
|
|
2310
|
+
};
|
|
2311
|
+
return this.convert(buffer, streamInfo);
|
|
2312
|
+
}
|
|
2313
|
+
async convert(input, streamInfo) {
|
|
2314
|
+
const errors = [];
|
|
2315
|
+
for (const converter of this.#converters) {
|
|
2316
|
+
if (!converter.accepts(streamInfo)) continue;
|
|
2317
|
+
try {
|
|
2318
|
+
return await converter.convert(input, streamInfo, this.#options);
|
|
2319
|
+
} catch (error) {
|
|
2320
|
+
errors.push({
|
|
2321
|
+
converter: converter.name,
|
|
2322
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
2323
|
+
});
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
if (errors.length > 0) {
|
|
2327
|
+
const details = errors.map((e) => ` ${e.converter}: ${e.error.message}`).join("\n");
|
|
2328
|
+
throw new Error(`Conversion failed:\n${details}`);
|
|
2329
|
+
}
|
|
2330
|
+
throw new Error(`Unsupported format: ${streamInfo.extension || streamInfo.mimetype || "unknown"}`);
|
|
2331
|
+
}
|
|
2332
|
+
};
|
|
2333
|
+
//#endregion
|
|
2334
|
+
export { Markit };
|