@llamaindex/liteparse 2.13.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/dist/cli.js +1236 -398
- package/dist/cli.js.map +1 -1
- package/dist/lib.cjs +712 -0
- package/dist/lib.cjs.map +1 -0
- package/dist/lib.d.cts +911 -0
- package/dist/lib.d.ts +339 -38
- package/dist/lib.js +654 -323
- package/dist/lib.js.map +1 -1
- package/dist/pool-worker.js +253 -0
- package/dist/pool-worker.js.map +1 -0
- package/liteparse.linux-x64-gnu.node +0 -0
- package/package.json +22 -12
- package/dist/cli-json.d.ts +0 -154
- package/dist/cli-json.d.ts.map +0 -1
- package/dist/cli-json.js +0 -241
- package/dist/cli-json.js.map +0 -1
- package/dist/cli.d.ts +0 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/lib.d.ts.map +0 -1
- package/dist/native.d.ts +0 -345
- package/dist/native.d.ts.map +0 -1
- package/dist/native.js +0 -71
- package/dist/native.js.map +0 -1
package/dist/lib.cjs
ADDED
|
@@ -0,0 +1,712 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/lib.ts
|
|
21
|
+
var lib_exports = {};
|
|
22
|
+
__export(lib_exports, {
|
|
23
|
+
LiteParse: () => LiteParse,
|
|
24
|
+
ParseTimeoutError: () => ParseTimeoutError,
|
|
25
|
+
default: () => lib_default,
|
|
26
|
+
searchItems: () => searchItems,
|
|
27
|
+
toParseResult: () => toParseResult
|
|
28
|
+
});
|
|
29
|
+
module.exports = __toCommonJS(lib_exports);
|
|
30
|
+
|
|
31
|
+
// node_modules/tsup/assets/cjs_shims.js
|
|
32
|
+
var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.tagName.toUpperCase() === "SCRIPT" ? document.currentScript.src : new URL("main.js", document.baseURI).href;
|
|
33
|
+
var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
34
|
+
|
|
35
|
+
// src/native.ts
|
|
36
|
+
var import_node_module = require("module");
|
|
37
|
+
var import_node_path = require("path");
|
|
38
|
+
var import_node_url = require("url");
|
|
39
|
+
var require2 = (0, import_node_module.createRequire)(importMetaUrl);
|
|
40
|
+
var __dirname = (0, import_node_path.dirname)((0, import_node_url.fileURLToPath)(importMetaUrl));
|
|
41
|
+
function loadNative() {
|
|
42
|
+
const triples = {
|
|
43
|
+
"darwin-x64": "@llamaindex/liteparse-darwin-x64",
|
|
44
|
+
"darwin-arm64": "@llamaindex/liteparse-darwin-arm64",
|
|
45
|
+
"linux-x64-gnu": "@llamaindex/liteparse-linux-x64-gnu",
|
|
46
|
+
"linux-x64-musl": "@llamaindex/liteparse-linux-x64-musl",
|
|
47
|
+
"linux-arm64-gnu": "@llamaindex/liteparse-linux-arm64-gnu",
|
|
48
|
+
"linux-arm64-musl": "@llamaindex/liteparse-linux-arm64-musl",
|
|
49
|
+
"win32-x64-msvc": "@llamaindex/liteparse-win32-x64-msvc",
|
|
50
|
+
"win32-arm64-msvc": "@llamaindex/liteparse-win32-arm64-msvc"
|
|
51
|
+
};
|
|
52
|
+
const platform = process.platform;
|
|
53
|
+
const arch = process.arch;
|
|
54
|
+
const candidates = [];
|
|
55
|
+
if (platform === "linux") {
|
|
56
|
+
candidates.push(`${platform}-${arch}-gnu`);
|
|
57
|
+
candidates.push(`${platform}-${arch}-musl`);
|
|
58
|
+
} else if (platform === "win32") {
|
|
59
|
+
candidates.push(`${platform}-${arch}-msvc`);
|
|
60
|
+
} else {
|
|
61
|
+
candidates.push(`${platform}-${arch}`);
|
|
62
|
+
}
|
|
63
|
+
for (const key of candidates) {
|
|
64
|
+
const pkg = triples[key];
|
|
65
|
+
if (pkg) {
|
|
66
|
+
try {
|
|
67
|
+
return require2(pkg);
|
|
68
|
+
} catch {
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const searchDirs = [__dirname, (0, import_node_path.join)(__dirname, ".."), (0, import_node_path.join)(__dirname, "..", "..")];
|
|
73
|
+
const fileNames = [
|
|
74
|
+
...candidates.map((c) => `liteparse.${c}.node`),
|
|
75
|
+
`liteparse.${platform}-${arch}.node`,
|
|
76
|
+
"liteparse.node"
|
|
77
|
+
];
|
|
78
|
+
for (const dir of searchDirs) {
|
|
79
|
+
for (const fileName of fileNames) {
|
|
80
|
+
try {
|
|
81
|
+
return require2((0, import_node_path.join)(dir, fileName));
|
|
82
|
+
} catch {
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
throw new Error(
|
|
87
|
+
`Failed to load native module for ${platform}-${arch}. Ensure the correct optional dependency is installed.`
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
var native = loadNative();
|
|
91
|
+
|
|
92
|
+
// src/pool.ts
|
|
93
|
+
var import_node_child_process = require("child_process");
|
|
94
|
+
var import_node_url2 = require("url");
|
|
95
|
+
var ParseTimeoutError = class extends Error {
|
|
96
|
+
source;
|
|
97
|
+
timeoutMs;
|
|
98
|
+
constructor(message, source, timeoutMs) {
|
|
99
|
+
super(message);
|
|
100
|
+
this.name = "ParseTimeoutError";
|
|
101
|
+
this.source = source;
|
|
102
|
+
this.timeoutMs = timeoutMs;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
var WorkerTimeout = class extends Error {
|
|
106
|
+
};
|
|
107
|
+
var WorkerCrashed = class extends Error {
|
|
108
|
+
};
|
|
109
|
+
var WORKER_PATH = (0, import_node_url2.fileURLToPath)(new URL("./pool-worker.js", importMetaUrl));
|
|
110
|
+
function reviveBuffers(result) {
|
|
111
|
+
for (const image of result.images ?? []) {
|
|
112
|
+
if (image.bytes && !Buffer.isBuffer(image.bytes)) {
|
|
113
|
+
const b = image.bytes;
|
|
114
|
+
image.bytes = Buffer.from(b.buffer, b.byteOffset, b.byteLength);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
for (const shot of result.screenshots ?? []) {
|
|
118
|
+
if (shot.imageBuffer && !Buffer.isBuffer(shot.imageBuffer)) {
|
|
119
|
+
const b = shot.imageBuffer;
|
|
120
|
+
shot.imageBuffer = Buffer.from(b.buffer, b.byteOffset, b.byteLength);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return result;
|
|
124
|
+
}
|
|
125
|
+
var WorkerHandle = class {
|
|
126
|
+
child;
|
|
127
|
+
readyPromise;
|
|
128
|
+
pending = null;
|
|
129
|
+
dead = false;
|
|
130
|
+
constructor(config) {
|
|
131
|
+
this.child = (0, import_node_child_process.fork)(WORKER_PATH, [], {
|
|
132
|
+
serialization: "advanced",
|
|
133
|
+
// stdout/stderr inherited: parse logs and crash traces stay visible.
|
|
134
|
+
stdio: ["ignore", "inherit", "inherit", "ipc"]
|
|
135
|
+
});
|
|
136
|
+
let readyResolve;
|
|
137
|
+
let readyReject;
|
|
138
|
+
this.readyPromise = new Promise((resolve, reject) => {
|
|
139
|
+
readyResolve = resolve;
|
|
140
|
+
readyReject = reject;
|
|
141
|
+
});
|
|
142
|
+
this.readyPromise.catch(() => {
|
|
143
|
+
});
|
|
144
|
+
this.child.on("message", (msg) => {
|
|
145
|
+
if (msg.type === "ready") {
|
|
146
|
+
readyResolve();
|
|
147
|
+
if (this.pending === null) this.idle();
|
|
148
|
+
} else if (msg.type === "initError") {
|
|
149
|
+
readyReject(new Error(msg.message));
|
|
150
|
+
} else if (this.pending) {
|
|
151
|
+
const { resolve, reject } = this.pending;
|
|
152
|
+
this.pending = null;
|
|
153
|
+
this.idle();
|
|
154
|
+
if (msg.type === "ok") resolve(reviveBuffers(msg.result));
|
|
155
|
+
else reject(new WorkerCrashed(msg.message));
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
const onGone = (cause) => {
|
|
159
|
+
this.dead = true;
|
|
160
|
+
readyReject(new WorkerCrashed(cause));
|
|
161
|
+
if (this.pending) {
|
|
162
|
+
const { reject } = this.pending;
|
|
163
|
+
this.pending = null;
|
|
164
|
+
reject(new WorkerCrashed(cause));
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
this.child.on("error", (e) => onGone(e.message));
|
|
168
|
+
this.child.on(
|
|
169
|
+
"exit",
|
|
170
|
+
(code, signal) => onGone(`worker exited (code=${code}, signal=${signal})`)
|
|
171
|
+
);
|
|
172
|
+
this.child.send({ type: "init", config });
|
|
173
|
+
}
|
|
174
|
+
/** Resolves when the worker's native parser is constructed. Init time
|
|
175
|
+
* never counts toward the parse deadline — the deadline is a promise about
|
|
176
|
+
* parsing, not about process startup. */
|
|
177
|
+
ready() {
|
|
178
|
+
return this.readyPromise;
|
|
179
|
+
}
|
|
180
|
+
/** An idle pool must not hold the parent's event loop open. */
|
|
181
|
+
idle() {
|
|
182
|
+
this.child.unref();
|
|
183
|
+
this.child.channel?.unref();
|
|
184
|
+
}
|
|
185
|
+
request(payload, timeoutMs) {
|
|
186
|
+
if (this.dead) {
|
|
187
|
+
return Promise.reject(new WorkerCrashed("worker already exited"));
|
|
188
|
+
}
|
|
189
|
+
this.child.ref();
|
|
190
|
+
this.child.channel?.ref();
|
|
191
|
+
return new Promise((resolve, reject) => {
|
|
192
|
+
let timer;
|
|
193
|
+
const settle = (fn) => (value) => {
|
|
194
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
195
|
+
fn(value);
|
|
196
|
+
};
|
|
197
|
+
this.pending = {
|
|
198
|
+
resolve: settle(resolve),
|
|
199
|
+
reject: settle(reject)
|
|
200
|
+
};
|
|
201
|
+
if (timeoutMs !== void 0) {
|
|
202
|
+
timer = setTimeout(() => {
|
|
203
|
+
if (this.pending) {
|
|
204
|
+
const { reject: rejectPending } = this.pending;
|
|
205
|
+
this.pending = null;
|
|
206
|
+
rejectPending(new WorkerTimeout());
|
|
207
|
+
}
|
|
208
|
+
}, timeoutMs);
|
|
209
|
+
}
|
|
210
|
+
this.child.send({ type: "parse", payload });
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
kill() {
|
|
214
|
+
this.dead = true;
|
|
215
|
+
this.child.kill("SIGKILL");
|
|
216
|
+
}
|
|
217
|
+
/** Graceful shutdown; escalates to SIGKILL if the worker doesn't exit. */
|
|
218
|
+
stop() {
|
|
219
|
+
if (this.dead) return;
|
|
220
|
+
this.dead = true;
|
|
221
|
+
try {
|
|
222
|
+
this.child.send({ type: "stop" });
|
|
223
|
+
} catch {
|
|
224
|
+
}
|
|
225
|
+
const escalate = setTimeout(() => this.child.kill("SIGKILL"), 5e3);
|
|
226
|
+
escalate.unref();
|
|
227
|
+
this.child.once("exit", () => clearTimeout(escalate));
|
|
228
|
+
this.idle();
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
var WorkerPool = class {
|
|
232
|
+
config;
|
|
233
|
+
timeoutMs;
|
|
234
|
+
workers = /* @__PURE__ */ new Set();
|
|
235
|
+
idle = [];
|
|
236
|
+
waiters = [];
|
|
237
|
+
closed = false;
|
|
238
|
+
constructor(config, poolSize, parseTimeoutMs) {
|
|
239
|
+
if (!Number.isInteger(poolSize) || poolSize < 1) {
|
|
240
|
+
throw new Error("poolSize must be an integer >= 1");
|
|
241
|
+
}
|
|
242
|
+
if (parseTimeoutMs !== void 0 && !(parseTimeoutMs > 0)) {
|
|
243
|
+
throw new Error("parseTimeoutMs must be > 0");
|
|
244
|
+
}
|
|
245
|
+
this.config = config;
|
|
246
|
+
this.timeoutMs = parseTimeoutMs;
|
|
247
|
+
for (let i = 0; i < poolSize; i++) {
|
|
248
|
+
this.spawnWorker();
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
spawnWorker() {
|
|
252
|
+
const worker = new WorkerHandle(this.config);
|
|
253
|
+
this.workers.add(worker);
|
|
254
|
+
this.release(worker);
|
|
255
|
+
}
|
|
256
|
+
acquire() {
|
|
257
|
+
const worker = this.idle.pop();
|
|
258
|
+
if (worker !== void 0) return Promise.resolve(worker);
|
|
259
|
+
return new Promise(
|
|
260
|
+
(resolve, reject) => this.waiters.push({ resolve, reject })
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
release(worker) {
|
|
264
|
+
if (this.closed) {
|
|
265
|
+
this.workers.delete(worker);
|
|
266
|
+
worker.stop();
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const waiter = this.waiters.shift();
|
|
270
|
+
if (waiter !== void 0) waiter.resolve(worker);
|
|
271
|
+
else this.idle.push(worker);
|
|
272
|
+
}
|
|
273
|
+
retire(worker) {
|
|
274
|
+
worker.kill();
|
|
275
|
+
this.workers.delete(worker);
|
|
276
|
+
if (!this.closed) this.spawnWorker();
|
|
277
|
+
}
|
|
278
|
+
/** Run one parse on an idle worker.
|
|
279
|
+
*
|
|
280
|
+
* Waits for a free worker first; `parseTimeoutMs` bounds the parse itself,
|
|
281
|
+
* not the wait. */
|
|
282
|
+
async parse(payload, source) {
|
|
283
|
+
if (this.closed) throw new Error("parser pool is closed");
|
|
284
|
+
const worker = await this.acquire();
|
|
285
|
+
try {
|
|
286
|
+
await worker.ready();
|
|
287
|
+
const result = await worker.request(payload, this.timeoutMs);
|
|
288
|
+
this.release(worker);
|
|
289
|
+
return result;
|
|
290
|
+
} catch (e) {
|
|
291
|
+
this.retire(worker);
|
|
292
|
+
if (e instanceof WorkerTimeout) {
|
|
293
|
+
throw new ParseTimeoutError(
|
|
294
|
+
`parse of ${source} exceeded ${this.timeoutMs}ms; the worker process was killed`,
|
|
295
|
+
source,
|
|
296
|
+
this.timeoutMs
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
if (e instanceof WorkerCrashed) {
|
|
300
|
+
throw new Error(
|
|
301
|
+
`liteparse worker process died while parsing ${source}: ${e.message}`
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
throw e;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
/** Resolves when every worker is initialized. Optional — the first parse
|
|
308
|
+
* per worker waits for init anyway. */
|
|
309
|
+
async warmUp() {
|
|
310
|
+
await Promise.all([...this.workers].map((w) => w.ready()));
|
|
311
|
+
}
|
|
312
|
+
/** Shut down all workers. Idempotent. Busy workers are stopped as their
|
|
313
|
+
* in-flight parses finish. */
|
|
314
|
+
close() {
|
|
315
|
+
if (this.closed) return;
|
|
316
|
+
this.closed = true;
|
|
317
|
+
for (const waiter of this.waiters.splice(0)) {
|
|
318
|
+
waiter.reject(new Error("parser pool is closed"));
|
|
319
|
+
}
|
|
320
|
+
for (const worker of this.idle.splice(0)) {
|
|
321
|
+
this.workers.delete(worker);
|
|
322
|
+
worker.stop();
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
// src/lib.ts
|
|
328
|
+
var LiteParse = class {
|
|
329
|
+
_native;
|
|
330
|
+
_config;
|
|
331
|
+
_pool = null;
|
|
332
|
+
constructor(userConfig = {}) {
|
|
333
|
+
const nativeConfig = {
|
|
334
|
+
ocrLanguage: userConfig.ocrLanguage,
|
|
335
|
+
ocrEnabled: userConfig.ocrEnabled,
|
|
336
|
+
ocrServerUrl: userConfig.ocrServerUrl,
|
|
337
|
+
ocrServerHeaders: userConfig.ocrServerHeaders,
|
|
338
|
+
tessdataPath: userConfig.tessdataPath,
|
|
339
|
+
maxPages: userConfig.maxPages,
|
|
340
|
+
targetPages: userConfig.targetPages,
|
|
341
|
+
extractScreenshots: userConfig.extractScreenshots,
|
|
342
|
+
continueOnPageError: userConfig.continueOnPageError,
|
|
343
|
+
dpi: userConfig.dpi,
|
|
344
|
+
outputFormat: userConfig.outputFormat,
|
|
345
|
+
imageMode: userConfig.imageMode,
|
|
346
|
+
extractImages: userConfig.extractImages,
|
|
347
|
+
imageOutputDir: userConfig.imageOutputDir,
|
|
348
|
+
extractLinks: userConfig.extractLinks,
|
|
349
|
+
keepHeadersFooters: userConfig.keepHeadersFooters,
|
|
350
|
+
extractAnnotations: userConfig.extractAnnotations,
|
|
351
|
+
extractFormFields: userConfig.extractFormFields,
|
|
352
|
+
extractStructureTree: userConfig.extractStructureTree,
|
|
353
|
+
extractBlocks: userConfig.extractBlocks,
|
|
354
|
+
extractXfaPackets: userConfig.extractXfaPackets,
|
|
355
|
+
extractDocumentMetadata: userConfig.extractDocumentMetadata,
|
|
356
|
+
extractContentBounds: userConfig.extractContentBounds,
|
|
357
|
+
detectScreenshotRects: userConfig.detectScreenshotRects,
|
|
358
|
+
renderFormFields: userConfig.renderFormFields,
|
|
359
|
+
preserveVerySmallText: userConfig.preserveVerySmallText,
|
|
360
|
+
password: userConfig.password,
|
|
361
|
+
quiet: userConfig.quiet,
|
|
362
|
+
numWorkers: userConfig.numWorkers,
|
|
363
|
+
ocrFailureFatal: userConfig.ocrFailureFatal,
|
|
364
|
+
ocrHedgeDelaysMs: userConfig.ocrHedgeDelaysMs,
|
|
365
|
+
emitWordBoxes: userConfig.emitWordBoxes,
|
|
366
|
+
extractTextMetadata: userConfig.extractTextMetadata,
|
|
367
|
+
cropBox: userConfig.cropBox,
|
|
368
|
+
skipDiagonalText: userConfig.skipDiagonalText,
|
|
369
|
+
includeComplexity: userConfig.includeComplexity,
|
|
370
|
+
extractVectorGraphics: userConfig.extractVectorGraphics
|
|
371
|
+
};
|
|
372
|
+
this._native = new native.LiteParse(nativeConfig);
|
|
373
|
+
if (userConfig.parseTimeoutMs !== void 0 && userConfig.poolSize === void 0) {
|
|
374
|
+
throw new Error(
|
|
375
|
+
"parseTimeoutMs requires poolSize"
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
if (userConfig.poolSize !== void 0) {
|
|
379
|
+
this._pool = new WorkerPool(
|
|
380
|
+
nativeConfig,
|
|
381
|
+
userConfig.poolSize,
|
|
382
|
+
userConfig.parseTimeoutMs
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
const resolved = this._native.config;
|
|
386
|
+
this._config = {
|
|
387
|
+
ocrLanguage: resolved.ocrLanguage ?? "eng",
|
|
388
|
+
ocrEnabled: resolved.ocrEnabled ?? true,
|
|
389
|
+
ocrServerUrl: resolved.ocrServerUrl ?? void 0,
|
|
390
|
+
ocrServerHeaders: resolved.ocrServerHeaders ?? void 0,
|
|
391
|
+
tessdataPath: resolved.tessdataPath ?? void 0,
|
|
392
|
+
maxPages: resolved.maxPages ?? 1e3,
|
|
393
|
+
targetPages: resolved.targetPages ?? void 0,
|
|
394
|
+
extractScreenshots: resolved.extractScreenshots ?? false,
|
|
395
|
+
continueOnPageError: resolved.continueOnPageError ?? false,
|
|
396
|
+
dpi: resolved.dpi ?? 150,
|
|
397
|
+
outputFormat: resolved.outputFormat ?? "json",
|
|
398
|
+
imageMode: resolved.imageMode ?? "placeholder",
|
|
399
|
+
extractImages: resolved.extractImages ?? false,
|
|
400
|
+
imageOutputDir: resolved.imageOutputDir ?? void 0,
|
|
401
|
+
extractLinks: resolved.extractLinks ?? true,
|
|
402
|
+
keepHeadersFooters: resolved.keepHeadersFooters ?? false,
|
|
403
|
+
extractAnnotations: resolved.extractAnnotations ?? false,
|
|
404
|
+
extractFormFields: resolved.extractFormFields ?? false,
|
|
405
|
+
extractStructureTree: resolved.extractStructureTree ?? false,
|
|
406
|
+
extractBlocks: resolved.extractBlocks ?? false,
|
|
407
|
+
extractXfaPackets: resolved.extractXfaPackets ?? false,
|
|
408
|
+
extractDocumentMetadata: resolved.extractDocumentMetadata ?? false,
|
|
409
|
+
extractContentBounds: resolved.extractContentBounds ?? false,
|
|
410
|
+
detectScreenshotRects: resolved.detectScreenshotRects ?? false,
|
|
411
|
+
renderFormFields: resolved.renderFormFields ?? false,
|
|
412
|
+
preserveVerySmallText: resolved.preserveVerySmallText ?? false,
|
|
413
|
+
password: resolved.password ?? void 0,
|
|
414
|
+
quiet: resolved.quiet ?? false,
|
|
415
|
+
numWorkers: resolved.numWorkers ?? 1,
|
|
416
|
+
ocrFailureFatal: resolved.ocrFailureFatal ?? true,
|
|
417
|
+
ocrHedgeDelaysMs: resolved.ocrHedgeDelaysMs ?? [],
|
|
418
|
+
emitWordBoxes: resolved.emitWordBoxes ?? false,
|
|
419
|
+
extractTextMetadata: resolved.extractTextMetadata ?? false,
|
|
420
|
+
cropBox: resolved.cropBox ?? void 0,
|
|
421
|
+
skipDiagonalText: resolved.skipDiagonalText ?? false,
|
|
422
|
+
includeComplexity: resolved.includeComplexity ?? false,
|
|
423
|
+
extractVectorGraphics: resolved.extractVectorGraphics ?? false
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
async parse(input) {
|
|
427
|
+
const nativeInput = typeof input === "string" ? input : Buffer.from(input);
|
|
428
|
+
if (this._pool !== null) {
|
|
429
|
+
const source = typeof nativeInput === "string" ? nativeInput : `<${nativeInput.byteLength} bytes>`;
|
|
430
|
+
return this._pool.parse(nativeInput, source);
|
|
431
|
+
}
|
|
432
|
+
const result = await this._native.parse(nativeInput);
|
|
433
|
+
return toParseResult(result);
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Resolves once every pool worker is initialized. No-op without `poolSize`.
|
|
437
|
+
*
|
|
438
|
+
* Optional: the first parse on each worker waits for its init anyway. Call
|
|
439
|
+
* this before latency-sensitive traffic to avoid paying worker startup on
|
|
440
|
+
* the first request.
|
|
441
|
+
*/
|
|
442
|
+
async warmUp() {
|
|
443
|
+
if (this._pool !== null) await this._pool.warmUp();
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Shut down pool workers, if pool mode is enabled. Idempotent.
|
|
447
|
+
*
|
|
448
|
+
* Without `poolSize` this is a no-op. An idle pool never keeps the event
|
|
449
|
+
* loop alive and workers exit when the parent does, so forgetting to call
|
|
450
|
+
* this leaks nothing past process exit.
|
|
451
|
+
*/
|
|
452
|
+
close() {
|
|
453
|
+
if (this._pool !== null) this._pool.close();
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Parse a document in bounded-memory page batches of `batchSize` pages.
|
|
457
|
+
*
|
|
458
|
+
* Each yielded result is independent and becomes collectible once the caller
|
|
459
|
+
* advances the iterator, so a consumer that does not retain batches never
|
|
460
|
+
* holds more than one batch of pages in memory. A non-PDF source is
|
|
461
|
+
* converted once when the iterator starts, not once per batch; its temporary
|
|
462
|
+
* file is released when iteration ends — including an early `break` or
|
|
463
|
+
* `throw`, which run the generator's cleanup.
|
|
464
|
+
*
|
|
465
|
+
* Cross-page passes see only the pages in their own batch, so repeated
|
|
466
|
+
* header/footer removal and image deduplication are batch-local and the
|
|
467
|
+
* output can differ from `parse()`. Prefer `parse()` unless the size of the
|
|
468
|
+
* materialized result is the problem.
|
|
469
|
+
*
|
|
470
|
+
* As with any async generator, work starts on the first `next()` call, so
|
|
471
|
+
* errors (an unreadable file, or a parser configured with `targetPages` —
|
|
472
|
+
* ambiguous with generated batch ranges) surface on the first iteration
|
|
473
|
+
* rather than when `parseBatches()` itself is called.
|
|
474
|
+
*/
|
|
475
|
+
async *parseBatches(input, options = {}) {
|
|
476
|
+
const nativeInput = typeof input === "string" ? input : Buffer.from(input);
|
|
477
|
+
const session = await this._native.openBatchSession(
|
|
478
|
+
nativeInput,
|
|
479
|
+
options.batchSize
|
|
480
|
+
);
|
|
481
|
+
try {
|
|
482
|
+
const totalPages = session.totalPages;
|
|
483
|
+
for (; ; ) {
|
|
484
|
+
const batch = await session.nextBatch();
|
|
485
|
+
if (batch == null) {
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
yield {
|
|
489
|
+
startPage: batch.startPage,
|
|
490
|
+
endPage: batch.endPage,
|
|
491
|
+
totalPages,
|
|
492
|
+
result: toParseResult(batch.result)
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
} finally {
|
|
496
|
+
await session.close();
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
/**
|
|
500
|
+
* Parse from pre-extracted pages, skipping PDFium text extraction. Runs only
|
|
501
|
+
* grid projection + the configured output formatter, so the caller's own
|
|
502
|
+
* text-extraction / font-recovery owns the text content. Synchronous: no
|
|
503
|
+
* PDFium load and no OCR on this path.
|
|
504
|
+
*/
|
|
505
|
+
parsePages(pages) {
|
|
506
|
+
const nativePages = pages.map((p) => ({
|
|
507
|
+
pageNumber: p.pageNumber,
|
|
508
|
+
pageWidth: p.pageWidth,
|
|
509
|
+
pageHeight: p.pageHeight,
|
|
510
|
+
textItems: p.textItems,
|
|
511
|
+
graphics: p.graphics
|
|
512
|
+
}));
|
|
513
|
+
const result = this._native.parsePages(nativePages);
|
|
514
|
+
return toParseResult(result);
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Determine per-page complexity without running a full parse. Returns one
|
|
518
|
+
* entry per page with signals and a `needsOcr` verdict — a cheap pre-OCR
|
|
519
|
+
* check to decide whether a document needs advanced parsing.
|
|
520
|
+
*/
|
|
521
|
+
async isComplex(input) {
|
|
522
|
+
const nativeInput = typeof input === "string" ? input : Buffer.from(input);
|
|
523
|
+
const stats = await this._native.isComplex(nativeInput);
|
|
524
|
+
return stats.map(toComplexity);
|
|
525
|
+
}
|
|
526
|
+
async screenshot(input, pageNumbers) {
|
|
527
|
+
const nativeInput = typeof input === "string" ? input : Buffer.from(input);
|
|
528
|
+
const results = await this._native.screenshot(
|
|
529
|
+
nativeInput,
|
|
530
|
+
pageNumbers ?? null
|
|
531
|
+
);
|
|
532
|
+
return results.map((r) => ({
|
|
533
|
+
pageNum: r.pageNum,
|
|
534
|
+
width: r.width,
|
|
535
|
+
height: r.height,
|
|
536
|
+
imageBuffer: r.imageBuffer,
|
|
537
|
+
isSolidFill: r.isSolidFill,
|
|
538
|
+
rects: r.rects
|
|
539
|
+
}));
|
|
540
|
+
}
|
|
541
|
+
getConfig() {
|
|
542
|
+
return { ...this._config };
|
|
543
|
+
}
|
|
544
|
+
};
|
|
545
|
+
function toComplexity(s) {
|
|
546
|
+
return {
|
|
547
|
+
pageNumber: s.pageNumber,
|
|
548
|
+
textLength: s.textLength,
|
|
549
|
+
textCoverage: s.textCoverage,
|
|
550
|
+
hasSubstantialImages: s.hasSubstantialImages,
|
|
551
|
+
imageBlockCount: s.imageBlockCount,
|
|
552
|
+
imageCoverage: s.imageCoverage,
|
|
553
|
+
largestImageCoverage: s.largestImageCoverage,
|
|
554
|
+
fullPageImage: s.fullPageImage,
|
|
555
|
+
uncoveredVectorArea: s.uncoveredVectorArea ?? void 0,
|
|
556
|
+
isGarbled: s.isGarbled,
|
|
557
|
+
pageArea: s.pageArea,
|
|
558
|
+
needsOcr: s.needsOcr,
|
|
559
|
+
reasons: s.reasons,
|
|
560
|
+
layout: s.layout ? {
|
|
561
|
+
columnCount: s.layout.columnCount,
|
|
562
|
+
ruledTableCount: s.layout.ruledTableCount,
|
|
563
|
+
ruledTableCoverage: s.layout.ruledTableCoverage,
|
|
564
|
+
textTableRunCount: s.layout.textTableRunCount,
|
|
565
|
+
figureCount: s.layout.figureCount,
|
|
566
|
+
figureCoverage: s.layout.figureCoverage,
|
|
567
|
+
isComplex: s.layout.isComplex,
|
|
568
|
+
reasons: s.layout.reasons
|
|
569
|
+
} : void 0
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
function toParseResult(result) {
|
|
573
|
+
return {
|
|
574
|
+
totalPages: result.totalPages,
|
|
575
|
+
pages: result.pages.map(toPage),
|
|
576
|
+
pageErrors: result.pageErrors ?? [],
|
|
577
|
+
text: result.text,
|
|
578
|
+
images: (result.images ?? []).map(toImage),
|
|
579
|
+
screenshots: (result.screenshots ?? []).map(toScreenshot),
|
|
580
|
+
imageErrorCount: result.imageErrorCount ?? 0,
|
|
581
|
+
formType: result.formType,
|
|
582
|
+
creator: result.creator,
|
|
583
|
+
producer: result.producer,
|
|
584
|
+
docMeta: result.docMeta,
|
|
585
|
+
xfaPackets: result.xfaPackets
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
function toPage(p) {
|
|
589
|
+
return {
|
|
590
|
+
pageNum: p.pageNum,
|
|
591
|
+
width: p.width,
|
|
592
|
+
height: p.height,
|
|
593
|
+
contentBounds: p.contentBounds,
|
|
594
|
+
text: p.text,
|
|
595
|
+
markdown: p.markdown,
|
|
596
|
+
textItems: p.textItems.map(toTextItem),
|
|
597
|
+
complexity: p.complexity ? toComplexity(p.complexity) : void 0,
|
|
598
|
+
vectorGraphics: p.vectorGraphics ?? void 0,
|
|
599
|
+
annotations: p.annotations,
|
|
600
|
+
formFields: p.formFields?.map((field) => ({
|
|
601
|
+
id: field.id,
|
|
602
|
+
type: field.fieldType,
|
|
603
|
+
page: field.page,
|
|
604
|
+
annotationIndex: field.annotationIndex,
|
|
605
|
+
widgetIndex: field.widgetIndex,
|
|
606
|
+
objectNumber: field.objectNumber,
|
|
607
|
+
name: field.name,
|
|
608
|
+
alternateName: field.alternateName,
|
|
609
|
+
value: field.value,
|
|
610
|
+
exportValue: field.exportValue,
|
|
611
|
+
fieldFlags: field.fieldFlags,
|
|
612
|
+
controlCount: field.controlCount,
|
|
613
|
+
controlIndex: field.controlIndex,
|
|
614
|
+
checked: field.checked,
|
|
615
|
+
rect: field.rect,
|
|
616
|
+
options: field.options,
|
|
617
|
+
selectedOptions: field.selectedOptions
|
|
618
|
+
})),
|
|
619
|
+
structureTree: p.structureTree ? { roots: p.structureTree.roots.map(toStructureTreeElement) } : void 0,
|
|
620
|
+
blocks: p.blocks
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
function toStructureTreeElement(element) {
|
|
624
|
+
const attributes = {};
|
|
625
|
+
for (const attribute of element.attributes) {
|
|
626
|
+
if (attribute.booleanValue !== void 0) {
|
|
627
|
+
attributes[attribute.name] = attribute.booleanValue;
|
|
628
|
+
} else if (attribute.numberValue !== void 0) {
|
|
629
|
+
attributes[attribute.name] = attribute.numberValue;
|
|
630
|
+
} else if (attribute.stringValue !== void 0) {
|
|
631
|
+
attributes[attribute.name] = attribute.stringValue;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
return {
|
|
635
|
+
type: element.elementType,
|
|
636
|
+
id: element.id,
|
|
637
|
+
actualText: element.actualText,
|
|
638
|
+
altText: element.altText,
|
|
639
|
+
title: element.title,
|
|
640
|
+
attributes,
|
|
641
|
+
markedContentIds: element.markedContentIds,
|
|
642
|
+
children: element.children.map(toStructureTreeElement),
|
|
643
|
+
annotations: element.annotations
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
function toImage(img) {
|
|
647
|
+
return {
|
|
648
|
+
id: img.id,
|
|
649
|
+
name: img.name,
|
|
650
|
+
path: img.path,
|
|
651
|
+
page: img.page,
|
|
652
|
+
bbox: img.bbox,
|
|
653
|
+
width: img.width,
|
|
654
|
+
height: img.height,
|
|
655
|
+
rotation: img.rotation,
|
|
656
|
+
format: img.format,
|
|
657
|
+
duplicateOf: img.duplicateOf,
|
|
658
|
+
bytes: img.bytes
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
function toScreenshot(result) {
|
|
662
|
+
return {
|
|
663
|
+
pageNum: result.pageNum,
|
|
664
|
+
width: result.width,
|
|
665
|
+
height: result.height,
|
|
666
|
+
imageBuffer: result.imageBuffer,
|
|
667
|
+
isSolidFill: result.isSolidFill,
|
|
668
|
+
rects: result.rects
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
function toTextItem(item) {
|
|
672
|
+
return {
|
|
673
|
+
text: item.text,
|
|
674
|
+
x: item.x,
|
|
675
|
+
y: item.y,
|
|
676
|
+
width: item.width,
|
|
677
|
+
height: item.height,
|
|
678
|
+
fontName: item.fontName,
|
|
679
|
+
fontSize: item.fontSize,
|
|
680
|
+
fontHeight: item.fontHeight,
|
|
681
|
+
fontAscent: item.fontAscent,
|
|
682
|
+
fontDescent: item.fontDescent,
|
|
683
|
+
fontWeight: item.fontWeight,
|
|
684
|
+
textWidth: item.textWidth,
|
|
685
|
+
fontIsBuggy: item.fontIsBuggy,
|
|
686
|
+
mcid: item.mcid,
|
|
687
|
+
fillColor: item.fillColor,
|
|
688
|
+
strokeColor: item.strokeColor,
|
|
689
|
+
charCodes: item.charCodes,
|
|
690
|
+
trailingSpaceGenerated: item.trailingSpaceGenerated,
|
|
691
|
+
confidence: item.confidence,
|
|
692
|
+
rotation: item.rotation,
|
|
693
|
+
words: item.words
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
function searchItems(items, options) {
|
|
697
|
+
const nativeResults = native.searchItems(
|
|
698
|
+
items,
|
|
699
|
+
options.phrase,
|
|
700
|
+
options.caseSensitive ?? false
|
|
701
|
+
);
|
|
702
|
+
return nativeResults.map(toTextItem);
|
|
703
|
+
}
|
|
704
|
+
var lib_default = LiteParse;
|
|
705
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
706
|
+
0 && (module.exports = {
|
|
707
|
+
LiteParse,
|
|
708
|
+
ParseTimeoutError,
|
|
709
|
+
searchItems,
|
|
710
|
+
toParseResult
|
|
711
|
+
});
|
|
712
|
+
//# sourceMappingURL=lib.cjs.map
|