@llamaindex/liteparse 2.13.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/dist/cli.js +1236 -398
- package/dist/cli.js.map +1 -1
- package/dist/lib.cjs +712 -0
- package/dist/lib.cjs.map +1 -0
- package/dist/lib.d.cts +911 -0
- package/dist/lib.d.ts +339 -38
- package/dist/lib.js +654 -323
- package/dist/lib.js.map +1 -1
- package/dist/pool-worker.js +253 -0
- package/dist/pool-worker.js.map +1 -0
- package/liteparse.linux-x64-gnu.node +0 -0
- package/package.json +22 -12
- package/dist/cli-json.d.ts +0 -154
- package/dist/cli-json.d.ts.map +0 -1
- package/dist/cli-json.js +0 -241
- package/dist/cli-json.js.map +0 -1
- package/dist/cli.d.ts +0 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/lib.d.ts.map +0 -1
- package/dist/native.d.ts +0 -345
- package/dist/native.d.ts.map +0 -1
- package/dist/native.js +0 -71
- package/dist/native.js.map +0 -1
package/README.md
CHANGED
|
@@ -26,6 +26,18 @@ for (const page of result.pages) {
|
|
|
26
26
|
}
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
+
CommonJS is supported as well:
|
|
30
|
+
|
|
31
|
+
```javascript
|
|
32
|
+
const { LiteParse } = require('@llamaindex/liteparse');
|
|
33
|
+
|
|
34
|
+
(async () => {
|
|
35
|
+
const parser = new LiteParse();
|
|
36
|
+
const result = await parser.parse('document.pdf');
|
|
37
|
+
console.log(result.text);
|
|
38
|
+
})();
|
|
39
|
+
```
|
|
40
|
+
|
|
29
41
|
### Bounded-memory parsing
|
|
30
42
|
|
|
31
43
|
For documents with many text items, consume page batches without retaining
|
|
@@ -150,6 +162,32 @@ const result = await parser.parse(pdfBytes);
|
|
|
150
162
|
console.log(result.text);
|
|
151
163
|
```
|
|
152
164
|
|
|
165
|
+
## Worker Pool and Hard Timeouts
|
|
166
|
+
|
|
167
|
+
PDFium is not thread-safe, so all parses inside one process serialize on a
|
|
168
|
+
process-global lock — even `Promise.all` over multiple `parse()` calls runs
|
|
169
|
+
them one at a time.
|
|
170
|
+
|
|
171
|
+
For high-throughput services, or for cases when you need to enforce a
|
|
172
|
+
runtime timeout, run parses in a pool of persistent worker processes instead:
|
|
173
|
+
|
|
174
|
+
```typescript
|
|
175
|
+
import { LiteParse, ParseTimeoutError } from '@llamaindex/liteparse';
|
|
176
|
+
|
|
177
|
+
const parser = new LiteParse({ poolSize: 4, parseTimeoutMs: 15_000 });
|
|
178
|
+
await parser.warmUp(); // optional: pre-initialize workers (~45ms total)
|
|
179
|
+
|
|
180
|
+
try {
|
|
181
|
+
const result = await parser.parse('document.pdf');
|
|
182
|
+
} catch (e) {
|
|
183
|
+
if (e instanceof ParseTimeoutError) {
|
|
184
|
+
console.warn(`killed rogue document: ${e.source} (deadline ${e.timeoutMs}ms)`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
parser.close(); // frees workers immediately; an idle pool never blocks exit
|
|
189
|
+
```
|
|
190
|
+
|
|
153
191
|
## Screenshots
|
|
154
192
|
|
|
155
193
|
Generate PNG screenshots of document pages:
|