@effortlessmotion/pdf-to-md 0.1.8 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +5 -5
  2. package/dist/cjs/analyze.d.cts +3 -1
  3. package/dist/cjs/analyze.d.ts +3 -1
  4. package/dist/cjs/analyze.js +38 -100
  5. package/dist/cjs/analyze.js.map +1 -1
  6. package/dist/cjs/audit.d.cts +6 -5
  7. package/dist/cjs/audit.d.ts +6 -5
  8. package/dist/cjs/audit.js +368 -60
  9. package/dist/cjs/audit.js.map +1 -1
  10. package/dist/cjs/cli/audit.js.map +1 -1
  11. package/dist/cjs/cli/extract.js +13 -48
  12. package/dist/cjs/cli/extract.js.map +1 -1
  13. package/dist/cjs/cli/index.js.map +1 -1
  14. package/dist/cjs/cli/render.js.map +1 -1
  15. package/dist/cjs/cli/types.js.map +1 -1
  16. package/dist/cjs/cli.js.map +1 -1
  17. package/dist/cjs/collect.js +91 -35
  18. package/dist/cjs/collect.js.map +1 -1
  19. package/dist/cjs/dict.js.map +1 -1
  20. package/dist/cjs/emit.js.map +1 -1
  21. package/dist/cjs/engines/tesseract.js.map +1 -1
  22. package/dist/cjs/errors.d.cts +2 -0
  23. package/dist/cjs/errors.d.ts +2 -0
  24. package/dist/cjs/errors.js.map +1 -1
  25. package/dist/cjs/extract.d.cts +20 -9
  26. package/dist/cjs/extract.d.ts +20 -9
  27. package/dist/cjs/extract.js +333 -106
  28. package/dist/cjs/extract.js.map +1 -1
  29. package/dist/cjs/geometry.js.map +1 -1
  30. package/dist/cjs/index.js.map +1 -1
  31. package/dist/cjs/lines.js +21 -10
  32. package/dist/cjs/lines.js.map +1 -1
  33. package/dist/cjs/markdown.js +140 -54
  34. package/dist/cjs/markdown.js.map +1 -1
  35. package/dist/cjs/page-marker.d.cts +1 -1
  36. package/dist/cjs/page-marker.d.ts +1 -1
  37. package/dist/cjs/page-marker.js +39 -8
  38. package/dist/cjs/page-marker.js.map +1 -1
  39. package/dist/cjs/pdf-open.js.map +1 -1
  40. package/dist/cjs/raster.js.map +1 -1
  41. package/dist/cjs/render-analysis.d.cts +4 -2
  42. package/dist/cjs/render-analysis.d.ts +4 -2
  43. package/dist/cjs/render-analysis.js +5 -3
  44. package/dist/cjs/render-analysis.js.map +1 -1
  45. package/dist/cjs/report.d.cts +2 -2
  46. package/dist/cjs/report.d.ts +2 -2
  47. package/dist/cjs/report.js +27 -8
  48. package/dist/cjs/report.js.map +1 -1
  49. package/dist/cjs/resolve.js.map +1 -1
  50. package/dist/cjs/text-analysis.d.cts +12 -0
  51. package/dist/cjs/text-analysis.d.ts +12 -0
  52. package/dist/cjs/text-analysis.js +175 -0
  53. package/dist/cjs/text-analysis.js.map +1 -0
  54. package/dist/cjs/types.d.cts +40 -3
  55. package/dist/cjs/types.d.ts +40 -3
  56. package/dist/cjs/types.js +2 -3
  57. package/dist/cjs/types.js.map +1 -1
  58. package/dist/cjs/validate.js.map +1 -1
  59. package/dist/esm/analyze.d.ts +3 -1
  60. package/dist/esm/analyze.js +18 -89
  61. package/dist/esm/analyze.js.map +1 -1
  62. package/dist/esm/audit.d.ts +6 -5
  63. package/dist/esm/audit.js +236 -29
  64. package/dist/esm/audit.js.map +1 -1
  65. package/dist/esm/cli/audit.js.map +1 -1
  66. package/dist/esm/cli/extract.js +10 -23
  67. package/dist/esm/cli/extract.js.map +1 -1
  68. package/dist/esm/cli/index.js.map +1 -1
  69. package/dist/esm/cli/render.js.map +1 -1
  70. package/dist/esm/cli/types.js.map +1 -1
  71. package/dist/esm/cli.js.map +1 -1
  72. package/dist/esm/collect.js +64 -15
  73. package/dist/esm/collect.js.map +1 -1
  74. package/dist/esm/dict.js.map +1 -1
  75. package/dist/esm/emit.js.map +1 -1
  76. package/dist/esm/engines/tesseract.js.map +1 -1
  77. package/dist/esm/errors.d.ts +2 -0
  78. package/dist/esm/errors.js.map +1 -1
  79. package/dist/esm/extract.d.ts +20 -9
  80. package/dist/esm/extract.js +167 -76
  81. package/dist/esm/extract.js.map +1 -1
  82. package/dist/esm/geometry.js.map +1 -1
  83. package/dist/esm/index.js.map +1 -1
  84. package/dist/esm/lines.js +14 -10
  85. package/dist/esm/lines.js.map +1 -1
  86. package/dist/esm/markdown.js +110 -27
  87. package/dist/esm/markdown.js.map +1 -1
  88. package/dist/esm/page-marker.d.ts +1 -1
  89. package/dist/esm/page-marker.js +15 -9
  90. package/dist/esm/page-marker.js.map +1 -1
  91. package/dist/esm/pdf-open.js.map +1 -1
  92. package/dist/esm/raster.js.map +1 -1
  93. package/dist/esm/render-analysis.d.ts +4 -2
  94. package/dist/esm/render-analysis.js +9 -5
  95. package/dist/esm/render-analysis.js.map +1 -1
  96. package/dist/esm/report.d.ts +2 -2
  97. package/dist/esm/report.js +12 -7
  98. package/dist/esm/report.js.map +1 -1
  99. package/dist/esm/resolve.js.map +1 -1
  100. package/dist/esm/text-analysis.d.ts +12 -0
  101. package/dist/esm/text-analysis.js +81 -0
  102. package/dist/esm/text-analysis.js.map +1 -0
  103. package/dist/esm/types.d.ts +40 -3
  104. package/dist/esm/types.js +4 -4
  105. package/dist/esm/types.js.map +1 -1
  106. package/dist/esm/validate.js.map +1 -1
  107. package/package.json +1 -1
  108. package/skills/pdf-to-md/SKILL.md +39 -66
@@ -1,5 +1,5 @@
1
- // Pass 1: load the PDF and collect positioned glyphs per page, plus the
2
- // document-wide char-weighted font-height histogram. Heights round to 0.1pt for stable histogram keys; y is flipped so sorting ascending reads top-down.
1
+ // Pass 1: load each PDF text run into viewport points, plus the document-wide
2
+ // char-weighted font-height histogram. Height rounds to 0.1pt for stable histogram keys.
3
3
  "use strict";
4
4
  Object.defineProperty(exports, "__esModule", {
5
5
  value: true
@@ -28,6 +28,9 @@ function _array_like_to_array(arr, len) {
28
28
  function _array_with_holes(arr) {
29
29
  if (Array.isArray(arr)) return arr;
30
30
  }
31
+ function _array_without_holes(arr) {
32
+ if (Array.isArray(arr)) return _array_like_to_array(arr);
33
+ }
31
34
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
32
35
  try {
33
36
  var info = gen[key](arg);
@@ -54,6 +57,11 @@ function _async_to_generator(fn) {
54
57
  });
55
58
  };
56
59
  }
60
+ function _iterable_to_array(iter) {
61
+ if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
62
+ return Array.from(iter);
63
+ }
64
+ }
57
65
  function _iterable_to_array_limit(arr, i) {
58
66
  var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
59
67
  if (_i == null) return;
@@ -81,9 +89,15 @@ function _iterable_to_array_limit(arr, i) {
81
89
  function _non_iterable_rest() {
82
90
  throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
83
91
  }
92
+ function _non_iterable_spread() {
93
+ throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
94
+ }
84
95
  function _sliced_to_array(arr, i) {
85
96
  return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
86
97
  }
98
+ function _to_consumable_array(arr) {
99
+ return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
100
+ }
87
101
  function _ts_generator(thisArg, body) {
88
102
  var f, y, t, _ = {
89
103
  label: 0,
@@ -194,12 +208,22 @@ function _unsupported_iterable_to_array(o, minLen) {
194
208
  // HYBRID-PAGE DETECTION (README's "Design contract"): a page can carry a
195
209
  // real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.
196
210
  //
197
- // Signal: scan the operator list for `OPS.paintImageXObject`, whose args are
198
- // `[objId, width, height]` — available synchronously, `large` is >=200x200px. Masks/nested Form XObjects aren't walked (accepted gap); missing dims fall back to "op present" as the signal.
211
+ // Image presence uses the paint operators for XObjects, inline images, and masks.
212
+ // The large-image threshold uses `paintImageXObject` dimensions when available.
199
213
  var LARGE_IMAGE_MIN_PX = 200;
200
- function hasLargeImage(page) {
214
+ var IMAGE_PAINT_OPS = new Set([
215
+ _pdf.OPS.paintImageMaskXObject,
216
+ _pdf.OPS.paintImageMaskXObjectGroup,
217
+ _pdf.OPS.paintImageMaskXObjectRepeat,
218
+ _pdf.OPS.paintImageXObject,
219
+ _pdf.OPS.paintImageXObjectRepeat,
220
+ _pdf.OPS.paintInlineImageXObject,
221
+ _pdf.OPS.paintInlineImageXObjectGroup,
222
+ _pdf.OPS.paintSolidColorImageMask
223
+ ]);
224
+ function getPageImageSignals(page, textChars) {
201
225
  return _async_to_generator(function() {
202
- var _ref, fnArray, argsArray, i, _argsArray_i, w, h;
226
+ var _ref, fnArray, argsArray, hasImage, largeImage, i, _argsArray_i, w, h;
203
227
  return _ts_generator(this, function(_state) {
204
228
  switch(_state.label){
205
229
  case 0:
@@ -209,22 +233,22 @@ function hasLargeImage(page) {
209
233
  ];
210
234
  case 1:
211
235
  _ref = _state.sent(), fnArray = _ref.fnArray, argsArray = _ref.argsArray;
236
+ hasImage = false;
237
+ largeImage = false;
212
238
  for(i = 0; i < fnArray.length; i++){
239
+ if (IMAGE_PAINT_OPS.has(fnArray[i])) hasImage = true;
213
240
  if (fnArray[i] !== _pdf.OPS.paintImageXObject) continue;
214
241
  _argsArray_i = _sliced_to_array(argsArray[i], 3), w = _argsArray_i[1], h = _argsArray_i[2];
215
242
  if (typeof w === 'number' && typeof h === 'number') {
216
- if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) return [
217
- 2,
218
- true
219
- ];
220
- } else return [
221
- 2,
222
- true
223
- ]; // dims unavailable: coarser "op present" signal
243
+ if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) largeImage = true;
244
+ } else largeImage = true; // dims unavailable: coarser "op present" signal
224
245
  }
225
246
  return [
226
247
  2,
227
- false
248
+ {
249
+ hasImage: hasImage,
250
+ largeImage: textChars > 80 && largeImage
251
+ }
228
252
  ];
229
253
  }
230
254
  });
@@ -252,7 +276,7 @@ function pageCount(src) {
252
276
  }
253
277
  function collect(_0) {
254
278
  return _async_to_generator(function(src) {
255
- var _ref, first, last, doc, from, to, pages, heightChars, p, page, items, vp, glyphs, textChars, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, it, h, largeImage, _tmp;
279
+ var _ref, first, last, doc, from, to, pages, heightChars, p, page, items, vp, viewportScale, glyphs, textChars, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, it, _Math, _Math1, _Math2, _Math3, _Math4, _Math5, transform, baselineLength, w, h, baselineX, baselineY, corners, boundsX, boundsY, _ref1, hasImage, largeImage;
256
280
  var _arguments = arguments;
257
281
  return _ts_generator(this, function(_state) {
258
282
  switch(_state.label){
@@ -273,7 +297,7 @@ function collect(_0) {
273
297
  case 2:
274
298
  if (!(p <= to)) return [
275
299
  3,
276
- 8
300
+ 7
277
301
  ];
278
302
  return [
279
303
  4,
@@ -290,22 +314,61 @@ function collect(_0) {
290
314
  vp = page.getViewport({
291
315
  scale: 1
292
316
  });
317
+ viewportScale = Math.hypot(vp.transform[0], vp.transform[1]);
293
318
  glyphs = [];
294
319
  textChars = 0;
295
320
  _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
296
321
  try {
297
322
  for(_iterator = items[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
298
323
  it = _step.value;
324
+ ;
299
325
  if (!('str' in it) || typeof it.str !== 'string') continue;
300
326
  textChars += it.str.length;
301
327
  if (!it.str.trim()) continue;
302
- h = Math.round(Math.abs(it.transform[3]) * 10) / 10;
328
+ transform = _pdf.Util.transform(vp.transform, it.transform);
329
+ baselineLength = Math.hypot(transform[0], transform[1]);
330
+ w = Math.abs(it.width) * viewportScale;
331
+ h = Math.round(Math.hypot(transform[2], transform[3]) * 10) / 10;
332
+ baselineX = baselineLength > 0 ? transform[0] / baselineLength * w : w;
333
+ baselineY = baselineLength > 0 ? transform[1] / baselineLength * w : 0;
334
+ corners = [
335
+ [
336
+ transform[4],
337
+ transform[5]
338
+ ],
339
+ [
340
+ transform[4] + baselineX,
341
+ transform[5] + baselineY
342
+ ],
343
+ [
344
+ transform[4] + baselineX + transform[2],
345
+ transform[5] + baselineY + transform[3]
346
+ ],
347
+ [
348
+ transform[4] + transform[2],
349
+ transform[5] + transform[3]
350
+ ]
351
+ ];
352
+ boundsX = corners.map(function(param) {
353
+ var _param = _sliced_to_array(param, 1), x = _param[0];
354
+ return x;
355
+ });
356
+ boundsY = corners.map(function(param) {
357
+ var _param = _sliced_to_array(param, 2), y = _param[1];
358
+ return y;
359
+ });
303
360
  glyphs.push({
304
361
  s: it.str,
305
- x: it.transform[4],
306
- y: vp.height - it.transform[5],
307
- w: it.width,
308
- h: h
362
+ x: transform[4],
363
+ y: transform[5],
364
+ w: w,
365
+ h: h,
366
+ bounds: {
367
+ x: (_Math = Math).min.apply(_Math, _to_consumable_array(boundsX)),
368
+ y: (_Math1 = Math).min.apply(_Math1, _to_consumable_array(boundsY)),
369
+ w: (_Math2 = Math).max.apply(_Math2, _to_consumable_array(boundsX)) - (_Math3 = Math).min.apply(_Math3, _to_consumable_array(boundsX)),
370
+ h: (_Math4 = Math).max.apply(_Math4, _to_consumable_array(boundsY)) - (_Math5 = Math).min.apply(_Math5, _to_consumable_array(boundsY))
371
+ }
309
372
  });
310
373
  heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);
311
374
  }
@@ -323,36 +386,29 @@ function collect(_0) {
323
386
  }
324
387
  }
325
388
  }
326
- _tmp = textChars > 80;
327
- if (!_tmp) return [
328
- 3,
329
- 6
330
- ];
331
389
  return [
332
390
  4,
333
- hasLargeImage(page)
391
+ getPageImageSignals(page, textChars)
334
392
  ];
335
393
  case 5:
336
- _tmp = _state.sent();
337
- _state.label = 6;
338
- case 6:
339
- largeImage = _tmp;
394
+ _ref1 = _state.sent(), hasImage = _ref1.hasImage, largeImage = _ref1.largeImage;
340
395
  pages.push({
341
396
  page: p,
342
397
  glyphs: glyphs,
343
398
  width: vp.width,
344
399
  height: vp.height,
345
400
  textChars: textChars,
401
+ hasImage: hasImage,
346
402
  largeImage: largeImage
347
403
  });
348
- _state.label = 7;
349
- case 7:
404
+ _state.label = 6;
405
+ case 6:
350
406
  p++;
351
407
  return [
352
408
  3,
353
409
  2
354
410
  ];
355
- case 8:
411
+ case 7:
356
412
  // `numPages` is the SELECTED count, not doc.numPages: the auto-OCR
357
413
  // chars/page floor must judge a `--pages` subset against its own page count, or a genuinely image-only page reads as merely "sparse".
358
414
  return [
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/collect.ts"],"sourcesContent":["// Pass 1: load the PDF and collect positioned glyphs per page, plus the\n// document-wide char-weighted font-height histogram. Heights round to 0.1pt for stable histogram keys; y is flipped so sorting ascending reads top-down.\nimport type { PDFPageProxy } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { OPS } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { openDocument } from './pdf-open.ts';\nimport type { CollectedPage, CollectResult, Glyph } from './types.ts';\n\n// HYBRID-PAGE DETECTION (README's \"Design contract\"): a page can carry a\n// real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.\n//\n// Signal: scan the operator list for `OPS.paintImageXObject`, whose args are\n// `[objId, width, height]` — available synchronously, `large` is >=200x200px. Masks/nested Form XObjects aren't walked (accepted gap); missing dims fall back to \"op present\" as the signal.\nconst LARGE_IMAGE_MIN_PX = 200;\nasync function hasLargeImage(page: PDFPageProxy): Promise<boolean> {\n const { fnArray, argsArray } = await page.getOperatorList();\n for (let i = 0; i < fnArray.length; i++) {\n if (fnArray[i] !== OPS.paintImageXObject) continue;\n const [, w, h] = argsArray[i];\n if (typeof w === 'number' && typeof h === 'number') {\n if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) return true;\n } else return true; // dims unavailable: coarser \"op present\" signal\n }\n return false;\n}\n\n// { first, last }: 1-based, inclusive page range (the CLI's --pages).\n// Out-of-range values CLAMP to the document's bounds; `page` on each returned entry is always the SOURCE page number, so markers/splicing stay correct.\n//\n// CAVEAT: `heightChars` is built ONLY from the pages actually collected, so\n// a `--pages` subset's heading levels can differ from a full run's — accepted since re-extracted pages get spliced back in for their prose, not diffed.\n\n// Cheap page-count-only load (no per-page text extraction) — the OCR path\n// needs it to resolve its own [first, last] clamping and print a \"### pN\" marker for every page in range, including ones with no recognized words.\nexport async function pageCount(src: string): Promise<number> {\n const doc = await openDocument(src);\n return doc.numPages;\n}\n\nexport async function collect(src: string, { first, last }: { first?: number; last?: number } = {}): Promise<CollectResult> {\n const doc = await openDocument(src);\n const from = Math.min(Math.max(1, first ?? 1), Math.max(1, doc.numPages));\n const to = Math.min(Math.max(from, last ?? doc.numPages), doc.numPages);\n const pages: CollectedPage[] = [];\n const heightChars = new Map<number, number>();\n for (let p = from; p <= to; p++) {\n const page = await doc.getPage(p);\n const { items } = await page.getTextContent();\n const vp = page.getViewport({ scale: 1 });\n const glyphs: Glyph[] = [];\n let textChars = 0;\n for (const it of items) {\n if (!('str' in it) || typeof it.str !== 'string') continue;\n textChars += it.str.length;\n if (!it.str.trim()) continue;\n const h = Math.round(Math.abs(it.transform[3]) * 10) / 10;\n glyphs.push({\n s: it.str,\n x: it.transform[4],\n y: vp.height - it.transform[5],\n w: it.width,\n h,\n });\n heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);\n }\n // \"text layer used\" for hybrid-page purposes (page text > 80 chars) — a\n // page with only a couple of stray scraps isn't \"using\" the text layer even if it also carries a large image.\n const largeImage = textChars > 80 && (await hasLargeImage(page));\n pages.push({\n page: p,\n glyphs,\n width: vp.width,\n height: vp.height,\n textChars,\n largeImage,\n });\n }\n // `numPages` is the SELECTED count, not doc.numPages: the auto-OCR\n // chars/page floor must judge a `--pages` subset against its own page count, or a genuinely image-only page reads as merely \"sparse\".\n return { numPages: pages.length, pages, heightChars };\n}\n"],"names":["collect","pageCount","LARGE_IMAGE_MIN_PX","hasLargeImage","page","fnArray","argsArray","i","w","h","getOperatorList","length","OPS","paintImageXObject","src","doc","openDocument","numPages","first","last","from","to","pages","heightChars","p","items","vp","glyphs","textChars","it","largeImage","Math","min","max","Map","getPage","getTextContent","getViewport","scale","str","trim","round","abs","transform","push","s","x","y","height","width","set","get"],"mappings":"AAAA,wEAAwE;AACxE,yJAAyJ;;;;;;;;;;;;QAqCnIA;eAAAA;;QALAC;eAAAA;;;mBA9BF;yBACS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG7B,yEAAyE;AACzE,2HAA2H;AAC3H,EAAE;AACF,6EAA6E;AAC7E,6LAA6L;AAC7L,IAAMC,qBAAqB;AAC3B,SAAeC,cAAcC,IAAkB;;YACd,MAAvBC,SAASC,WACRC,GAEUD,cAARE,GAAGC;;;;oBAHiB;;wBAAML,KAAKM,eAAe;;;oBAA1B,OAAA,eAAvBL,UAAuB,KAAvBA,SAASC,YAAc,KAAdA;oBACjB,IAASC,IAAI,GAAGA,IAAIF,QAAQM,MAAM,EAAEJ,IAAK;wBACvC,IAAIF,OAAO,CAACE,EAAE,KAAKK,QAAG,CAACC,iBAAiB,EAAE;wBACzBP,gCAAAA,SAAS,CAACC,EAAE,MAApBC,IAAQF,iBAALG,IAAKH;wBACjB,IAAI,OAAOE,MAAM,YAAY,OAAOC,MAAM,UAAU;4BAClD,IAAID,KAAKN,sBAAsBO,KAAKP,oBAAoB;;gCAAO;;wBACjE,OAAO;;4BAAO;2BAAM,gDAAgD;oBACtE;oBACA;;wBAAO;;;;IACT;;AAUO,SAAeD,UAAUa,GAAW;;YACnCC;;;;oBAAM;;wBAAMC,IAAAA,uBAAY,EAACF;;;oBAAzBC,MAAM;oBACZ;;wBAAOA,IAAIE,QAAQ;;;;IACrB;;AAEO,SAAejB;wCAAQc,GAAW;YAAE,MAAEI,OAAOC,MAC5CJ,KACAK,MACAC,IACAC,OACAC,aACGC,GACDpB,MACEqB,OACFC,IACAC,QACFC,WACC,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,IAIHpB,GAYFqB;;;;;oBA5BiC,OAAA,oEAAqD,CAAC,GAApDZ,QAAF,KAAEA,OAAOC,OAAT,KAASA;oBACtC;;wBAAMH,IAAAA,uBAAY,EAACF;;;oBAAzBC,MAAM;oBACNK,OAAOW,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,GAAGf,kBAAAA,mBAAAA,QAAS,IAAIa,KAAKE,GAAG,CAAC,GAAGlB,IAAIE,QAAQ;oBACjEI,KAAKU,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAACb,MAAMD,iBAAAA,kBAAAA,OAAQJ,IAAIE,QAAQ,GAAGF,IAAIE,QAAQ;oBAChEK;oBACAC,cAAc,IAAIW;oBACfV,IAAIJ;;;yBAAMI,CAAAA,KAAKH,EAAC;;;;oBACV;;wBAAMN,IAAIoB,OAAO,CAACX;;;oBAAzBpB,OAAO;oBACK;;wBAAMA,KAAKgC,cAAc;;;oBAAnCX,QAAU,cAAVA;oBACFC,KAAKtB,KAAKiC,WAAW,CAAC;wBAAEC,OAAO;oBAAE;oBACjCX;oBACFC,YAAY;oBACX,kCAAA,2BAAA;;wBAAL,IAAK,YAAYH,4BAAZ,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,KAAN;4BACH,IAAI,CAAE,CAAA,SAASA,EAAC,KAAM,OAAOA,GAAGU,GAAG,KAAK,UAAU;4BAClDX,aAAaC,GAAGU,GAAG,CAAC5B,MAAM;4BAC1B,IAAI,CAACkB,GAAGU,GAAG,CAACC,IAAI,IAAI;4BACd/B,IAAIsB,KAAKU,KAAK,CAACV,KAAKW,GAAG,CAACb,GAAGc,SAAS,CAAC,EAAE,IAAI,MAAM;4BACvDhB,OAAOiB,IAAI,CAAC;gCACVC,GAAGhB,GAAGU,GAAG;gCACTO,GAAGjB,GAAGc,SAAS,CAAC,EAAE;gCAClBI,GAAGrB,GAAGsB,MAAM,GAAGnB,GAAGc,SAAS,CAAC,EAAE;gCAC9BnC,GAAGqB,GAAGoB,KAAK;gCACXxC,GAAAA;4BACF;4BACAc,YAAY2B,GAAG,CAACzC,GAAG,AAACc,CAAAA,YAAY4B,GAAG,CAAC1C,MAAM,CAAA,IAAKoB,GAAGU,GAAG,CAAC5B,MAAM;wBAC9D;;wBAbK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;2BAgBciB,YAAY;+BAAZA;;;;oBAAmB;;wBAAMzB,cAAcC;;;2BAApB;;;oBAAhC0B;oBACNR,MAAMsB,IAAI,CAAC;wBACTxC,MAAMoB;wBACNG,QAAAA;wBACAsB,OAAOvB,GAAGuB,KAAK;wBACfD,QAAQtB,GAAGsB,MAAM;wBACjBpB,WAAAA;wBACAE,YAAAA;oBACF;;;oBA9B0BN;;;;;;oBAgC5B,mEAAmE;oBACnE,sIAAsI;oBACtI;;wBAAO;4BAAEP,UAAUK,MAAMX,MAAM;4BAAEW,OAAAA;4BAAOC,aAAAA;wBAAY;;;;IACtD"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/collect.ts"],"sourcesContent":["// Pass 1: load each PDF text run into viewport points, plus the document-wide\n// char-weighted font-height histogram. Height rounds to 0.1pt for stable histogram keys.\nimport type { PDFPageProxy } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { OPS, Util } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { openDocument } from './pdf-open.ts';\nimport type { CollectedPage, CollectResult, Glyph } from './types.ts';\n\n// HYBRID-PAGE DETECTION (README's \"Design contract\"): a page can carry a\n// real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.\n//\n// Image presence uses the paint operators for XObjects, inline images, and masks.\n// The large-image threshold uses `paintImageXObject` dimensions when available.\nconst LARGE_IMAGE_MIN_PX = 200;\ninterface PageImageSignals {\n hasImage: boolean;\n largeImage: boolean;\n}\n\nconst IMAGE_PAINT_OPS = new Set([OPS.paintImageMaskXObject, OPS.paintImageMaskXObjectGroup, OPS.paintImageMaskXObjectRepeat, OPS.paintImageXObject, OPS.paintImageXObjectRepeat, OPS.paintInlineImageXObject, OPS.paintInlineImageXObjectGroup, OPS.paintSolidColorImageMask]);\n\nasync function getPageImageSignals(page: PDFPageProxy, textChars: number): Promise<PageImageSignals> {\n const { fnArray, argsArray } = await page.getOperatorList();\n let hasImage = false;\n let largeImage = false;\n for (let i = 0; i < fnArray.length; i++) {\n if (IMAGE_PAINT_OPS.has(fnArray[i])) hasImage = true;\n if (fnArray[i] !== OPS.paintImageXObject) continue;\n const [, w, h] = argsArray[i];\n if (typeof w === 'number' && typeof h === 'number') {\n if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) largeImage = true;\n } else largeImage = true; // dims unavailable: coarser \"op present\" signal\n }\n return { hasImage, largeImage: textChars > 80 && largeImage };\n}\n\n// { first, last }: 1-based, inclusive page range (the CLI's --pages).\n// Out-of-range values CLAMP to the document's bounds; `page` on each returned entry is always the SOURCE page number, so markers/splicing stay correct.\n//\n// CAVEAT: `heightChars` is built ONLY from the pages actually collected, so\n// a `--pages` subset's heading levels can differ from a full run's — accepted since re-extracted pages get spliced back in for their prose, not diffed.\n\n// Cheap page-count-only load (no per-page text extraction) — the OCR path\n// needs it to resolve its own [first, last] clamping and print a \"### pN\" marker for every page in range, including ones with no recognized words.\nexport async function pageCount(src: string): Promise<number> {\n const doc = await openDocument(src);\n return doc.numPages;\n}\n\nexport async function collect(src: string, { first, last }: { first?: number; last?: number } = {}): Promise<CollectResult> {\n const doc = await openDocument(src);\n const from = Math.min(Math.max(1, first ?? 1), Math.max(1, doc.numPages));\n const to = Math.min(Math.max(from, last ?? doc.numPages), doc.numPages);\n const pages: CollectedPage[] = [];\n const heightChars = new Map<number, number>();\n for (let p = from; p <= to; p++) {\n const page = await doc.getPage(p);\n const { items } = await page.getTextContent();\n const vp = page.getViewport({ scale: 1 });\n const viewportScale = Math.hypot(vp.transform[0], vp.transform[1]);\n const glyphs: Glyph[] = [];\n let textChars = 0;\n for (const it of items) {\n if (!('str' in it) || typeof it.str !== 'string') continue;\n textChars += it.str.length;\n if (!it.str.trim()) continue;\n const transform = Util.transform(vp.transform, it.transform);\n const baselineLength = Math.hypot(transform[0], transform[1]);\n const w = Math.abs(it.width) * viewportScale;\n const h = Math.round(Math.hypot(transform[2], transform[3]) * 10) / 10;\n const baselineX = baselineLength > 0 ? (transform[0] / baselineLength) * w : w;\n const baselineY = baselineLength > 0 ? (transform[1] / baselineLength) * w : 0;\n const corners = [\n [transform[4], transform[5]],\n [transform[4] + baselineX, transform[5] + baselineY],\n [transform[4] + baselineX + transform[2], transform[5] + baselineY + transform[3]],\n [transform[4] + transform[2], transform[5] + transform[3]],\n ];\n const boundsX = corners.map(([x]) => x);\n const boundsY = corners.map(([, y]) => y);\n glyphs.push({\n s: it.str,\n x: transform[4],\n y: transform[5],\n w,\n h,\n bounds: {\n x: Math.min(...boundsX),\n y: Math.min(...boundsY),\n w: Math.max(...boundsX) - Math.min(...boundsX),\n h: Math.max(...boundsY) - Math.min(...boundsY),\n },\n });\n heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);\n }\n // \"text layer used\" for hybrid-page purposes (page text > 80 chars) — a\n // page with only a couple of stray scraps isn't \"using\" the text layer even if it also carries a large image.\n const { hasImage, largeImage } = await getPageImageSignals(page, textChars);\n pages.push({\n page: p,\n glyphs,\n width: vp.width,\n height: vp.height,\n textChars,\n hasImage,\n largeImage,\n });\n }\n // `numPages` is the SELECTED count, not doc.numPages: the auto-OCR\n // chars/page floor must judge a `--pages` subset against its own page count, or a genuinely image-only page reads as merely \"sparse\".\n return { numPages: pages.length, pages, heightChars };\n}\n"],"names":["collect","pageCount","LARGE_IMAGE_MIN_PX","IMAGE_PAINT_OPS","Set","OPS","paintImageMaskXObject","paintImageMaskXObjectGroup","paintImageMaskXObjectRepeat","paintImageXObject","paintImageXObjectRepeat","paintInlineImageXObject","paintInlineImageXObjectGroup","paintSolidColorImageMask","getPageImageSignals","page","textChars","fnArray","argsArray","hasImage","largeImage","i","w","h","getOperatorList","length","has","src","doc","openDocument","numPages","first","last","from","to","pages","heightChars","p","items","vp","viewportScale","glyphs","it","Math","transform","baselineLength","baselineX","baselineY","corners","boundsX","boundsY","min","max","Map","getPage","getTextContent","getViewport","scale","hypot","str","trim","Util","abs","width","round","map","x","y","push","s","bounds","set","get","height"],"mappings":"AAAA,8EAA8E;AAC9E,yFAAyF;;;;;;;;;;;;QA+CnEA;eAAAA;;QALAC;eAAAA;;;mBAxCI;yBACG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG7B,yEAAyE;AACzE,2HAA2H;AAC3H,EAAE;AACF,kFAAkF;AAClF,gFAAgF;AAChF,IAAMC,qBAAqB;AAM3B,IAAMC,kBAAkB,IAAIC,IAAI;IAACC,QAAG,CAACC,qBAAqB;IAAED,QAAG,CAACE,0BAA0B;IAAEF,QAAG,CAACG,2BAA2B;IAAEH,QAAG,CAACI,iBAAiB;IAAEJ,QAAG,CAACK,uBAAuB;IAAEL,QAAG,CAACM,uBAAuB;IAAEN,QAAG,CAACO,4BAA4B;IAAEP,QAAG,CAACQ,wBAAwB;CAAC;AAE7Q,SAAeC,oBAAoBC,IAAkB,EAAEC,SAAiB;;YACvC,MAAvBC,SAASC,WACbC,UACAC,YACKC,GAGUH,cAARI,GAAGC;;;;oBANiB;;wBAAMR,KAAKS,eAAe;;;oBAA1B,OAAA,eAAvBP,UAAuB,KAAvBA,SAASC,YAAc,KAAdA;oBACbC,WAAW;oBACXC,aAAa;oBACjB,IAASC,IAAI,GAAGA,IAAIJ,QAAQQ,MAAM,EAAEJ,IAAK;wBACvC,IAAIlB,gBAAgBuB,GAAG,CAACT,OAAO,CAACI,EAAE,GAAGF,WAAW;wBAChD,IAAIF,OAAO,CAACI,EAAE,KAAKhB,QAAG,CAACI,iBAAiB,EAAE;wBACzBS,gCAAAA,SAAS,CAACG,EAAE,MAApBC,IAAQJ,iBAALK,IAAKL;wBACjB,IAAI,OAAOI,MAAM,YAAY,OAAOC,MAAM,UAAU;4BAClD,IAAID,KAAKpB,sBAAsBqB,KAAKrB,oBAAoBkB,aAAa;wBACvE,OAAOA,aAAa,MAAM,gDAAgD;oBAC5E;oBACA;;wBAAO;4BAAED,UAAAA;4BAAUC,YAAYJ,YAAY,MAAMI;wBAAW;;;;IAC9D;;AAUO,SAAenB,UAAU0B,GAAW;;YACnCC;;;;oBAAM;;wBAAMC,IAAAA,uBAAY,EAACF;;;oBAAzBC,MAAM;oBACZ;;wBAAOA,IAAIE,QAAQ;;;;IACrB;;AAEO,SAAe9B;wCAAQ2B,GAAW;YAAE,MAAEI,OAAOC,MAC5CJ,KACAK,MACAC,IACAC,OACAC,aACGC,GACDtB,MACEuB,OACFC,IACAC,eACAC,QACFzB,WACC,2BAAA,mBAAA,gBAAA,WAAA,OAAM0B,IAyBFC,OACAA,QACAA,QAAuBA,QACvBA,QAAuBA,QAxBxBC,WACAC,gBACAvB,GACAC,GACAuB,WACAC,WACAC,SAMAC,SACAC,SAkByB,OAAzB/B,UAAUC;;;;;oBAhDqB,OAAA,oEAAqD,CAAC,GAApDW,QAAF,KAAEA,OAAOC,OAAT,KAASA;oBACtC;;wBAAMH,IAAAA,uBAAY,EAACF;;;oBAAzBC,MAAM;oBACNK,OAAOU,KAAKQ,GAAG,CAACR,KAAKS,GAAG,CAAC,GAAGrB,kBAAAA,mBAAAA,QAAS,IAAIY,KAAKS,GAAG,CAAC,GAAGxB,IAAIE,QAAQ;oBACjEI,KAAKS,KAAKQ,GAAG,CAACR,KAAKS,GAAG,CAACnB,MAAMD,iBAAAA,kBAAAA,OAAQJ,IAAIE,QAAQ,GAAGF,IAAIE,QAAQ;oBAChEK;oBACAC,cAAc,IAAIiB;oBACfhB,IAAIJ;;;yBAAMI,CAAAA,KAAKH,EAAC;;;;oBACV;;wBAAMN,IAAI0B,OAAO,CAACjB;;;oBAAzBtB,OAAO;oBACK;;wBAAMA,KAAKwC,cAAc;;;oBAAnCjB,QAAU,cAAVA;oBACFC,KAAKxB,KAAKyC,WAAW,CAAC;wBAAEC,OAAO;oBAAE;oBACjCjB,gBAAgBG,KAAKe,KAAK,CAACnB,GAAGK,SAAS,CAAC,EAAE,EAAEL,GAAGK,SAAS,CAAC,EAAE;oBAC3DH;oBACFzB,YAAY;oBACX,kCAAA,2BAAA;;wBAAL,IAAK,YAAYsB,4BAAZ,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbI,KAAN;;4BACH,IAAI,CAAE,CAAA,SAASA,EAAC,KAAM,OAAOA,GAAGiB,GAAG,KAAK,UAAU;4BAClD3C,aAAa0B,GAAGiB,GAAG,CAAClC,MAAM;4BAC1B,IAAI,CAACiB,GAAGiB,GAAG,CAACC,IAAI,IAAI;4BACdhB,YAAYiB,SAAI,CAACjB,SAAS,CAACL,GAAGK,SAAS,EAAEF,GAAGE,SAAS;4BACrDC,iBAAiBF,KAAKe,KAAK,CAACd,SAAS,CAAC,EAAE,EAAEA,SAAS,CAAC,EAAE;4BACtDtB,IAAIqB,KAAKmB,GAAG,CAACpB,GAAGqB,KAAK,IAAIvB;4BACzBjB,IAAIoB,KAAKqB,KAAK,CAACrB,KAAKe,KAAK,CAACd,SAAS,CAAC,EAAE,EAAEA,SAAS,CAAC,EAAE,IAAI,MAAM;4BAC9DE,YAAYD,iBAAiB,IAAI,AAACD,SAAS,CAAC,EAAE,GAAGC,iBAAkBvB,IAAIA;4BACvEyB,YAAYF,iBAAiB,IAAI,AAACD,SAAS,CAAC,EAAE,GAAGC,iBAAkBvB,IAAI;4BACvE0B;;oCACHJ,SAAS,CAAC,EAAE;oCAAEA,SAAS,CAAC,EAAE;;;oCAC1BA,SAAS,CAAC,EAAE,GAAGE;oCAAWF,SAAS,CAAC,EAAE,GAAGG;;;oCACzCH,SAAS,CAAC,EAAE,GAAGE,YAAYF,SAAS,CAAC,EAAE;oCAAEA,SAAS,CAAC,EAAE,GAAGG,YAAYH,SAAS,CAAC,EAAE;;;oCAChFA,SAAS,CAAC,EAAE,GAAGA,SAAS,CAAC,EAAE;oCAAEA,SAAS,CAAC,EAAE,GAAGA,SAAS,CAAC,EAAE;;;4BAErDK,UAAUD,QAAQiB,GAAG,CAAC;yEAAEC;uCAAOA;;4BAC/BhB,UAAUF,QAAQiB,GAAG,CAAC;yEAAIE;uCAAOA;;4BACvC1B,OAAO2B,IAAI,CAAC;gCACVC,GAAG3B,GAAGiB,GAAG;gCACTO,GAAGtB,SAAS,CAAC,EAAE;gCACfuB,GAAGvB,SAAS,CAAC,EAAE;gCACftB,GAAAA;gCACAC,GAAAA;gCACA+C,QAAQ;oCACNJ,GAAGvB,CAAAA,QAAAA,MAAKQ,GAAG,OAARR,OAAS,qBAAGM;oCACfkB,GAAGxB,CAAAA,SAAAA,MAAKQ,GAAG,OAARR,QAAS,qBAAGO;oCACf5B,GAAGqB,CAAAA,SAAAA,MAAKS,GAAG,OAART,QAAS,qBAAGM,YAAWN,CAAAA,SAAAA,MAAKQ,GAAG,OAARR,QAAS,qBAAGM;oCACtC1B,GAAGoB,CAAAA,SAAAA,MAAKS,GAAG,OAART,QAAS,qBAAGO,YAAWP,CAAAA,SAAAA,MAAKQ,GAAG,OAARR,QAAS,qBAAGO;gCACxC;4BACF;4BACAd,YAAYmC,GAAG,CAAChD,GAAG,AAACa,CAAAA,YAAYoC,GAAG,CAACjD,MAAM,CAAA,IAAKmB,GAAGiB,GAAG,CAAClC,MAAM;wBAC9D;;wBAhCK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAmC4B;;wBAAMX,oBAAoBC,MAAMC;;;oBAAhC,QAAA,eAAzBG,WAAyB,MAAzBA,UAAUC,aAAe,MAAfA;oBAClBe,MAAMiC,IAAI,CAAC;wBACTrD,MAAMsB;wBACNI,QAAAA;wBACAsB,OAAOxB,GAAGwB,KAAK;wBACfU,QAAQlC,GAAGkC,MAAM;wBACjBzD,WAAAA;wBACAG,UAAAA;wBACAC,YAAAA;oBACF;;;oBAnD0BiB;;;;;;oBAqD5B,mEAAmE;oBACnE,sIAAsI;oBACtI;;wBAAO;4BAAEP,UAAUK,MAAMV,MAAM;4BAAEU,OAAAA;4BAAOC,aAAAA;wBAAY;;;;IACtD"}
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/dict.ts"],"sourcesContent":["// The hyphen-rejoin dictionary, shared by src/emit.ts and src/markdown.ts.\n// Loaded once, lazily, on first use — never at import — so importing the library does no file IO on its own.\nimport { existsSync, readFileSync } from 'node:fs';\nimport wordListPath from 'word-list';\n\n// Bundled list (not the OS's /usr/share/dict/words) → identical rejoin decisions on every\n// OS/machine. PDF_TO_MD_DICT_PATH overrides it — the test seam for simulating no dictionary.\nexport const DICT_PATH = process.env.PDF_TO_MD_DICT_PATH || wordListPath;\n\nlet cache: { words: Set<string>; available: boolean } | null = null;\n\nfunction load(): { words: Set<string>; available: boolean } {\n if (!cache) {\n const available = existsSync(DICT_PATH);\n cache = { words: available ? new Set(readFileSync(DICT_PATH, 'utf8').toLowerCase().split('\\n')) : new Set(), available };\n }\n return cache;\n}\n\n/** The hyphen-rejoin dictionary; an empty set if none was found. */\nexport function getDict(): Set<string> {\n return load().words;\n}\n\n/** False when no dictionary file exists at DICT_PATH — every line-final\n * hyphen then stays unresolved. */\nexport function dictAvailable(): boolean {\n return load().available;\n}\n\n/** The notice callers surface via warn() when dictAvailable() is false. */\nexport function dictUnavailableWarning(): string {\n return `no dictionary at ${DICT_PATH} — line-final hyphens kept unresolved`;\n}\n"],"names":["DICT_PATH","dictAvailable","dictUnavailableWarning","getDict","process","env","PDF_TO_MD_DICT_PATH","wordListPath","cache","load","available","existsSync","words","Set","readFileSync","toLowerCase","split"],"mappings":"AAAA,2EAA2E;AAC3E,6GAA6G;;;;;;;;;;;;QAMhGA;eAAAA;;QAmBGC;eAAAA;;QAKAC;eAAAA;;QAXAC;eAAAA;;;sBAlByB;+DAChB;;;;;;AAIlB,IAAMH,YAAYI,QAAQC,GAAG,CAACC,mBAAmB,IAAIC,iBAAY;AAExE,IAAIC,QAA2D;AAE/D,SAASC;IACP,IAAI,CAACD,OAAO;QACV,IAAME,YAAYC,IAAAA,kBAAU,EAACX;QAC7BQ,QAAQ;YAAEI,OAAOF,YAAY,IAAIG,IAAIC,IAAAA,oBAAY,EAACd,WAAW,QAAQe,WAAW,GAAGC,KAAK,CAAC,SAAS,IAAIH;YAAOH,WAAAA;QAAU;IACzH;IACA,OAAOF;AACT;AAGO,SAASL;IACd,OAAOM,OAAOG,KAAK;AACrB;AAIO,SAASX;IACd,OAAOQ,OAAOC,SAAS;AACzB;AAGO,SAASR;IACd,OAAO,AAAC,oBAA6B,OAAVF,WAAU;AACvC"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/dict.ts"],"sourcesContent":["// The hyphen-rejoin dictionary, shared by src/emit.ts and src/markdown.ts.\n// Loaded once, lazily, on first use — never at import — so importing the library does no file IO on its own.\nimport { existsSync, readFileSync } from 'node:fs';\nimport wordListPath from 'word-list';\n\n// Bundled list (not the OS's /usr/share/dict/words) → identical rejoin decisions on every\n// OS/machine. PDF_TO_MD_DICT_PATH overrides it — the test seam for simulating no dictionary.\nexport const DICT_PATH = process.env.PDF_TO_MD_DICT_PATH || wordListPath;\n\nlet cache: { words: Set<string>; available: boolean } | null = null;\n\nfunction load(): { words: Set<string>; available: boolean } {\n if (!cache) {\n const available = existsSync(DICT_PATH);\n cache = { words: available ? new Set(readFileSync(DICT_PATH, 'utf8').toLowerCase().split('\\n')) : new Set(), available };\n }\n return cache;\n}\n\n/** The hyphen-rejoin dictionary; an empty set if none was found. */\nexport function getDict(): Set<string> {\n return load().words;\n}\n\n/** False when no dictionary file exists at DICT_PATH — every line-final\n * hyphen then stays unresolved. */\nexport function dictAvailable(): boolean {\n return load().available;\n}\n\n/** The notice callers surface via warn() when dictAvailable() is false. */\nexport function dictUnavailableWarning(): string {\n return `no dictionary at ${DICT_PATH} — line-final hyphens kept unresolved`;\n}\n"],"names":["DICT_PATH","dictAvailable","dictUnavailableWarning","getDict","process","env","PDF_TO_MD_DICT_PATH","wordListPath","cache","load","available","existsSync","words","Set","readFileSync","toLowerCase","split"],"mappings":"AAAA,2EAA2E;AAC3E,6GAA6G;;;;;;;;;;;;QAMhGA;eAAAA;;QAmBGC;eAAAA;;QAKAC;eAAAA;;QAXAC;eAAAA;;;sBAlByB;+DAChB;;;;;;AAIlB,IAAMH,YAAYI,QAAQC,GAAG,CAACC,mBAAmB,IAAIC,iBAAY;AAExE,IAAIC,QAA2D;AAE/D,SAASC;IACP,IAAI,CAACD,OAAO;QACV,IAAME,YAAYC,IAAAA,kBAAU,EAACX;QAC7BQ,QAAQ;YAAEI,OAAOF,YAAY,IAAIG,IAAIC,IAAAA,oBAAY,EAACd,WAAW,QAAQe,WAAW,GAAGC,KAAK,CAAC,SAAS,IAAIH;YAAOH,WAAAA;QAAU;IACzH;IACA,OAAOF;AACT;AAGO,SAASL;IACd,OAAOM,OAAOG,KAAK;AACrB;AAIO,SAASX;IACd,OAAOQ,OAAOC,SAAS;AACzB;AAGO,SAASR;IACd,OAAO,AAAC,oBAA6B,OAAVF,WAAU;AACvC"}
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/emit.ts"],"sourcesContent":["// emit.ts — OCR-path pass 3: lines -> markdown blocks.\nimport { getDict } from './dict.ts';\nimport type { OcrLine, PageOCR, Tuning, Word } from './geometry.ts';\nimport { bucket } from './geometry.ts';\n\n// Join a wrapped line onto the open paragraph, closing a line-final hyphen\n// only when the dictionary confirms the joined form is a word.\nfunction hyphenJoin(acc: string, next: string): string {\n if (acc.endsWith('-')) {\n const withoutDash = acc.slice(0, -1);\n const pen = withoutDash.at(-1);\n if (pen && /[A-Za-z0-9]/.test(pen)) {\n const head = withoutDash.split(' ').pop() ?? '';\n const firstTok = next.split(' ')[0] ?? '';\n const filtered = [...firstTok].filter((c) => /[A-Za-z-]/.test(c)).join('');\n const tail = filtered.split('-')[0] ?? '';\n const close = getDict().has((head + tail).toLowerCase());\n return (close ? withoutDash : acc) + next;\n }\n }\n return `${acc} ${next}`;\n}\n\n// Blocks with too few word-like tokens to trust as prose (see\n// UNSTRUCTURED_WORDLIKE_FRACTION below) are labeled `> [review: reason] text` instead of guessed at.\nconst REVIEW_REASON = {\n unstructuredLabels: 'unstructured labels',\n} as const;\n\n/** Share of word-like tokens (3+ letter run) below which a block counts as\n * unstructured. Measured (`review-blocks.test.ts`): junk 0.000-0.100 vs prose 0.743-0.857, cut 0.4. */\nexport const UNSTRUCTURED_WORDLIKE_FRACTION = 0.4;\nfunction isUnstructured(text: string): boolean {\n const tokens = text.split(/\\s+/).filter(Boolean);\n if (tokens.length === 0) return false; // never mark an empty block\n const wordLike = tokens.filter((t) => /[A-Za-z]{3}/.test(t)).length;\n return wordLike / tokens.length < UNSTRUCTURED_WORDLIKE_FRACTION;\n}\n\nfunction reviewLine(reason: string, text: string): string {\n return `> [review: ${reason}] ${text}`;\n}\n\n// One classified block, carrying constituent words for analyze()'s\n// block->word linkage; discriminated on `kind` so `reason`/`level` are compile-time guaranteed present on their variants.\nexport type EmitBlock =\n | { kind: 'heading'; text: string; level: number; words: Word[] }\n | { kind: 'paragraph'; text: string; words: Word[] }\n | { kind: 'quote'; text: string; words: Word[] }\n | {\n kind: 'review';\n text: string;\n /** The parseable reason inside `> [review: ...]`, carried on the block\n * so more reasons can be added without changing the render format. */\n reason: string;\n words: Word[];\n };\n\n// Renders one EmitBlock to the exact markdown emitPage used to push\n// directly — shared so extract.ts and analyze.ts produce byte-identical text.\nexport function renderEmitBlock(b: EmitBlock): string {\n switch (b.kind) {\n case 'heading':\n return `${'#'.repeat(b.level)} ${b.text}`;\n case 'quote':\n return `> ${b.text}`;\n case 'review':\n return reviewLine(b.reason, b.text);\n default:\n return b.text;\n }\n}\n\nexport function emitPage(p: PageOCR, levelOf: Map<number, number>, tuning: Tuning): EmitBlock[] {\n const blocks: EmitBlock[] = [];\n let para: OcrLine[] = [];\n const flush = () => {\n if (!para.length) return;\n const texts = para.map((l) => l.text);\n const merged = texts.slice(1).reduce(hyphenJoin, texts[0]);\n const words = para.flatMap((l) => l.words);\n // Same review-marker rule as the heading path below: no word may be\n // dropped or reordered here, only how the block renders changes.\n blocks.push(isUnstructured(merged) ? { kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words } : { kind: 'paragraph', text: merged, words });\n para = [];\n };\n\n let i = 0;\n while (i < p.lines.length) {\n const line: OcrLine = p.lines[i];\n const prev = i > 0 ? p.lines[i - 1] : null;\n if (prev && (line.col !== prev.col || prev.midY - line.midY > p.pitch * tuning.paraGap)) {\n // Join across a column break (instead of flushing) only when the column\n // changed, the paragraph dangles (no terminal punct), and the new text starts lowercase; a miss stays visible via pageStats.danglingLong.\n const danglingEnd = para.length > 0 && !/[.!?:;\"'”’)\\]]$/.test(para[para.length - 1].text);\n const continuationStart = /^[a-z]/.test(line.text);\n const joinAcrossColumns = line.col !== prev.col && danglingEnd && continuationStart && !levelOf.has(bucket(line.h));\n if (!joinAcrossColumns) flush();\n }\n\n if (levelOf.has(bucket(line.h))) {\n // Gather the run of consecutive heading-size lines: a wrapped title is\n // one heading, a long run is a pull quote in a bigger face.\n flush();\n const run: OcrLine[] = [line];\n let maxB = bucket(line.h);\n while (i + 1 < p.lines.length && levelOf.has(bucket(p.lines[i + 1].h)) && p.lines[i + 1].col === p.lines[i].col && p.lines[i].midY - p.lines[i + 1].midY < Math.max(p.lines[i].h, p.lines[i + 1].h) * 2.2) {\n i++;\n run.push(p.lines[i]);\n maxB = Math.max(maxB, bucket(p.lines[i].h));\n }\n const runTexts = run.map((l) => l.text);\n const merged = runTexts.slice(1).reduce(hyphenJoin, runTexts[0]);\n const words = run.flatMap((l) => l.words);\n // Below UNSTRUCTURED_WORDLIKE_FRACTION = chart labels/stray numerals,\n // not a heading; checked FIRST so a long numeral run isn't asserted as a pull quote.\n const junk = isUnstructured(merged);\n if (junk) blocks.push({ kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words });\n else if (merged.length > 150) blocks.push({ kind: 'quote', text: merged, words });\n else blocks.push({ kind: 'heading', text: merged, level: levelOf.get(maxB) as number, words });\n } else {\n para.push(line);\n }\n i++;\n }\n flush();\n return blocks;\n}\n"],"names":["UNSTRUCTURED_WORDLIKE_FRACTION","emitPage","renderEmitBlock","hyphenJoin","acc","next","endsWith","withoutDash","slice","pen","at","test","filtered","head","split","pop","firstTok","filter","c","join","tail","close","getDict","has","toLowerCase","REVIEW_REASON","unstructuredLabels","isUnstructured","text","tokens","Boolean","length","wordLike","t","reviewLine","reason","b","kind","repeat","level","p","levelOf","tuning","blocks","para","flush","texts","map","l","merged","reduce","words","flatMap","push","i","lines","line","prev","col","midY","pitch","paraGap","danglingEnd","continuationStart","joinAcrossColumns","bucket","h","run","maxB","Math","max","runTexts","junk","get"],"mappings":"AAAA,uDAAuD;;;;;;;;;;;;QA+B1CA;eAAAA;;QA0CGC;eAAAA;;QAbAC;eAAAA;;;sBA3DQ;0BAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvB,2EAA2E;AAC3E,+DAA+D;AAC/D,SAASC,WAAWC,GAAW,EAAEC,IAAY;IAC3C,IAAID,IAAIE,QAAQ,CAAC,MAAM;QACrB,IAAMC,cAAcH,IAAII,KAAK,CAAC,GAAG,CAAC;QAClC,IAAMC,MAAMF,YAAYG,EAAE,CAAC,CAAC;QAC5B,IAAID,OAAO,cAAcE,IAAI,CAACF,MAAM;gBACrBF,wBACIF,cAEJO;YAHb,IAAMC,QAAON,yBAAAA,YAAYO,KAAK,CAAC,KAAKC,GAAG,gBAA1BR,oCAAAA,yBAAgC;YAC7C,IAAMS,YAAWX,eAAAA,KAAKS,KAAK,CAAC,IAAI,CAAC,EAAE,cAAlBT,0BAAAA,eAAsB;YACvC,IAAMO,WAAW,AAAC,qBAAGI,UAAUC,MAAM,CAAC,SAACC;uBAAM,YAAYP,IAAI,CAACO;eAAIC,IAAI,CAAC;YACvE,IAAMC,QAAOR,mBAAAA,SAASE,KAAK,CAAC,IAAI,CAAC,EAAE,cAAtBF,8BAAAA,mBAA0B;YACvC,IAAMS,QAAQC,IAAAA,eAAO,IAAGC,GAAG,CAAC,AAACV,CAAAA,OAAOO,IAAG,EAAGI,WAAW;YACrD,OAAO,AAACH,CAAAA,QAAQd,cAAcH,GAAE,IAAKC;QACvC;IACF;IACA,OAAO,AAAC,GAASA,OAAPD,KAAI,KAAQ,OAALC;AACnB;AAEA,8DAA8D;AAC9D,qGAAqG;AACrG,IAAMoB,gBAAgB;IACpBC,oBAAoB;AACtB;AAIO,IAAM1B,iCAAiC;AAC9C,SAAS2B,eAAeC,IAAY;IAClC,IAAMC,SAASD,KAAKd,KAAK,CAAC,OAAOG,MAAM,CAACa;IACxC,IAAID,OAAOE,MAAM,KAAK,GAAG,OAAO,OAAO,4BAA4B;IACnE,IAAMC,WAAWH,OAAOZ,MAAM,CAAC,SAACgB;eAAM,cAActB,IAAI,CAACsB;OAAIF,MAAM;IACnE,OAAOC,WAAWH,OAAOE,MAAM,GAAG/B;AACpC;AAEA,SAASkC,WAAWC,MAAc,EAAEP,IAAY;IAC9C,OAAO,AAAC,cAAwBA,OAAXO,QAAO,MAAS,OAALP;AAClC;AAmBO,SAAS1B,gBAAgBkC,CAAY;IAC1C,OAAQA,EAAEC,IAAI;QACZ,KAAK;YACH,OAAO,AAAC,GAAyBD,OAAvB,IAAIE,MAAM,CAACF,EAAEG,KAAK,GAAE,KAAU,OAAPH,EAAER,IAAI;QACzC,KAAK;YACH,OAAO,AAAC,KAAW,OAAPQ,EAAER,IAAI;QACpB,KAAK;YACH,OAAOM,WAAWE,EAAED,MAAM,EAAEC,EAAER,IAAI;QACpC;YACE,OAAOQ,EAAER,IAAI;IACjB;AACF;AAEO,SAAS3B,SAASuC,CAAU,EAAEC,OAA4B,EAAEC,MAAc;IAC/E,IAAMC,SAAsB,EAAE;IAC9B,IAAIC,OAAkB,EAAE;IACxB,IAAMC,QAAQ;QACZ,IAAI,CAACD,KAAKb,MAAM,EAAE;QAClB,IAAMe,QAAQF,KAAKG,GAAG,CAAC,SAACC;mBAAMA,EAAEpB,IAAI;;QACpC,IAAMqB,SAASH,MAAMtC,KAAK,CAAC,GAAG0C,MAAM,CAAC/C,YAAY2C,KAAK,CAAC,EAAE;QACzD,IAAMK,QAAQP,KAAKQ,OAAO,CAAC,SAACJ;mBAAMA,EAAEG,KAAK;;QACzC,oEAAoE;QACpE,iEAAiE;QACjER,OAAOU,IAAI,CAAC1B,eAAesB,UAAU;YAAEZ,MAAM;YAAUT,MAAMqB;YAAQd,QAAQV,cAAcC,kBAAkB;YAAEyB,OAAAA;QAAM,IAAI;YAAEd,MAAM;YAAaT,MAAMqB;YAAQE,OAAAA;QAAM;QAClKP,OAAO,EAAE;IACX;IAEA,IAAIU,IAAI;IACR,MAAOA,IAAId,EAAEe,KAAK,CAACxB,MAAM,CAAE;QACzB,IAAMyB,OAAgBhB,EAAEe,KAAK,CAACD,EAAE;QAChC,IAAMG,OAAOH,IAAI,IAAId,EAAEe,KAAK,CAACD,IAAI,EAAE,GAAG;QACtC,IAAIG,QAASD,CAAAA,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAID,KAAKE,IAAI,GAAGH,KAAKG,IAAI,GAAGnB,EAAEoB,KAAK,GAAGlB,OAAOmB,OAAO,AAAD,GAAI;YACvF,wEAAwE;YACxE,0IAA0I;YAC1I,IAAMC,cAAclB,KAAKb,MAAM,GAAG,KAAK,CAAC,kBAAkBpB,IAAI,CAACiC,IAAI,CAACA,KAAKb,MAAM,GAAG,EAAE,CAACH,IAAI;YACzF,IAAMmC,oBAAoB,SAASpD,IAAI,CAAC6C,KAAK5B,IAAI;YACjD,IAAMoC,oBAAoBR,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAII,eAAeC,qBAAqB,CAACtB,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACT,KAAKU,CAAC;YACjH,IAAI,CAACF,mBAAmBnB;QAC1B;QAEA,IAAIJ,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACT,KAAKU,CAAC,IAAI;YAC/B,uEAAuE;YACvE,4DAA4D;YAC5DrB;YACA,IAAMsB,MAAiB;gBAACX;aAAK;YAC7B,IAAIY,OAAOH,IAAAA,kBAAM,EAACT,KAAKU,CAAC;YACxB,MAAOZ,IAAI,IAAId,EAAEe,KAAK,CAACxB,MAAM,IAAIU,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACzB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACY,CAAC,MAAM1B,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACI,GAAG,KAAKlB,EAAEe,KAAK,CAACD,EAAE,CAACI,GAAG,IAAIlB,EAAEe,KAAK,CAACD,EAAE,CAACK,IAAI,GAAGnB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACK,IAAI,GAAGU,KAAKC,GAAG,CAAC9B,EAAEe,KAAK,CAACD,EAAE,CAACY,CAAC,EAAE1B,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACY,CAAC,IAAI,IAAK;gBACzMZ;gBACAa,IAAId,IAAI,CAACb,EAAEe,KAAK,CAACD,EAAE;gBACnBc,OAAOC,KAAKC,GAAG,CAACF,MAAMH,IAAAA,kBAAM,EAACzB,EAAEe,KAAK,CAACD,EAAE,CAACY,CAAC;YAC3C;YACA,IAAMK,WAAWJ,IAAIpB,GAAG,CAAC,SAACC;uBAAMA,EAAEpB,IAAI;;YACtC,IAAMqB,SAASsB,SAAS/D,KAAK,CAAC,GAAG0C,MAAM,CAAC/C,YAAYoE,QAAQ,CAAC,EAAE;YAC/D,IAAMpB,QAAQgB,IAAIf,OAAO,CAAC,SAACJ;uBAAMA,EAAEG,KAAK;;YACxC,sEAAsE;YACtE,qFAAqF;YACrF,IAAMqB,OAAO7C,eAAesB;YAC5B,IAAIuB,MAAM7B,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAUT,MAAMqB;gBAAQd,QAAQV,cAAcC,kBAAkB;gBAAEyB,OAAAA;YAAM;iBACjG,IAAIF,OAAOlB,MAAM,GAAG,KAAKY,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAST,MAAMqB;gBAAQE,OAAAA;YAAM;iBAC1ER,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAWT,MAAMqB;gBAAQV,OAAOE,QAAQgC,GAAG,CAACL;gBAAiBjB,OAAAA;YAAM;QAC9F,OAAO;YACLP,KAAKS,IAAI,CAACG;QACZ;QACAF;IACF;IACAT;IACA,OAAOF;AACT"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/emit.ts"],"sourcesContent":["// emit.ts — OCR-path pass 3: lines -> markdown blocks.\nimport { getDict } from './dict.ts';\nimport type { OcrLine, PageOCR, Tuning, Word } from './geometry.ts';\nimport { bucket } from './geometry.ts';\n\n// Join a wrapped line onto the open paragraph, closing a line-final hyphen\n// only when the dictionary confirms the joined form is a word.\nfunction hyphenJoin(acc: string, next: string): string {\n if (acc.endsWith('-')) {\n const withoutDash = acc.slice(0, -1);\n const pen = withoutDash.at(-1);\n if (pen && /[A-Za-z0-9]/.test(pen)) {\n const head = withoutDash.split(' ').pop() ?? '';\n const firstTok = next.split(' ')[0] ?? '';\n const filtered = [...firstTok].filter((c) => /[A-Za-z-]/.test(c)).join('');\n const tail = filtered.split('-')[0] ?? '';\n const close = getDict().has((head + tail).toLowerCase());\n return (close ? withoutDash : acc) + next;\n }\n }\n return `${acc} ${next}`;\n}\n\n// Blocks with too few word-like tokens to trust as prose (see\n// UNSTRUCTURED_WORDLIKE_FRACTION below) are labeled `> [review: reason] text` instead of guessed at.\nconst REVIEW_REASON = {\n unstructuredLabels: 'unstructured labels',\n} as const;\n\n/** Share of word-like tokens (3+ letter run) below which a block counts as\n * unstructured. Measured (`review-blocks.test.ts`): junk 0.000-0.100 vs prose 0.743-0.857, cut 0.4. */\nexport const UNSTRUCTURED_WORDLIKE_FRACTION = 0.4;\nfunction isUnstructured(text: string): boolean {\n const tokens = text.split(/\\s+/).filter(Boolean);\n if (tokens.length === 0) return false; // never mark an empty block\n const wordLike = tokens.filter((t) => /[A-Za-z]{3}/.test(t)).length;\n return wordLike / tokens.length < UNSTRUCTURED_WORDLIKE_FRACTION;\n}\n\nfunction reviewLine(reason: string, text: string): string {\n return `> [review: ${reason}] ${text}`;\n}\n\n// One classified block, carrying constituent words for analyze()'s\n// block->word linkage; discriminated on `kind` so `reason`/`level` are compile-time guaranteed present on their variants.\nexport type EmitBlock =\n | { kind: 'heading'; text: string; level: number; words: Word[] }\n | { kind: 'paragraph'; text: string; words: Word[] }\n | { kind: 'quote'; text: string; words: Word[] }\n | {\n kind: 'review';\n text: string;\n /** The parseable reason inside `> [review: ...]`, carried on the block\n * so more reasons can be added without changing the render format. */\n reason: string;\n words: Word[];\n };\n\n// Renders one EmitBlock to the exact markdown emitPage used to push\n// directly — shared so extract.ts and analyze.ts produce byte-identical text.\nexport function renderEmitBlock(b: EmitBlock): string {\n switch (b.kind) {\n case 'heading':\n return `${'#'.repeat(b.level)} ${b.text}`;\n case 'quote':\n return `> ${b.text}`;\n case 'review':\n return reviewLine(b.reason, b.text);\n default:\n return b.text;\n }\n}\n\nexport function emitPage(p: PageOCR, levelOf: Map<number, number>, tuning: Tuning): EmitBlock[] {\n const blocks: EmitBlock[] = [];\n let para: OcrLine[] = [];\n const flush = () => {\n if (!para.length) return;\n const texts = para.map((l) => l.text);\n const merged = texts.slice(1).reduce(hyphenJoin, texts[0]);\n const words = para.flatMap((l) => l.words);\n // Same review-marker rule as the heading path below: no word may be\n // dropped or reordered here, only how the block renders changes.\n blocks.push(isUnstructured(merged) ? { kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words } : { kind: 'paragraph', text: merged, words });\n para = [];\n };\n\n let i = 0;\n while (i < p.lines.length) {\n const line: OcrLine = p.lines[i];\n const prev = i > 0 ? p.lines[i - 1] : null;\n if (prev && (line.col !== prev.col || prev.midY - line.midY > p.pitch * tuning.paraGap)) {\n // Join across a column break (instead of flushing) only when the column\n // changed, the paragraph dangles (no terminal punct), and the new text starts lowercase; a miss stays visible via pageStats.danglingLong.\n const danglingEnd = para.length > 0 && !/[.!?:;\"'”’)\\]]$/.test(para[para.length - 1].text);\n const continuationStart = /^[a-z]/.test(line.text);\n const joinAcrossColumns = line.col !== prev.col && danglingEnd && continuationStart && !levelOf.has(bucket(line.h));\n if (!joinAcrossColumns) flush();\n }\n\n if (levelOf.has(bucket(line.h))) {\n // Gather the run of consecutive heading-size lines: a wrapped title is\n // one heading, a long run is a pull quote in a bigger face.\n flush();\n const run: OcrLine[] = [line];\n let maxB = bucket(line.h);\n while (i + 1 < p.lines.length && levelOf.has(bucket(p.lines[i + 1].h)) && p.lines[i + 1].col === p.lines[i].col && p.lines[i].midY - p.lines[i + 1].midY < Math.max(p.lines[i].h, p.lines[i + 1].h) * 2.2) {\n i++;\n run.push(p.lines[i]);\n maxB = Math.max(maxB, bucket(p.lines[i].h));\n }\n const runTexts = run.map((l) => l.text);\n const merged = runTexts.slice(1).reduce(hyphenJoin, runTexts[0]);\n const words = run.flatMap((l) => l.words);\n // Below UNSTRUCTURED_WORDLIKE_FRACTION = chart labels/stray numerals,\n // not a heading; checked FIRST so a long numeral run isn't asserted as a pull quote.\n const junk = isUnstructured(merged);\n if (junk) blocks.push({ kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words });\n else if (merged.length > 150) blocks.push({ kind: 'quote', text: merged, words });\n else blocks.push({ kind: 'heading', text: merged, level: levelOf.get(maxB) as number, words });\n } else {\n para.push(line);\n }\n i++;\n }\n flush();\n return blocks;\n}\n"],"names":["UNSTRUCTURED_WORDLIKE_FRACTION","emitPage","renderEmitBlock","hyphenJoin","acc","next","endsWith","withoutDash","slice","pen","at","test","filtered","head","split","pop","firstTok","filter","c","join","tail","close","getDict","has","toLowerCase","REVIEW_REASON","unstructuredLabels","isUnstructured","text","tokens","Boolean","length","wordLike","t","reviewLine","reason","b","kind","repeat","level","p","levelOf","tuning","blocks","para","flush","texts","map","l","merged","reduce","words","flatMap","push","i","lines","line","prev","col","midY","pitch","paraGap","danglingEnd","continuationStart","joinAcrossColumns","bucket","h","run","maxB","Math","max","runTexts","junk","get"],"mappings":"AAAA,uDAAuD;;;;;;;;;;;;QA+B1CA;eAAAA;;QA0CGC;eAAAA;;QAbAC;eAAAA;;;sBA3DQ;0BAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvB,2EAA2E;AAC3E,+DAA+D;AAC/D,SAASC,WAAWC,GAAW,EAAEC,IAAY;IAC3C,IAAID,IAAIE,QAAQ,CAAC,MAAM;QACrB,IAAMC,cAAcH,IAAII,KAAK,CAAC,GAAG,CAAC;QAClC,IAAMC,MAAMF,YAAYG,EAAE,CAAC,CAAC;QAC5B,IAAID,OAAO,cAAcE,IAAI,CAACF,MAAM;gBACrBF,wBACIF,cAEJO;YAHb,IAAMC,QAAON,yBAAAA,YAAYO,KAAK,CAAC,KAAKC,GAAG,gBAA1BR,oCAAAA,yBAAgC;YAC7C,IAAMS,YAAWX,eAAAA,KAAKS,KAAK,CAAC,IAAI,CAAC,EAAE,cAAlBT,0BAAAA,eAAsB;YACvC,IAAMO,WAAW,AAAC,qBAAGI,UAAUC,MAAM,CAAC,SAACC;uBAAM,YAAYP,IAAI,CAACO;eAAIC,IAAI,CAAC;YACvE,IAAMC,QAAOR,mBAAAA,SAASE,KAAK,CAAC,IAAI,CAAC,EAAE,cAAtBF,8BAAAA,mBAA0B;YACvC,IAAMS,QAAQC,IAAAA,eAAO,IAAGC,GAAG,CAAC,AAACV,CAAAA,OAAOO,IAAG,EAAGI,WAAW;YACrD,OAAO,AAACH,CAAAA,QAAQd,cAAcH,GAAE,IAAKC;QACvC;IACF;IACA,OAAO,AAAC,GAASA,OAAPD,KAAI,KAAQ,OAALC;AACnB;AAEA,8DAA8D;AAC9D,qGAAqG;AACrG,IAAMoB,gBAAgB;IACpBC,oBAAoB;AACtB;AAIO,IAAM1B,iCAAiC;AAC9C,SAAS2B,eAAeC,IAAY;IAClC,IAAMC,SAASD,KAAKd,KAAK,CAAC,OAAOG,MAAM,CAACa;IACxC,IAAID,OAAOE,MAAM,KAAK,GAAG,OAAO,OAAO,4BAA4B;IACnE,IAAMC,WAAWH,OAAOZ,MAAM,CAAC,SAACgB;eAAM,cAActB,IAAI,CAACsB;OAAIF,MAAM;IACnE,OAAOC,WAAWH,OAAOE,MAAM,GAAG/B;AACpC;AAEA,SAASkC,WAAWC,MAAc,EAAEP,IAAY;IAC9C,OAAO,AAAC,cAAwBA,OAAXO,QAAO,MAAS,OAALP;AAClC;AAmBO,SAAS1B,gBAAgBkC,CAAY;IAC1C,OAAQA,EAAEC,IAAI;QACZ,KAAK;YACH,OAAO,AAAC,GAAyBD,OAAvB,IAAIE,MAAM,CAACF,EAAEG,KAAK,GAAE,KAAU,OAAPH,EAAER,IAAI;QACzC,KAAK;YACH,OAAO,AAAC,KAAW,OAAPQ,EAAER,IAAI;QACpB,KAAK;YACH,OAAOM,WAAWE,EAAED,MAAM,EAAEC,EAAER,IAAI;QACpC;YACE,OAAOQ,EAAER,IAAI;IACjB;AACF;AAEO,SAAS3B,SAASuC,CAAU,EAAEC,OAA4B,EAAEC,MAAc;IAC/E,IAAMC,SAAsB,EAAE;IAC9B,IAAIC,OAAkB,EAAE;IACxB,IAAMC,QAAQ;QACZ,IAAI,CAACD,KAAKb,MAAM,EAAE;QAClB,IAAMe,QAAQF,KAAKG,GAAG,CAAC,SAACC;mBAAMA,EAAEpB,IAAI;;QACpC,IAAMqB,SAASH,MAAMtC,KAAK,CAAC,GAAG0C,MAAM,CAAC/C,YAAY2C,KAAK,CAAC,EAAE;QACzD,IAAMK,QAAQP,KAAKQ,OAAO,CAAC,SAACJ;mBAAMA,EAAEG,KAAK;;QACzC,oEAAoE;QACpE,iEAAiE;QACjER,OAAOU,IAAI,CAAC1B,eAAesB,UAAU;YAAEZ,MAAM;YAAUT,MAAMqB;YAAQd,QAAQV,cAAcC,kBAAkB;YAAEyB,OAAAA;QAAM,IAAI;YAAEd,MAAM;YAAaT,MAAMqB;YAAQE,OAAAA;QAAM;QAClKP,OAAO,EAAE;IACX;IAEA,IAAIU,IAAI;IACR,MAAOA,IAAId,EAAEe,KAAK,CAACxB,MAAM,CAAE;QACzB,IAAMyB,OAAgBhB,EAAEe,KAAK,CAACD,EAAE;QAChC,IAAMG,OAAOH,IAAI,IAAId,EAAEe,KAAK,CAACD,IAAI,EAAE,GAAG;QACtC,IAAIG,QAASD,CAAAA,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAID,KAAKE,IAAI,GAAGH,KAAKG,IAAI,GAAGnB,EAAEoB,KAAK,GAAGlB,OAAOmB,OAAO,AAAD,GAAI;YACvF,wEAAwE;YACxE,0IAA0I;YAC1I,IAAMC,cAAclB,KAAKb,MAAM,GAAG,KAAK,CAAC,kBAAkBpB,IAAI,CAACiC,IAAI,CAACA,KAAKb,MAAM,GAAG,EAAE,CAACH,IAAI;YACzF,IAAMmC,oBAAoB,SAASpD,IAAI,CAAC6C,KAAK5B,IAAI;YACjD,IAAMoC,oBAAoBR,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAII,eAAeC,qBAAqB,CAACtB,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACT,KAAKU,CAAC;YACjH,IAAI,CAACF,mBAAmBnB;QAC1B;QAEA,IAAIJ,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACT,KAAKU,CAAC,IAAI;YAC/B,uEAAuE;YACvE,4DAA4D;YAC5DrB;YACA,IAAMsB,MAAiB;gBAACX;aAAK;YAC7B,IAAIY,OAAOH,IAAAA,kBAAM,EAACT,KAAKU,CAAC;YACxB,MAAOZ,IAAI,IAAId,EAAEe,KAAK,CAACxB,MAAM,IAAIU,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACzB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACY,CAAC,MAAM1B,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACI,GAAG,KAAKlB,EAAEe,KAAK,CAACD,EAAE,CAACI,GAAG,IAAIlB,EAAEe,KAAK,CAACD,EAAE,CAACK,IAAI,GAAGnB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACK,IAAI,GAAGU,KAAKC,GAAG,CAAC9B,EAAEe,KAAK,CAACD,EAAE,CAACY,CAAC,EAAE1B,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACY,CAAC,IAAI,IAAK;gBACzMZ;gBACAa,IAAId,IAAI,CAACb,EAAEe,KAAK,CAACD,EAAE;gBACnBc,OAAOC,KAAKC,GAAG,CAACF,MAAMH,IAAAA,kBAAM,EAACzB,EAAEe,KAAK,CAACD,EAAE,CAACY,CAAC;YAC3C;YACA,IAAMK,WAAWJ,IAAIpB,GAAG,CAAC,SAACC;uBAAMA,EAAEpB,IAAI;;YACtC,IAAMqB,SAASsB,SAAS/D,KAAK,CAAC,GAAG0C,MAAM,CAAC/C,YAAYoE,QAAQ,CAAC,EAAE;YAC/D,IAAMpB,QAAQgB,IAAIf,OAAO,CAAC,SAACJ;uBAAMA,EAAEG,KAAK;;YACxC,sEAAsE;YACtE,qFAAqF;YACrF,IAAMqB,OAAO7C,eAAesB;YAC5B,IAAIuB,MAAM7B,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAUT,MAAMqB;gBAAQd,QAAQV,cAAcC,kBAAkB;gBAAEyB,OAAAA;YAAM;iBACjG,IAAIF,OAAOlB,MAAM,GAAG,KAAKY,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAST,MAAMqB;gBAAQE,OAAAA;YAAM;iBAC1ER,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAWT,MAAMqB;gBAAQV,OAAOE,QAAQgC,GAAG,CAACL;gBAAiBjB,OAAAA;YAAM;QAC9F,OAAO;YACLP,KAAKS,IAAI,CAACG;QACZ;QACAF;IACF;IACAT;IACA,OAAOF;AACT"}
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/engines/tesseract.ts"],"sourcesContent":["// engines/tesseract.ts — THE OCR engine: tesseract.js on every platform.\n//\n// Seam contract: words come back in normalized 0-1 page coordinates, origin\n// bottom-left, y increasing upward — the y-flip happens here, and nowhere else.\n//\n// Pure WASM, in-process, no native dependency: no crash containment needed.\nimport { mkdirSync } from 'node:fs';\nimport os from 'node:os';\nimport path from 'node:path';\nimport type { Page as TesseractPage } from 'tesseract.js';\nimport { createWorker, PSM } from 'tesseract.js';\nimport type { PageWord, Word } from '../geometry.ts';\nimport { foldTypographic } from '../lines.ts';\nimport { awaitPdfOpen, openPdfForRender } from '../pdf-open.ts';\nimport { renderLoadedPage } from '../raster.ts';\nimport { resolvePackagePath } from '../resolve.ts';\n\n// 288 dpi: the render resolution the engine's word-overlap numbers were\n// measured at (test/fixtures/ocr-engine-poc/) — the known-good operating point.\nexport const OCR_RENDER_DPI = 288;\n\n// Chart pages starve PSM.AUTO (unrecognized text can't be marked for review).\n// Prose-fraction retry trigger, measured (sparse-retry.test.ts): chart 0.442 vs prose 0.767-0.938, cut 0.6.\nexport const SPARSE_RETRY_PROSE_FRACTION = 0.6;\n\n/** Share of `words` whose text contains a run of 3+ letters (`src/emit.ts`'s isUnstructured canary, applied per-page instead of per-block). */\nfunction proseFraction(words: { text: string }[]): number {\n if (words.length === 0) return 0;\n return words.filter((w) => /[A-Za-z]{3}/.test(w.text)).length / words.length;\n}\n\n/** Does this page's PSM.AUTO pass look chart-like enough to re-recognize\n * with PSM.SPARSE_TEXT? A page with 0 words never retries (nothing to take a prose fraction of). */\nexport function shouldRetrySparse(words: { text: string }[]): boolean {\n return words.length > 0 && proseFraction(words) < SPARSE_RETRY_PROSE_FRACTION;\n}\n\n// The locally installed traineddata (npm package @tesseract.js-data/eng),\n// resolved dynamically so depcheck won't flag it \"unused\" — it's a genuine runtime dependency, listed in .depcheckrc's `ignores`.\nconst TESSDATA_DIR = path.join(path.dirname(resolvePackagePath('@tesseract.js-data/eng/package.json')), '4.0.0');\n\n/** Per-page recognition progress, delivered once per page as it finishes.\n * Engine chatter goes through `onWarning`/`warnings` instead — this is timing/count data only. */\nexport type OcrProgressEvent = { type: 'page'; page: number; words: number; ms: number };\n\nexport interface RecognizeResult {\n words: PageWord[];\n /** pages (1-based) where the engine returned zero words */\n emptyPages: number[];\n /** wall-clock ms per page (render + recognize), keyed by page number */\n msPerPage: Map<number, number>;\n}\n\n/** tesseract.js's nested blocks/paragraphs/lines/words -> this repo's flat Word contract (normalize + y-flip). Same shape regardless of which PSM produced `page`, so both the AUTO pass and the SPARSE_TEXT retry share it. */\nfunction wordsFromPage(page: TesseractPage, width: number, height: number): Word[] {\n const pageWords: Word[] = [];\n for (const block of page.blocks ?? []) {\n for (const para of block.paragraphs ?? []) {\n for (const line of para.lines ?? []) {\n for (const w of line.words ?? []) {\n const { x0, y0, x1, y1 } = w.bbox;\n pageWords.push({\n // Same typographic folding as the text path (src/lines.ts) — both\n // paths must agree on how a printed apostrophe is spelled.\n text: foldTypographic(w.text),\n x: x0 / width,\n y: (height - y1) / height, // y-flip: top-left px -> bottom-left normalized\n w: (x1 - x0) / width,\n h: (y1 - y0) / height,\n // Plumbed through unmodified — tesseract.js's own 0-100\n // per-word confidence. See src/report.ts's LOW_CONFIDENCE_THRESHOLD.\n confidence: w.confidence,\n });\n }\n }\n }\n }\n return pageWords;\n}\n\n// tesseract.js-core's WASM build writes native diagnostics straight to this\n// process's stdout/stderr, with no override to silence them at the source.\n//\n// Suppression must span the whole run: worker-thread stdio arrives\n// asynchronously, on later event-loop turns; the synchronous carve-out below (`withRealStdio`) is the only safe exit.\nlet suppressDepth = 0;\nlet savedStdoutWrite: typeof process.stdout.write | null = null;\nlet savedStderrWrite: typeof process.stderr.write | null = null;\nasync function withSuppressedStdio<T>(fn: () => Promise<T>): Promise<T> {\n if (suppressDepth === 0) {\n savedStdoutWrite = process.stdout.write.bind(process.stdout);\n savedStderrWrite = process.stderr.write.bind(process.stderr);\n process.stdout.write = (() => true) as typeof process.stdout.write;\n process.stderr.write = (() => true) as typeof process.stderr.write;\n }\n suppressDepth++;\n try {\n return await fn();\n } finally {\n suppressDepth--;\n if (suppressDepth === 0) {\n if (savedStdoutWrite) process.stdout.write = savedStdoutWrite;\n if (savedStderrWrite) process.stderr.write = savedStderrWrite;\n savedStdoutWrite = null;\n savedStderrWrite = null;\n }\n }\n}\n\n/** Pierces an active `withSuppressedStdio` region SYNCHRONOUSLY to let this\n * module's own onProgress/onWarning calls through. No-op outside any suppressed region. */\nfunction withRealStdio<T>(fn: () => T): T {\n if (suppressDepth === 0) return fn();\n const realOut = savedStdoutWrite;\n const realErr = savedStderrWrite;\n if (realOut) process.stdout.write = realOut;\n if (realErr) process.stderr.write = realErr;\n try {\n return fn();\n } finally {\n process.stdout.write = (() => true) as typeof process.stdout.write;\n process.stderr.write = (() => true) as typeof process.stderr.write;\n }\n}\n\n/** Recognize words on pages [first..last] of a PDF (renders at `dpi`,\n * default OCR_RENDER_DPI). Progress via `onProgress`; diagnostics via `onWarning` — this module never writes to stdout/stderr itself. */\nexport async function recognizePdf(pdfPath: string, first: number, last: number, { dpi = OCR_RENDER_DPI, onProgress, onWarning }: { dpi?: number; onProgress?: (e: OcrProgressEvent) => void; onWarning?: (message: string) => void } = {}): Promise<RecognizeResult> {\n // tesseract.js writes its decompressed traineddata cache here by default,\n // dropping a model file into the user's cwd — redirect it under tmpdir() instead.\n const cachePath = path.join(os.tmpdir(), 'pdf-to-md-engine');\n mkdirSync(cachePath, { recursive: true });\n\n const words: PageWord[] = [];\n const emptyPages: number[] = [];\n const msPerPage = new Map<number, number>();\n\n // ONE continuous suppressed region for the whole run; onProgress/onWarning\n // below go through `withRealStdio`, the only sanctioned way out of it.\n await withSuppressedStdio(async () => {\n // OEM 1 (LSTM only, tesseract.js's own default): measured 99.0% recall\n // on the hard benchmark pages (2026-08-16).\n const worker = await createWorker('eng', 1, {\n langPath: TESSDATA_DIR,\n cachePath,\n });\n try {\n // NOT optional: tesseract.js's own default PSM (SINGLE_BLOCK) measured\n // 3-8 percentage points worse than AUTO on two-column pages.\n await worker.setParameters({ tessedit_pageseg_mode: PSM.AUTO });\n\n const loadingTask = openPdfForRender(pdfPath);\n try {\n const pdf = await awaitPdfOpen(loadingTask, pdfPath);\n for (let pn = first; pn <= Math.min(last, pdf.numPages); pn++) {\n const t0 = Date.now();\n const { png, width, height } = await renderLoadedPage(pdf, pn, dpi);\n const { data } = await worker.recognize(png, {}, { blocks: true });\n let pageWords: Word[] = wordsFromPage(data, width, height);\n\n // Adaptive second pass: re-recognizes the SAME rendered PNG, so\n // this costs one extra recognize() call only on chart-like pages.\n if (shouldRetrySparse(pageWords)) {\n const autoFraction = proseFraction(pageWords);\n const autoWords = pageWords;\n // setParameters on the shared worker is reversible, so one\n // in-process worker can alternate PSM per page instead of paying for a second worker.\n await worker.setParameters({ tessedit_pageseg_mode: PSM.SPARSE_TEXT });\n const sparse = await worker.recognize(png, {}, { blocks: true });\n const sparseWords = wordsFromPage(sparse.data, width, height);\n await worker.setParameters({ tessedit_pageseg_mode: PSM.AUTO });\n\n if (sparseWords.length > autoWords.length) {\n pageWords = sparseWords;\n const autoCount = autoWords.length;\n const sparseCount = sparseWords.length;\n withRealStdio(() => onWarning?.(`p${pn}: chart-like page (prose fraction ${autoFraction.toFixed(2)}) — retried with sparse-text segmentation, ${autoCount} -> ${sparseCount} words`));\n }\n }\n\n const ms = Date.now() - t0;\n for (const w of pageWords) words.push({ page: pn, ...w });\n if (pageWords.length === 0) emptyPages.push(pn);\n msPerPage.set(pn, ms);\n const wordCount = pageWords.length;\n withRealStdio(() => onProgress?.({ type: 'page', page: pn, words: wordCount, ms }));\n }\n } finally {\n await loadingTask.destroy();\n }\n } finally {\n await worker.terminate();\n }\n });\n\n return { words, emptyPages, msPerPage };\n}\n"],"names":["OCR_RENDER_DPI","SPARSE_RETRY_PROSE_FRACTION","recognizePdf","shouldRetrySparse","proseFraction","words","length","filter","w","test","text","TESSDATA_DIR","path","join","dirname","resolvePackagePath","wordsFromPage","page","width","height","pageWords","blocks","block","paragraphs","para","lines","line","bbox","x0","y0","x1","y1","push","foldTypographic","x","y","h","confidence","suppressDepth","savedStdoutWrite","savedStderrWrite","withSuppressedStdio","fn","process","stdout","write","bind","stderr","withRealStdio","realOut","realErr","pdfPath","first","last","dpi","onProgress","onWarning","cachePath","emptyPages","msPerPage","os","tmpdir","mkdirSync","recursive","Map","worker","loadingTask","pdf","pn","createWorker","langPath","setParameters","tessedit_pageseg_mode","PSM","AUTO","openPdfForRender","t0","png","data","autoFraction","autoWords","sparse","sparseWords","autoCount","sparseCount","ms","wordCount","Date","now","renderLoadedPage","recognize","SPARSE_TEXT","toFixed","set","type","awaitPdfOpen","Math","min","numPages","destroy","terminate"],"mappings":"AAAA,yEAAyE;AACzE,EAAE;AACF,4EAA4E;AAC5E,gFAAgF;AAChF,EAAE;AACF,4EAA4E;;;;;;;;;;;;QAc/DA;eAAAA;;QAIAC;eAAAA;;QAwGSC;eAAAA;;QA9FNC;eAAAA;;;sBA3BU;6DACX;+DACE;yBAEiB;uBAEF;yBACe;wBACd;yBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAI5B,IAAMH,iBAAiB;AAIvB,IAAMC,8BAA8B;AAE3C,6IAA6I,GAC7I,SAASG,cAAcC,KAAyB;IAC9C,IAAIA,MAAMC,MAAM,KAAK,GAAG,OAAO;IAC/B,OAAOD,MAAME,MAAM,CAAC,SAACC;eAAM,cAAcC,IAAI,CAACD,EAAEE,IAAI;OAAGJ,MAAM,GAAGD,MAAMC,MAAM;AAC9E;AAIO,SAASH,kBAAkBE,KAAyB;IACzD,OAAOA,MAAMC,MAAM,GAAG,KAAKF,cAAcC,SAASJ;AACpD;AAEA,0EAA0E;AAC1E,kIAAkI;AAClI,IAAMU,eAAeC,iBAAI,CAACC,IAAI,CAACD,iBAAI,CAACE,OAAO,CAACC,IAAAA,6BAAkB,EAAC,yCAAyC;AAcxG,8NAA8N,GAC9N,SAASC,cAAcC,IAAmB,EAAEC,KAAa,EAAEC,MAAc;QAEnDF;IADpB,IAAMG,YAAoB,EAAE;QACvB,kCAAA,2BAAA;;QAAL,QAAK,cAAeH,eAAAA,KAAKI,MAAM,cAAXJ,0BAAAA,eAAe,EAAE,sBAAhC,SAAA,6BAAA,QAAA,yBAAA,iCAAkC;YAAlC,IAAMK,QAAN;gBACgBA;gBAAd,mCAAA,4BAAA;;gBAAL,QAAK,eAAcA,oBAAAA,MAAMC,UAAU,cAAhBD,+BAAAA,oBAAoB,EAAE,sBAApC,UAAA,8BAAA,SAAA,0BAAA,kCAAsC;oBAAtC,IAAME,OAAN;wBACgBA;wBAAd,mCAAA,4BAAA;;wBAAL,QAAK,eAAcA,cAAAA,KAAKC,KAAK,cAAVD,yBAAAA,cAAc,EAAE,sBAA9B,UAAA,8BAAA,SAAA,0BAAA,kCAAgC;4BAAhC,IAAME,OAAN;gCACaA;gCAAX,mCAAA,4BAAA;;gCAAL,QAAK,eAAWA,cAAAA,KAAKrB,KAAK,cAAVqB,yBAAAA,cAAc,EAAE,sBAA3B,UAAA,8BAAA,SAAA,0BAAA,kCAA6B;oCAA7B,IAAMlB,IAAN;oCACH,IAA2BA,UAAAA,EAAEmB,IAAI,EAAzBC,KAAmBpB,QAAnBoB,IAAIC,KAAerB,QAAfqB,IAAIC,KAAWtB,QAAXsB,IAAIC,KAAOvB,QAAPuB;oCACpBX,UAAUY,IAAI,CAAC;wCACb,kEAAkE;wCAClE,2DAA2D;wCAC3DtB,MAAMuB,IAAAA,wBAAe,EAACzB,EAAEE,IAAI;wCAC5BwB,GAAGN,KAAKV;wCACRiB,GAAG,AAAChB,CAAAA,SAASY,EAAC,IAAKZ;wCACnBX,GAAG,AAACsB,CAAAA,KAAKF,EAAC,IAAKV;wCACfkB,GAAG,AAACL,CAAAA,KAAKF,EAAC,IAAKV;wCACf,wDAAwD;wCACxD,qEAAqE;wCACrEkB,YAAY7B,EAAE6B,UAAU;oCAC1B;gCACF;;gCAdK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAeP;;wBAhBK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBAiBP;;gBAlBK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAmBP;;QApBK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAqBL,OAAOjB;AACT;AAEA,4EAA4E;AAC5E,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,sHAAsH;AACtH,IAAIkB,gBAAgB;AACpB,IAAIC,mBAAuD;AAC3D,IAAIC,mBAAuD;AAC3D,SAAeC,oBAAuBC,EAAoB;;;;;oBACxD,IAAIJ,kBAAkB,GAAG;wBACvBC,mBAAmBI,QAAQC,MAAM,CAACC,KAAK,CAACC,IAAI,CAACH,QAAQC,MAAM;wBAC3DJ,mBAAmBG,QAAQI,MAAM,CAACF,KAAK,CAACC,IAAI,CAACH,QAAQI,MAAM;wBAC3DJ,QAAQC,MAAM,CAACC,KAAK,GAAI;mCAAM;;wBAC9BF,QAAQI,MAAM,CAACF,KAAK,GAAI;mCAAM;;oBAChC;oBACAP;;;;;;;;;oBAES;;wBAAMI;;;oBAAb;;wBAAO;;;oBAEPJ;oBACA,IAAIA,kBAAkB,GAAG;wBACvB,IAAIC,kBAAkBI,QAAQC,MAAM,CAACC,KAAK,GAAGN;wBAC7C,IAAIC,kBAAkBG,QAAQI,MAAM,CAACF,KAAK,GAAGL;wBAC7CD,mBAAmB;wBACnBC,mBAAmB;oBACrB;;;;;;;;;;IAEJ;;AAEA;yFACyF,GACzF,SAASQ,cAAiBN,EAAW;IACnC,IAAIJ,kBAAkB,GAAG,OAAOI;IAChC,IAAMO,UAAUV;IAChB,IAAMW,UAAUV;IAChB,IAAIS,SAASN,QAAQC,MAAM,CAACC,KAAK,GAAGI;IACpC,IAAIC,SAASP,QAAQI,MAAM,CAACF,KAAK,GAAGK;IACpC,IAAI;QACF,OAAOR;IACT,SAAU;QACRC,QAAQC,MAAM,CAACC,KAAK,GAAI;mBAAM;;QAC9BF,QAAQI,MAAM,CAACF,KAAK,GAAI;mBAAM;;IAChC;AACF;AAIO,SAAe3C;wCAAaiD,OAAe,EAAEC,KAAa,EAAEC,IAAY;YAAE,gBAAEC,KAAsBC,YAAYC,WAG7GC,WAGApD,OACAqD,YACAC;;;;;oBARyE,OAAA,oEAAuJ,CAAC,cAAxJ,KAAEL,KAAAA,4BAAMtD,2BAAgBuD,aAAxB,KAAwBA,YAAYC,YAApC,KAAoCA;oBACnH,0EAA0E;oBAC1E,kFAAkF;oBAC5EC,YAAY7C,iBAAI,CAACC,IAAI,CAAC+C,eAAE,CAACC,MAAM,IAAI;oBACzCC,IAAAA,iBAAS,EAACL,WAAW;wBAAEM,WAAW;oBAAK;oBAEjC1D;oBACAqD;oBACAC,YAAY,IAAIK;oBAEtB,2EAA2E;oBAC3E,uEAAuE;oBACvE;;wBAAMvB,oBAAoB;;oCAGlBwB,QASEC,oBAEEC,KACGC;;;;4CAZE;;gDAAMC,IAAAA,uBAAY,EAAC,OAAO,GAAG;oDAC1CC,UAAU3D;oDACV8C,WAAAA;gDACF;;;4CAHMQ,SAAS;;;;;;;;;4CAKb,uEAAuE;4CACvE,6DAA6D;4CAC7D;;gDAAMA,OAAOM,aAAa,CAAC;oDAAEC,uBAAuBC,cAAG,CAACC,IAAI;gDAAC;;;4CAA7D;4CAEMR,cAAcS,IAAAA,2BAAgB,EAACxB;;;;;;;;;;oDAI3ByB,IACyB,MAAvBC,KAAK3D,OAAOC,QACZ2D,MACJ1D,WAKI2D,cACAC,WAIAC,QACAC,aAKEC,WACAC,aAKJC,IACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM7E,GAGL8E;;;;4DA7BAV,KAAKW,KAAKC,GAAG;4DACY;;gEAAMC,IAAAA,0BAAgB,EAACtB,KAAKC,IAAId;;;4DAAhC,OAAA,eAAvBuB,MAAuB,KAAvBA,KAAK3D,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;4DACH;;gEAAM8C,OAAOyB,SAAS,CAACb,KAAK,CAAC,GAAG;oEAAExD,QAAQ;gEAAK;;;4DAAxDyD,OAAS,cAATA;4DACJ1D,YAAoBJ,cAAc8D,MAAM5D,OAAOC;iEAI/ChB,kBAAkBiB,YAAlBjB;;;;4DACI4E,eAAe3E,cAAcgB;4DAC7B4D,YAAY5D;4DAClB,2DAA2D;4DAC3D,sFAAsF;4DACtF;;gEAAM6C,OAAOM,aAAa,CAAC;oEAAEC,uBAAuBC,cAAG,CAACkB,WAAW;gEAAC;;;4DAApE;4DACe;;gEAAM1B,OAAOyB,SAAS,CAACb,KAAK,CAAC,GAAG;oEAAExD,QAAQ;gEAAK;;;4DAAxD4D,SAAS;4DACTC,cAAclE,cAAciE,OAAOH,IAAI,EAAE5D,OAAOC;4DACtD;;gEAAM8C,OAAOM,aAAa,CAAC;oEAAEC,uBAAuBC,cAAG,CAACC,IAAI;gEAAC;;;4DAA7D;4DAEA,IAAIQ,YAAY5E,MAAM,GAAG0E,UAAU1E,MAAM,EAAE;gEACzCc,YAAY8D;gEACNC,YAAYH,UAAU1E,MAAM;gEAC5B8E,cAAcF,YAAY5E,MAAM;gEACtC0C,cAAc;2EAAMQ,sBAAAA,gCAAAA,UAAY,AAAC,IAA0CuB,OAAvCX,IAAG,sCAAyGe,OAArEJ,aAAaa,OAAO,CAAC,IAAG,+CAA6DR,OAAhBD,WAAU,QAAkB,OAAZC,aAAY;;4DAC9K;;;4DAGIC,KAAKE,KAAKC,GAAG,KAAKZ;4DACnB,kCAAA,2BAAA;;gEAAL,IAAK,YAAWxD,gCAAX,6BAAA,QAAA,yBAAA;oEAAMZ,IAAN;oEAAsBH,MAAM2B,IAAI,CAAC;wEAAEf,MAAMmD;uEAAO5D;;;gEAAhD;gEAAA;;;yEAAA,6BAAA;wEAAA;;;wEAAA;8EAAA;;;;4DACL,IAAIY,UAAUd,MAAM,KAAK,GAAGoD,WAAW1B,IAAI,CAACoC;4DAC5CT,UAAUkC,GAAG,CAACzB,IAAIiB;4DACZC,YAAYlE,UAAUd,MAAM;4DAClC0C,cAAc;uEAAMO,uBAAAA,iCAAAA,WAAa;oEAAEuC,MAAM;oEAAQ7E,MAAMmD;oEAAI/D,OAAOiF;oEAAWD,IAAAA;gEAAG;;;;;;;4CAClF;4CAjCY;;gDAAMU,IAAAA,uBAAY,EAAC7B,aAAaf;;;4CAAtCgB,MAAM;4CACHC,KAAKhB;;;iDAAOgB,CAAAA,MAAM4B,KAAKC,GAAG,CAAC5C,MAAMc,IAAI+B,QAAQ,CAAA;;;;;;;;;;;;4CAAG9B;;;;;;;;;;;4CAkCzD;;gDAAMF,YAAYiC,OAAO;;;4CAAzB;;;;;;;;;;4CAGF;;gDAAMlC,OAAOmC,SAAS;;;4CAAtB;;;;;;;;;;4BAEJ;;;;oBAtDA;oBAwDA;;wBAAO;4BAAE/F,OAAAA;4BAAOqD,YAAAA;4BAAYC,WAAAA;wBAAU;;;;IACxC"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/engines/tesseract.ts"],"sourcesContent":["// engines/tesseract.ts — THE OCR engine: tesseract.js on every platform.\n//\n// Seam contract: words come back in normalized 0-1 page coordinates, origin\n// bottom-left, y increasing upward — the y-flip happens here, and nowhere else.\n//\n// Pure WASM, in-process, no native dependency: no crash containment needed.\nimport { mkdirSync } from 'node:fs';\nimport os from 'node:os';\nimport path from 'node:path';\nimport type { Page as TesseractPage } from 'tesseract.js';\nimport { createWorker, PSM } from 'tesseract.js';\nimport type { PageWord, Word } from '../geometry.ts';\nimport { foldTypographic } from '../lines.ts';\nimport { awaitPdfOpen, openPdfForRender } from '../pdf-open.ts';\nimport { renderLoadedPage } from '../raster.ts';\nimport { resolvePackagePath } from '../resolve.ts';\n\n// 288 dpi: the render resolution the engine's word-overlap numbers were\n// measured at (test/fixtures/ocr-engine-poc/) — the known-good operating point.\nexport const OCR_RENDER_DPI = 288;\n\n// Chart pages starve PSM.AUTO (unrecognized text can't be marked for review).\n// Prose-fraction retry trigger, measured (sparse-retry.test.ts): chart 0.442 vs prose 0.767-0.938, cut 0.6.\nexport const SPARSE_RETRY_PROSE_FRACTION = 0.6;\n\n/** Share of `words` whose text contains a run of 3+ letters (`src/emit.ts`'s isUnstructured canary, applied per-page instead of per-block). */\nfunction proseFraction(words: { text: string }[]): number {\n if (words.length === 0) return 0;\n return words.filter((w) => /[A-Za-z]{3}/.test(w.text)).length / words.length;\n}\n\n/** Does this page's PSM.AUTO pass look chart-like enough to re-recognize\n * with PSM.SPARSE_TEXT? A page with 0 words never retries (nothing to take a prose fraction of). */\nexport function shouldRetrySparse(words: { text: string }[]): boolean {\n return words.length > 0 && proseFraction(words) < SPARSE_RETRY_PROSE_FRACTION;\n}\n\n// The locally installed traineddata (npm package @tesseract.js-data/eng),\n// resolved dynamically so depcheck won't flag it \"unused\" — it's a genuine runtime dependency, listed in .depcheckrc's `ignores`.\nconst TESSDATA_DIR = path.join(path.dirname(resolvePackagePath('@tesseract.js-data/eng/package.json')), '4.0.0');\n\n/** Per-page recognition progress, delivered once per page as it finishes.\n * Engine chatter goes through `onWarning`/`warnings` instead — this is timing/count data only. */\nexport type OcrProgressEvent = { type: 'page'; page: number; words: number; ms: number };\n\nexport interface RecognizeResult {\n words: PageWord[];\n /** pages (1-based) where the engine returned zero words */\n emptyPages: number[];\n /** wall-clock ms per page (render + recognize), keyed by page number */\n msPerPage: Map<number, number>;\n}\n\n/** tesseract.js's nested blocks/paragraphs/lines/words -> this repo's flat Word contract (normalize + y-flip). Same shape regardless of which PSM produced `page`, so both the AUTO pass and the SPARSE_TEXT retry share it. */\nfunction wordsFromPage(page: TesseractPage, width: number, height: number): Word[] {\n const pageWords: Word[] = [];\n for (const block of page.blocks ?? []) {\n for (const para of block.paragraphs ?? []) {\n for (const line of para.lines ?? []) {\n for (const w of line.words ?? []) {\n const { x0, y0, x1, y1 } = w.bbox;\n pageWords.push({\n // Same typographic folding as the text path (src/lines.ts) — both\n // paths must agree on how a printed apostrophe is spelled.\n text: foldTypographic(w.text),\n x: x0 / width,\n y: (height - y1) / height, // y-flip: top-left px -> bottom-left normalized\n w: (x1 - x0) / width,\n h: (y1 - y0) / height,\n // Plumbed through unmodified — tesseract.js's own 0-100\n // per-word confidence. See src/report.ts's LOW_CONFIDENCE_THRESHOLD.\n confidence: w.confidence,\n });\n }\n }\n }\n }\n return pageWords;\n}\n\n// tesseract.js-core's WASM build writes native diagnostics straight to this\n// process's stdout/stderr, with no override to silence them at the source.\n//\n// Suppression must span the whole run: worker-thread stdio arrives\n// asynchronously, on later event-loop turns; the synchronous carve-out below (`withRealStdio`) is the only safe exit.\nlet suppressDepth = 0;\nlet savedStdoutWrite: typeof process.stdout.write | null = null;\nlet savedStderrWrite: typeof process.stderr.write | null = null;\nasync function withSuppressedStdio<T>(fn: () => Promise<T>): Promise<T> {\n if (suppressDepth === 0) {\n savedStdoutWrite = process.stdout.write.bind(process.stdout);\n savedStderrWrite = process.stderr.write.bind(process.stderr);\n process.stdout.write = (() => true) as typeof process.stdout.write;\n process.stderr.write = (() => true) as typeof process.stderr.write;\n }\n suppressDepth++;\n try {\n return await fn();\n } finally {\n suppressDepth--;\n if (suppressDepth === 0) {\n if (savedStdoutWrite) process.stdout.write = savedStdoutWrite;\n if (savedStderrWrite) process.stderr.write = savedStderrWrite;\n savedStdoutWrite = null;\n savedStderrWrite = null;\n }\n }\n}\n\n/** Pierces an active `withSuppressedStdio` region SYNCHRONOUSLY to let this\n * module's own onProgress/onWarning calls through. No-op outside any suppressed region. */\nfunction withRealStdio<T>(fn: () => T): T {\n if (suppressDepth === 0) return fn();\n const realOut = savedStdoutWrite;\n const realErr = savedStderrWrite;\n if (realOut) process.stdout.write = realOut;\n if (realErr) process.stderr.write = realErr;\n try {\n return fn();\n } finally {\n process.stdout.write = (() => true) as typeof process.stdout.write;\n process.stderr.write = (() => true) as typeof process.stderr.write;\n }\n}\n\n/** Recognize words on pages [first..last] of a PDF (renders at `dpi`,\n * default OCR_RENDER_DPI). Progress via `onProgress`; diagnostics via `onWarning` — this module never writes to stdout/stderr itself. */\nexport async function recognizePdf(pdfPath: string, first: number, last: number, { dpi = OCR_RENDER_DPI, onProgress, onWarning }: { dpi?: number; onProgress?: (e: OcrProgressEvent) => void; onWarning?: (message: string) => void } = {}): Promise<RecognizeResult> {\n // tesseract.js writes its decompressed traineddata cache here by default,\n // dropping a model file into the user's cwd — redirect it under tmpdir() instead.\n const cachePath = path.join(os.tmpdir(), 'pdf-to-md-engine');\n mkdirSync(cachePath, { recursive: true });\n\n const words: PageWord[] = [];\n const emptyPages: number[] = [];\n const msPerPage = new Map<number, number>();\n\n // ONE continuous suppressed region for the whole run; onProgress/onWarning\n // below go through `withRealStdio`, the only sanctioned way out of it.\n await withSuppressedStdio(async () => {\n // OEM 1 (LSTM only, tesseract.js's own default): measured 99.0% recall\n // on the hard benchmark pages (2026-08-16).\n const worker = await createWorker('eng', 1, {\n langPath: TESSDATA_DIR,\n cachePath,\n });\n try {\n // NOT optional: tesseract.js's own default PSM (SINGLE_BLOCK) measured\n // 3-8 percentage points worse than AUTO on two-column pages.\n await worker.setParameters({ tessedit_pageseg_mode: PSM.AUTO });\n\n const loadingTask = openPdfForRender(pdfPath);\n try {\n const pdf = await awaitPdfOpen(loadingTask, pdfPath);\n for (let pn = first; pn <= Math.min(last, pdf.numPages); pn++) {\n const t0 = Date.now();\n const { png, width, height } = await renderLoadedPage(pdf, pn, dpi);\n const { data } = await worker.recognize(png, {}, { blocks: true });\n let pageWords: Word[] = wordsFromPage(data, width, height);\n\n // Adaptive second pass: re-recognizes the SAME rendered PNG, so\n // this costs one extra recognize() call only on chart-like pages.\n if (shouldRetrySparse(pageWords)) {\n const autoFraction = proseFraction(pageWords);\n const autoWords = pageWords;\n // setParameters on the shared worker is reversible, so one\n // in-process worker can alternate PSM per page instead of paying for a second worker.\n await worker.setParameters({ tessedit_pageseg_mode: PSM.SPARSE_TEXT });\n const sparse = await worker.recognize(png, {}, { blocks: true });\n const sparseWords = wordsFromPage(sparse.data, width, height);\n await worker.setParameters({ tessedit_pageseg_mode: PSM.AUTO });\n\n if (sparseWords.length > autoWords.length) {\n pageWords = sparseWords;\n const autoCount = autoWords.length;\n const sparseCount = sparseWords.length;\n withRealStdio(() => onWarning?.(`p${pn}: chart-like page (prose fraction ${autoFraction.toFixed(2)}) — retried with sparse-text segmentation, ${autoCount} -> ${sparseCount} words`));\n }\n }\n\n const ms = Date.now() - t0;\n for (const w of pageWords) words.push({ page: pn, ...w });\n if (pageWords.length === 0) emptyPages.push(pn);\n msPerPage.set(pn, ms);\n const wordCount = pageWords.length;\n withRealStdio(() => onProgress?.({ type: 'page', page: pn, words: wordCount, ms }));\n }\n } finally {\n await loadingTask.destroy();\n }\n } finally {\n await worker.terminate();\n }\n });\n\n return { words, emptyPages, msPerPage };\n}\n"],"names":["OCR_RENDER_DPI","SPARSE_RETRY_PROSE_FRACTION","recognizePdf","shouldRetrySparse","proseFraction","words","length","filter","w","test","text","TESSDATA_DIR","path","join","dirname","resolvePackagePath","wordsFromPage","page","width","height","pageWords","blocks","block","paragraphs","para","lines","line","bbox","x0","y0","x1","y1","push","foldTypographic","x","y","h","confidence","suppressDepth","savedStdoutWrite","savedStderrWrite","withSuppressedStdio","fn","process","stdout","write","bind","stderr","withRealStdio","realOut","realErr","pdfPath","first","last","dpi","onProgress","onWarning","cachePath","emptyPages","msPerPage","os","tmpdir","mkdirSync","recursive","Map","worker","loadingTask","pdf","pn","createWorker","langPath","setParameters","tessedit_pageseg_mode","PSM","AUTO","openPdfForRender","t0","png","data","autoFraction","autoWords","sparse","sparseWords","autoCount","sparseCount","ms","wordCount","Date","now","renderLoadedPage","recognize","SPARSE_TEXT","toFixed","set","type","awaitPdfOpen","Math","min","numPages","destroy","terminate"],"mappings":"AAAA,yEAAyE;AACzE,EAAE;AACF,4EAA4E;AAC5E,gFAAgF;AAChF,EAAE;AACF,4EAA4E;;;;;;;;;;;;QAc/DA;eAAAA;;QAIAC;eAAAA;;QAwGSC;eAAAA;;QA9FNC;eAAAA;;;sBA3BU;6DACX;+DACE;yBAEiB;uBAEF;yBACe;wBACd;yBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAI5B,IAAMH,iBAAiB;AAIvB,IAAMC,8BAA8B;AAE3C,6IAA6I,GAC7I,SAASG,cAAcC,KAAyB;IAC9C,IAAIA,MAAMC,MAAM,KAAK,GAAG,OAAO;IAC/B,OAAOD,MAAME,MAAM,CAAC,SAACC;eAAM,cAAcC,IAAI,CAACD,EAAEE,IAAI;OAAGJ,MAAM,GAAGD,MAAMC,MAAM;AAC9E;AAIO,SAASH,kBAAkBE,KAAyB;IACzD,OAAOA,MAAMC,MAAM,GAAG,KAAKF,cAAcC,SAASJ;AACpD;AAEA,0EAA0E;AAC1E,kIAAkI;AAClI,IAAMU,eAAeC,iBAAI,CAACC,IAAI,CAACD,iBAAI,CAACE,OAAO,CAACC,IAAAA,6BAAkB,EAAC,yCAAyC;AAcxG,8NAA8N,GAC9N,SAASC,cAAcC,IAAmB,EAAEC,KAAa,EAAEC,MAAc;QAEnDF;IADpB,IAAMG,YAAoB,EAAE;QACvB,kCAAA,2BAAA;;QAAL,QAAK,cAAeH,eAAAA,KAAKI,MAAM,cAAXJ,0BAAAA,eAAe,EAAE,sBAAhC,SAAA,6BAAA,QAAA,yBAAA,iCAAkC;YAAlC,IAAMK,QAAN;gBACgBA;gBAAd,mCAAA,4BAAA;;gBAAL,QAAK,eAAcA,oBAAAA,MAAMC,UAAU,cAAhBD,+BAAAA,oBAAoB,EAAE,sBAApC,UAAA,8BAAA,SAAA,0BAAA,kCAAsC;oBAAtC,IAAME,OAAN;wBACgBA;wBAAd,mCAAA,4BAAA;;wBAAL,QAAK,eAAcA,cAAAA,KAAKC,KAAK,cAAVD,yBAAAA,cAAc,EAAE,sBAA9B,UAAA,8BAAA,SAAA,0BAAA,kCAAgC;4BAAhC,IAAME,OAAN;gCACaA;gCAAX,mCAAA,4BAAA;;gCAAL,QAAK,eAAWA,cAAAA,KAAKrB,KAAK,cAAVqB,yBAAAA,cAAc,EAAE,sBAA3B,UAAA,8BAAA,SAAA,0BAAA,kCAA6B;oCAA7B,IAAMlB,IAAN;oCACH,IAA2BA,UAAAA,EAAEmB,IAAI,EAAzBC,KAAmBpB,QAAnBoB,IAAIC,KAAerB,QAAfqB,IAAIC,KAAWtB,QAAXsB,IAAIC,KAAOvB,QAAPuB;oCACpBX,UAAUY,IAAI,CAAC;wCACb,kEAAkE;wCAClE,2DAA2D;wCAC3DtB,MAAMuB,IAAAA,wBAAe,EAACzB,EAAEE,IAAI;wCAC5BwB,GAAGN,KAAKV;wCACRiB,GAAG,AAAChB,CAAAA,SAASY,EAAC,IAAKZ;wCACnBX,GAAG,AAACsB,CAAAA,KAAKF,EAAC,IAAKV;wCACfkB,GAAG,AAACL,CAAAA,KAAKF,EAAC,IAAKV;wCACf,wDAAwD;wCACxD,qEAAqE;wCACrEkB,YAAY7B,EAAE6B,UAAU;oCAC1B;gCACF;;gCAdK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBAeP;;wBAhBK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBAiBP;;gBAlBK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAmBP;;QApBK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAqBL,OAAOjB;AACT;AAEA,4EAA4E;AAC5E,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,sHAAsH;AACtH,IAAIkB,gBAAgB;AACpB,IAAIC,mBAAuD;AAC3D,IAAIC,mBAAuD;AAC3D,SAAeC,oBAAuBC,EAAoB;;;;;oBACxD,IAAIJ,kBAAkB,GAAG;wBACvBC,mBAAmBI,QAAQC,MAAM,CAACC,KAAK,CAACC,IAAI,CAACH,QAAQC,MAAM;wBAC3DJ,mBAAmBG,QAAQI,MAAM,CAACF,KAAK,CAACC,IAAI,CAACH,QAAQI,MAAM;wBAC3DJ,QAAQC,MAAM,CAACC,KAAK,GAAI;mCAAM;;wBAC9BF,QAAQI,MAAM,CAACF,KAAK,GAAI;mCAAM;;oBAChC;oBACAP;;;;;;;;;oBAES;;wBAAMI;;;oBAAb;;wBAAO;;;oBAEPJ;oBACA,IAAIA,kBAAkB,GAAG;wBACvB,IAAIC,kBAAkBI,QAAQC,MAAM,CAACC,KAAK,GAAGN;wBAC7C,IAAIC,kBAAkBG,QAAQI,MAAM,CAACF,KAAK,GAAGL;wBAC7CD,mBAAmB;wBACnBC,mBAAmB;oBACrB;;;;;;;;;;IAEJ;;AAEA;yFACyF,GACzF,SAASQ,cAAiBN,EAAW;IACnC,IAAIJ,kBAAkB,GAAG,OAAOI;IAChC,IAAMO,UAAUV;IAChB,IAAMW,UAAUV;IAChB,IAAIS,SAASN,QAAQC,MAAM,CAACC,KAAK,GAAGI;IACpC,IAAIC,SAASP,QAAQI,MAAM,CAACF,KAAK,GAAGK;IACpC,IAAI;QACF,OAAOR;IACT,SAAU;QACRC,QAAQC,MAAM,CAACC,KAAK,GAAI;mBAAM;;QAC9BF,QAAQI,MAAM,CAACF,KAAK,GAAI;mBAAM;;IAChC;AACF;AAIO,SAAe3C;wCAAaiD,OAAe,EAAEC,KAAa,EAAEC,IAAY;YAAE,gBAAEC,KAAsBC,YAAYC,WAG7GC,WAGApD,OACAqD,YACAC;;;;;oBARyE,OAAA,oEAAuJ,CAAC,cAAxJ,KAAEL,KAAAA,4BAAMtD,2BAAgBuD,aAAxB,KAAwBA,YAAYC,YAApC,KAAoCA;oBACnH,0EAA0E;oBAC1E,kFAAkF;oBAC5EC,YAAY7C,iBAAI,CAACC,IAAI,CAAC+C,eAAE,CAACC,MAAM,IAAI;oBACzCC,IAAAA,iBAAS,EAACL,WAAW;wBAAEM,WAAW;oBAAK;oBAEjC1D;oBACAqD;oBACAC,YAAY,IAAIK;oBAEtB,2EAA2E;oBAC3E,uEAAuE;oBACvE;;wBAAMvB,oBAAoB;;oCAGlBwB,QASEC,oBAEEC,KACGC;;;;4CAZE;;gDAAMC,IAAAA,uBAAY,EAAC,OAAO,GAAG;oDAC1CC,UAAU3D;oDACV8C,WAAAA;gDACF;;;4CAHMQ,SAAS;;;;;;;;;4CAKb,uEAAuE;4CACvE,6DAA6D;4CAC7D;;gDAAMA,OAAOM,aAAa,CAAC;oDAAEC,uBAAuBC,cAAG,CAACC,IAAI;gDAAC;;;4CAA7D;4CAEMR,cAAcS,IAAAA,2BAAgB,EAACxB;;;;;;;;;;oDAI3ByB,IACyB,MAAvBC,KAAK3D,OAAOC,QACZ2D,MACJ1D,WAKI2D,cACAC,WAIAC,QACAC,aAKEC,WACAC,aAKJC,IACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM7E,GAGL8E;;;;4DA7BAV,KAAKW,KAAKC,GAAG;4DACY;;gEAAMC,IAAAA,0BAAgB,EAACtB,KAAKC,IAAId;;;4DAAhC,OAAA,eAAvBuB,MAAuB,KAAvBA,KAAK3D,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;4DACH;;gEAAM8C,OAAOyB,SAAS,CAACb,KAAK,CAAC,GAAG;oEAAExD,QAAQ;gEAAK;;;4DAAxDyD,OAAS,cAATA;4DACJ1D,YAAoBJ,cAAc8D,MAAM5D,OAAOC;iEAI/ChB,kBAAkBiB,YAAlBjB;;;;4DACI4E,eAAe3E,cAAcgB;4DAC7B4D,YAAY5D;4DAClB,2DAA2D;4DAC3D,sFAAsF;4DACtF;;gEAAM6C,OAAOM,aAAa,CAAC;oEAAEC,uBAAuBC,cAAG,CAACkB,WAAW;gEAAC;;;4DAApE;4DACe;;gEAAM1B,OAAOyB,SAAS,CAACb,KAAK,CAAC,GAAG;oEAAExD,QAAQ;gEAAK;;;4DAAxD4D,SAAS;4DACTC,cAAclE,cAAciE,OAAOH,IAAI,EAAE5D,OAAOC;4DACtD;;gEAAM8C,OAAOM,aAAa,CAAC;oEAAEC,uBAAuBC,cAAG,CAACC,IAAI;gEAAC;;;4DAA7D;4DAEA,IAAIQ,YAAY5E,MAAM,GAAG0E,UAAU1E,MAAM,EAAE;gEACzCc,YAAY8D;gEACNC,YAAYH,UAAU1E,MAAM;gEAC5B8E,cAAcF,YAAY5E,MAAM;gEACtC0C,cAAc;2EAAMQ,sBAAAA,gCAAAA,UAAY,AAAC,IAA0CuB,OAAvCX,IAAG,sCAAyGe,OAArEJ,aAAaa,OAAO,CAAC,IAAG,+CAA6DR,OAAhBD,WAAU,QAAkB,OAAZC,aAAY;;4DAC9K;;;4DAGIC,KAAKE,KAAKC,GAAG,KAAKZ;4DACnB,kCAAA,2BAAA;;gEAAL,IAAK,YAAWxD,gCAAX,6BAAA,QAAA,yBAAA;oEAAMZ,IAAN;oEAAsBH,MAAM2B,IAAI,CAAC;wEAAEf,MAAMmD;uEAAO5D;;;gEAAhD;gEAAA;;;yEAAA,6BAAA;wEAAA;;;wEAAA;8EAAA;;;;4DACL,IAAIY,UAAUd,MAAM,KAAK,GAAGoD,WAAW1B,IAAI,CAACoC;4DAC5CT,UAAUkC,GAAG,CAACzB,IAAIiB;4DACZC,YAAYlE,UAAUd,MAAM;4DAClC0C,cAAc;uEAAMO,uBAAAA,iCAAAA,WAAa;oEAAEuC,MAAM;oEAAQ7E,MAAMmD;oEAAI/D,OAAOiF;oEAAWD,IAAAA;gEAAG;;;;;;;4CAClF;4CAjCY;;gDAAMU,IAAAA,uBAAY,EAAC7B,aAAaf;;;4CAAtCgB,MAAM;4CACHC,KAAKhB;;;iDAAOgB,CAAAA,MAAM4B,KAAKC,GAAG,CAAC5C,MAAMc,IAAI+B,QAAQ,CAAA;;;;;;;;;;;;4CAAG9B;;;;;;;;;;;4CAkCzD;;gDAAMF,YAAYiC,OAAO;;;4CAAzB;;;;;;;;;;4CAGF;;gDAAMlC,OAAOmC,SAAS;;;4CAAtB;;;;;;;;;;4BAEJ;;;;oBAtDA;oBAwDA;;wBAAO;4BAAE/F,OAAAA;4BAAOqD,YAAAA;4BAAYC,WAAAA;wBAAU;;;;IACxC"}
@@ -11,6 +11,8 @@ export type PdfToMdErrorCode =
11
11
  | 'OCR_PAGE_FAILED'
12
12
  /** malformed word-dump input (--words-json / extractOcr words) */
13
13
  | 'WORDS_INPUT'
14
+ /** malformed or insufficient evidence passed to auditWords */
15
+ | 'AUDIT_INPUT'
14
16
  /** malformed Analysis handed to toMarkdown/toText (e.g. an out-of-range
15
17
  * wordIndexes entry) — validated fail-fast, whole object checked before any output. */
16
18
  | 'ANALYSIS_INPUT';
@@ -11,6 +11,8 @@ export type PdfToMdErrorCode =
11
11
  | 'OCR_PAGE_FAILED'
12
12
  /** malformed word-dump input (--words-json / extractOcr words) */
13
13
  | 'WORDS_INPUT'
14
+ /** malformed or insufficient evidence passed to auditWords */
15
+ | 'AUDIT_INPUT'
14
16
  /** malformed Analysis handed to toMarkdown/toText (e.g. an out-of-range
15
17
  * wordIndexes entry) — validated fail-fast, whole object checked before any output. */
16
18
  | 'ANALYSIS_INPUT';
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/errors.ts"],"sourcesContent":["// Library code throws; only src/cli.ts prints and exits.\n\n/** The closed set of error codes every `PdfToMdError` carries — switch on\n * `code` instead of parsing `message`. */\nexport type PdfToMdErrorCode =\n /** input missing/unreadable/not a PDF */\n | 'PDF_OPEN'\n /** malformed page selection (validated before any work) */\n | 'PAGE_RANGE'\n /** rendering an opened, in-range page to PNG failed */\n | 'PAGE_RENDER'\n /** the OCR engine crashed twice on one page (message names it) */\n | 'OCR_PAGE_FAILED'\n /** malformed word-dump input (--words-json / extractOcr words) */\n | 'WORDS_INPUT'\n /** malformed Analysis handed to toMarkdown/toText (e.g. an out-of-range\n * wordIndexes entry) — validated fail-fast, whole object checked before any output. */\n | 'ANALYSIS_INPUT';\n\n/** The error every entry point throws — never a bare `Error`. */\nexport class PdfToMdError extends Error {\n code: PdfToMdErrorCode;\n\n constructor(code: PdfToMdErrorCode, message: string) {\n super(message);\n this.name = 'PdfToMdError';\n this.code = code;\n }\n}\n"],"names":["PdfToMdError","code","message","name","Error"],"mappings":"AAAA,yDAAyD;AAEzD;wCACwC;;;;+BAiB3BA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,6BAAN;;cAAMA;aAAAA,aAGCC,IAAsB,EAAEC,OAAe;gCAHxCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAqBI"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/errors.ts"],"sourcesContent":["// Library code throws; only src/cli.ts prints and exits.\n\n/** The closed set of error codes every `PdfToMdError` carries — switch on\n * `code` instead of parsing `message`. */\nexport type PdfToMdErrorCode =\n /** input missing/unreadable/not a PDF */\n | 'PDF_OPEN'\n /** malformed page selection (validated before any work) */\n | 'PAGE_RANGE'\n /** rendering an opened, in-range page to PNG failed */\n | 'PAGE_RENDER'\n /** the OCR engine crashed twice on one page (message names it) */\n | 'OCR_PAGE_FAILED'\n /** malformed word-dump input (--words-json / extractOcr words) */\n | 'WORDS_INPUT'\n /** malformed or insufficient evidence passed to auditWords */\n | 'AUDIT_INPUT'\n /** malformed Analysis handed to toMarkdown/toText (e.g. an out-of-range\n * wordIndexes entry) — validated fail-fast, whole object checked before any output. */\n | 'ANALYSIS_INPUT';\n\n/** The error every entry point throws — never a bare `Error`. */\nexport class PdfToMdError extends Error {\n code: PdfToMdErrorCode;\n\n constructor(code: PdfToMdErrorCode, message: string) {\n super(message);\n this.name = 'PdfToMdError';\n this.code = code;\n }\n}\n"],"names":["PdfToMdError","code","message","name","Error"],"mappings":"AAAA,yDAAyD;AAEzD;wCACwC;;;;+BAmB3BA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,6BAAN;;cAAMA;aAAAA,aAGCC,IAAsB,EAAEC,OAAe;gCAHxCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAqBI"}
@@ -1,6 +1,6 @@
1
1
  import type { OcrProgressEvent } from './engines/tesseract.js';
2
2
  import type { PageWord, Tuning } from './geometry.js';
3
- import type { Report } from './types.js';
3
+ import type { Analysis, Report } from './types.js';
4
4
  export type { OcrProgressEvent } from './engines/tesseract.js';
5
5
  /** One recognized word with its page — the OCR replay input contract (same
6
6
  * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */
@@ -34,6 +34,9 @@ export interface ExtractTextOptions extends CommonOptions {
34
34
  /** emit a `### pN` marker before each page's content (output formatting
35
35
  * only — `report.pageStats` is always populated regardless) */
36
36
  pageMarkers?: boolean;
37
+ /** source PDF text runs entering layout, delivered once for the selected
38
+ * final result. Runs retain measured bounds and are not fabricated words. */
39
+ onWords?: (words: OcrWordInput[]) => void;
37
40
  }
38
41
  /** Options for {@link extractOcr}. */
39
42
  export interface ExtractOcrOptions extends CommonOptions {
@@ -59,6 +62,10 @@ export interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOc
59
62
  /** never fall back to OCR, even for image-only pages */
60
63
  noOcr?: boolean;
61
64
  }
65
+ /** Extract embedded text with a QA report and warnings; never runs OCR.
66
+ * @param pdfPath path to the PDF file
67
+ * @param opts see {@link ExtractTextOptions}
68
+ */
62
69
  export declare function extractText(pdfPath: string, opts?: ExtractTextOptions): Promise<ExtractResult>;
63
70
  /**
64
71
  * OCR extraction. Input is either a PDF (pages render at `dpi` and the
@@ -72,15 +79,19 @@ export declare function extractOcr(input: {
72
79
  } | {
73
80
  words: OcrWordInput[];
74
81
  }, opts?: ExtractOcrOptions): Promise<ExtractResult>;
75
- /** pdfToMarkdown's auto-OCR fallback predicate, shared with the CLI's
76
- * --format=raw|txt path so the two call sites can never drift apart. Module-internal: not re-exported by `src/index.ts`. */
77
- export declare function needsOcrFallback(charsPerPage: number): boolean;
78
- /** The exact stderr/warnings notice printed when needsOcrFallback fires —
79
- * one copy of the string for both call sites (asserted verbatim by the suite). */
80
- export declare function ocrFallbackNotice(charsPerPage: number): string;
82
+ interface AutoAnalysisResult {
83
+ analysis: Analysis;
84
+ markdown: string;
85
+ warnings: string[];
86
+ }
87
+ /** Shared automatic route for the library's Markdown result and the CLI's
88
+ * raw/plain-text formats. A page changes producer only when it has image
89
+ * evidence and an unusable text layer, so a blank or vector-only page never
90
+ * becomes speculative OCR input. */
91
+ export declare function analyzePdfAuto(pdfPath: string, opts?: PdfToMarkdownOptions): Promise<AutoAnalysisResult>;
81
92
  /**
82
- * The headline API: extract a PDF to markdown, text layer first, falling
83
- * back to OCR when the text layer is empty enough the pages are certainly images of text (< `AUTO_OCR_CHARS_PER_PAGE` chars/page) — the same decision the CLI makes.
93
+ * Extract each page from embedded text, using OCR only when the page has
94
+ * image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
84
95
  *
85
96
  * @param pdfPath path to the PDF file
86
97
  * @param opts see {@link PdfToMarkdownOptions}
@@ -1,6 +1,6 @@
1
1
  import type { OcrProgressEvent } from './engines/tesseract.js';
2
2
  import type { PageWord, Tuning } from './geometry.js';
3
- import type { Report } from './types.js';
3
+ import type { Analysis, Report } from './types.js';
4
4
  export type { OcrProgressEvent } from './engines/tesseract.js';
5
5
  /** One recognized word with its page — the OCR replay input contract (same
6
6
  * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */
@@ -34,6 +34,9 @@ export interface ExtractTextOptions extends CommonOptions {
34
34
  /** emit a `### pN` marker before each page's content (output formatting
35
35
  * only — `report.pageStats` is always populated regardless) */
36
36
  pageMarkers?: boolean;
37
+ /** source PDF text runs entering layout, delivered once for the selected
38
+ * final result. Runs retain measured bounds and are not fabricated words. */
39
+ onWords?: (words: OcrWordInput[]) => void;
37
40
  }
38
41
  /** Options for {@link extractOcr}. */
39
42
  export interface ExtractOcrOptions extends CommonOptions {
@@ -59,6 +62,10 @@ export interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOc
59
62
  /** never fall back to OCR, even for image-only pages */
60
63
  noOcr?: boolean;
61
64
  }
65
+ /** Extract embedded text with a QA report and warnings; never runs OCR.
66
+ * @param pdfPath path to the PDF file
67
+ * @param opts see {@link ExtractTextOptions}
68
+ */
62
69
  export declare function extractText(pdfPath: string, opts?: ExtractTextOptions): Promise<ExtractResult>;
63
70
  /**
64
71
  * OCR extraction. Input is either a PDF (pages render at `dpi` and the
@@ -72,15 +79,19 @@ export declare function extractOcr(input: {
72
79
  } | {
73
80
  words: OcrWordInput[];
74
81
  }, opts?: ExtractOcrOptions): Promise<ExtractResult>;
75
- /** pdfToMarkdown's auto-OCR fallback predicate, shared with the CLI's
76
- * --format=raw|txt path so the two call sites can never drift apart. Module-internal: not re-exported by `src/index.ts`. */
77
- export declare function needsOcrFallback(charsPerPage: number): boolean;
78
- /** The exact stderr/warnings notice printed when needsOcrFallback fires —
79
- * one copy of the string for both call sites (asserted verbatim by the suite). */
80
- export declare function ocrFallbackNotice(charsPerPage: number): string;
82
+ interface AutoAnalysisResult {
83
+ analysis: Analysis;
84
+ markdown: string;
85
+ warnings: string[];
86
+ }
87
+ /** Shared automatic route for the library's Markdown result and the CLI's
88
+ * raw/plain-text formats. A page changes producer only when it has image
89
+ * evidence and an unusable text layer, so a blank or vector-only page never
90
+ * becomes speculative OCR input. */
91
+ export declare function analyzePdfAuto(pdfPath: string, opts?: PdfToMarkdownOptions): Promise<AutoAnalysisResult>;
81
92
  /**
82
- * The headline API: extract a PDF to markdown, text layer first, falling
83
- * back to OCR when the text layer is empty enough the pages are certainly images of text (< `AUTO_OCR_CHARS_PER_PAGE` chars/page) — the same decision the CLI makes.
93
+ * Extract each page from embedded text, using OCR only when the page has
94
+ * image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
84
95
  *
85
96
  * @param pdfPath path to the PDF file
86
97
  * @param opts see {@link PdfToMarkdownOptions}