@json-to-office/jto-ops 4.4.2 → 4.4.4

package/dist/index.d.ts

@@ -294,6 +294,57 @@ declare function createLibreOfficePptxBatchRasterizer(options?: {
     cacheDir?: string | null;
 }): PptxBatchRasterizer;
 
+/**
+ * Where the ink is on a rendered page, row by row.
+ *
+ * Text geometry says where the words are; it says nothing about a chart, a
+ * table rule or a full-bleed image, and a page that holds only those would
+ * read as blank. A low-resolution grayscale raster of the same PDF answers
+ * the other question — how far down the page anything at all was painted —
+ * and that is what the page-fill rule needs. `pdftoppm -gray` writes one
+ * PGM (P5) file per page: an ASCII header, then one byte per pixel, so
+ * reading it needs no image library.
+ */
+/** One page's ink profile, as cached beside its text geometry. */
+interface PdfPageInk {
+    /** Raster rows the page was sampled into; row `r` spans `heightPt * r / rows` to `heightPt * (r + 1) / rows`. */
+    rows: number;
+    /** Row indices, ascending, where at least one pixel is darker than the ink threshold. */
+    inked: number[];
+}
+interface Pgm {
+    width: number;
+    height: number;
+    maxValue: number;
+    /** Row-major, one value per pixel, `width * height` long. */
+    pixels: Uint8Array;
+}
+/** Pixels at or below this value count as ink: dark enough to be text, a rule or a chart bar, not anti-aliasing haze. */
+declare const INK_THRESHOLD = 200;
+/** Dots per inch the fill measurement rasterizes at: an A4 page is ~280 rows, enough to place content to within 3pt. */
+declare const INK_DPI = 24;
+/** Parse a binary PGM (`P5`) as pdftoppm writes it: header tokens, one whitespace byte, then the pixels. */
+declare function parsePgm(buffer: Buffer): Pgm;
+/** Rows carrying at least one pixel at or below `threshold`, ascending. */
+declare function inkedRows(pgm: Pgm, threshold?: number): number[];
+declare function pageInkFromPgm(buffer: Buffer, threshold?: number): PdfPageInk;
+interface ExtractPdfPageInkOptions {
+    dpi?: number;
+    threshold?: number;
+    /** A resolved pdftoppm path, when the caller already found one. */
+    binary?: string;
+    timeoutMs?: number;
+    /** Aborting kills the pdftoppm process and rejects. */
+    signal?: AbortSignal;
+}
+/**
+ * Rasterize every page of a PDF in grayscale at a low resolution and read
+ * back which rows carry ink. One pdftoppm spawn into a private temp dir,
+ * removed before returning; the result is small enough to cache with the
+ * text geometry.
+ */
+declare function extractPdfPageInk(pdfPath: string, options?: ExtractPdfPageInkOptions): Promise<PdfPageInk[]>;
+
 /**
  * Text geometry from a rendered PDF — the ground truth the quality
  * estimators are guessing at (#216 follow-up).
@@ -314,6 +365,7 @@ declare function createLibreOfficePptxBatchRasterizer(options?: {
  * Consumers: the quality ground-truth harness (estimator calibration) today;
  * a `rendered`-certainty analysis pass tomorrow.
  */
+
 /** One word as laid out on the page, in PDF points, top-left origin. */
 interface PdfTextWord {
     text: string;
@@ -341,6 +393,11 @@ interface PdfTextPage {
     words: PdfTextWord[];
     /** Line grouping; empty for plain `-bbox` output, which groups nothing. */
     lines: PdfTextLine[];
+    /**
+     * Which raster rows carry ink, when the host sampled the page (see
+     * `pdf-page-ink.ts`). Absent, the rules that need it stay silent.
+     */
+    ink?: PdfPageInk;
 }
 /**
  * Parse `pdftotext -bbox` or `-bbox-layout` output (XHTML with `<page>`,
@@ -412,7 +469,7 @@ declare function extractPdfFonts(pdfPath: string, options?: {
  * pass over it, memoised per fact, so eight rules cost one search.
  */
 
-type RenderedRuleId = 'rendered/clip' | 'rendered/spill' | 'rendered/overlap' | 'rendered/text-missing' | 'rendered/font-substituted' | 'rendered/empty-page' | 'rendered/heading-stranded' | 'rendered/paragraph-split';
+type RenderedRuleId = 'rendered/clip' | 'rendered/spill' | 'rendered/overlap' | 'rendered/text-missing' | 'rendered/font-substituted' | 'rendered/empty-page' | 'rendered/heading-stranded' | 'rendered/paragraph-split' | 'rendered/page-underfilled';
 declare const RENDERED_QUALITY_RULES: QualityRulePack;
 
 /**
@@ -946,4 +1003,4 @@ declare function emitDiagnostic(text: string, tone?: DiagnosticTone): void;
  */
 declare const stderrDiagnosticSink: DiagnosticSink;
 
-export { type BlockMatrixCase, type BlockMatrixDefinition, type BlockMatrixOptions, type CaseConditions, type DiagnosticSink, type DiagnosticTone, DocxFormatAdapter, FALLBACK_FONTS, type FontStageHandle, type FontStageOptions, type FontStager, FontconfigStager, type FormatAdapter, type FormatName, type GeneratorOptions, type GeneratorResult, type InventoryEntry, type InventoryMatch, MATRIX_IMAGE, MacOSCoreTextStager, type MappingStatus, type MatrixCanvas, type MatrixEdge, type MatrixFont, NoopFontStager, type PdfFontInfo, type PdfTextLine, type PdfTextPage, type PdfTextWord, PptxFormatAdapter, RENDERED_QUALITY_RULES, type RasterizerCacheStats, type RenderedAnalysis, type RenderedAnalysisInput, type RenderedAnalysisSummary, type RenderedMapping, type RenderedRuleId, type RenderedTextEntry, type RequestedFont, type TextOccurrence, VISIBLE_SPILL_PT, WindowsFontStager, analyzeRenderedDocument, assignInventory, authoredTextForMatch, blockCaseDocument, boundaryInvocation, boundarySlotValue, clearRasterizerCache, createAdapter, createLibreOfficePptxBatchRasterizer, createLibreOfficePptxRasterizer, emitDiagnostic, enumerateBlockDefinitions, extractPdfFonts, extractPdfTextGeometry, familyRendered, generateBlockMatrix, getFontStager, getRasterizerCacheStats, normalizeForMatch, overBudgetInvocation, parsePdfFonts, parsePdfTextBbox, pdffontsAvailable, pdftotextAvailable, readingOrder, reportCaseDocument, runWithDiagnosticSink, stderrDiagnosticSink, widestNumber };
+export { type BlockMatrixCase, type BlockMatrixDefinition, type BlockMatrixOptions, type CaseConditions, type DiagnosticSink, type DiagnosticTone, DocxFormatAdapter, type ExtractPdfPageInkOptions, FALLBACK_FONTS, type FontStageHandle, type FontStageOptions, type FontStager, FontconfigStager, type FormatAdapter, type FormatName, type GeneratorOptions, type GeneratorResult, INK_DPI, INK_THRESHOLD, type InventoryEntry, type InventoryMatch, MATRIX_IMAGE, MacOSCoreTextStager, type MappingStatus, type MatrixCanvas, type MatrixEdge, type MatrixFont, NoopFontStager, type PdfFontInfo, type PdfPageInk, type PdfTextLine, type PdfTextPage, type PdfTextWord, type Pgm, PptxFormatAdapter, RENDERED_QUALITY_RULES, type RasterizerCacheStats, type RenderedAnalysis, type RenderedAnalysisInput, type RenderedAnalysisSummary, type RenderedMapping, type RenderedRuleId, type RenderedTextEntry, type RequestedFont, type TextOccurrence, VISIBLE_SPILL_PT, WindowsFontStager, analyzeRenderedDocument, assignInventory, authoredTextForMatch, blockCaseDocument, boundaryInvocation, boundarySlotValue, clearRasterizerCache, createAdapter, createLibreOfficePptxBatchRasterizer, createLibreOfficePptxRasterizer, emitDiagnostic, enumerateBlockDefinitions, extractPdfFonts, extractPdfPageInk, extractPdfTextGeometry, familyRendered, generateBlockMatrix, getFontStager, getRasterizerCacheStats, inkedRows, normalizeForMatch, overBudgetInvocation, pageInkFromPgm, parsePdfFonts, parsePdfTextBbox, parsePgm, pdffontsAvailable, pdftotextAvailable, readingOrder, reportCaseDocument, runWithDiagnosticSink, stderrDiagnosticSink, widestNumber };

package/dist/index.js

@@ -1850,9 +1850,126 @@ async function extractPdfTextGeometry(pdfPath, options = {}) {
   return parsePdfTextBbox(stdout);
 }
 
-// src/pdf-fonts.ts
+// src/pdf-page-ink.ts
 import { execFile as execFile3 } from "child_process";
-import * as path7 from "path";
+import { promises as fs6 } from "fs";
+import os2 from "os";
+import path7 from "path";
+var INK_THRESHOLD = 200;
+var INK_DPI = 24;
+function parsePgm(buffer) {
+  const header = /^P5\s+(?:#[^\n]*\n\s*)*(\d+)\s+(\d+)\s+(\d+)\s/.exec(
+    buffer.subarray(0, 64).toString("latin1")
+  );
+  if (!header) throw new Error("Not a binary PGM (P5) image.");
+  const width = Number(header[1]);
+  const height = Number(header[2]);
+  const maxValue = Number(header[3]);
+  if (maxValue > 255) throw new Error("16-bit PGM is not supported.");
+  const start = header[0].length;
+  if (buffer.length - start < width * height) {
+    throw new Error("PGM shorter than its declared size.");
+  }
+  const pixels = new Uint8Array(
+    buffer.buffer,
+    buffer.byteOffset + start,
+    width * height
+  );
+  return { width, height, maxValue, pixels };
+}
+function inkedRows(pgm, threshold = INK_THRESHOLD) {
+  const rows = [];
+  for (let row = 0; row < pgm.height; row += 1) {
+    const start = row * pgm.width;
+    const end = start + pgm.width;
+    for (let i = start; i < end; i += 1) {
+      if (pgm.pixels[i] <= threshold) {
+        rows.push(row);
+        break;
+      }
+    }
+  }
+  return rows;
+}
+function pageInkFromPgm(buffer, threshold) {
+  const pgm = parsePgm(buffer);
+  return { rows: pgm.height, inked: inkedRows(pgm, threshold) };
+}
+function pdftoppmCandidates2() {
+  const configured = process.env.PDFTOPPM_PATH?.trim();
+  return [.../* @__PURE__ */ new Set([...configured ? [configured] : [], "pdftoppm"])];
+}
+async function run2(binary, args, timeoutMs, signal) {
+  return new Promise((resolve2, reject) => {
+    execFile3(
+      binary,
+      args,
+      {
+        timeout: timeoutMs,
+        maxBuffer: 16 * 1024 * 1024,
+        windowsHide: true,
+        ...signal && { signal }
+      },
+      (error) => error ? reject(error) : resolve2()
+    );
+  });
+}
+var pdftoppmPromise2;
+async function resolvePdftoppm2() {
+  if (!pdftoppmPromise2) {
+    pdftoppmPromise2 = (async () => {
+      for (const candidate of pdftoppmCandidates2()) {
+        try {
+          await run2(candidate, ["-v"], 1e4);
+          return candidate;
+        } catch (error) {
+          const code = error.code;
+          if (code !== "ENOENT" && code !== "EACCES") return candidate;
+        }
+      }
+      throw new Error(
+        `Page ink extraction needs pdftoppm (poppler), which was not found. Install poppler-utils or set PDFTOPPM_PATH (searched: ${pdftoppmCandidates2().join(", ")}).`
+      );
+    })().catch((error) => {
+      pdftoppmPromise2 = void 0;
+      throw error;
+    });
+  }
+  return pdftoppmPromise2;
+}
+async function extractPdfPageInk(pdfPath, options = {}) {
+  const binary = options.binary ?? await resolvePdftoppm2();
+  const dpi = options.dpi ?? INK_DPI;
+  const dir = await fs6.mkdtemp(path7.join(os2.tmpdir(), "jto-page-ink-"));
+  try {
+    const prefix = path7.join(dir, "page");
+    await run2(
+      binary,
+      ["-gray", "-r", String(dpi), pdfPath, prefix],
+      options.timeoutMs ?? 6e4,
+      options.signal
+    );
+    const files = (await fs6.readdir(dir)).filter((name) => /^page-\d+\.pgm$/.test(name)).sort(
+      (a, b) => Number(/\d+/.exec(a)?.[0] ?? 0) - Number(/\d+/.exec(b)?.[0] ?? 0)
+    );
+    const pages = [];
+    for (const file of files) {
+      pages.push(
+        pageInkFromPgm(
+          await fs6.readFile(path7.join(dir, file)),
+          options.threshold
+        )
+      );
+    }
+    return pages;
+  } finally {
+    await fs6.rm(dir, { recursive: true, force: true });
+  }
+}
+
+// src/pdf-fonts.ts
+import { execFile as execFile4 } from "child_process";
+import * as path8 from "path";
 var ROW_PATTERN = /^(\S+)\s+(.+?)\s+(\S+)\s+(yes|no)\s+(yes|no)\s+(yes|no)\s+/;
 function parsePdfFonts(stdout) {
   const fonts = [];
@@ -1887,8 +2004,8 @@ function pdffontsCandidates() {
   const sibling = process.env.PDFTOTEXT_PATH?.trim();
   if (sibling) {
     candidates.push(
-      path7.join(
-        path7.dirname(sibling),
+      path8.join(
+        path8.dirname(sibling),
         process.platform === "win32" ? "pdffonts.exe" : "pdffonts"
       )
     );
@@ -1896,9 +2013,9 @@ function pdffontsCandidates() {
   candidates.push("pdffonts");
   return [...new Set(candidates)];
 }
-async function run2(binary, args, timeoutMs) {
+async function run3(binary, args, timeoutMs) {
   return new Promise((resolve2, reject) => {
-    execFile3(
+    execFile4(
       binary,
       args,
       { timeout: timeoutMs, maxBuffer: 16 * 1024 * 1024 },
@@ -1915,7 +2032,7 @@ async function resolvePdffonts() {
     pdffontsPromise = (async () => {
       for (const candidate of pdffontsCandidates()) {
         try {
-          await run2(candidate, ["-v"], 1e4);
+          await run3(candidate, ["-v"], 1e4);
           return candidate;
         } catch (error) {
           const code = error.code;
@@ -1942,7 +2059,7 @@ async function pdffontsAvailable() {
 }
 async function extractPdfFonts(pdfPath, options = {}) {
   const binary = await resolvePdffonts();
-  const stdout = await run2(binary, [pdfPath], options.timeoutMs ?? 3e4);
+  const stdout = await run3(binary, [pdfPath], options.timeoutMs ?? 3e4);
   return parsePdfFonts(stdout);
 }
 
@@ -2016,6 +2133,14 @@ var RENDERED_QUALITY_RULES = {
       category: "integrity",
       defaultSeverity: "info"
     }),
+    renderedRule({
+      id: "rendered/page-underfilled",
+      code: QUALITY_CODES.RENDERED_PAGE_UNDERFILLED,
+      description: "A docx page, other than the first and the last, whose ink stops less than halfway down the body area before the next page begins (`context.fill` is the share used): a section that ends early on its own page, argued in a paragraph where a table or chart was owed.",
+      category: "composition",
+      defaultSeverity: "info",
+      formats: ["docx"]
+    }),
     renderedRule({
       id: "rendered/heading-stranded",
       code: QUALITY_CODES.RENDERED_HEADING_STRANDED,
@@ -2271,6 +2396,24 @@ function inChromeBand(page, box) {
   return box.yMax <= page.heightPt * CHROME_BAND || box.yMin >= page.heightPt * (1 - CHROME_BAND);
 }
 var PAGE_NUMBER_ROW = /^[0-9\s./|-]*$/;
+var CHROME_ROW_TOLERANCE_PT = 6;
+function chromeOccurrences(occurrences, pages) {
+  const nearestPerPage = /* @__PURE__ */ new Map();
+  const edgeDistance = (o) => Math.min(o.parts[0].yMin, pages[o.pageIndex].heightPt - o.parts[0].yMax);
+  for (const o of occurrences) {
+    const held = nearestPerPage.get(o.pageIndex);
+    if (!held || edgeDistance(o) < edgeDistance(held)) {
+      nearestPerPage.set(o.pageIndex, o);
+    }
+  }
+  const nearest = [...nearestPerPage.values()];
+  if (nearest.length < 2) return nearest;
+  return nearest.filter(
+    (o) => nearest.some(
+      (other) => other.pageIndex !== o.pageIndex && Math.abs(other.parts[0].yMin - o.parts[0].yMin) <= CHROME_ROW_TOLERANCE_PT
+    )
+  );
+}
 function assignInventory(pages, inventory) {
   const results = /* @__PURE__ */ new Map();
   const chromeWords = /* @__PURE__ */ new Set();
@@ -2294,9 +2437,10 @@ function assignInventory(pages, inventory) {
         });
         continue;
       }
-      const occurrences = findOccurrences(full, segments).filter(
+      const inBand = findOccurrences(full, segments).filter(
         (o) => o.pageIndex === o.endPageIndex && inChromeBand(pages[o.pageIndex], o.parts[0])
       );
+      const occurrences = chromeOccurrences(inBand, pages);
       for (const o of occurrences) claimRow(o.pageIndex, o.parts[0]);
       results.set(entry, {
         entry,
@@ -2387,6 +2531,7 @@ function assignInventory(pages, inventory) {
 }
 
 // src/rendered-analysis.ts
+var MINIMUM_PAGE_FILL = 0.5;
 function isRenderedGeometryFact(fact) {
   return fact.kind === "rendered/geometry";
 }
@@ -2674,6 +2819,61 @@ function draftRenderedFindings(input) {
       })
     );
   });
+  if (format === "docx") {
+    pages.forEach((page, pageIndex) => {
+      if (!page.ink || pageIndex === 0 || pageIndex === pages.length - 1) {
+        return;
+      }
+      const rowPt = page.heightPt / page.ink.rows;
+      let headerBottom = 0;
+      let footerTop = page.heightPt;
+      page.words.forEach((w, i) => {
+        if (!chromeWords.has(`${pageIndex}:${i}`)) return;
+        if (w.yMax <= page.heightPt * CHROME_BAND) {
+          headerBottom = Math.max(headerBottom, w.yMax);
+        } else if (w.yMin >= page.heightPt * (1 - CHROME_BAND)) {
+          footerTop = Math.min(footerTop, w.yMin);
+        }
+      });
+      const body = page.ink.inked.filter(
+        (row) => row * rowPt > headerBottom + rowPt && (row + 1) * rowPt < footerTop
+      );
+      if (body.length === 0) return;
+      const top = body[0] * rowPt;
+      const bottom = (body[body.length - 1] + 1) * rowPt;
+      const fill = Math.round((bottom - top) / (footerTop - top) * 100) / 100;
+      if (fill >= MINIMUM_PAGE_FILL) return;
+      let owner;
+      for (const match of matches) {
+        if (match.status !== "mapped" || match.entry.repeats) continue;
+        for (const o of match.occurrences) {
+          for (const part of o.parts) {
+            if (part.pageIndex !== pageIndex) continue;
+            if (!owner || part.yMax > owner.yMax) {
+              owner = { path: match.entry.path, yMax: part.yMax };
+            }
+          }
+        }
+      }
+      const section = owner ? /^\/children\/\d+/.exec(owner.path)?.[0] : void 0;
+      findings.push(
+        finding({
+          ruleId: "rendered/page-underfilled",
+          mapping: section ? "mapped" : "unmapped",
+          page: pageIndex + 1,
+          path: section ?? "",
+          message: `Page ${pageIndex + 1} is ${Math.round(fill * 100)}% filled: its content stops ${Math.round(footerTop - bottom)}pt above the footer and the next page begins anyway.`,
+          suggestion: "Let the next section continue on this page, or give the page the table or chart its argument owes; a short section on its own page reads as unfinished.",
+          evidence: {
+            summary: "Share of the body area the page paints, header to footer",
+            expected: `\u2265 ${Math.round(MINIMUM_PAGE_FILL * 100)}%`,
+            actual: fill
+          },
+          context: { fill }
+        })
+      );
+    });
+  }
   if (format === "docx") {
     for (const match of matches) {
       if (match.entry.role !== "heading" || match.status !== "mapped") continue;
@@ -3227,6 +3427,8 @@ export {
   DocxFormatAdapter,
   FALLBACK_FONTS,
   FontconfigStager,
+  INK_DPI,
+  INK_THRESHOLD,
   MATRIX_IMAGE,
   MacOSCoreTextStager,
   NoopFontStager,
@@ -3247,15 +3449,19 @@ export {
   emitDiagnostic,
   enumerateBlockDefinitions,
   extractPdfFonts,
+  extractPdfPageInk,
   extractPdfTextGeometry,
   familyRendered,
   generateBlockMatrix,
   getFontStager,
   getRasterizerCacheStats,
+  inkedRows,
   normalizeForMatch,
   overBudgetInvocation,
+  pageInkFromPgm,
   parsePdfFonts,
   parsePdfTextBbox,
+  parsePgm,
   pdffontsAvailable,
   pdftotextAvailable,
   readingOrder,