dsh-ab-ocr 0.1.0

package/python/assemble.py

@@ -0,0 +1,358 @@
+"""Assemble cleaned pages into one Markdown document.
+
+The merge is deliberately a separate pass over the per-page records: the OCR
+pass only has to produce page files, and this pass only has to read them, so the
+resident cost of a long document stays flat and an interrupted run keeps every
+page it already finished.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from statistics import median
+from typing import Mapping, Sequence
+
+from clean import Page, ends_sentence, join_wrapped, running_labels, strip_noise
+from layout import (
+    Line,
+    body_height,
+    heading_level,
+    is_cjk,
+    is_list_item,
+    is_named_section,
+    merge_list_markers,
+    normalize_heading,
+    section_number,
+)
+
+
+@dataclass
+class MergeOptions:
+    """Document-level switches and thresholds for the merge pass."""
+
+    remove_page_numbers: bool = True
+    remove_running_heads: bool = True
+    running_head_ratio: float = 0.6
+    running_head_min_pages: int = 3
+    detect_headings: bool = True
+    heading_min_ratio: float = 1.18
+    #: Left-edge offset, in body glyph heights, that marks an indented first line.
+    indent_ratio: float = 1.0
+    #: Vertical gap, in body glyph heights, that separates two paragraphs.
+    paragraph_gap_ratio: float = 0.85
+    #: Glyph height over body height that makes a line an outline candidate.
+    outline_candidate_ratio: float = 1.05
+    #: Most candidates one document may carry; a model pass reads the first ones.
+    max_outline_candidates: int = 400
+    #: Candidate id to heading level, assigned by a model-assisted outline pass.
+    #: A level below 1, like the absence of one, makes the line body text.
+    level_overrides: Mapping[str, int | None] = field(default_factory=dict)
+
+
+#: A line taller than this multiple of the body glyph height is display type,
+#: which is content even when it reprints the running head on every page.
+FURNITURE_HEIGHT_RATIO = 1.15
+
+
+@dataclass
+class MergeStats:
+    """What the merge pass produced, for the tool's result line."""
+
+    pages: int
+    lines: int
+    headings: int
+    dropped_page_numbers: int
+    dropped_running_heads: int
+    joined_across_pages: int
+
+
+@dataclass
+class OutlineCandidate:
+    """One line a model-assisted outline pass may re-level.
+
+    The heuristic reads geometry and numbering; a candidate is a line where that
+    reading may be wrong in either direction, so the caller can have a model
+    assign the level instead.
+    """
+
+    id: str
+    page: int
+    text: str
+    level: int | None
+    #: Glyph height over body glyph height, rounded for the caller's prompt.
+    ratio: float
+    #: The section number the line opens with, when it opens with one.
+    numbered: str | None
+
+
+def page_text(lines: Sequence[Line]) -> str:
+    """Render one page's lines as the plain text stored beside its record."""
+    return '\n'.join(line.text.strip() for line in lines if line.text.strip() != '')
+
+
+def _left_margin(lines: Sequence[Line]) -> float:
+    """Estimate the left margin from the lines that are not indented."""
+    if not lines:
+        return 0.0
+    return median([line.x0 for line in lines])
+
+
+def _dedupe_repeats(lines: Sequence[Line]) -> list[Line]:
+    """Drop runs of identical consecutive lines, which OCR sometimes emits."""
+    kept: list[Line] = []
+    for line in lines:
+        if kept and kept[-1].text == line.text:
+            continue
+        kept.append(line)
+    return kept
+
+
+def _override_level(candidate_id: str, value: object) -> int | None:
+    """Read the level a model pass assigned to one outline candidate.
+
+    A level below 1, or no level at all, makes the line body text: that is how a
+    candidate the heuristic wrongly called a heading is demoted.
+    @param candidate_id: the candidate the caller named.
+    @param value: the level as it arrived from the caller.
+    @returns: a Markdown level from 1 to 6, or None for body text.
+    """
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ValueError('outline level override for ' + candidate_id + ' is not an integer: ' + repr(value))
+    if value < 1:
+        return None
+    return min(value, 6)
+
+
+def _outline_pass(
+    pages: Sequence[Page],
+    kept: Mapping[int, list[Line]],
+    base: float,
+    options: MergeOptions,
+) -> tuple[list[OutlineCandidate], dict[tuple[int, int], int | None], bool]:
+    """Collect the lines a model-assisted outline pass may re-level.
+
+    The walk follows the merge's own order, so an id names the line the block
+    loop reaches at the same page number and position.
+    @param pages: every page of the document, in order.
+    @param kept: each page number's kept lines.
+    @param base: the document's body glyph height.
+    @param options: the deployment's thresholds and the caller's level overrides.
+    @returns: the capped candidate list, the levels the overrides settle, keyed
+        by page number and line position, and whether the cap dropped a
+        candidate.
+    """
+    settled = {
+        candidate_id: _override_level(candidate_id, value)
+        for candidate_id, value in options.level_overrides.items()
+    }
+    candidates: list[OutlineCandidate] = []
+    overrides: dict[tuple[int, int], int | None] = {}
+    truncated = False
+    for page in pages:
+        for position, line in enumerate(kept[page.index]):
+            text = line.text.strip()
+            # The candidate list is what the caller's model pass reads, so it
+            # does not depend on whether the heuristic may render headings on
+            # its own: a candidate the overrides leave alone keeps that
+            # behaviour exactly.
+            level = heading_level(text, line.height, base, options.heading_min_ratio)
+            numbered = section_number(text)
+            if is_list_item(text) and numbered is None:
+                continue
+            tall = base > 0 and line.height / base >= options.outline_candidate_ratio
+            if level is None and numbered is None and not tall and not is_named_section(text):
+                continue
+            if len(candidates) >= options.max_outline_candidates:
+                truncated = True
+                continue
+            candidate_id = 'h' + str(len(candidates) + 1)
+            if candidate_id in settled:
+                overrides[(page.index, position)] = settled[candidate_id]
+            candidates.append(OutlineCandidate(
+                id=candidate_id,
+                page=page.index,
+                text=normalize_heading(text),
+                level=level,
+                ratio=round(line.height / base, 2) if base > 0 else 0.0,
+                numbered=None if numbered is None else text[:numbered[1]].strip(),
+            ))
+    return candidates, overrides, truncated
+
+
+def build_markdown(
+    pages: Sequence[Page],
+    options: MergeOptions,
+) -> tuple[str, MergeStats, list[OutlineCandidate], bool]:
+    """Turn the per-page records of one document into a single Markdown text.
+
+    @param pages: every page of the document, in order.
+    @param options: the deployment's cleaning and structure thresholds.
+    @returns: the Markdown document, the counts the tool reports, the outline
+        candidates a model-assisted pass may re-level, and whether the candidate
+        list hit its cap.
+    """
+    # The size ceiling that separates furniture from display type comes from a
+    # first pass over the raw lines, because the lines it removes must not be
+    # allowed to shape the body estimate they are measured against.
+    raw_lines = [line for page in pages for line in page.lines]
+    raw_base = body_height(raw_lines)
+    furniture_ceiling = raw_base * FURNITURE_HEIGHT_RATIO if raw_base > 0 else 0.0
+    labels = (
+        running_labels(
+            pages,
+            options.running_head_ratio,
+            options.running_head_min_pages,
+            max_height=furniture_ceiling,
+        )
+        if options.remove_running_heads
+        else set()
+    )
+    # Kept lines are keyed by page number, not by position: a selection may
+    # start at any page, and a repeated page number would otherwise be read
+    # twice.
+    kept: dict[int, list[Line]] = {}
+    for page in pages:
+        if page.index in kept:
+            raise ValueError('page index ' + str(page.index) + ' appears twice in the document')
+        kept[page.index] = _dedupe_repeats(
+            strip_noise(
+                page,
+                labels,
+                options.remove_page_numbers,
+                options.remove_running_heads,
+                max_height=furniture_ceiling,
+            )
+        )
+    all_lines = [line for page in pages for line in kept[page.index]]
+    base = body_height(all_lines)
+    margin = _left_margin([line for line in all_lines if not is_list_item(line.text)])
+
+    after_numbers = sum(
+        len(strip_noise(page, set(), True, False)) for page in pages
+    )
+    dropped_numbers = len(raw_lines) - after_numbers
+    dropped_heads = max(0, after_numbers - len(all_lines))
+
+    candidates, overrides, truncated = _outline_pass(pages, kept, base, options)
+
+    blocks: list[list[str]] = []
+    paragraph: list[tuple[Page, Line]] = []
+    headings = 0
+    joined = 0
+
+    def flush() -> None:
+        if paragraph:
+            text = paragraph[0][1].text.strip()
+            for _page, line in paragraph[1:]:
+                text = join_wrapped(text, line.text.strip())
+            blocks.append(['text', text])
+            paragraph.clear()
+
+    for page in pages:
+        lines = kept[page.index]
+        previous: Line | None = None
+        for position, line in enumerate(lines):
+            text = line.text.strip()
+            level = (
+                heading_level(text, line.height, base, options.heading_min_ratio)
+                if options.detect_headings
+                else None
+            )
+            # An override to None means body text, so the key's presence decides.
+            if (page.index, position) in overrides:
+                level = overrides[(page.index, position)]
+            if level is not None:
+                flush()
+                blocks.append(['heading', '#' * level + ' ' + normalize_heading(text)])
+                headings += 1
+                previous = line
+                continue
+            if is_list_item(text):
+                flush()
+                blocks.append(['item', merge_list_markers(text)])
+                previous = line
+                continue
+
+            first_on_page = position == 0
+            if paragraph and first_on_page:
+                open_text = paragraph[0][1].text.strip()
+                for _page, earlier in paragraph[1:]:
+                    open_text = join_wrapped(open_text, earlier.text.strip())
+                indented = line.x0 > margin + options.indent_ratio * base
+                if not indented and not ends_sentence(open_text):
+                    paragraph.append((page, line))
+                    joined += 1
+                    previous = line
+                    continue
+                flush()
+            elif paragraph:
+                gap = 0.0 if previous is None else line.y0 - previous.y1
+                indented = line.x0 > margin + options.indent_ratio * base
+                if indented or gap > options.paragraph_gap_ratio * base:
+                    flush()
+            paragraph.append((page, line))
+            previous = line
+    flush()
+
+    markdown = _render_blocks(blocks)
+    return markdown, MergeStats(
+        pages=len(pages),
+        lines=len(all_lines),
+        headings=headings,
+        dropped_page_numbers=dropped_numbers,
+        dropped_running_heads=dropped_heads,
+        joined_across_pages=joined,
+    ), candidates, truncated
+
+
+def _render_blocks(blocks: Sequence[Sequence[str]]) -> str:
+    """Join the merged blocks into Markdown, grouping consecutive list items."""
+    parts: list[str] = []
+    pending_items: list[str] = []
+    for block in blocks:
+        if block[0] == 'item':
+            pending_items.append(block[1])
+            continue
+        if pending_items:
+            parts.append('\n'.join(pending_items))
+            pending_items = []
+        parts.append(block[1])
+    if pending_items:
+        parts.append('\n'.join(pending_items))
+    return normalize_markdown('\n\n'.join(parts))
+
+
+#: A space between two CJK characters is a line-box artifact, never text. Only
+#: blanks within a line are removed: a newline still separates two blocks.
+_CJK_SPACE = re.compile(r'(?<=[\u3400-\u9fff\uf900-\ufaff\u3040-\u30ff])[ \t]+(?=[\u3400-\u9fff\uf900-\ufaff\u3040-\u30ff])')
+
+#: Spaces that OCR inserts before closing punctuation or after opening punctuation.
+_BEFORE_CLOSER = re.compile(r'[ \t]+(?=[，。、；：？！）》」』】”’])')
+_AFTER_OPENER = re.compile(r'(?<=[（《「『【“‘])[ \t]+')
+
+
+def normalize_markdown(text: str) -> str:
+    """Remove the whitespace artifacts a page-by-page merge leaves behind.
+
+    @param text: the joined Markdown body.
+    @returns: the same body with collapsed blank runs, no trailing spaces, and no
+        spaces that would only exist because of a line box.
+    """
+    lines = [line.rstrip() for line in text.replace('\r\n', '\n').split('\n')]
+    collapsed: list[str] = []
+    for line in lines:
+        if line == '' and collapsed and collapsed[-1] == '':
+            continue
+        collapsed.append(line)
+    # A heading's spacing is deliberate, so the line-box cleanup skips it.
+    cleaned = [
+        line if line.startswith('#') else _CJK_SPACE.sub('', line)
+        for line in collapsed
+    ]
+    body = '\n'.join(cleaned).strip('\n')
+    body = _BEFORE_CLOSER.sub('', body)
+    body = _AFTER_OPENER.sub('', body)
+    return body.rstrip() + '\n' if body.strip() != '' else ''

package/python/clean.py

@@ -0,0 +1,197 @@
+"""Page-level noise removal and cross-page paragraph stitching.
+
+A page of OCR output carries furniture that is not part of the document body:
+the folio, the running head or foot, and the whitespace the page break leaves
+behind. This module identifies that furniture from geometry and from repetition
+across pages, so a bound volume reads as continuous prose.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass, field
+from math import ceil
+from typing import Sequence
+
+from layout import Line, is_cjk
+
+#: Fraction of page height at each edge that counts as the header/footer band.
+BAND_FRACTION = 0.14
+
+#: Patterns that identify a folio on their own, wherever the line sits. Each
+#: carries a decoration, a caption, or a total, so plain content cannot match it.
+STRONG_PAGE_NUMBER = (
+    re.compile(r'^[-—–=~·]\s*\d{1,5}\s*[-—–=~·]$'),
+    re.compile(r'^\d{1,5}\s*/\s*\d{1,5}$'),
+    re.compile(r'^[第頁页]\s*\d{1,5}\s*[頁页]?$'),
+    re.compile(r'^第\s*\d+\s*页\s*共\s*\d+\s*页$'),
+    re.compile(r'^(?:page|page\s+no\.?|p\.)\s*\d+(?:\s*(?:of|/)\s*\d+)?\.?$', re.IGNORECASE),
+)
+
+#: Patterns that identify a folio only inside the header or footer band, because
+#: a bare number in the middle of a page is ordinary content.
+WEAK_PAGE_NUMBER = (
+    re.compile(r'^\d{1,5}$'),
+    re.compile(r'^[ivxlcdm]{1,8}$', re.IGNORECASE),
+    re.compile(r'^[-—–=~·]\s*\d{1,5}$'),
+)
+
+#: Characters a folio may carry around its number without becoming content.
+_FOLIO_FILLER = re.compile(r'^[\s\-—–=~·.()\[\]{}<>|/\\]+$')
+
+
+@dataclass
+class Page:
+    """One rendered page: its size and the lines recognized on it."""
+
+    index: int
+    width: float
+    height: float
+    lines: list[Line] = field(default_factory=list)
+
+
+def normalize_label(text: str) -> str:
+    """Fold a line to the key used for cross-page repetition counting.
+
+    Digits and Roman numerals are replaced by a placeholder so that a running
+    head whose only variation is the folio counts as one repeated label.
+    """
+    folded = unicodedata.normalize('NFKC', text).strip().lower()
+    folded = re.sub(r'\d+', '#', folded)
+    folded = re.sub(r'\s+', ' ', folded)
+    return folded
+
+
+def band_of(line: Line, page_height: float, fraction: float = BAND_FRACTION) -> str | None:
+    """Report which edge band, if any, contains a line."""
+    if page_height <= 0:
+        return None
+    if line.y1 <= fraction * page_height:
+        return 'top'
+    if line.y0 >= (1.0 - fraction) * page_height:
+        return 'bottom'
+    return None
+
+
+def is_page_number(line: Line, page_height: float, fraction: float = BAND_FRACTION) -> bool:
+    """Report whether a line is a folio rather than document content."""
+    text = line.text.strip()
+    if text == '' or len(text) > 32:
+        return False
+    bare = text.strip('-—–=~·. ')
+    if bare == '' or _FOLIO_FILLER.match(text):
+        return True
+    for pattern in STRONG_PAGE_NUMBER:
+        if pattern.match(text):
+            return True
+    if band_of(line, page_height, fraction) is not None:
+        for pattern in WEAK_PAGE_NUMBER:
+            if pattern.match(text):
+                return True
+    return False
+
+
+def running_labels(
+    pages: Sequence[Page],
+    ratio: float,
+    min_pages: int,
+    fraction: float = BAND_FRACTION,
+    max_height: float = 0.0,
+) -> set[str]:
+    """Collect the labels that repeat in the edge bands of most pages.
+
+    A label repeated in the same band on enough pages is furniture (a running
+    head, a journal name, a download stamp), not content. Requiring an absolute
+    page minimum keeps a two-page document from losing its own repeated lines,
+    and the height ceiling keeps a title set in display type from voting: a
+    document whose title is reprinted as its running head must keep the title on
+    the page that opens it.
+
+    @param pages: every page of the document, in order.
+    @param ratio: fraction of pages a label must appear on.
+    @param min_pages: fewest pages a label must appear on, whatever the ratio.
+    @param fraction: size of each edge band as a fraction of page height.
+    @param max_height: tallest line that may count as furniture; 0 disables it.
+    @returns: the folded labels to drop.
+    """
+    if not pages:
+        return set()
+    seen: dict[tuple[str, str], set[int]] = {}
+    for page in pages:
+        for line in page.lines:
+            if max_height > 0 and line.height > max_height:
+                continue
+            band = band_of(line, page.height, fraction)
+            if band is None:
+                continue
+            label = normalize_label(line.text)
+            if label == '' or len(label) > 80:
+                continue
+            seen.setdefault((band, label), set()).add(page.index)
+    threshold = max(min_pages, ceil(ratio * len(pages)))
+    return {label for (_band, label), indexes in seen.items() if len(indexes) >= threshold}
+
+
+def strip_noise(
+    page: Page,
+    labels: set[str],
+    remove_page_numbers: bool,
+    remove_running_heads: bool,
+    fraction: float = BAND_FRACTION,
+    max_height: float = 0.0,
+) -> list[Line]:
+    """Drop a page's furniture and return the lines that are document content.
+
+    @param page: the page to clean.
+    @param labels: running-head labels collected from the whole document.
+    @param remove_page_numbers: whether folios are dropped.
+    @param remove_running_heads: whether repeated edge labels are dropped.
+    @param fraction: size of each edge band as a fraction of page height.
+    @param max_height: tallest line the label rule may drop; 0 disables it.
+    @returns: the remaining lines, in reading order.
+    """
+    kept: list[Line] = []
+    for line in page.lines:
+        if line.text.strip() == '':
+            continue
+        if remove_page_numbers and is_page_number(line, page.height, fraction):
+            continue
+        if remove_running_heads and (max_height <= 0 or line.height <= max_height):
+            band = band_of(line, page.height, fraction)
+            if band is not None and normalize_label(line.text) in labels:
+                continue
+        kept.append(line)
+    return kept
+
+
+#: Characters that close a sentence, so the next line starts a new one.
+TERMINAL = '。！？…；!?;'
+
+#: Characters that close a bracket run, which also ends the sentence.
+CLOSERS = '”"』」）》】)\']'
+
+
+def ends_sentence(text: str) -> bool:
+    """Report whether a line finishes the sentence it belongs to."""
+    stripped = text.rstrip()
+    while stripped and stripped[-1] in CLOSERS:
+        stripped = stripped[:-1].rstrip()
+    return stripped.endswith(tuple(TERMINAL))
+
+
+def join_wrapped(previous: str, following: str) -> str:
+    """Join two lines of one paragraph the way the source language demands.
+
+    Latin text keeps a single space and repairs a hyphenated break; CJK text is
+    joined with nothing, because the space would be an artifact of the line box.
+    """
+    left = previous.rstrip()
+    right = following.lstrip()
+    if left.endswith('-') and right[:1].islower():
+        return left[:-1] + right
+    if left.endswith(tuple('([{“‘《〈「『（【')):
+        return left + right
+    if left and right and (is_cjk(left[-1]) or is_cjk(right[0])):
+        return left + right
+    return left + ' ' + right