document-ocr 0.0.0-stage → 3.1.3

package/hosting/core/document_input.py

@@ -0,0 +1,135 @@
+"""Bounded PDF page loading and native text geometry in rendered-page coordinates.
+
+PDFium is not thread-safe. Every PDF operation, including object disposal, runs
+under the same process-wide lock. Native text is extraction evidence, not a
+claim that a PDF is authentic or that its hidden text matches its visible ink.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import threading
+import math
+
+import cv2
+import numpy as np
+
+from .ocr_engine import TextRegion
+
+PDF_LOCK = threading.RLock()
+MAX_PAGES = 10
+MAX_PAGE_PIXELS = 20_000_000
+MAX_DOCUMENT_PIXELS = 40_000_000
+MAX_TEXT_CHARS = 100_000
+PDF_WIDTH = 1600
+
+
+@dataclass
+class InputPage:
+    image: np.ndarray
+    regions: list[TextRegion]
+    number: int
+    total: int
+
+
+def input_bytes(source: str | bytes | Path) -> bytes:
+    return Path(source).read_bytes() if isinstance(source, (str, Path)) else source
+
+
+def pdf_pages(data: bytes, *, first_only: bool = False) -> list[InputPage]:
+    from .preprocessor import ImageQualityError
+    import pypdfium2 as pdfium
+
+    with PDF_LOCK:
+        try:
+            doc = pdfium.PdfDocument(data)
+        except pdfium.PdfiumError as exc:
+            raise ImageQualityError('INVALID_PDF') from exc
+        try:
+            total = len(doc)
+            if not total:
+                raise ImageQualityError('PDF_HAS_NO_PAGES')
+            if total > MAX_PAGES and not first_only:
+                raise ImageQualityError('TOO_MANY_PAGES')
+            output = []
+            total_pixels = 0
+            for index in range(1 if first_only else total):
+                page = doc[index]
+                textpage = bitmap = None
+                try:
+                    width, height = page.get_size()
+                    if width <= 0 or height <= 0:
+                        raise ImageQualityError('INVALID_PDF_PAGE_SIZE')
+                    scale = PDF_WIDTH / width
+                    pixels = math.ceil(width * scale) * math.ceil(height * scale)
+                    if pixels > MAX_PAGE_PIXELS:
+                        raise ImageQualityError('PDF_PAGE_TOO_LARGE')
+                    total_pixels += pixels
+                    if total_pixels > MAX_DOCUMENT_PIXELS:
+                        raise ImageQualityError('PDF_DOCUMENT_TOO_LARGE')
+                    bitmap = page.render(scale=scale)
+                    image = cv2.cvtColor(np.array(bitmap.to_pil().convert('RGB')), cv2.COLOR_RGB2BGR)
+                    textpage = page.get_textpage()
+                    regions = _native_regions(textpage, bitmap.get_posconv(page), image.shape[1], image.shape[0])
+                    # Character boxes are in unrotated PDF space. On a rotated
+                    # page fall back to rendered OCR until its transform is known.
+                    if page.get_rotation():
+                        regions = []
+                    output.append(InputPage(image, regions, index + 1, total))
+                finally:
+                    if textpage is not None:
+                        textpage.close()
+                    if bitmap is not None:
+                        bitmap.close()
+                    page.close()
+            return output
+        except pdfium.PdfiumError as exc:
+            raise ImageQualityError('INVALID_PDF') from exc
+        finally:
+            doc.close()
+
+
+def _native_regions(textpage, converter, width: int, height: int) -> list[TextRegion]:
+    from .preprocessor import ImageQualityError
+    count = textpage.count_chars()
+    if count > MAX_TEXT_CHARS:
+        raise ImageQualityError('PDF_TEXT_TOO_LARGE')
+    regions, chars, boxes = [], [], []
+    line_top = line_bottom = None
+
+    def flush():
+        nonlocal line_top, line_bottom
+        text = ''.join(chars).strip()
+        if text and boxes:
+            x0, y0 = min(b[0] for b in boxes), min(b[1] for b in boxes)
+            x1, y1 = max(b[2] for b in boxes), max(b[3] for b in boxes)
+            regions.append(TextRegion(text, [[x0,y0],[x1,y0],[x1,y1],[x0,y1]], 1.0))
+        chars.clear()
+        boxes.clear()
+        line_top = line_bottom = None
+
+    for index in range(count):
+        char = textpage.get_text_range(index, 1)
+        if not char or char in '\r\n':
+            flush()
+            continue
+        if char.isspace():
+            chars.append(' ')
+            continue
+        left, bottom, right, top = textpage.get_charbox(index)
+        # PDFium's own transform handles CropBox/MediaBox offsets and bitmap
+        # rounding. Scaling page coordinates ourselves misaligns redaction.
+        corners = [converter.to_bitmap(x, y) for x, y in
+                   ((left, bottom), (right, bottom), (right, top), (left, top))]
+        box = [max(0, min(p[0] for p in corners)), max(0, min(p[1] for p in corners)),
+               min(width, max(p[0] for p in corners)), min(height, max(p[1] for p in corners))]
+        if box[0] >= box[2] or box[1] >= box[3]:
+            continue
+        if boxes and (box[1] > line_bottom+3 or box[3] < line_top-3 or box[0] < boxes[-1][0]-5):
+            flush()
+        chars.append(char)
+        boxes.append(box)
+        line_top = min(line_top, box[1]) if line_top is not None else box[1]
+        line_bottom = max(line_bottom, box[3]) if line_bottom is not None else box[3]
+    flush()
+    return regions

package/hosting/core/document_registry.py

@@ -0,0 +1,143 @@
+"""Explicit document capabilities and normalized country/type hints."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pycountry
+
+
+# ISO 3166-1 plus Doc 9303 Part 3, section 5 A–G (2024 consolidated edition):
+# https://www.icao.int/sites/default/files/publications/DocSeries/9303_p3_cons_en.pdf
+# Includes unspecified nationality XXX and deprecated codes for old documents.
+# Part H's IAO signs ICAO master lists; it is not a travel-document country.
+MRZ_COUNTRY_CODES = frozenset(country.alpha_3 for country in pycountry.countries) | {
+    "D", "GBD", "GBN", "GBO", "GBP", "GBS", "RKS", "EUE",
+    "UNO", "UNA", "UNK", "XBA", "XIM", "XCC", "XCE", "XCO", "XEC",
+    "XPO", "XES", "XMP", "XOM", "XDC", "XXA", "XXB", "XXC", "XXX",
+    "ANT", "NTZ", "UTO",
+}
+
+
+def is_known_mrz_country(code: str | None) -> bool:
+    """Validate an observed MRZ code without substituting a different code."""
+    return isinstance(code, str) and code.upper().rstrip("<") in MRZ_COUNTRY_CODES
+
+
+@dataclass(frozen=True)
+class DocumentProfile:
+    document_type: str
+    name: str
+    countries: tuple[str, ...]
+    fields: tuple[str, ...]
+    required_fields: tuple[str, ...]
+    extraction_methods: tuple[str, ...]
+    experimental: bool = True
+    legacy: bool = False
+
+
+_IDENTITY = ("surname", "given_names", "date_of_birth", "document_number", "expiry_date")
+_ADDRESS = ("address", "city", "state", "postal_code")
+_MRZ = ("country_code", "mrz_raw")
+_PROFILES = (
+    DocumentProfile("passport", "Passport book", ("*",), ("surname", "given_names", "full_name", "passport_number", "nationality", "date_of_birth", "sex", "expiry_date", "issue_date", "place_of_birth", "country_code"), ("surname", "passport_number"), ("mrz", "ocr"), False, True),
+    DocumentProfile("pan", "Indian PAN card", ("IN",), ("pan_number", "name", "father_name", "date_of_birth"), ("pan_number", "name", "date_of_birth"), ("ocr",), False, True),
+    DocumentProfile("aadhaar", "Indian Aadhaar card", ("IN",), ("aadhaar_number", "name", "date_of_birth", "year_of_birth", "gender", "address", "pincode", "checksum_valid", "aadhaar_masked", "aadhaar_last4", "vid"), ("aadhaar_number_or_masked_last4", "name", "date_or_year_of_birth", "gender"), ("ocr",), False, True),
+    DocumentProfile("driving_licence", "Indian driving licence", ("IN",), ("dl_number", "name", "date_of_birth", "issue_date", "validity_date", "address", "relation_name", "blood_group", "class_of_vehicle", "validity_date_transport"), ("dl_number", "name", "date_of_birth", "issue_date", "validity_date"), ("ocr",), False, True),
+    DocumentProfile("voter_id", "Indian voter ID", ("IN",), ("epic_number", "name", "relation_name", "relation_type", "gender", "date_of_birth", "age"), ("epic_number", "name", "date_of_birth_or_age"), ("ocr",), False, True),
+    DocumentProfile("nrega_job_card", "Indian NREGA job card", ("IN",), ("job_card_number", "head_of_household", "category", "registration_date", "validity_from", "validity_to", "address", "village", "gram_panchayat", "block", "district", "state", "bpl_status", "family_id", "members"), ("job_card_number", "household_or_member_name", "location"), ("ocr",), True, True),
+    DocumentProfile("npr_letter", "Indian NPR letter", ("IN",), ("reference_number", "name", "address", "pincode", "issue_date"), ("name", "address"), ("ocr",), True, True),
+    DocumentProfile("us_driver_license", "US driver license", ("US",), _IDENTITY + _ADDRESS + ("first_name", "middle_names", "issue_date", "sex", "license_class", "restrictions", "endorsements"), ("surname", "document_number", "date_of_birth", "expiry_date"), ("pdf417", "ocr")),
+    DocumentProfile("us_state_id", "US state identification card", ("US",), _IDENTITY + _ADDRESS + ("first_name", "middle_names", "issue_date", "sex"), ("surname", "document_number", "date_of_birth", "expiry_date"), ("pdf417", "ocr")),
+    DocumentProfile("passport_card", "Passport card", ("*",), _IDENTITY + ("nationality", "sex", "issue_date", "address") + _MRZ, ("surname", "document_number", "date_of_birth", "expiry_date"), ("mrz", "ocr")),
+    DocumentProfile("us_green_card", "US Permanent Resident Card (I-551)", ("US",), ("surname", "given_names", "uscis_number", "card_number", "date_of_birth", "expiry_date", "category", "resident_since", "country_of_birth", "sex") + _MRZ, ("surname", "uscis_number", "card_number", "date_of_birth"), ("mrz", "ocr")),
+    DocumentProfile("us_ead", "US Employment Authorization Document (I-766)", ("US",), ("surname", "given_names", "uscis_number", "card_number", "date_of_birth", "expiry_date", "category", "valid_from", "country_of_birth", "sex") + _MRZ, ("surname", "uscis_number", "card_number", "date_of_birth", "expiry_date"), ("mrz", "ocr")),
+    DocumentProfile("visa", "Machine-readable visa (MRV-A / MRV-B)", ("*",), _IDENTITY + ("nationality", "sex", "mrz_format") + _MRZ, ("surname", "document_number", "date_of_birth", "expiry_date"), ("mrz", "ocr")),
+    DocumentProfile("us_i94", "US I-94 arrival/departure record", ("US",), ("i94_number", "surname", "given_names", "date_of_birth", "admission_date", "admit_until", "class_of_admission", "passport_number", "country_of_citizenship"), ("i94_number", "surname", "class_of_admission", "admit_until"), ("ocr",)),
+    DocumentProfile("us_w9", "US Form W-9", ("US",), ("name", "business_name", "taxpayer_id", "taxpayer_id_type", "address", "city_state_postal_code"), ("name", "taxpayer_id", "taxpayer_id_type"), ("ocr",)),
+)
+DOCUMENT_PROFILES = {profile.document_type: profile for profile in _PROFILES}
+
+
+def normalize_country(country: str | None) -> str | None:
+    if country is None:
+        return None
+    if not isinstance(country, str):
+        raise ValueError("INVALID_COUNTRY_HINT_TYPE")
+    value = country.strip().upper()
+    aliases = {"USA": "US", "UNITED STATES": "US", "UNITED STATES OF AMERICA": "US", "IND": "IN", "INDIA": "IN", "UK": "GB"}
+    value = aliases.get(value, value)
+    result = pycountry.countries.get(alpha_2=value) if len(value) == 2 else pycountry.countries.get(alpha_3=value)
+    if result is None:
+        raise ValueError(f"UNSUPPORTED_COUNTRY: {country}")
+    return result.alpha_2
+
+
+def validate_document_hint(document_type: str | None, country: str | None = None) -> tuple[str | None, str | None]:
+    """Return canonical hints or fail explicitly instead of ignoring a conflict."""
+    country = normalize_country(country)
+    if document_type is None:
+        return None, country
+    if not isinstance(document_type, str):
+        raise ValueError("INVALID_DOCUMENT_TYPE_HINT_TYPE")
+    value = document_type.strip().lower().replace("-", "_").replace(" ", "_")
+    aliases = {
+        "driver_license": "driving_licence", "driving_license": "driving_licence",
+        "drivers_license": "driving_licence", "state_id": "us_state_id",
+        "green_card": "us_green_card", "i551": "us_green_card", "i_551": "us_green_card",
+        "ead": "us_ead", "i766": "us_ead", "i_766": "us_ead",
+        "i94": "us_i94", "i_94": "us_i94", "w9": "us_w9", "w_9": "us_w9",
+        "us_passport": "passport", "passport_book": "passport",
+    }
+    value = aliases.get(value, value)
+    if value == "driving_licence" and country == "US":
+        value = "us_driver_license"
+    profile = DOCUMENT_PROFILES.get(value)
+    if profile is None:
+        raise ValueError(f"UNSUPPORTED_DOCUMENT_TYPE: {document_type}")
+    if country is not None and "*" not in profile.countries and country not in profile.countries:
+        raise ValueError(f"DOCUMENT_COUNTRY_MISMATCH: {value}/{country}")
+    if country is None and len(profile.countries) == 1 and profile.countries[0] != "*":
+        country = profile.countries[0]
+    return value, country
+
+
+def list_document_profiles() -> list[dict]:
+    """Return JSON-safe metadata, including experimental support boundaries."""
+    def camel(value: str) -> str:
+        head, *tail = value.split("_")
+        return head + "".join(part.title() for part in tail)
+
+    profiles = [
+        {
+            "documentType": profile.document_type,
+            "label": profile.name,
+            "countries": list(profile.countries),
+            "fields": [camel(name) for name in profile.fields],
+            "requiredFields": [camel(name) for name in profile.required_fields],
+            "sources": list(profile.extraction_methods),
+            "experimental": profile.experimental,
+        }
+        for profile in _PROFILES
+    ]
+    # The legacy completion rules contain alternatives and, for Aadhaar,
+    # different front/address-side requirements. Expose those explicitly rather
+    # than pretending every logical requirement is one physical output field.
+    for profile in profiles:
+        if profile["documentType"] == "aadhaar":
+            profile["requiredFieldsByVariant"] = {
+                "front": profile["requiredFields"].copy(),
+                "address": ["aadhaarNumberOrMaskedLast4", "address", "pincode"],
+            }
+            profile["requirementAlternatives"] = {
+                "aadhaarNumberOrMaskedLast4": [["aadhaarNumber"], ["aadhaarMasked", "aadhaarLast4"]],
+                "dateOrYearOfBirth": [["dateOfBirth"], ["yearOfBirth"]],
+            }
+        elif profile["documentType"] == "voter_id":
+            profile["requirementAlternatives"] = {"dateOfBirthOrAge": [["dateOfBirth"], ["age"]]}
+        elif profile["documentType"] == "nrega_job_card":
+            profile["requirementAlternatives"] = {
+                "householdOrMemberName": [["headOfHousehold"], ["members[].name"]],
+                "location": [["village"], ["gramPanchayat"], ["district"], ["address"]],
+            }
+    return profiles