document-ocr 0.0.0-stage → 3.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +260 -2
  3. package/dist/index.cjs +659 -0
  4. package/dist/index.cjs.map +1 -0
  5. package/dist/index.d.cts +293 -0
  6. package/dist/index.d.ts +293 -0
  7. package/dist/index.js +616 -0
  8. package/dist/index.js.map +1 -0
  9. package/hosting/.dockerignore +17 -0
  10. package/hosting/.gcloudignore +14 -0
  11. package/hosting/FEATURES.md +172 -0
  12. package/hosting/HOSTING.md +268 -0
  13. package/hosting/LICENSE +21 -0
  14. package/hosting/README.md +369 -0
  15. package/hosting/core/__init__.py +0 -0
  16. package/hosting/core/aadhaar_extractor.py +298 -0
  17. package/hosting/core/back_page_extractor.py +492 -0
  18. package/hosting/core/barcodes.py +125 -0
  19. package/hosting/core/card_recovery.py +244 -0
  20. package/hosting/core/document_bundle.py +82 -0
  21. package/hosting/core/document_classifier.py +233 -0
  22. package/hosting/core/document_input.py +135 -0
  23. package/hosting/core/document_registry.py +143 -0
  24. package/hosting/core/driving_licence_extractor.py +510 -0
  25. package/hosting/core/ead_recovery.py +54 -0
  26. package/hosting/core/evidence.py +54 -0
  27. package/hosting/core/form_recovery.py +61 -0
  28. package/hosting/core/jobs.py +232 -0
  29. package/hosting/core/kyc_ocr.py +132 -0
  30. package/hosting/core/kyc_validation.py +411 -0
  31. package/hosting/core/model_setup.py +13 -0
  32. package/hosting/core/mrz_parser.py +336 -0
  33. package/hosting/core/mrz_recovery.py +193 -0
  34. package/hosting/core/npr_extractor.py +410 -0
  35. package/hosting/core/nrega_extractor.py +827 -0
  36. package/hosting/core/ocr_engine.py +207 -0
  37. package/hosting/core/page_classifier.py +137 -0
  38. package/hosting/core/pan_extractor.py +183 -0
  39. package/hosting/core/pipeline.py +712 -0
  40. package/hosting/core/preprocessor.py +297 -0
  41. package/hosting/core/recovery_budget.py +67 -0
  42. package/hosting/core/redaction.py +48 -0
  43. package/hosting/core/review.html +244 -0
  44. package/hosting/core/structured_documents.py +422 -0
  45. package/hosting/core/travel_mrz.py +164 -0
  46. package/hosting/core/travel_recovery.py +187 -0
  47. package/hosting/core/validator.py +396 -0
  48. package/hosting/core/validators.py +179 -0
  49. package/hosting/core/voter_id_extractor.py +363 -0
  50. package/hosting/deploy/cloudflare/deploy.sh +44 -0
  51. package/hosting/deploy/cloudflare/package-lock.json +3088 -0
  52. package/hosting/deploy/cloudflare/package.json +29 -0
  53. package/hosting/deploy/cloudflare/src/gateway.ts +42 -0
  54. package/hosting/deploy/cloudflare/src/index.ts +21 -0
  55. package/hosting/deploy/cloudflare/test/gateway.test.ts +59 -0
  56. package/hosting/deploy/cloudflare/tsconfig.json +12 -0
  57. package/hosting/deploy/cloudflare/worker-configuration.d.ts +16253 -0
  58. package/hosting/deploy/cloudflare/wrangler.jsonc +26 -0
  59. package/hosting/deploy/cloudrun/cloudbuild.yaml +18 -0
  60. package/hosting/deploy/cloudrun/deploy.sh +71 -0
  61. package/hosting/deploy/cloudrun/render_service.py +47 -0
  62. package/hosting/deploy/common.sh +14 -0
  63. package/hosting/deploy/docker/Dockerfile +16 -0
  64. package/hosting/deploy/docker/server.py +313 -0
  65. package/hosting/deploy/lambda/Dockerfile.lambda +11 -0
  66. package/hosting/deploy/lambda/deploy.sh +22 -0
  67. package/hosting/deploy/lambda/handler.py +132 -0
  68. package/hosting/deploy/lambda/template.yaml +59 -0
  69. package/hosting/deploy/local/dev.sh +18 -0
  70. package/hosting/deploy/server/Caddyfile +5 -0
  71. package/hosting/deploy/server/compose.jobs.yaml +36 -0
  72. package/hosting/deploy/server/compose.yaml +38 -0
  73. package/hosting/deploy/server/deploy.sh +23 -0
  74. package/hosting/infra/cloudrun/main.tf +104 -0
  75. package/hosting/infra/cloudrun/terraform.tfvars.example +7 -0
  76. package/hosting/infra/cloudrun/variables.tf +55 -0
  77. package/hosting/infra/lambda/main.tf +63 -0
  78. package/hosting/infra/lambda/terraform.tfvars.example +6 -0
  79. package/hosting/infra/lambda/variables.tf +37 -0
  80. package/hosting/pyproject.toml +65 -0
  81. package/hosting/requirements.lock +134 -0
  82. package/package.json +76 -4
  83. package/python/core/__init__.py +0 -0
  84. package/python/core/aadhaar_extractor.py +298 -0
  85. package/python/core/back_page_extractor.py +492 -0
  86. package/python/core/barcodes.py +125 -0
  87. package/python/core/card_recovery.py +244 -0
  88. package/python/core/document_bundle.py +82 -0
  89. package/python/core/document_classifier.py +233 -0
  90. package/python/core/document_input.py +135 -0
  91. package/python/core/document_registry.py +143 -0
  92. package/python/core/driving_licence_extractor.py +510 -0
  93. package/python/core/ead_recovery.py +54 -0
  94. package/python/core/evidence.py +54 -0
  95. package/python/core/form_recovery.py +61 -0
  96. package/python/core/jobs.py +232 -0
  97. package/python/core/kyc_ocr.py +132 -0
  98. package/python/core/kyc_validation.py +411 -0
  99. package/python/core/model_setup.py +13 -0
  100. package/python/core/mrz_parser.py +336 -0
  101. package/python/core/mrz_recovery.py +193 -0
  102. package/python/core/npr_extractor.py +410 -0
  103. package/python/core/nrega_extractor.py +827 -0
  104. package/python/core/ocr_engine.py +207 -0
  105. package/python/core/page_classifier.py +137 -0
  106. package/python/core/pan_extractor.py +183 -0
  107. package/python/core/pipeline.py +712 -0
  108. package/python/core/preprocessor.py +297 -0
  109. package/python/core/recovery_budget.py +67 -0
  110. package/python/core/redaction.py +48 -0
  111. package/python/core/review.html +244 -0
  112. package/python/core/structured_documents.py +422 -0
  113. package/python/core/travel_mrz.py +164 -0
  114. package/python/core/travel_recovery.py +187 -0
  115. package/python/core/validator.py +396 -0
  116. package/python/core/validators.py +179 -0
  117. package/python/core/voter_id_extractor.py +363 -0
  118. package/python/pyproject.toml +34 -0
  119. package/python/requirements.lock +134 -0
  120. package/python/server.py +313 -0
  121. package/scripts/dococr.mjs +163 -0
  122. package/scripts/postinstall.mjs +173 -0
  123. package/scripts/sync-hosting.mjs +35 -0
  124. package/scripts/sync-python.sh +20 -0
@@ -0,0 +1,422 @@
1
+ """Country-aware extraction for explicitly registered document profiles.
2
+
3
+ Machine-readable data and visual OCR remain separate evidence. Conflicting
4
+ values are surfaced instead of silently letting a confident source overwrite
5
+ another. These extraction checks do not authenticate a document or its holder.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ import math
12
+ from dataclasses import dataclass, field
13
+ from datetime import date, datetime
14
+ from typing import Callable
15
+
16
+ import pycountry
17
+
18
+ from .barcodes import BarcodeResult, parse_aamva
19
+ from .document_registry import DOCUMENT_PROFILES, normalize_country, validate_document_hint
20
+ from .ocr_engine import TextRegion
21
+ from .travel_mrz import parse_travel_mrz
22
+
23
+
24
+ @dataclass
25
+ class StructuredExtraction:
26
+ document_type: str
27
+ issuing_country: str | None = None
28
+ issuing_region: str | None = None
29
+ fields: dict = field(default_factory=dict)
30
+ field_evidence: dict = field(default_factory=dict)
31
+ checks: dict = field(default_factory=dict)
32
+ missing_required_fields: list[str] = field(default_factory=list)
33
+ errors: list[str] = field(default_factory=list)
34
+ warnings: list[str] = field(default_factory=list)
35
+ confidence: float = 0.0
36
+
37
+ @property
38
+ def complete(self) -> bool:
39
+ return bool(self.fields) and not self.missing_required_fields and not self.errors
40
+
41
+
42
+ _US_STATES = {item.code.split("-")[1]: item.name.upper() for item in pycountry.subdivisions.get(country_code="US")}
43
+ _FIELD_LABEL = re.compile(
44
+ r"^(?:\d{1,2}[a-z]?\s+)?(?:3[ab]\b|surname|family name|given names?|first name|middle name|date of birth|birth date|dob|"
45
+ r"date of (?:issue|expiry)|expires(?:\s*on)?|exp(?:iry|iration)?(?: date)?|iss|issue(?:d| date)?|sex|nationality|"
46
+ r"uscis|card (?:number|expires)|category|country of birth|resident since|valid from|"
47
+ r"class of admission|admit until|admission.*record number|passport number|"
48
+ r"name(?: of entity/individual)?|business name|address|city|social security number|employer identification number)\b",
49
+ re.I,
50
+ )
51
+
52
+
53
+ def _compact(value: str) -> str:
54
+ return re.sub(r"[\s-]", "", value.upper())
55
+
56
+
57
+ def _us_date(value: str) -> str | None:
58
+ value = value.strip().replace(".", "/")
59
+ if re.fullmatch(r"[0-9\s/-]+", value):
60
+ value = re.sub(r"\s+", "", value)
61
+ # OCR often joins a printed month to its neighbouring day or year.
62
+ value = re.sub(r"(?<=[A-Za-z])(?=[0-9])|(?<=[0-9])(?=[A-Za-z])", " ", value)
63
+ for fmt in ("%m/%d/%Y", "%m-%d-%Y", "%Y-%m-%d", "%m%d%Y", "%d %b %Y", "%d %B %Y", "%d-%b-%Y",
64
+ "%m/%d/%y", "%m-%d-%y", "%Y %B %d", "%Y %b %d"):
65
+ try:
66
+ return datetime.strptime(value, fmt).date().isoformat()
67
+ except ValueError:
68
+ continue
69
+ return None
70
+
71
+
72
+ def _name(value: str) -> str | None:
73
+ value = " ".join(value.strip(" :").split())
74
+ if not 1 <= len(value) <= 100 or not re.search(r"[A-Za-z]", value):
75
+ return None
76
+ if re.search(r"\b(?:required|enter|instructions|certification|requester|taxpayer|identification|security)\b|shown on|line [0-9]", value, re.I):
77
+ return None
78
+ return value
79
+
80
+
81
+ def _person_name(value: str) -> str | None:
82
+ return _name(value) if not re.search(r"[0-9]", value) else None
83
+
84
+
85
+ def _pattern(pattern: str) -> Callable[[str], str | None]:
86
+ return lambda value: _compact(value) if re.fullmatch(pattern, _compact(value)) else None
87
+
88
+
89
+ def _document_number(value: str) -> str | None:
90
+ value = _compact(value)
91
+ return value if re.fullmatch(r"[A-Z0-9]{4,25}", value) and re.search(r"[0-9]", value) else None
92
+
93
+
94
+ def _country_of_birth(value: str) -> str | None:
95
+ try:
96
+ return pycountry.countries.lookup(value.strip()).alpha_3
97
+ except LookupError:
98
+ return None
99
+
100
+
101
+ def _evidence(region: TextRegion, text: str | None = None) -> dict:
102
+ return {"text": text if text is not None else region.text, "bbox": region.bbox, "source": "ocr", "confidence": region.confidence}
103
+
104
+
105
+ def _label_values(regions: list[TextRegion], labels: tuple[str, ...]):
106
+ """Yield explicit inline values, then spatially adjacent non-label regions."""
107
+ for label in labels:
108
+ expression = re.compile(r"^\s*(?:[0-9]{1,2}[a-z]?\s+)?(?:" + label + r")(?![A-Za-z])\s*[:#]?\s*", re.I)
109
+ for region in regions:
110
+ match = expression.match(region.text)
111
+ if not match:
112
+ continue
113
+ inline = region.text[match.end():].strip()
114
+ if inline:
115
+ yield inline, region
116
+ if not region.bbox:
117
+ continue
118
+ origin, end = region.bbox[:2]
119
+ dx, dy = end[0] - origin[0], end[1] - origin[1]
120
+ length = math.hypot(dx, dy)
121
+ if length < 1:
122
+ continue
123
+ ux, uy = dx / length, dy / length
124
+
125
+ def bounds(other):
126
+ xs = [(p[0] - origin[0]) * ux + (p[1] - origin[1]) * uy for p in other.bbox]
127
+ ys = [-(p[0] - origin[0]) * uy + (p[1] - origin[1]) * ux for p in other.bbox]
128
+ return min(xs), min(ys), max(xs), max(ys)
129
+
130
+ # Match fields in the label's own axes, including tilted photos.
131
+ left, top, right, bottom = bounds(region)
132
+ height = max(bottom - top, 1)
133
+ nearby = []
134
+ for candidate in regions:
135
+ if candidate is region or not candidate.bbox or _FIELD_LABEL.match(candidate.text):
136
+ continue
137
+ c_left, c_top, c_right, c_bottom = bounds(candidate)
138
+ if c_bottom - c_top > height * 2:
139
+ # Large background words/watermarks can overlap a small
140
+ # label without being that field's value.
141
+ continue
142
+ overlap = min(bottom, c_bottom) - max(top, c_top)
143
+ gap = c_left - right
144
+ # A long horizontal value can have a slightly different OCR
145
+ # angle from its short label. Its furthest corner exaggerates
146
+ # vertical overlap; the top-edge centre follows the text row.
147
+ value_top = sum(-(p[0] - origin[0]) * uy + (p[1] - origin[1]) * ux
148
+ for p in candidate.bbox[:2]) / 2
149
+ # OCR boxes can overlap even when the printed value is to
150
+ # the right. Limit tolerance to a small part of the label,
151
+ # and require the value's centre beyond the label's edge.
152
+ overlap_allowance = min(height * .75, (right - left) * .25)
153
+ same_row = (overlap >= height * .5
154
+ and -overlap_allowance <= gap < 450
155
+ and (c_left + c_right) / 2 > right)
156
+ if same_row:
157
+ crossed_label = any(
158
+ other is not region
159
+ and other is not candidate
160
+ and other.bbox
161
+ and _FIELD_LABEL.match(other.text)
162
+ and (left + right) / 2 < bounds(other)[0] < (c_left + c_right) / 2
163
+ and min(bottom, bounds(other)[3]) - max(top, bounds(other)[1]) >= height * .5
164
+ for other in regions
165
+ )
166
+ if crossed_label:
167
+ continue
168
+ nearby.append((max(gap, 0), candidate))
169
+ elif -max(5, height * .35) <= value_top - bottom < max(90, height * 3) and abs(c_left - left) < 220:
170
+ # A blank field must not borrow the value belonging to the
171
+ # next labeled field underneath it (for example W-9 lines
172
+ # 1 and 2). A label in the same column closes this field.
173
+ crossed_label = any(
174
+ other is not region
175
+ and other.bbox
176
+ and _FIELD_LABEL.match(other.text)
177
+ and bottom - 5 <= bounds(other)[1] <= value_top
178
+ and abs(bounds(other)[0] - left) < 220
179
+ for other in regions
180
+ )
181
+ if crossed_label:
182
+ continue
183
+ nearby.append((500 + max(value_top - bottom, 0) + abs(c_left - left), candidate))
184
+ for _, candidate in sorted(nearby, key=lambda item: item[0]):
185
+ yield candidate.text, candidate
186
+
187
+
188
+ def _put(result: StructuredExtraction, key: str, value, evidence: dict | list[dict]) -> None:
189
+ if value is None or value == "":
190
+ return
191
+ evidence = evidence if isinstance(evidence, list) else [evidence]
192
+ result.field_evidence.setdefault(key, []).extend(evidence)
193
+ previous = result.fields.get(key)
194
+ if previous is not None and re.sub(r"\W", "", str(previous)).casefold() != re.sub(r"\W", "", str(value)).casefold():
195
+ error = f"FIELD_MISMATCH_{key.upper()}"
196
+ if error not in result.errors:
197
+ result.errors.append(error)
198
+ result.checks[f"{key}_agreement"] = False
199
+ return
200
+ result.fields[key] = value
201
+
202
+
203
+ def _visual(result: StructuredExtraction, regions: list[TextRegion], key: str, labels: tuple[str, ...], parser: Callable[[str], str | None] = _name) -> None:
204
+ for raw, region in _label_values(regions, labels):
205
+ value = parser(raw)
206
+ if value is not None:
207
+ _put(result, key, value, _evidence(region, raw))
208
+ return
209
+
210
+
211
+ def _state(text: str) -> str | None:
212
+ upper = text.upper()
213
+ for code, name in _US_STATES.items():
214
+ if re.search(r"\b" + re.escape(name) + r"\b", upper):
215
+ return code
216
+ match = re.search(r"\bUSA\s*([A-Z]{2})\b", upper)
217
+ if match and match[1] in _US_STATES:
218
+ return match[1]
219
+ return None
220
+
221
+
222
+ def _detect_text_type(regions: list[TextRegion]) -> tuple[str | None, str | None]:
223
+ text = "\n".join(region.text.upper() for region in regions)
224
+ state = _state(text)
225
+ if state and re.search(r"\bDRIVER(?:'S|S)? LICENSE\b", text):
226
+ return "us_driver_license", state
227
+ if state and re.search(r"\b(?:IDENTIFICATION|IDENTITY) CARD\b", text):
228
+ return "us_state_id", state
229
+ american = bool(re.search(r"\bUSCIS\b|UNITED\s*STATES", text))
230
+ if american and "PERMANENT RESIDENT" in text:
231
+ return "us_green_card", None
232
+ if american and "EMPLOYMENT AUTHORIZATION" in text:
233
+ return "us_ead", None
234
+ if american and "PASSPORT CARD" in text:
235
+ return "passport_card", None
236
+ if re.search(r"\bI[ -]?94\b", text) and "CLASS OF ADMISSION" in text and re.search(r"ADMISSION.*RECORD NUMBER|CUSTOMS AND BORDER|HOMELAND SECURITY", text):
237
+ return "us_i94", None
238
+ if (re.search(r"\bW[ -]?9\b", text) and "REQUEST FOR TAXPAYER" in text
239
+ and "IDENTIFICATION NUMBER AND CERTIFICATION" in text):
240
+ return "us_w9", None
241
+ return None, None
242
+
243
+
244
+ def _extract_aamva(barcodes: list[BarcodeResult]) -> StructuredExtraction | None:
245
+ found: list[StructuredExtraction] = []
246
+ for barcode in barcodes:
247
+ if "PDF417" not in re.sub(r"[^A-Z0-9]", "", barcode.format.upper()):
248
+ continue
249
+ try:
250
+ parsed = parse_aamva(barcode)
251
+ except ValueError as error:
252
+ # A corrupt payload cannot confidently identify DL versus ID.
253
+ result = StructuredExtraction("us_driver_license", errors=[str(error)])
254
+ result.checks.update({"barcode_decoded": True, "barcode_structure_valid": False})
255
+ found.append(result)
256
+ continue
257
+ if parsed is None:
258
+ continue
259
+ raw = parsed.elements
260
+ raw_country = raw.get("DCG")
261
+ country = normalize_country(raw_country) if raw_country in ("US", "USA", "CA", "CAN") else None
262
+ state = raw.get("DAJ")
263
+ if country != "US" and not (raw_country is None and state in _US_STATES):
264
+ continue
265
+ result = StructuredExtraction(parsed.document_type, "US", state, errors=parsed.errors.copy())
266
+ result.checks.update({"barcode_decoded": True, "barcode_structure_valid": not parsed.errors, "aamva_version": parsed.version, "aamva_jurisdiction_version": parsed.jurisdiction_version, "aamva_issuer_id": parsed.issuer_id, "issuer_authenticated": False, "signature_verified": False})
267
+ evidence = {"text": barcode.text, "bbox": barcode.bbox, "source": "pdf417", "confidence": 1.0}
268
+ keys = {"DAQ": "document_number", "DCS": "surname", "DAB": "surname", "DAC": "first_name", "DAD": "middle_names", "DAG": "address", "DAI": "city", "DAJ": "state", "DAK": "postal_code", "DCA": "license_class", "DCB": "restrictions", "DCD": "endorsements"}
269
+ for element, key in keys.items():
270
+ if parsed.document_type == "us_state_id" and element in ("DCA", "DCB", "DCD"):
271
+ continue
272
+ if raw.get(element):
273
+ _put(result, key, raw[element], evidence)
274
+ if raw.get("DAC"):
275
+ _put(result, "given_names", " ".join(part for part in (raw["DAC"], raw.get("DAD")) if part), evidence)
276
+ for element, key in (("DBB", "date_of_birth"), ("DBA", "expiry_date"), ("DBD", "issue_date")):
277
+ if raw.get(element):
278
+ value = _us_date(raw[element])
279
+ if value is None:
280
+ result.errors.append(f"INVALID_{key.upper()}")
281
+ else:
282
+ _put(result, key, value, evidence)
283
+ if raw.get("DBC"):
284
+ sex = {"1": "M", "2": "F", "9": "X"}.get(raw["DBC"])
285
+ if sex is None:
286
+ result.errors.append("INVALID_SEX")
287
+ else:
288
+ _put(result, "sex", sex, evidence)
289
+ if state not in _US_STATES:
290
+ result.errors.append("INVALID_ISSUING_REGION")
291
+ for element in ("DDE", "DDF", "DDG"):
292
+ if raw.get(element) == "T":
293
+ result.warnings.append(f"AAMVA_NAME_TRUNCATED_{element}")
294
+ found.append(result)
295
+ if len(found) > 1:
296
+ found[0].errors.append("MULTIPLE_AAMVA_DOCUMENTS")
297
+ return found[0] if found else None
298
+
299
+
300
+ def _identity_visual(result: StructuredExtraction, regions: list[TextRegion]) -> None:
301
+ _visual(result, regions, "surname", (r"surname", r"family name", r"last name", r"LN"), _person_name)
302
+ if result.document_type in ("us_driver_license", "us_state_id"):
303
+ _visual(result, regions, "given_names", (r"given names?", r"FN"), _person_name)
304
+ _visual(result, regions, "first_name", (r"first name",), _person_name)
305
+ _visual(result, regions, "middle_names", (r"middle names?",), _person_name)
306
+ if not result.fields.get("surname"):
307
+ _visual(result, regions, "surname", (r"1",), _person_name)
308
+ if not result.fields.get("given_names"):
309
+ _visual(result, regions, "given_names", (r"2",), _person_name)
310
+ else:
311
+ _visual(result, regions, "given_names", (r"given names?", r"first(?: \(given\))? name", r"FN"), _person_name)
312
+ _visual(result, regions, "date_of_birth", (r"(?:USA\s*)?date of birth", r"birth date(?: \([^)]*\))?", r"DOB"), _us_date)
313
+ _visual(result, regions, "expiry_date", (r"card expires", r"expires(?:\s*on)?", r"date of expiry", r"expiration date", r"expiry date", r"EXP"), _us_date)
314
+ _visual(result, regions, "sex", (r"sex",), lambda value: value.upper() if value.upper() in ("M", "F", "X") else None)
315
+
316
+
317
+ def _extract_visual(result: StructuredExtraction, regions: list[TextRegion]) -> None:
318
+ kind = result.document_type
319
+ if kind in ("us_driver_license", "us_state_id", "passport_card", "us_green_card", "us_ead", "visa"):
320
+ _identity_visual(result, regions)
321
+ if kind in ("us_driver_license", "us_state_id", "passport_card"):
322
+ labels = (r"passport\s*(?:card[\s'’]*)?(?:number|no\.?)", r"card (?:number|no\.?)") if kind == "passport_card" else (r"(?:driver(?:'s)? license|license|licence|lic\.?|identification|id|dl)\s*(?:number|no\.?|#)", r"DL", r"ID")
323
+ _visual(result, regions, "document_number", labels, _document_number)
324
+ _visual(result, regions, "issue_date", (r"date of issue", r"issued(?: on)?", r"ISS"), _us_date)
325
+ _visual(result, regions, "address", (r"address",), lambda value: value.strip() if re.search(r"[0-9]", value) else None)
326
+ elif kind in ("us_green_card", "us_ead"):
327
+ _visual(result, regions, "uscis_number", (r"USCIS\s*#?", r"A[ -]?(?:number|#)"), _pattern(r"[0-9]{9}"))
328
+ _visual(result, regions, "card_number", (r"(?:category\s+)?card\s*(?:#|number|no\.?)",), _pattern(r"[A-Z]{3}[0-9]{10}"))
329
+ _visual(result, regions, "category", (r"category",), _pattern(r"[A-Z]{1,2}[0-9]{1,2}"))
330
+ _visual(result, regions, "country_of_birth", (r"country of birth",), _country_of_birth)
331
+ _visual(result, regions, "resident_since" if kind == "us_green_card" else "valid_from", (r"resident since",) if kind == "us_green_card" else (r"valid from",), _us_date)
332
+ elif kind == "us_i94":
333
+ _visual(result, regions, "i94_number", (r"admission\s*\(I[ -]?94\)\s*record number", r"I[ -]?94\s*(?:admission|record)?\s*(?:number|no\.?)", r"admission record number"), _pattern(r"[0-9]{9}[A-Z0-9][0-9]"))
334
+ _visual(result, regions, "surname", (r"last/surname", r"family name", r"surname"))
335
+ _visual(result, regions, "given_names", (r"first(?: \(given\))? name", r"given name"))
336
+ _visual(result, regions, "date_of_birth", (r"birth date(?: \([^)]*\))?", r"date of birth"), _us_date)
337
+ _visual(result, regions, "class_of_admission", (r"class of admission",), _pattern(r"[A-Z][A-Z0-9]{0,4}"))
338
+ _visual(result, regions, "admit_until", (r"admit until date(?: \([^)]*\))?", r"admit until"), lambda value: "D/S" if re.sub(r"\s", "", value.upper()) in ("D/S", "DS") else _us_date(value))
339
+ _visual(result, regions, "admission_date", (r"most recent date of entry", r"date of entry(?: \([^)]*\))?", r"admission date"), _us_date)
340
+ _visual(result, regions, "passport_number", (r"passport number",), _pattern(r"[A-Z0-9]{4,20}"))
341
+ _visual(result, regions, "country_of_citizenship", (r"country of citizenship",), _name)
342
+ elif kind == "us_w9":
343
+ _visual(result, regions, "name", (r"1\s+name(?: of entity/individual)?(?:\s*\([^)]*\))?", r"name(?:\s*\([^)]*\))?"))
344
+ _visual(result, regions, "business_name", (r"(?:2\s+)?business name(?:/disregarded entity name)?(?:,?\s+if different from above)?",))
345
+ _visual(result, regions, "address", (r"5\s+address(?:\s*\([^)]*\))?", r"address"), lambda value: value.strip() if re.search(r"[0-9]", value) else None)
346
+ _visual(result, regions, "city_state_postal_code", (r"6\s+city,?\s+state,?\s+and ZIP code", r"city,?\s+state,?\s+and ZIP code"), lambda value: value.strip() if re.search(r"\b[0-9]{5}(?:-[0-9]{4})?\b", value) else None)
347
+ for labels, id_type in (((r"social security number", r"SSN"), "ssn"), ((r"employer identification number", r"EIN"), "ein")):
348
+ for raw, region in _label_values(regions, labels):
349
+ digits = _compact(raw)
350
+ if re.fullmatch(r"[0-9]{9}", digits) and digits != "000000000":
351
+ _put(result, "taxpayer_id", digits, _evidence(region, raw))
352
+ _put(result, "taxpayer_id_type", id_type, _evidence(region, raw))
353
+ break
354
+
355
+ if kind == "passport_card" and not result.fields.get("date_of_birth"):
356
+ # Some cards print sex and DOB on one baseline; OCR can merge both
357
+ # values into a single region. Require both labels and a strict date.
358
+ has_sex = any(re.fullmatch(r"sex", r.text.strip(), re.I) for r in regions)
359
+ if has_sex:
360
+ for raw, region in _label_values(regions, (r"date of birth",)):
361
+ match = re.fullmatch(r"([MFX])\s*(\d{1,2}\s*[A-Za-z]{3,9}\s*\d{4})", raw)
362
+ if match and (birth := _us_date(match[2])):
363
+ _put(result, "date_of_birth", birth, _evidence(region))
364
+ _put(result, "sex", match[1], _evidence(region))
365
+ break
366
+
367
+
368
+ def _finish(result: StructuredExtraction) -> StructuredExtraction:
369
+ profile = DOCUMENT_PROFILES[result.document_type]
370
+ result.missing_required_fields = [name for name in profile.required_fields if not result.fields.get(name)]
371
+ result.checks.setdefault("issuer_authenticated", False)
372
+ result.checks["required_fields_present"] = not result.missing_required_fields
373
+ birth, expiry = result.fields.get("date_of_birth"), result.fields.get("expiry_date")
374
+ if birth and date.fromisoformat(birth) > date.today():
375
+ result.errors.append("DATE_OF_BIRTH_IN_FUTURE")
376
+ if birth and expiry and expiry <= birth:
377
+ result.errors.append("EXPIRY_BEFORE_DATE_OF_BIRTH")
378
+ if expiry:
379
+ result.checks["printed_expiry_in_past"] = expiry < date.today().isoformat()
380
+ scores = [item["confidence"] for items in result.field_evidence.values() for item in items]
381
+ confidence = sum(scores) / len(scores) if scores else 0.0
382
+ result.confidence = round(min(confidence, 0.49 if result.errors else 0.69 if result.missing_required_fields else 0.99), 3)
383
+ result.errors = list(dict.fromkeys(result.errors))
384
+ result.warnings = list(dict.fromkeys(["EXPERIMENTAL_DOCUMENT_PROFILE", *result.warnings]))
385
+ return result
386
+
387
+
388
+ def extract_structured_document(
389
+ regions: list[TextRegion],
390
+ *,
391
+ document_type: str | None = None,
392
+ country: str | None = None,
393
+ barcodes: list[BarcodeResult] | None = None,
394
+ ) -> StructuredExtraction | None:
395
+ """Extract a registered new profile; return None for legacy/unrecognized input."""
396
+ document_type, country = validate_document_hint(document_type, country)
397
+ if document_type and DOCUMENT_PROFILES[document_type].legacy:
398
+ return None
399
+ result = _extract_aamva(barcodes or [])
400
+ mrz = parse_travel_mrz(regions)
401
+ if result is not None and mrz is not None:
402
+ result.errors.append("MULTIPLE_MACHINE_READABLE_DOCUMENTS")
403
+ if result is None and mrz is not None:
404
+ result = StructuredExtraction(mrz.document_type, mrz.issuing_country, fields=mrz.fields, field_evidence=mrz.field_evidence, checks=mrz.checks, errors=mrz.errors, warnings=mrz.warnings)
405
+ text_type, state = _detect_text_type(regions)
406
+ if result is None and text_type:
407
+ result = StructuredExtraction(text_type, "US", state)
408
+ if result is None:
409
+ if document_type is None:
410
+ return None
411
+ result = StructuredExtraction(document_type, country, errors=["DOCUMENT_TYPE_NOT_CONFIRMED"])
412
+ return _finish(result)
413
+ if document_type is not None and result.document_type != document_type:
414
+ result.errors.append("DOCUMENT_TYPE_MISMATCH")
415
+ if country is not None and result.issuing_country is not None and result.issuing_country != country:
416
+ result.errors.append("DOCUMENT_COUNTRY_MISMATCH")
417
+ if text_type is not None and text_type != result.document_type:
418
+ result.errors.append("DOCUMENT_TYPE_EVIDENCE_CONFLICT")
419
+ _extract_visual(result, regions)
420
+ if result.document_type in ("us_driver_license", "us_state_id") and not result.checks.get("barcode_decoded"):
421
+ result.warnings.append("PDF417_NOT_DECODED")
422
+ return _finish(result)
@@ -0,0 +1,164 @@
1
+ """Strict TD1 and machine-readable visa extraction, with US card profiles.
2
+
3
+ Field positions follow ICAO Doc 9303 parts 5 and 7. USCIS cards expose an
4
+ A-number in the main number field and a separate card number in optional data;
5
+ these are intentionally not mapped to the same public field.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from dataclasses import dataclass, field
12
+
13
+ from .document_registry import is_known_mrz_country, normalize_country
14
+ from .mrz_parser import _DIGIT_CORRECTIONS, _clean_mrz_text, _parse_mrz_date, verify_check_digit
15
+ from .ocr_engine import TextRegion
16
+
17
+
18
+ @dataclass
19
+ class TravelMRZResult:
20
+ document_type: str
21
+ issuing_country: str | None
22
+ fields: dict
23
+ field_evidence: dict
24
+ checks: dict
25
+ errors: list[str] = field(default_factory=list)
26
+ warnings: list[str] = field(default_factory=list)
27
+
28
+
29
+ def _name(text: str) -> str:
30
+ return " ".join(text.replace("<", " ").split())
31
+
32
+
33
+ def _correct_digits(text: str, positions: list[int]) -> str:
34
+ values = list(text)
35
+ for index in positions:
36
+ values[index] = values[index].translate(_DIGIT_CORRECTIONS)
37
+ return "".join(values)
38
+
39
+
40
+ def _country(code: str) -> str | None:
41
+ if code == "D<<":
42
+ return "DE"
43
+ try:
44
+ return normalize_country(code.replace("<", ""))
45
+ except ValueError:
46
+ return None
47
+
48
+
49
+ def _evidence(regions: list[TextRegion]) -> list[dict]:
50
+ return [{"text": region.text, "bbox": region.bbox, "source": "mrz", "confidence": region.confidence} for region in regions]
51
+
52
+
53
+ def parse_travel_mrz(regions: list[TextRegion]) -> TravelMRZResult | None:
54
+ """Parse a supported exact-width MRZ without padding or guessed characters."""
55
+ candidates: list[tuple[int, str, TextRegion]] = []
56
+ for region in regions:
57
+ for line in region.text.splitlines():
58
+ text = _clean_mrz_text(line)
59
+ if re.fullmatch(r"[A-Z0-9<]{30}|[A-Z0-9<]{36}|[A-Z0-9<]{44}", text):
60
+ y = min((point[1] for point in region.bbox), default=0)
61
+ candidates.append((y, text, region))
62
+ candidates.sort(key=lambda candidate: candidate[0])
63
+ parsed: list[TravelMRZResult] = []
64
+ for index, (_, first, region) in enumerate(candidates):
65
+ if len(first) == 30 and index + 2 < len(candidates):
66
+ following = candidates[index:index + 3]
67
+ if all(len(item[1]) == 30 for item in following):
68
+ prefix, issuer = first[:2], first[2:5]
69
+ document_type = "passport_card" if prefix == "IP" else "us_green_card" if issuer == "USA" and prefix in ("C1", "C2") else "us_ead" if issuer == "USA" and prefix == "IA" else None
70
+ if document_type and re.fullmatch(r"[A-Z<]{30}", following[2][1]):
71
+ parsed.append(_parse_td1(document_type, [item[1] for item in following], [item[2] for item in following]))
72
+ elif len(first) in (36, 44) and first.startswith("V") and index + 1 < len(candidates):
73
+ second = candidates[index + 1]
74
+ if len(second[1]) == len(first) and re.fullmatch(r"V[A-Z<][A-Z<]{3}[A-Z<]+", first):
75
+ parsed.append(_parse_visa([first, second[1]], [region, second[2]]))
76
+ if not parsed:
77
+ return None
78
+ # Multiple independent documents cannot safely be collapsed into one result.
79
+ if len(parsed) > 1:
80
+ parsed[0].errors.append("MULTIPLE_MRZ_DOCUMENTS")
81
+ return parsed[0]
82
+
83
+
84
+ def _finish(document_type: str, lines: list[str], regions: list[TextRegion], fields: dict, checks: dict, name_line: str) -> TravelMRZResult:
85
+ surname, _, given = name_line.partition("<<")
86
+ fields.update({"surname": _name(surname) or None, "given_names": _name(given) or None, "country_code": lines[0][2:5], "mrz_raw": lines})
87
+ errors = [f"MRZ_{key.upper()}_FAILED" for key, value in checks.items() if value is False]
88
+ warnings = []
89
+ for field_name in ("date_of_birth", "expiry_date"):
90
+ if fields.get(field_name) is None:
91
+ errors.append(f"MISSING_OR_INVALID_{field_name.upper()}")
92
+ if fields.get("sex") not in ("M", "F", "X"):
93
+ errors.append("INVALID_SEX")
94
+ issuing_country = _country(fields["country_code"])
95
+ if issuing_country is None:
96
+ errors.append("UNKNOWN_ISSUING_COUNTRY")
97
+ if "nationality" in fields and not is_known_mrz_country(fields["nationality"]):
98
+ errors.append("UNKNOWN_NATIONALITY")
99
+ checks["mrz_checksums_valid"] = all(checks.values())
100
+ checks["issuer_authenticated"] = False
101
+ evidence = {name: _evidence(regions) for name, value in fields.items() if value is not None}
102
+ return TravelMRZResult(document_type, issuing_country, fields, evidence, checks, errors, warnings)
103
+
104
+
105
+ def _parse_td1(document_type: str, lines: list[str], regions: list[TextRegion]) -> TravelMRZResult:
106
+ first = _correct_digits(lines[0], [14])
107
+ second = _correct_digits(lines[1], list(range(0, 7)) + list(range(8, 15)) + [29])
108
+ surname_line = lines[2]
109
+ number = first[5:14]
110
+ number_check = first[14]
111
+ fields: dict = {
112
+ "document_number": number.rstrip("<") or None,
113
+ "date_of_birth": _parse_mrz_date(second[:6]),
114
+ "expiry_date": _parse_mrz_date(second[8:14], is_expiry=True),
115
+ "sex": "X" if second[7] == "<" else second[7],
116
+ "nationality": second[15:18].replace("<", "") or None,
117
+ }
118
+ # ICAO TD1 extended document numbers move their check digit into optional
119
+ # data, immediately after the continuation and before a filler.
120
+ if document_type == "passport_card" and number_check == "<":
121
+ continuation = first[15:30].split("<", 1)[0]
122
+ if len(continuation) >= 2 and continuation[-1].isdigit():
123
+ number += continuation[:-1]
124
+ number_check = continuation[-1]
125
+ fields["document_number"] = number
126
+ checks = {
127
+ "document_number_checksum": verify_check_digit(number, number_check),
128
+ "date_of_birth_checksum": verify_check_digit(second[:6], second[6]),
129
+ "expiry_date_checksum": verify_check_digit(second[8:14], second[14]),
130
+ "composite_checksum": verify_check_digit(first[5:30] + second[:7] + second[8:15] + second[18:29], second[29]),
131
+ }
132
+ if document_type in ("us_green_card", "us_ead"):
133
+ fields.pop("document_number")
134
+ fields.pop("nationality")
135
+ fields["uscis_number"] = number.rstrip("<")
136
+ fields["card_number"] = first[15:30].rstrip("<") or None
137
+ # US profiles encode country of birth here, not citizenship.
138
+ fields["country_of_birth"] = second[15:18].replace("<", "") or None
139
+ result = _finish(document_type, [first, second, surname_line], regions, fields, checks, surname_line)
140
+ if document_type in ("us_green_card", "us_ead"):
141
+ if not re.fullmatch(r"[0-9]{9}", fields["uscis_number"] or ""):
142
+ result.errors.append("INVALID_USCIS_NUMBER")
143
+ if not re.fullmatch(r"[A-Z]{3}[0-9]{10}", fields["card_number"] or ""):
144
+ result.errors.append("INVALID_CARD_NUMBER")
145
+ return result
146
+
147
+
148
+ def _parse_visa(lines: list[str], regions: list[TextRegion]) -> TravelMRZResult:
149
+ first, second = lines
150
+ second = _correct_digits(second, [9] + list(range(13, 20)) + list(range(21, 28)))
151
+ fields = {
152
+ "document_number": second[:9].rstrip("<") or None,
153
+ "date_of_birth": _parse_mrz_date(second[13:19]),
154
+ "expiry_date": _parse_mrz_date(second[21:27], is_expiry=True),
155
+ "nationality": second[10:13].replace("<", "") or None,
156
+ "sex": "X" if second[20] == "<" else second[20],
157
+ "mrz_format": "MRV_A" if len(first) == 44 else "MRV_B",
158
+ }
159
+ checks = {
160
+ "document_number_checksum": verify_check_digit(second[:9], second[9]),
161
+ "date_of_birth_checksum": verify_check_digit(second[13:19], second[19]),
162
+ "expiry_date_checksum": verify_check_digit(second[21:27], second[27]),
163
+ }
164
+ return _finish("visa", [first, second], regions, fields, checks, first[5:])