document-ocr 0.0.0-stage → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +260 -2
- package/dist/index.cjs +659 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +293 -0
- package/dist/index.d.ts +293 -0
- package/dist/index.js +616 -0
- package/dist/index.js.map +1 -0
- package/hosting/.dockerignore +17 -0
- package/hosting/.gcloudignore +14 -0
- package/hosting/FEATURES.md +172 -0
- package/hosting/HOSTING.md +268 -0
- package/hosting/LICENSE +21 -0
- package/hosting/README.md +369 -0
- package/hosting/core/__init__.py +0 -0
- package/hosting/core/aadhaar_extractor.py +298 -0
- package/hosting/core/back_page_extractor.py +492 -0
- package/hosting/core/barcodes.py +125 -0
- package/hosting/core/card_recovery.py +244 -0
- package/hosting/core/document_bundle.py +82 -0
- package/hosting/core/document_classifier.py +233 -0
- package/hosting/core/document_input.py +135 -0
- package/hosting/core/document_registry.py +143 -0
- package/hosting/core/driving_licence_extractor.py +510 -0
- package/hosting/core/ead_recovery.py +54 -0
- package/hosting/core/evidence.py +54 -0
- package/hosting/core/form_recovery.py +61 -0
- package/hosting/core/jobs.py +232 -0
- package/hosting/core/kyc_ocr.py +132 -0
- package/hosting/core/kyc_validation.py +411 -0
- package/hosting/core/model_setup.py +13 -0
- package/hosting/core/mrz_parser.py +336 -0
- package/hosting/core/mrz_recovery.py +193 -0
- package/hosting/core/npr_extractor.py +410 -0
- package/hosting/core/nrega_extractor.py +827 -0
- package/hosting/core/ocr_engine.py +207 -0
- package/hosting/core/page_classifier.py +137 -0
- package/hosting/core/pan_extractor.py +183 -0
- package/hosting/core/pipeline.py +712 -0
- package/hosting/core/preprocessor.py +297 -0
- package/hosting/core/recovery_budget.py +67 -0
- package/hosting/core/redaction.py +48 -0
- package/hosting/core/review.html +244 -0
- package/hosting/core/structured_documents.py +422 -0
- package/hosting/core/travel_mrz.py +164 -0
- package/hosting/core/travel_recovery.py +187 -0
- package/hosting/core/validator.py +396 -0
- package/hosting/core/validators.py +179 -0
- package/hosting/core/voter_id_extractor.py +363 -0
- package/hosting/deploy/cloudflare/deploy.sh +44 -0
- package/hosting/deploy/cloudflare/package-lock.json +3088 -0
- package/hosting/deploy/cloudflare/package.json +29 -0
- package/hosting/deploy/cloudflare/src/gateway.ts +42 -0
- package/hosting/deploy/cloudflare/src/index.ts +21 -0
- package/hosting/deploy/cloudflare/test/gateway.test.ts +59 -0
- package/hosting/deploy/cloudflare/tsconfig.json +12 -0
- package/hosting/deploy/cloudflare/worker-configuration.d.ts +16253 -0
- package/hosting/deploy/cloudflare/wrangler.jsonc +26 -0
- package/hosting/deploy/cloudrun/cloudbuild.yaml +18 -0
- package/hosting/deploy/cloudrun/deploy.sh +71 -0
- package/hosting/deploy/cloudrun/render_service.py +47 -0
- package/hosting/deploy/common.sh +14 -0
- package/hosting/deploy/docker/Dockerfile +16 -0
- package/hosting/deploy/docker/server.py +313 -0
- package/hosting/deploy/lambda/Dockerfile.lambda +11 -0
- package/hosting/deploy/lambda/deploy.sh +22 -0
- package/hosting/deploy/lambda/handler.py +132 -0
- package/hosting/deploy/lambda/template.yaml +59 -0
- package/hosting/deploy/local/dev.sh +18 -0
- package/hosting/deploy/server/Caddyfile +5 -0
- package/hosting/deploy/server/compose.jobs.yaml +36 -0
- package/hosting/deploy/server/compose.yaml +38 -0
- package/hosting/deploy/server/deploy.sh +23 -0
- package/hosting/infra/cloudrun/main.tf +104 -0
- package/hosting/infra/cloudrun/terraform.tfvars.example +7 -0
- package/hosting/infra/cloudrun/variables.tf +55 -0
- package/hosting/infra/lambda/main.tf +63 -0
- package/hosting/infra/lambda/terraform.tfvars.example +6 -0
- package/hosting/infra/lambda/variables.tf +37 -0
- package/hosting/pyproject.toml +65 -0
- package/hosting/requirements.lock +134 -0
- package/package.json +76 -4
- package/python/core/__init__.py +0 -0
- package/python/core/aadhaar_extractor.py +298 -0
- package/python/core/back_page_extractor.py +492 -0
- package/python/core/barcodes.py +125 -0
- package/python/core/card_recovery.py +244 -0
- package/python/core/document_bundle.py +82 -0
- package/python/core/document_classifier.py +233 -0
- package/python/core/document_input.py +135 -0
- package/python/core/document_registry.py +143 -0
- package/python/core/driving_licence_extractor.py +510 -0
- package/python/core/ead_recovery.py +54 -0
- package/python/core/evidence.py +54 -0
- package/python/core/form_recovery.py +61 -0
- package/python/core/jobs.py +232 -0
- package/python/core/kyc_ocr.py +132 -0
- package/python/core/kyc_validation.py +411 -0
- package/python/core/model_setup.py +13 -0
- package/python/core/mrz_parser.py +336 -0
- package/python/core/mrz_recovery.py +193 -0
- package/python/core/npr_extractor.py +410 -0
- package/python/core/nrega_extractor.py +827 -0
- package/python/core/ocr_engine.py +207 -0
- package/python/core/page_classifier.py +137 -0
- package/python/core/pan_extractor.py +183 -0
- package/python/core/pipeline.py +712 -0
- package/python/core/preprocessor.py +297 -0
- package/python/core/recovery_budget.py +67 -0
- package/python/core/redaction.py +48 -0
- package/python/core/review.html +244 -0
- package/python/core/structured_documents.py +422 -0
- package/python/core/travel_mrz.py +164 -0
- package/python/core/travel_recovery.py +187 -0
- package/python/core/validator.py +396 -0
- package/python/core/validators.py +179 -0
- package/python/core/voter_id_extractor.py +363 -0
- package/python/pyproject.toml +34 -0
- package/python/requirements.lock +134 -0
- package/python/server.py +313 -0
- package/scripts/dococr.mjs +163 -0
- package/scripts/postinstall.mjs +173 -0
- package/scripts/sync-hosting.mjs +35 -0
- package/scripts/sync-python.sh +20 -0
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"""Country-aware extraction for explicitly registered document profiles.
|
|
2
|
+
|
|
3
|
+
Machine-readable data and visual OCR remain separate evidence. Conflicting
|
|
4
|
+
values are surfaced instead of silently letting a confident source overwrite
|
|
5
|
+
another. These extraction checks do not authenticate a document or its holder.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import math
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from datetime import date, datetime
|
|
14
|
+
from typing import Callable
|
|
15
|
+
|
|
16
|
+
import pycountry
|
|
17
|
+
|
|
18
|
+
from .barcodes import BarcodeResult, parse_aamva
|
|
19
|
+
from .document_registry import DOCUMENT_PROFILES, normalize_country, validate_document_hint
|
|
20
|
+
from .ocr_engine import TextRegion
|
|
21
|
+
from .travel_mrz import parse_travel_mrz
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class StructuredExtraction:
|
|
26
|
+
document_type: str
|
|
27
|
+
issuing_country: str | None = None
|
|
28
|
+
issuing_region: str | None = None
|
|
29
|
+
fields: dict = field(default_factory=dict)
|
|
30
|
+
field_evidence: dict = field(default_factory=dict)
|
|
31
|
+
checks: dict = field(default_factory=dict)
|
|
32
|
+
missing_required_fields: list[str] = field(default_factory=list)
|
|
33
|
+
errors: list[str] = field(default_factory=list)
|
|
34
|
+
warnings: list[str] = field(default_factory=list)
|
|
35
|
+
confidence: float = 0.0
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def complete(self) -> bool:
|
|
39
|
+
return bool(self.fields) and not self.missing_required_fields and not self.errors
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
_US_STATES = {item.code.split("-")[1]: item.name.upper() for item in pycountry.subdivisions.get(country_code="US")}
|
|
43
|
+
_FIELD_LABEL = re.compile(
|
|
44
|
+
r"^(?:\d{1,2}[a-z]?\s+)?(?:3[ab]\b|surname|family name|given names?|first name|middle name|date of birth|birth date|dob|"
|
|
45
|
+
r"date of (?:issue|expiry)|expires(?:\s*on)?|exp(?:iry|iration)?(?: date)?|iss|issue(?:d| date)?|sex|nationality|"
|
|
46
|
+
r"uscis|card (?:number|expires)|category|country of birth|resident since|valid from|"
|
|
47
|
+
r"class of admission|admit until|admission.*record number|passport number|"
|
|
48
|
+
r"name(?: of entity/individual)?|business name|address|city|social security number|employer identification number)\b",
|
|
49
|
+
re.I,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _compact(value: str) -> str:
|
|
54
|
+
return re.sub(r"[\s-]", "", value.upper())
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _us_date(value: str) -> str | None:
|
|
58
|
+
value = value.strip().replace(".", "/")
|
|
59
|
+
if re.fullmatch(r"[0-9\s/-]+", value):
|
|
60
|
+
value = re.sub(r"\s+", "", value)
|
|
61
|
+
# OCR often joins a printed month to its neighbouring day or year.
|
|
62
|
+
value = re.sub(r"(?<=[A-Za-z])(?=[0-9])|(?<=[0-9])(?=[A-Za-z])", " ", value)
|
|
63
|
+
for fmt in ("%m/%d/%Y", "%m-%d-%Y", "%Y-%m-%d", "%m%d%Y", "%d %b %Y", "%d %B %Y", "%d-%b-%Y",
|
|
64
|
+
"%m/%d/%y", "%m-%d-%y", "%Y %B %d", "%Y %b %d"):
|
|
65
|
+
try:
|
|
66
|
+
return datetime.strptime(value, fmt).date().isoformat()
|
|
67
|
+
except ValueError:
|
|
68
|
+
continue
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _name(value: str) -> str | None:
|
|
73
|
+
value = " ".join(value.strip(" :").split())
|
|
74
|
+
if not 1 <= len(value) <= 100 or not re.search(r"[A-Za-z]", value):
|
|
75
|
+
return None
|
|
76
|
+
if re.search(r"\b(?:required|enter|instructions|certification|requester|taxpayer|identification|security)\b|shown on|line [0-9]", value, re.I):
|
|
77
|
+
return None
|
|
78
|
+
return value
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _person_name(value: str) -> str | None:
|
|
82
|
+
return _name(value) if not re.search(r"[0-9]", value) else None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _pattern(pattern: str) -> Callable[[str], str | None]:
|
|
86
|
+
return lambda value: _compact(value) if re.fullmatch(pattern, _compact(value)) else None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _document_number(value: str) -> str | None:
|
|
90
|
+
value = _compact(value)
|
|
91
|
+
return value if re.fullmatch(r"[A-Z0-9]{4,25}", value) and re.search(r"[0-9]", value) else None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _country_of_birth(value: str) -> str | None:
|
|
95
|
+
try:
|
|
96
|
+
return pycountry.countries.lookup(value.strip()).alpha_3
|
|
97
|
+
except LookupError:
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _evidence(region: TextRegion, text: str | None = None) -> dict:
|
|
102
|
+
return {"text": text if text is not None else region.text, "bbox": region.bbox, "source": "ocr", "confidence": region.confidence}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _label_values(regions: list[TextRegion], labels: tuple[str, ...]):
|
|
106
|
+
"""Yield explicit inline values, then spatially adjacent non-label regions."""
|
|
107
|
+
for label in labels:
|
|
108
|
+
expression = re.compile(r"^\s*(?:[0-9]{1,2}[a-z]?\s+)?(?:" + label + r")(?![A-Za-z])\s*[:#]?\s*", re.I)
|
|
109
|
+
for region in regions:
|
|
110
|
+
match = expression.match(region.text)
|
|
111
|
+
if not match:
|
|
112
|
+
continue
|
|
113
|
+
inline = region.text[match.end():].strip()
|
|
114
|
+
if inline:
|
|
115
|
+
yield inline, region
|
|
116
|
+
if not region.bbox:
|
|
117
|
+
continue
|
|
118
|
+
origin, end = region.bbox[:2]
|
|
119
|
+
dx, dy = end[0] - origin[0], end[1] - origin[1]
|
|
120
|
+
length = math.hypot(dx, dy)
|
|
121
|
+
if length < 1:
|
|
122
|
+
continue
|
|
123
|
+
ux, uy = dx / length, dy / length
|
|
124
|
+
|
|
125
|
+
def bounds(other):
|
|
126
|
+
xs = [(p[0] - origin[0]) * ux + (p[1] - origin[1]) * uy for p in other.bbox]
|
|
127
|
+
ys = [-(p[0] - origin[0]) * uy + (p[1] - origin[1]) * ux for p in other.bbox]
|
|
128
|
+
return min(xs), min(ys), max(xs), max(ys)
|
|
129
|
+
|
|
130
|
+
# Match fields in the label's own axes, including tilted photos.
|
|
131
|
+
left, top, right, bottom = bounds(region)
|
|
132
|
+
height = max(bottom - top, 1)
|
|
133
|
+
nearby = []
|
|
134
|
+
for candidate in regions:
|
|
135
|
+
if candidate is region or not candidate.bbox or _FIELD_LABEL.match(candidate.text):
|
|
136
|
+
continue
|
|
137
|
+
c_left, c_top, c_right, c_bottom = bounds(candidate)
|
|
138
|
+
if c_bottom - c_top > height * 2:
|
|
139
|
+
# Large background words/watermarks can overlap a small
|
|
140
|
+
# label without being that field's value.
|
|
141
|
+
continue
|
|
142
|
+
overlap = min(bottom, c_bottom) - max(top, c_top)
|
|
143
|
+
gap = c_left - right
|
|
144
|
+
# A long horizontal value can have a slightly different OCR
|
|
145
|
+
# angle from its short label. Its furthest corner exaggerates
|
|
146
|
+
# vertical overlap; the top-edge centre follows the text row.
|
|
147
|
+
value_top = sum(-(p[0] - origin[0]) * uy + (p[1] - origin[1]) * ux
|
|
148
|
+
for p in candidate.bbox[:2]) / 2
|
|
149
|
+
# OCR boxes can overlap even when the printed value is to
|
|
150
|
+
# the right. Limit tolerance to a small part of the label,
|
|
151
|
+
# and require the value's centre beyond the label's edge.
|
|
152
|
+
overlap_allowance = min(height * .75, (right - left) * .25)
|
|
153
|
+
same_row = (overlap >= height * .5
|
|
154
|
+
and -overlap_allowance <= gap < 450
|
|
155
|
+
and (c_left + c_right) / 2 > right)
|
|
156
|
+
if same_row:
|
|
157
|
+
crossed_label = any(
|
|
158
|
+
other is not region
|
|
159
|
+
and other is not candidate
|
|
160
|
+
and other.bbox
|
|
161
|
+
and _FIELD_LABEL.match(other.text)
|
|
162
|
+
and (left + right) / 2 < bounds(other)[0] < (c_left + c_right) / 2
|
|
163
|
+
and min(bottom, bounds(other)[3]) - max(top, bounds(other)[1]) >= height * .5
|
|
164
|
+
for other in regions
|
|
165
|
+
)
|
|
166
|
+
if crossed_label:
|
|
167
|
+
continue
|
|
168
|
+
nearby.append((max(gap, 0), candidate))
|
|
169
|
+
elif -max(5, height * .35) <= value_top - bottom < max(90, height * 3) and abs(c_left - left) < 220:
|
|
170
|
+
# A blank field must not borrow the value belonging to the
|
|
171
|
+
# next labeled field underneath it (for example W-9 lines
|
|
172
|
+
# 1 and 2). A label in the same column closes this field.
|
|
173
|
+
crossed_label = any(
|
|
174
|
+
other is not region
|
|
175
|
+
and other.bbox
|
|
176
|
+
and _FIELD_LABEL.match(other.text)
|
|
177
|
+
and bottom - 5 <= bounds(other)[1] <= value_top
|
|
178
|
+
and abs(bounds(other)[0] - left) < 220
|
|
179
|
+
for other in regions
|
|
180
|
+
)
|
|
181
|
+
if crossed_label:
|
|
182
|
+
continue
|
|
183
|
+
nearby.append((500 + max(value_top - bottom, 0) + abs(c_left - left), candidate))
|
|
184
|
+
for _, candidate in sorted(nearby, key=lambda item: item[0]):
|
|
185
|
+
yield candidate.text, candidate
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _put(result: StructuredExtraction, key: str, value, evidence: dict | list[dict]) -> None:
|
|
189
|
+
if value is None or value == "":
|
|
190
|
+
return
|
|
191
|
+
evidence = evidence if isinstance(evidence, list) else [evidence]
|
|
192
|
+
result.field_evidence.setdefault(key, []).extend(evidence)
|
|
193
|
+
previous = result.fields.get(key)
|
|
194
|
+
if previous is not None and re.sub(r"\W", "", str(previous)).casefold() != re.sub(r"\W", "", str(value)).casefold():
|
|
195
|
+
error = f"FIELD_MISMATCH_{key.upper()}"
|
|
196
|
+
if error not in result.errors:
|
|
197
|
+
result.errors.append(error)
|
|
198
|
+
result.checks[f"{key}_agreement"] = False
|
|
199
|
+
return
|
|
200
|
+
result.fields[key] = value
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _visual(result: StructuredExtraction, regions: list[TextRegion], key: str, labels: tuple[str, ...], parser: Callable[[str], str | None] = _name) -> None:
|
|
204
|
+
for raw, region in _label_values(regions, labels):
|
|
205
|
+
value = parser(raw)
|
|
206
|
+
if value is not None:
|
|
207
|
+
_put(result, key, value, _evidence(region, raw))
|
|
208
|
+
return
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _state(text: str) -> str | None:
|
|
212
|
+
upper = text.upper()
|
|
213
|
+
for code, name in _US_STATES.items():
|
|
214
|
+
if re.search(r"\b" + re.escape(name) + r"\b", upper):
|
|
215
|
+
return code
|
|
216
|
+
match = re.search(r"\bUSA\s*([A-Z]{2})\b", upper)
|
|
217
|
+
if match and match[1] in _US_STATES:
|
|
218
|
+
return match[1]
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _detect_text_type(regions: list[TextRegion]) -> tuple[str | None, str | None]:
|
|
223
|
+
text = "\n".join(region.text.upper() for region in regions)
|
|
224
|
+
state = _state(text)
|
|
225
|
+
if state and re.search(r"\bDRIVER(?:'S|S)? LICENSE\b", text):
|
|
226
|
+
return "us_driver_license", state
|
|
227
|
+
if state and re.search(r"\b(?:IDENTIFICATION|IDENTITY) CARD\b", text):
|
|
228
|
+
return "us_state_id", state
|
|
229
|
+
american = bool(re.search(r"\bUSCIS\b|UNITED\s*STATES", text))
|
|
230
|
+
if american and "PERMANENT RESIDENT" in text:
|
|
231
|
+
return "us_green_card", None
|
|
232
|
+
if american and "EMPLOYMENT AUTHORIZATION" in text:
|
|
233
|
+
return "us_ead", None
|
|
234
|
+
if american and "PASSPORT CARD" in text:
|
|
235
|
+
return "passport_card", None
|
|
236
|
+
if re.search(r"\bI[ -]?94\b", text) and "CLASS OF ADMISSION" in text and re.search(r"ADMISSION.*RECORD NUMBER|CUSTOMS AND BORDER|HOMELAND SECURITY", text):
|
|
237
|
+
return "us_i94", None
|
|
238
|
+
if (re.search(r"\bW[ -]?9\b", text) and "REQUEST FOR TAXPAYER" in text
|
|
239
|
+
and "IDENTIFICATION NUMBER AND CERTIFICATION" in text):
|
|
240
|
+
return "us_w9", None
|
|
241
|
+
return None, None
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _extract_aamva(barcodes: list[BarcodeResult]) -> StructuredExtraction | None:
|
|
245
|
+
found: list[StructuredExtraction] = []
|
|
246
|
+
for barcode in barcodes:
|
|
247
|
+
if "PDF417" not in re.sub(r"[^A-Z0-9]", "", barcode.format.upper()):
|
|
248
|
+
continue
|
|
249
|
+
try:
|
|
250
|
+
parsed = parse_aamva(barcode)
|
|
251
|
+
except ValueError as error:
|
|
252
|
+
# A corrupt payload cannot confidently identify DL versus ID.
|
|
253
|
+
result = StructuredExtraction("us_driver_license", errors=[str(error)])
|
|
254
|
+
result.checks.update({"barcode_decoded": True, "barcode_structure_valid": False})
|
|
255
|
+
found.append(result)
|
|
256
|
+
continue
|
|
257
|
+
if parsed is None:
|
|
258
|
+
continue
|
|
259
|
+
raw = parsed.elements
|
|
260
|
+
raw_country = raw.get("DCG")
|
|
261
|
+
country = normalize_country(raw_country) if raw_country in ("US", "USA", "CA", "CAN") else None
|
|
262
|
+
state = raw.get("DAJ")
|
|
263
|
+
if country != "US" and not (raw_country is None and state in _US_STATES):
|
|
264
|
+
continue
|
|
265
|
+
result = StructuredExtraction(parsed.document_type, "US", state, errors=parsed.errors.copy())
|
|
266
|
+
result.checks.update({"barcode_decoded": True, "barcode_structure_valid": not parsed.errors, "aamva_version": parsed.version, "aamva_jurisdiction_version": parsed.jurisdiction_version, "aamva_issuer_id": parsed.issuer_id, "issuer_authenticated": False, "signature_verified": False})
|
|
267
|
+
evidence = {"text": barcode.text, "bbox": barcode.bbox, "source": "pdf417", "confidence": 1.0}
|
|
268
|
+
keys = {"DAQ": "document_number", "DCS": "surname", "DAB": "surname", "DAC": "first_name", "DAD": "middle_names", "DAG": "address", "DAI": "city", "DAJ": "state", "DAK": "postal_code", "DCA": "license_class", "DCB": "restrictions", "DCD": "endorsements"}
|
|
269
|
+
for element, key in keys.items():
|
|
270
|
+
if parsed.document_type == "us_state_id" and element in ("DCA", "DCB", "DCD"):
|
|
271
|
+
continue
|
|
272
|
+
if raw.get(element):
|
|
273
|
+
_put(result, key, raw[element], evidence)
|
|
274
|
+
if raw.get("DAC"):
|
|
275
|
+
_put(result, "given_names", " ".join(part for part in (raw["DAC"], raw.get("DAD")) if part), evidence)
|
|
276
|
+
for element, key in (("DBB", "date_of_birth"), ("DBA", "expiry_date"), ("DBD", "issue_date")):
|
|
277
|
+
if raw.get(element):
|
|
278
|
+
value = _us_date(raw[element])
|
|
279
|
+
if value is None:
|
|
280
|
+
result.errors.append(f"INVALID_{key.upper()}")
|
|
281
|
+
else:
|
|
282
|
+
_put(result, key, value, evidence)
|
|
283
|
+
if raw.get("DBC"):
|
|
284
|
+
sex = {"1": "M", "2": "F", "9": "X"}.get(raw["DBC"])
|
|
285
|
+
if sex is None:
|
|
286
|
+
result.errors.append("INVALID_SEX")
|
|
287
|
+
else:
|
|
288
|
+
_put(result, "sex", sex, evidence)
|
|
289
|
+
if state not in _US_STATES:
|
|
290
|
+
result.errors.append("INVALID_ISSUING_REGION")
|
|
291
|
+
for element in ("DDE", "DDF", "DDG"):
|
|
292
|
+
if raw.get(element) == "T":
|
|
293
|
+
result.warnings.append(f"AAMVA_NAME_TRUNCATED_{element}")
|
|
294
|
+
found.append(result)
|
|
295
|
+
if len(found) > 1:
|
|
296
|
+
found[0].errors.append("MULTIPLE_AAMVA_DOCUMENTS")
|
|
297
|
+
return found[0] if found else None
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _identity_visual(result: StructuredExtraction, regions: list[TextRegion]) -> None:
|
|
301
|
+
_visual(result, regions, "surname", (r"surname", r"family name", r"last name", r"LN"), _person_name)
|
|
302
|
+
if result.document_type in ("us_driver_license", "us_state_id"):
|
|
303
|
+
_visual(result, regions, "given_names", (r"given names?", r"FN"), _person_name)
|
|
304
|
+
_visual(result, regions, "first_name", (r"first name",), _person_name)
|
|
305
|
+
_visual(result, regions, "middle_names", (r"middle names?",), _person_name)
|
|
306
|
+
if not result.fields.get("surname"):
|
|
307
|
+
_visual(result, regions, "surname", (r"1",), _person_name)
|
|
308
|
+
if not result.fields.get("given_names"):
|
|
309
|
+
_visual(result, regions, "given_names", (r"2",), _person_name)
|
|
310
|
+
else:
|
|
311
|
+
_visual(result, regions, "given_names", (r"given names?", r"first(?: \(given\))? name", r"FN"), _person_name)
|
|
312
|
+
_visual(result, regions, "date_of_birth", (r"(?:USA\s*)?date of birth", r"birth date(?: \([^)]*\))?", r"DOB"), _us_date)
|
|
313
|
+
_visual(result, regions, "expiry_date", (r"card expires", r"expires(?:\s*on)?", r"date of expiry", r"expiration date", r"expiry date", r"EXP"), _us_date)
|
|
314
|
+
_visual(result, regions, "sex", (r"sex",), lambda value: value.upper() if value.upper() in ("M", "F", "X") else None)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _extract_visual(result: StructuredExtraction, regions: list[TextRegion]) -> None:
|
|
318
|
+
kind = result.document_type
|
|
319
|
+
if kind in ("us_driver_license", "us_state_id", "passport_card", "us_green_card", "us_ead", "visa"):
|
|
320
|
+
_identity_visual(result, regions)
|
|
321
|
+
if kind in ("us_driver_license", "us_state_id", "passport_card"):
|
|
322
|
+
labels = (r"passport\s*(?:card[\s'’]*)?(?:number|no\.?)", r"card (?:number|no\.?)") if kind == "passport_card" else (r"(?:driver(?:'s)? license|license|licence|lic\.?|identification|id|dl)\s*(?:number|no\.?|#)", r"DL", r"ID")
|
|
323
|
+
_visual(result, regions, "document_number", labels, _document_number)
|
|
324
|
+
_visual(result, regions, "issue_date", (r"date of issue", r"issued(?: on)?", r"ISS"), _us_date)
|
|
325
|
+
_visual(result, regions, "address", (r"address",), lambda value: value.strip() if re.search(r"[0-9]", value) else None)
|
|
326
|
+
elif kind in ("us_green_card", "us_ead"):
|
|
327
|
+
_visual(result, regions, "uscis_number", (r"USCIS\s*#?", r"A[ -]?(?:number|#)"), _pattern(r"[0-9]{9}"))
|
|
328
|
+
_visual(result, regions, "card_number", (r"(?:category\s+)?card\s*(?:#|number|no\.?)",), _pattern(r"[A-Z]{3}[0-9]{10}"))
|
|
329
|
+
_visual(result, regions, "category", (r"category",), _pattern(r"[A-Z]{1,2}[0-9]{1,2}"))
|
|
330
|
+
_visual(result, regions, "country_of_birth", (r"country of birth",), _country_of_birth)
|
|
331
|
+
_visual(result, regions, "resident_since" if kind == "us_green_card" else "valid_from", (r"resident since",) if kind == "us_green_card" else (r"valid from",), _us_date)
|
|
332
|
+
elif kind == "us_i94":
|
|
333
|
+
_visual(result, regions, "i94_number", (r"admission\s*\(I[ -]?94\)\s*record number", r"I[ -]?94\s*(?:admission|record)?\s*(?:number|no\.?)", r"admission record number"), _pattern(r"[0-9]{9}[A-Z0-9][0-9]"))
|
|
334
|
+
_visual(result, regions, "surname", (r"last/surname", r"family name", r"surname"))
|
|
335
|
+
_visual(result, regions, "given_names", (r"first(?: \(given\))? name", r"given name"))
|
|
336
|
+
_visual(result, regions, "date_of_birth", (r"birth date(?: \([^)]*\))?", r"date of birth"), _us_date)
|
|
337
|
+
_visual(result, regions, "class_of_admission", (r"class of admission",), _pattern(r"[A-Z][A-Z0-9]{0,4}"))
|
|
338
|
+
_visual(result, regions, "admit_until", (r"admit until date(?: \([^)]*\))?", r"admit until"), lambda value: "D/S" if re.sub(r"\s", "", value.upper()) in ("D/S", "DS") else _us_date(value))
|
|
339
|
+
_visual(result, regions, "admission_date", (r"most recent date of entry", r"date of entry(?: \([^)]*\))?", r"admission date"), _us_date)
|
|
340
|
+
_visual(result, regions, "passport_number", (r"passport number",), _pattern(r"[A-Z0-9]{4,20}"))
|
|
341
|
+
_visual(result, regions, "country_of_citizenship", (r"country of citizenship",), _name)
|
|
342
|
+
elif kind == "us_w9":
|
|
343
|
+
_visual(result, regions, "name", (r"1\s+name(?: of entity/individual)?(?:\s*\([^)]*\))?", r"name(?:\s*\([^)]*\))?"))
|
|
344
|
+
_visual(result, regions, "business_name", (r"(?:2\s+)?business name(?:/disregarded entity name)?(?:,?\s+if different from above)?",))
|
|
345
|
+
_visual(result, regions, "address", (r"5\s+address(?:\s*\([^)]*\))?", r"address"), lambda value: value.strip() if re.search(r"[0-9]", value) else None)
|
|
346
|
+
_visual(result, regions, "city_state_postal_code", (r"6\s+city,?\s+state,?\s+and ZIP code", r"city,?\s+state,?\s+and ZIP code"), lambda value: value.strip() if re.search(r"\b[0-9]{5}(?:-[0-9]{4})?\b", value) else None)
|
|
347
|
+
for labels, id_type in (((r"social security number", r"SSN"), "ssn"), ((r"employer identification number", r"EIN"), "ein")):
|
|
348
|
+
for raw, region in _label_values(regions, labels):
|
|
349
|
+
digits = _compact(raw)
|
|
350
|
+
if re.fullmatch(r"[0-9]{9}", digits) and digits != "000000000":
|
|
351
|
+
_put(result, "taxpayer_id", digits, _evidence(region, raw))
|
|
352
|
+
_put(result, "taxpayer_id_type", id_type, _evidence(region, raw))
|
|
353
|
+
break
|
|
354
|
+
|
|
355
|
+
if kind == "passport_card" and not result.fields.get("date_of_birth"):
|
|
356
|
+
# Some cards print sex and DOB on one baseline; OCR can merge both
|
|
357
|
+
# values into a single region. Require both labels and a strict date.
|
|
358
|
+
has_sex = any(re.fullmatch(r"sex", r.text.strip(), re.I) for r in regions)
|
|
359
|
+
if has_sex:
|
|
360
|
+
for raw, region in _label_values(regions, (r"date of birth",)):
|
|
361
|
+
match = re.fullmatch(r"([MFX])\s*(\d{1,2}\s*[A-Za-z]{3,9}\s*\d{4})", raw)
|
|
362
|
+
if match and (birth := _us_date(match[2])):
|
|
363
|
+
_put(result, "date_of_birth", birth, _evidence(region))
|
|
364
|
+
_put(result, "sex", match[1], _evidence(region))
|
|
365
|
+
break
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _finish(result: StructuredExtraction) -> StructuredExtraction:
|
|
369
|
+
profile = DOCUMENT_PROFILES[result.document_type]
|
|
370
|
+
result.missing_required_fields = [name for name in profile.required_fields if not result.fields.get(name)]
|
|
371
|
+
result.checks.setdefault("issuer_authenticated", False)
|
|
372
|
+
result.checks["required_fields_present"] = not result.missing_required_fields
|
|
373
|
+
birth, expiry = result.fields.get("date_of_birth"), result.fields.get("expiry_date")
|
|
374
|
+
if birth and date.fromisoformat(birth) > date.today():
|
|
375
|
+
result.errors.append("DATE_OF_BIRTH_IN_FUTURE")
|
|
376
|
+
if birth and expiry and expiry <= birth:
|
|
377
|
+
result.errors.append("EXPIRY_BEFORE_DATE_OF_BIRTH")
|
|
378
|
+
if expiry:
|
|
379
|
+
result.checks["printed_expiry_in_past"] = expiry < date.today().isoformat()
|
|
380
|
+
scores = [item["confidence"] for items in result.field_evidence.values() for item in items]
|
|
381
|
+
confidence = sum(scores) / len(scores) if scores else 0.0
|
|
382
|
+
result.confidence = round(min(confidence, 0.49 if result.errors else 0.69 if result.missing_required_fields else 0.99), 3)
|
|
383
|
+
result.errors = list(dict.fromkeys(result.errors))
|
|
384
|
+
result.warnings = list(dict.fromkeys(["EXPERIMENTAL_DOCUMENT_PROFILE", *result.warnings]))
|
|
385
|
+
return result
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def extract_structured_document(
|
|
389
|
+
regions: list[TextRegion],
|
|
390
|
+
*,
|
|
391
|
+
document_type: str | None = None,
|
|
392
|
+
country: str | None = None,
|
|
393
|
+
barcodes: list[BarcodeResult] | None = None,
|
|
394
|
+
) -> StructuredExtraction | None:
|
|
395
|
+
"""Extract a registered new profile; return None for legacy/unrecognized input."""
|
|
396
|
+
document_type, country = validate_document_hint(document_type, country)
|
|
397
|
+
if document_type and DOCUMENT_PROFILES[document_type].legacy:
|
|
398
|
+
return None
|
|
399
|
+
result = _extract_aamva(barcodes or [])
|
|
400
|
+
mrz = parse_travel_mrz(regions)
|
|
401
|
+
if result is not None and mrz is not None:
|
|
402
|
+
result.errors.append("MULTIPLE_MACHINE_READABLE_DOCUMENTS")
|
|
403
|
+
if result is None and mrz is not None:
|
|
404
|
+
result = StructuredExtraction(mrz.document_type, mrz.issuing_country, fields=mrz.fields, field_evidence=mrz.field_evidence, checks=mrz.checks, errors=mrz.errors, warnings=mrz.warnings)
|
|
405
|
+
text_type, state = _detect_text_type(regions)
|
|
406
|
+
if result is None and text_type:
|
|
407
|
+
result = StructuredExtraction(text_type, "US", state)
|
|
408
|
+
if result is None:
|
|
409
|
+
if document_type is None:
|
|
410
|
+
return None
|
|
411
|
+
result = StructuredExtraction(document_type, country, errors=["DOCUMENT_TYPE_NOT_CONFIRMED"])
|
|
412
|
+
return _finish(result)
|
|
413
|
+
if document_type is not None and result.document_type != document_type:
|
|
414
|
+
result.errors.append("DOCUMENT_TYPE_MISMATCH")
|
|
415
|
+
if country is not None and result.issuing_country is not None and result.issuing_country != country:
|
|
416
|
+
result.errors.append("DOCUMENT_COUNTRY_MISMATCH")
|
|
417
|
+
if text_type is not None and text_type != result.document_type:
|
|
418
|
+
result.errors.append("DOCUMENT_TYPE_EVIDENCE_CONFLICT")
|
|
419
|
+
_extract_visual(result, regions)
|
|
420
|
+
if result.document_type in ("us_driver_license", "us_state_id") and not result.checks.get("barcode_decoded"):
|
|
421
|
+
result.warnings.append("PDF417_NOT_DECODED")
|
|
422
|
+
return _finish(result)
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Strict TD1 and machine-readable visa extraction, with US card profiles.
|
|
2
|
+
|
|
3
|
+
Field positions follow ICAO Doc 9303 parts 5 and 7. USCIS cards expose an
|
|
4
|
+
A-number in the main number field and a separate card number in optional data;
|
|
5
|
+
these are intentionally not mapped to the same public field.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
from .document_registry import is_known_mrz_country, normalize_country
|
|
14
|
+
from .mrz_parser import _DIGIT_CORRECTIONS, _clean_mrz_text, _parse_mrz_date, verify_check_digit
|
|
15
|
+
from .ocr_engine import TextRegion
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class TravelMRZResult:
|
|
20
|
+
document_type: str
|
|
21
|
+
issuing_country: str | None
|
|
22
|
+
fields: dict
|
|
23
|
+
field_evidence: dict
|
|
24
|
+
checks: dict
|
|
25
|
+
errors: list[str] = field(default_factory=list)
|
|
26
|
+
warnings: list[str] = field(default_factory=list)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _name(text: str) -> str:
|
|
30
|
+
return " ".join(text.replace("<", " ").split())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _correct_digits(text: str, positions: list[int]) -> str:
|
|
34
|
+
values = list(text)
|
|
35
|
+
for index in positions:
|
|
36
|
+
values[index] = values[index].translate(_DIGIT_CORRECTIONS)
|
|
37
|
+
return "".join(values)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _country(code: str) -> str | None:
|
|
41
|
+
if code == "D<<":
|
|
42
|
+
return "DE"
|
|
43
|
+
try:
|
|
44
|
+
return normalize_country(code.replace("<", ""))
|
|
45
|
+
except ValueError:
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _evidence(regions: list[TextRegion]) -> list[dict]:
|
|
50
|
+
return [{"text": region.text, "bbox": region.bbox, "source": "mrz", "confidence": region.confidence} for region in regions]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def parse_travel_mrz(regions: list[TextRegion]) -> TravelMRZResult | None:
|
|
54
|
+
"""Parse a supported exact-width MRZ without padding or guessed characters."""
|
|
55
|
+
candidates: list[tuple[int, str, TextRegion]] = []
|
|
56
|
+
for region in regions:
|
|
57
|
+
for line in region.text.splitlines():
|
|
58
|
+
text = _clean_mrz_text(line)
|
|
59
|
+
if re.fullmatch(r"[A-Z0-9<]{30}|[A-Z0-9<]{36}|[A-Z0-9<]{44}", text):
|
|
60
|
+
y = min((point[1] for point in region.bbox), default=0)
|
|
61
|
+
candidates.append((y, text, region))
|
|
62
|
+
candidates.sort(key=lambda candidate: candidate[0])
|
|
63
|
+
parsed: list[TravelMRZResult] = []
|
|
64
|
+
for index, (_, first, region) in enumerate(candidates):
|
|
65
|
+
if len(first) == 30 and index + 2 < len(candidates):
|
|
66
|
+
following = candidates[index:index + 3]
|
|
67
|
+
if all(len(item[1]) == 30 for item in following):
|
|
68
|
+
prefix, issuer = first[:2], first[2:5]
|
|
69
|
+
document_type = "passport_card" if prefix == "IP" else "us_green_card" if issuer == "USA" and prefix in ("C1", "C2") else "us_ead" if issuer == "USA" and prefix == "IA" else None
|
|
70
|
+
if document_type and re.fullmatch(r"[A-Z<]{30}", following[2][1]):
|
|
71
|
+
parsed.append(_parse_td1(document_type, [item[1] for item in following], [item[2] for item in following]))
|
|
72
|
+
elif len(first) in (36, 44) and first.startswith("V") and index + 1 < len(candidates):
|
|
73
|
+
second = candidates[index + 1]
|
|
74
|
+
if len(second[1]) == len(first) and re.fullmatch(r"V[A-Z<][A-Z<]{3}[A-Z<]+", first):
|
|
75
|
+
parsed.append(_parse_visa([first, second[1]], [region, second[2]]))
|
|
76
|
+
if not parsed:
|
|
77
|
+
return None
|
|
78
|
+
# Multiple independent documents cannot safely be collapsed into one result.
|
|
79
|
+
if len(parsed) > 1:
|
|
80
|
+
parsed[0].errors.append("MULTIPLE_MRZ_DOCUMENTS")
|
|
81
|
+
return parsed[0]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _finish(document_type: str, lines: list[str], regions: list[TextRegion], fields: dict, checks: dict, name_line: str) -> TravelMRZResult:
|
|
85
|
+
surname, _, given = name_line.partition("<<")
|
|
86
|
+
fields.update({"surname": _name(surname) or None, "given_names": _name(given) or None, "country_code": lines[0][2:5], "mrz_raw": lines})
|
|
87
|
+
errors = [f"MRZ_{key.upper()}_FAILED" for key, value in checks.items() if value is False]
|
|
88
|
+
warnings = []
|
|
89
|
+
for field_name in ("date_of_birth", "expiry_date"):
|
|
90
|
+
if fields.get(field_name) is None:
|
|
91
|
+
errors.append(f"MISSING_OR_INVALID_{field_name.upper()}")
|
|
92
|
+
if fields.get("sex") not in ("M", "F", "X"):
|
|
93
|
+
errors.append("INVALID_SEX")
|
|
94
|
+
issuing_country = _country(fields["country_code"])
|
|
95
|
+
if issuing_country is None:
|
|
96
|
+
errors.append("UNKNOWN_ISSUING_COUNTRY")
|
|
97
|
+
if "nationality" in fields and not is_known_mrz_country(fields["nationality"]):
|
|
98
|
+
errors.append("UNKNOWN_NATIONALITY")
|
|
99
|
+
checks["mrz_checksums_valid"] = all(checks.values())
|
|
100
|
+
checks["issuer_authenticated"] = False
|
|
101
|
+
evidence = {name: _evidence(regions) for name, value in fields.items() if value is not None}
|
|
102
|
+
return TravelMRZResult(document_type, issuing_country, fields, evidence, checks, errors, warnings)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _parse_td1(document_type: str, lines: list[str], regions: list[TextRegion]) -> TravelMRZResult:
|
|
106
|
+
first = _correct_digits(lines[0], [14])
|
|
107
|
+
second = _correct_digits(lines[1], list(range(0, 7)) + list(range(8, 15)) + [29])
|
|
108
|
+
surname_line = lines[2]
|
|
109
|
+
number = first[5:14]
|
|
110
|
+
number_check = first[14]
|
|
111
|
+
fields: dict = {
|
|
112
|
+
"document_number": number.rstrip("<") or None,
|
|
113
|
+
"date_of_birth": _parse_mrz_date(second[:6]),
|
|
114
|
+
"expiry_date": _parse_mrz_date(second[8:14], is_expiry=True),
|
|
115
|
+
"sex": "X" if second[7] == "<" else second[7],
|
|
116
|
+
"nationality": second[15:18].replace("<", "") or None,
|
|
117
|
+
}
|
|
118
|
+
# ICAO TD1 extended document numbers move their check digit into optional
|
|
119
|
+
# data, immediately after the continuation and before a filler.
|
|
120
|
+
if document_type == "passport_card" and number_check == "<":
|
|
121
|
+
continuation = first[15:30].split("<", 1)[0]
|
|
122
|
+
if len(continuation) >= 2 and continuation[-1].isdigit():
|
|
123
|
+
number += continuation[:-1]
|
|
124
|
+
number_check = continuation[-1]
|
|
125
|
+
fields["document_number"] = number
|
|
126
|
+
checks = {
|
|
127
|
+
"document_number_checksum": verify_check_digit(number, number_check),
|
|
128
|
+
"date_of_birth_checksum": verify_check_digit(second[:6], second[6]),
|
|
129
|
+
"expiry_date_checksum": verify_check_digit(second[8:14], second[14]),
|
|
130
|
+
"composite_checksum": verify_check_digit(first[5:30] + second[:7] + second[8:15] + second[18:29], second[29]),
|
|
131
|
+
}
|
|
132
|
+
if document_type in ("us_green_card", "us_ead"):
|
|
133
|
+
fields.pop("document_number")
|
|
134
|
+
fields.pop("nationality")
|
|
135
|
+
fields["uscis_number"] = number.rstrip("<")
|
|
136
|
+
fields["card_number"] = first[15:30].rstrip("<") or None
|
|
137
|
+
# US profiles encode country of birth here, not citizenship.
|
|
138
|
+
fields["country_of_birth"] = second[15:18].replace("<", "") or None
|
|
139
|
+
result = _finish(document_type, [first, second, surname_line], regions, fields, checks, surname_line)
|
|
140
|
+
if document_type in ("us_green_card", "us_ead"):
|
|
141
|
+
if not re.fullmatch(r"[0-9]{9}", fields["uscis_number"] or ""):
|
|
142
|
+
result.errors.append("INVALID_USCIS_NUMBER")
|
|
143
|
+
if not re.fullmatch(r"[A-Z]{3}[0-9]{10}", fields["card_number"] or ""):
|
|
144
|
+
result.errors.append("INVALID_CARD_NUMBER")
|
|
145
|
+
return result
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _parse_visa(lines: list[str], regions: list[TextRegion]) -> TravelMRZResult:
|
|
149
|
+
first, second = lines
|
|
150
|
+
second = _correct_digits(second, [9] + list(range(13, 20)) + list(range(21, 28)))
|
|
151
|
+
fields = {
|
|
152
|
+
"document_number": second[:9].rstrip("<") or None,
|
|
153
|
+
"date_of_birth": _parse_mrz_date(second[13:19]),
|
|
154
|
+
"expiry_date": _parse_mrz_date(second[21:27], is_expiry=True),
|
|
155
|
+
"nationality": second[10:13].replace("<", "") or None,
|
|
156
|
+
"sex": "X" if second[20] == "<" else second[20],
|
|
157
|
+
"mrz_format": "MRV_A" if len(first) == 44 else "MRV_B",
|
|
158
|
+
}
|
|
159
|
+
checks = {
|
|
160
|
+
"document_number_checksum": verify_check_digit(second[:9], second[9]),
|
|
161
|
+
"date_of_birth_checksum": verify_check_digit(second[13:19], second[19]),
|
|
162
|
+
"expiry_date_checksum": verify_check_digit(second[21:27], second[27]),
|
|
163
|
+
}
|
|
164
|
+
return _finish("visa", [first, second], regions, fields, checks, first[5:])
|