document-ocr 0.0.0-stage → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +260 -2
- package/dist/index.cjs +659 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +293 -0
- package/dist/index.d.ts +293 -0
- package/dist/index.js +616 -0
- package/dist/index.js.map +1 -0
- package/hosting/.dockerignore +17 -0
- package/hosting/.gcloudignore +14 -0
- package/hosting/FEATURES.md +172 -0
- package/hosting/HOSTING.md +268 -0
- package/hosting/LICENSE +21 -0
- package/hosting/README.md +369 -0
- package/hosting/core/__init__.py +0 -0
- package/hosting/core/aadhaar_extractor.py +298 -0
- package/hosting/core/back_page_extractor.py +492 -0
- package/hosting/core/barcodes.py +125 -0
- package/hosting/core/card_recovery.py +244 -0
- package/hosting/core/document_bundle.py +82 -0
- package/hosting/core/document_classifier.py +233 -0
- package/hosting/core/document_input.py +135 -0
- package/hosting/core/document_registry.py +143 -0
- package/hosting/core/driving_licence_extractor.py +510 -0
- package/hosting/core/ead_recovery.py +54 -0
- package/hosting/core/evidence.py +54 -0
- package/hosting/core/form_recovery.py +61 -0
- package/hosting/core/jobs.py +232 -0
- package/hosting/core/kyc_ocr.py +132 -0
- package/hosting/core/kyc_validation.py +411 -0
- package/hosting/core/model_setup.py +13 -0
- package/hosting/core/mrz_parser.py +336 -0
- package/hosting/core/mrz_recovery.py +193 -0
- package/hosting/core/npr_extractor.py +410 -0
- package/hosting/core/nrega_extractor.py +827 -0
- package/hosting/core/ocr_engine.py +207 -0
- package/hosting/core/page_classifier.py +137 -0
- package/hosting/core/pan_extractor.py +183 -0
- package/hosting/core/pipeline.py +712 -0
- package/hosting/core/preprocessor.py +297 -0
- package/hosting/core/recovery_budget.py +67 -0
- package/hosting/core/redaction.py +48 -0
- package/hosting/core/review.html +244 -0
- package/hosting/core/structured_documents.py +422 -0
- package/hosting/core/travel_mrz.py +164 -0
- package/hosting/core/travel_recovery.py +187 -0
- package/hosting/core/validator.py +396 -0
- package/hosting/core/validators.py +179 -0
- package/hosting/core/voter_id_extractor.py +363 -0
- package/hosting/deploy/cloudflare/deploy.sh +44 -0
- package/hosting/deploy/cloudflare/package-lock.json +3088 -0
- package/hosting/deploy/cloudflare/package.json +29 -0
- package/hosting/deploy/cloudflare/src/gateway.ts +42 -0
- package/hosting/deploy/cloudflare/src/index.ts +21 -0
- package/hosting/deploy/cloudflare/test/gateway.test.ts +59 -0
- package/hosting/deploy/cloudflare/tsconfig.json +12 -0
- package/hosting/deploy/cloudflare/worker-configuration.d.ts +16253 -0
- package/hosting/deploy/cloudflare/wrangler.jsonc +26 -0
- package/hosting/deploy/cloudrun/cloudbuild.yaml +18 -0
- package/hosting/deploy/cloudrun/deploy.sh +71 -0
- package/hosting/deploy/cloudrun/render_service.py +47 -0
- package/hosting/deploy/common.sh +14 -0
- package/hosting/deploy/docker/Dockerfile +16 -0
- package/hosting/deploy/docker/server.py +313 -0
- package/hosting/deploy/lambda/Dockerfile.lambda +11 -0
- package/hosting/deploy/lambda/deploy.sh +22 -0
- package/hosting/deploy/lambda/handler.py +132 -0
- package/hosting/deploy/lambda/template.yaml +59 -0
- package/hosting/deploy/local/dev.sh +18 -0
- package/hosting/deploy/server/Caddyfile +5 -0
- package/hosting/deploy/server/compose.jobs.yaml +36 -0
- package/hosting/deploy/server/compose.yaml +38 -0
- package/hosting/deploy/server/deploy.sh +23 -0
- package/hosting/infra/cloudrun/main.tf +104 -0
- package/hosting/infra/cloudrun/terraform.tfvars.example +7 -0
- package/hosting/infra/cloudrun/variables.tf +55 -0
- package/hosting/infra/lambda/main.tf +63 -0
- package/hosting/infra/lambda/terraform.tfvars.example +6 -0
- package/hosting/infra/lambda/variables.tf +37 -0
- package/hosting/pyproject.toml +65 -0
- package/hosting/requirements.lock +134 -0
- package/package.json +76 -4
- package/python/core/__init__.py +0 -0
- package/python/core/aadhaar_extractor.py +298 -0
- package/python/core/back_page_extractor.py +492 -0
- package/python/core/barcodes.py +125 -0
- package/python/core/card_recovery.py +244 -0
- package/python/core/document_bundle.py +82 -0
- package/python/core/document_classifier.py +233 -0
- package/python/core/document_input.py +135 -0
- package/python/core/document_registry.py +143 -0
- package/python/core/driving_licence_extractor.py +510 -0
- package/python/core/ead_recovery.py +54 -0
- package/python/core/evidence.py +54 -0
- package/python/core/form_recovery.py +61 -0
- package/python/core/jobs.py +232 -0
- package/python/core/kyc_ocr.py +132 -0
- package/python/core/kyc_validation.py +411 -0
- package/python/core/model_setup.py +13 -0
- package/python/core/mrz_parser.py +336 -0
- package/python/core/mrz_recovery.py +193 -0
- package/python/core/npr_extractor.py +410 -0
- package/python/core/nrega_extractor.py +827 -0
- package/python/core/ocr_engine.py +207 -0
- package/python/core/page_classifier.py +137 -0
- package/python/core/pan_extractor.py +183 -0
- package/python/core/pipeline.py +712 -0
- package/python/core/preprocessor.py +297 -0
- package/python/core/recovery_budget.py +67 -0
- package/python/core/redaction.py +48 -0
- package/python/core/review.html +244 -0
- package/python/core/structured_documents.py +422 -0
- package/python/core/travel_mrz.py +164 -0
- package/python/core/travel_recovery.py +187 -0
- package/python/core/validator.py +396 -0
- package/python/core/validators.py +179 -0
- package/python/core/voter_id_extractor.py +363 -0
- package/python/pyproject.toml +34 -0
- package/python/requirements.lock +134 -0
- package/python/server.py +313 -0
- package/scripts/dococr.mjs +163 -0
- package/scripts/postinstall.mjs +173 -0
- package/scripts/sync-hosting.mjs +35 -0
- package/scripts/sync-python.sh +20 -0
|
@@ -0,0 +1,827 @@
|
|
|
1
|
+
"""Extract identity fields from Indian MGNREGA/NREGA job cards.
|
|
2
|
+
|
|
3
|
+
The Ministry of Rural Development's MGNREGA Operational Guidelines 2013,
|
|
4
|
+
Annexure 5, defines a household-level job-card number, household particulars,
|
|
5
|
+
location fields, and a repeating table of adult household members willing to
|
|
6
|
+
work. State-issued cards vary in language and layout, so this extractor is
|
|
7
|
+
label- and table-driven rather than tied to one card design.
|
|
8
|
+
|
|
9
|
+
This module intentionally extracts only the minimum identity/location fields
|
|
10
|
+
needed to identify a job card. Although some card generations also print
|
|
11
|
+
Aadhaar, bank/post-office, insurance and voter-card numbers, those high-risk
|
|
12
|
+
identifiers are deliberately outside this extractor's contract.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
import unicodedata
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from difflib import SequenceMatcher
|
|
21
|
+
from typing import Optional
|
|
22
|
+
|
|
23
|
+
from .ocr_engine import TextRegion
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class NregaMember:
|
|
28
|
+
"""One adult household member listed as willing to work."""
|
|
29
|
+
|
|
30
|
+
serial_number: Optional[str] = None
|
|
31
|
+
name: Optional[str] = None
|
|
32
|
+
father_or_husband_name: Optional[str] = None
|
|
33
|
+
gender: Optional[str] = None
|
|
34
|
+
age: Optional[int] = None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class NregaFields:
|
|
39
|
+
"""Stable household particulars shared by major job-card generations."""
|
|
40
|
+
|
|
41
|
+
job_card_number: Optional[str] = None
|
|
42
|
+
head_of_household: Optional[str] = None
|
|
43
|
+
category: Optional[str] = None
|
|
44
|
+
registration_date: Optional[str] = None
|
|
45
|
+
validity_from: Optional[str] = None
|
|
46
|
+
validity_to: Optional[str] = None
|
|
47
|
+
address: Optional[str] = None
|
|
48
|
+
village: Optional[str] = None
|
|
49
|
+
gram_panchayat: Optional[str] = None
|
|
50
|
+
block: Optional[str] = None
|
|
51
|
+
district: Optional[str] = None
|
|
52
|
+
state: Optional[str] = None
|
|
53
|
+
bpl_status: Optional[bool] = None
|
|
54
|
+
family_id: Optional[str] = None
|
|
55
|
+
members: list[NregaMember] = field(default_factory=list)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# Label vocabularies include common English/Hindi card wording and regional
|
|
59
|
+
# variants observed in state forms. Values are always returned as printed;
|
|
60
|
+
# only categorical values (gender/category/BPL) are canonicalised.
|
|
61
|
+
_JOB_CARD_LABELS = (
|
|
62
|
+
"JOB CARD NUMBER",
|
|
63
|
+
"JOB CARD NO",
|
|
64
|
+
"JOB CARD",
|
|
65
|
+
"NREGA JOB CARD NO",
|
|
66
|
+
"MGNREGA JOB CARD NO",
|
|
67
|
+
"REGISTRATION NUMBER",
|
|
68
|
+
"REGISTRATION NO",
|
|
69
|
+
"जॉब कार्ड संख्या",
|
|
70
|
+
"जॉब कार्ड क्रमांक",
|
|
71
|
+
"रोजगार कार्ड संख्या",
|
|
72
|
+
"নরেগা জব কার্ড নং",
|
|
73
|
+
"জব কার্ড নং",
|
|
74
|
+
"ఉపాధి హామీ కార్డు సంఖ్య",
|
|
75
|
+
"జాబ్ కార్డ్ నంబర్",
|
|
76
|
+
"வேலை அட்டை எண்",
|
|
77
|
+
)
|
|
78
|
+
_HEAD_LABELS = (
|
|
79
|
+
"NAME OF HEAD OF HOUSEHOLD",
|
|
80
|
+
"HEAD OF HOUSEHOLD",
|
|
81
|
+
"HOUSEHOLD HEAD NAME",
|
|
82
|
+
"NAME OF APPLICANT",
|
|
83
|
+
"APPLICANT NAME",
|
|
84
|
+
"परिवार के मुखिया का नाम",
|
|
85
|
+
"मुखिया का नाम",
|
|
86
|
+
"परिवार प्रमुख का नाम",
|
|
87
|
+
"পরিবারের প্রধানের নাম",
|
|
88
|
+
"కుటుంబ యజమాని పేరు",
|
|
89
|
+
"குடும்பத் தலைவர் பெயர்",
|
|
90
|
+
)
|
|
91
|
+
_CATEGORY_LABELS = (
|
|
92
|
+
"CATEGORY",
|
|
93
|
+
"SOCIAL CATEGORY",
|
|
94
|
+
"CASTE CATEGORY",
|
|
95
|
+
"वर्ग",
|
|
96
|
+
"श्रेणी",
|
|
97
|
+
"সামাজিক শ্রেণী",
|
|
98
|
+
"వర్గం",
|
|
99
|
+
"வகை",
|
|
100
|
+
)
|
|
101
|
+
_REGISTRATION_DATE_LABELS = (
|
|
102
|
+
"DATE OF REGISTRATION",
|
|
103
|
+
"REGISTRATION DATE",
|
|
104
|
+
"DATE OF ISSUE",
|
|
105
|
+
"ISSUE DATE",
|
|
106
|
+
"पंजीकरण की तारीख",
|
|
107
|
+
"पंजीकरण दिनांक",
|
|
108
|
+
"जारी करने की तारीख",
|
|
109
|
+
"নিবন্ধনের তারিখ",
|
|
110
|
+
"నమోదు తేదీ",
|
|
111
|
+
"பதிவு தேதி",
|
|
112
|
+
)
|
|
113
|
+
_VALIDITY_LABELS = (
|
|
114
|
+
"VALIDITY PERIOD",
|
|
115
|
+
"VALIDITY",
|
|
116
|
+
"वैधता अवधि",
|
|
117
|
+
"বৈধতার সময়কাল",
|
|
118
|
+
"చెల్లుబాటు కాలం",
|
|
119
|
+
"செல்லுபடியாகும் காலம்",
|
|
120
|
+
)
|
|
121
|
+
_VALID_FROM_LABELS = ("VALID FROM", "FROM", "से", "হইতে", "నుండి", "முதல்")
|
|
122
|
+
_VALID_TO_LABELS = ("VALID TO", "TO", "तक", "পর্যন্ত", "వరకు", "வரை")
|
|
123
|
+
_ADDRESS_LABELS = (
|
|
124
|
+
"HOUSEHOLD ADDRESS",
|
|
125
|
+
"ADDRESS",
|
|
126
|
+
"पता",
|
|
127
|
+
"घर का पता",
|
|
128
|
+
"ঠিকানা",
|
|
129
|
+
"చిరునామా",
|
|
130
|
+
"முகவரி",
|
|
131
|
+
)
|
|
132
|
+
_VILLAGE_LABELS = (
|
|
133
|
+
"NAME OF VILLAGE",
|
|
134
|
+
"VILLAGE NAME",
|
|
135
|
+
"VILLAGE",
|
|
136
|
+
"गांव का नाम",
|
|
137
|
+
"गाँव",
|
|
138
|
+
"ग्राम",
|
|
139
|
+
"গ্রাম",
|
|
140
|
+
"గ్రామం",
|
|
141
|
+
"கிராமம்",
|
|
142
|
+
)
|
|
143
|
+
_PANCHAYAT_LABELS = (
|
|
144
|
+
"NAME OF GRAM PANCHAYAT",
|
|
145
|
+
"GRAM PANCHAYAT",
|
|
146
|
+
"G P NAME",
|
|
147
|
+
"PANCHAYAT",
|
|
148
|
+
"ग्राम पंचायत का नाम",
|
|
149
|
+
"ग्राम पंचायत",
|
|
150
|
+
"পঞ্চায়েত",
|
|
151
|
+
"গ্রাম পঞ্চায়েত",
|
|
152
|
+
"గ్రామ పంచాయతీ",
|
|
153
|
+
"ஊராட்சி",
|
|
154
|
+
)
|
|
155
|
+
_BLOCK_LABELS = (
|
|
156
|
+
"NAME OF BLOCK",
|
|
157
|
+
"DEVELOPMENT BLOCK",
|
|
158
|
+
"BLOCK",
|
|
159
|
+
"MANDAL",
|
|
160
|
+
"विकास खंड",
|
|
161
|
+
"विकासखण्ड",
|
|
162
|
+
"ब्लॉक",
|
|
163
|
+
"खंड",
|
|
164
|
+
"ব্লক",
|
|
165
|
+
"మండలం",
|
|
166
|
+
"ஒன்றியம்",
|
|
167
|
+
"வட்டாரம்",
|
|
168
|
+
)
|
|
169
|
+
_DISTRICT_LABELS = (
|
|
170
|
+
"DISTRICT NAME",
|
|
171
|
+
"DISTRICT",
|
|
172
|
+
"जिला",
|
|
173
|
+
"जनपद",
|
|
174
|
+
"জেলা",
|
|
175
|
+
"జిల్లా",
|
|
176
|
+
"மாவட்டம்",
|
|
177
|
+
)
|
|
178
|
+
_STATE_LABELS = (
|
|
179
|
+
"STATE NAME",
|
|
180
|
+
"STATE",
|
|
181
|
+
"राज्य",
|
|
182
|
+
"প্রদেশ",
|
|
183
|
+
"রাজ্য",
|
|
184
|
+
"రాష్ట్రం",
|
|
185
|
+
"மாநிலம்",
|
|
186
|
+
)
|
|
187
|
+
_BPL_LABELS = (
|
|
188
|
+
"WHETHER BPL",
|
|
189
|
+
"BPL STATUS",
|
|
190
|
+
"BPL FAMILY",
|
|
191
|
+
"BELOW POVERTY LINE",
|
|
192
|
+
"क्या बीपीएल",
|
|
193
|
+
"बीपीएल परिवार",
|
|
194
|
+
"দারিদ্র্য সীমার নিচে",
|
|
195
|
+
"బిపిఎల్ కుటుంబం",
|
|
196
|
+
"வறுமைக் கோட்டிற்கு கீழ்",
|
|
197
|
+
)
|
|
198
|
+
_FAMILY_ID_LABELS = (
|
|
199
|
+
"FAMILY ID",
|
|
200
|
+
"FAMILY IDENTIFICATION NUMBER",
|
|
201
|
+
"HOUSEHOLD ID",
|
|
202
|
+
"परिवार आईडी",
|
|
203
|
+
"परिवार पहचान संख्या",
|
|
204
|
+
"পারিবারিক আইডি",
|
|
205
|
+
"కుటుంబ ఐడి",
|
|
206
|
+
"குடும்ப அடையாள எண்",
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
_MEMBER_SECTION_LABELS = (
|
|
210
|
+
"DETAILS OF THE APPLICANTS OF THE HOUSEHOLD WILLING TO WORK",
|
|
211
|
+
"DETAILS OF APPLICANTS",
|
|
212
|
+
"HOUSEHOLD MEMBERS WILLING TO WORK",
|
|
213
|
+
"ADULT MEMBERS WILLING TO WORK",
|
|
214
|
+
"काम करने के इच्छुक परिवार के सदस्यों का विवरण",
|
|
215
|
+
"आवेदकों का विवरण",
|
|
216
|
+
"কাজ করতে ইচ্ছুক সদস্যদের বিবরণ",
|
|
217
|
+
"పని చేయడానికి సిద్ధంగా ఉన్న సభ్యుల వివరాలు",
|
|
218
|
+
"வேலை செய்ய விரும்பும் உறுப்பினர்கள் விவரம்",
|
|
219
|
+
)
|
|
220
|
+
_SERIAL_HEADER_LABELS = (
|
|
221
|
+
"SL NO",
|
|
222
|
+
"SERIAL NO",
|
|
223
|
+
"S NO",
|
|
224
|
+
"क्रम संख्या",
|
|
225
|
+
"क्र सं",
|
|
226
|
+
"ক্রমিক নং",
|
|
227
|
+
"క్రమ సంఖ్య",
|
|
228
|
+
"வ எண்",
|
|
229
|
+
)
|
|
230
|
+
_MEMBER_NAME_HEADER_LABELS = (
|
|
231
|
+
"MEMBER NAME",
|
|
232
|
+
"NAME OF MEMBER",
|
|
233
|
+
"NAME OF APPLICANT",
|
|
234
|
+
"NAME",
|
|
235
|
+
"सदस्य का नाम",
|
|
236
|
+
"आवेदक का नाम",
|
|
237
|
+
"नाम",
|
|
238
|
+
"সদস্যের নাম",
|
|
239
|
+
"নাম",
|
|
240
|
+
"సభ్యుని పేరు",
|
|
241
|
+
"పేరు",
|
|
242
|
+
"உறுப்பினர் பெயர்",
|
|
243
|
+
"பெயர்",
|
|
244
|
+
)
|
|
245
|
+
_RELATION_HEADER_LABELS = (
|
|
246
|
+
"FATHER HUSBAND NAME",
|
|
247
|
+
"FATHERS HUSBANDS NAME",
|
|
248
|
+
"FATHER S HUSBAND S NAME",
|
|
249
|
+
"FATHER OR HUSBAND NAME",
|
|
250
|
+
"PARENT SPOUSE NAME",
|
|
251
|
+
"पिता पति का नाम",
|
|
252
|
+
"पिता या पति का नाम",
|
|
253
|
+
"পিতা স্বামীর নাম",
|
|
254
|
+
"తండ్రి భర్త పేరు",
|
|
255
|
+
"தந்தை கணவர் பெயர்",
|
|
256
|
+
)
|
|
257
|
+
_GENDER_HEADER_LABELS = (
|
|
258
|
+
"MALE FEMALE",
|
|
259
|
+
"GENDER",
|
|
260
|
+
"SEX",
|
|
261
|
+
"लिंग",
|
|
262
|
+
"पुरुष महिला",
|
|
263
|
+
"লিঙ্গ",
|
|
264
|
+
"స్త్రీ పురుష",
|
|
265
|
+
"లింగం",
|
|
266
|
+
"பாலினம்",
|
|
267
|
+
)
|
|
268
|
+
_AGE_HEADER_LABELS = (
|
|
269
|
+
"AGE ON DATE OF REGISTRATION",
|
|
270
|
+
"AGE AT REGISTRATION",
|
|
271
|
+
"AGE",
|
|
272
|
+
"पंजीकरण के समय आयु",
|
|
273
|
+
"उम्र",
|
|
274
|
+
"आयु",
|
|
275
|
+
"বয়স",
|
|
276
|
+
"వయస్సు",
|
|
277
|
+
"வயது",
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
_ALL_FIELD_LABEL_GROUPS = (
|
|
281
|
+
_JOB_CARD_LABELS,
|
|
282
|
+
_HEAD_LABELS,
|
|
283
|
+
_CATEGORY_LABELS,
|
|
284
|
+
_REGISTRATION_DATE_LABELS,
|
|
285
|
+
_VALIDITY_LABELS,
|
|
286
|
+
_ADDRESS_LABELS,
|
|
287
|
+
_VILLAGE_LABELS,
|
|
288
|
+
_PANCHAYAT_LABELS,
|
|
289
|
+
_BLOCK_LABELS,
|
|
290
|
+
_DISTRICT_LABELS,
|
|
291
|
+
_STATE_LABELS,
|
|
292
|
+
_BPL_LABELS,
|
|
293
|
+
_FAMILY_ID_LABELS,
|
|
294
|
+
_MEMBER_SECTION_LABELS,
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
_DATE_RE = re.compile(
|
|
298
|
+
r"\b(?:"
|
|
299
|
+
r"\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}"
|
|
300
|
+
r"|"
|
|
301
|
+
r"\d{1,2}\s+(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)"
|
|
302
|
+
r"(?:UARY|RUARY|CH|IL|E|Y|UST|TEMBER|OBER|EMBER)?\s+\d{4}"
|
|
303
|
+
r")\b",
|
|
304
|
+
re.IGNORECASE,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
# Job-card numbers are state-prefixed and contain multiple numeric hierarchy
|
|
308
|
+
# components. Requiring at least three numeric components avoids confusing a
|
|
309
|
+
# PAN, Aadhaar, phone number or the common one-component DL format for a JC.
|
|
310
|
+
_JOB_CARD_CANDIDATE_RE = re.compile(
|
|
311
|
+
r"(?<![A-Z0-9])("
|
|
312
|
+
r"[A-Z]{2}\s*[-–—]\s*[0-9OILSBZGQD]{1,10}"
|
|
313
|
+
r"(?:\s*[-/–—]\s*[0-9OILSBZGQD]{1,10}){2,7}"
|
|
314
|
+
r")(?![A-Z0-9])",
|
|
315
|
+
re.IGNORECASE,
|
|
316
|
+
)
|
|
317
|
+
_NUMERIC_OCR_TRANSLATION = str.maketrans(
|
|
318
|
+
{"O": "0", "Q": "0", "D": "0", "I": "1", "L": "1", "Z": "2",
|
|
319
|
+
"S": "5", "G": "6", "B": "8"}
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
_MALE_WORDS = {
|
|
323
|
+
"M", "MALE", "MAN", "पुरुष", "नर", "পুরুষ", "ಪುರುಷ", "పురుషుడు",
|
|
324
|
+
"ஆண்",
|
|
325
|
+
}
|
|
326
|
+
_FEMALE_WORDS = {
|
|
327
|
+
"F", "FEMALE", "WOMAN", "महिला", "स्त्री", "मादा", "মহিলা", "নারী",
|
|
328
|
+
"ಹೆಣ್ಣು", "స్త్రీ", "பெண்",
|
|
329
|
+
}
|
|
330
|
+
_TRANSGENDER_WORDS = {
|
|
331
|
+
"T", "TG", "TRANSGENDER", "THIRD GENDER", "किन्नर", "ट्रांसजेंडर",
|
|
332
|
+
"তৃতীয় লিঙ্গ", "ట్రాన్స్ జెండర్", "திருநங்கை",
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _bounds(region: TextRegion) -> tuple[int, int, int, int]:
|
|
337
|
+
return (
|
|
338
|
+
min(point[0] for point in region.bbox),
|
|
339
|
+
min(point[1] for point in region.bbox),
|
|
340
|
+
max(point[0] for point in region.bbox),
|
|
341
|
+
max(point[1] for point in region.bbox),
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _normalise(text: str, *, label: bool = False) -> str:
|
|
346
|
+
normalised = unicodedata.normalize("NFKC", text).casefold()
|
|
347
|
+
normalised = normalised.replace("&", " and ")
|
|
348
|
+
normalised = re.sub(r"['’`]", "", normalised)
|
|
349
|
+
# ``re``'s ``\w`` omits combining vowel signs used by Indic scripts.
|
|
350
|
+
# Preserve letters, numbers and combining marks explicitly so values such
|
|
351
|
+
# as "महिला" and "नहीं" survive normalisation intact.
|
|
352
|
+
normalised = "".join(
|
|
353
|
+
character
|
|
354
|
+
if (
|
|
355
|
+
character.isalnum()
|
|
356
|
+
or unicodedata.category(character).startswith(("L", "N", "M"))
|
|
357
|
+
)
|
|
358
|
+
else " "
|
|
359
|
+
for character in normalised
|
|
360
|
+
)
|
|
361
|
+
normalised = " ".join(normalised.split())
|
|
362
|
+
if label:
|
|
363
|
+
# These substitutions are only for comparing known label text. They
|
|
364
|
+
# are never applied to extracted values or identifiers.
|
|
365
|
+
normalised = normalised.translate(str.maketrans({"0": "o", "1": "i"}))
|
|
366
|
+
return normalised
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _label_score(text: str, aliases: tuple[str, ...]) -> float:
|
|
370
|
+
"""Return a conservative label-match score, including mild OCR noise."""
|
|
371
|
+
candidate = _normalise(text, label=True)
|
|
372
|
+
if not candidate:
|
|
373
|
+
return 0.0
|
|
374
|
+
|
|
375
|
+
# The text before a delimiter is usually the pure label; compare both it
|
|
376
|
+
# and the whole region so inline values remain discoverable.
|
|
377
|
+
prefix = re.split(r"\s*[:=]\s*|\s+[–—-]\s+", text, maxsplit=1)[0]
|
|
378
|
+
candidate_prefix = _normalise(prefix, label=True)
|
|
379
|
+
best = 0.0
|
|
380
|
+
|
|
381
|
+
for alias in aliases:
|
|
382
|
+
expected = _normalise(alias, label=True)
|
|
383
|
+
if not expected:
|
|
384
|
+
continue
|
|
385
|
+
if candidate == expected or candidate_prefix == expected:
|
|
386
|
+
best = max(best, 1.0 + min(len(expected), 100) / 1000)
|
|
387
|
+
continue
|
|
388
|
+
if f" {expected} " in f" {candidate} ":
|
|
389
|
+
best = max(best, 0.99 + min(len(expected), 100) / 1000)
|
|
390
|
+
continue
|
|
391
|
+
# Fuzzy matching is limited to similarly-sized label prefixes. This
|
|
392
|
+
# accepts OCR errors such as "Grarn Panchayat" but does not turn an
|
|
393
|
+
# arbitrary value into a label.
|
|
394
|
+
if len(expected) >= 5 and candidate_prefix:
|
|
395
|
+
size_ratio = len(candidate_prefix) / len(expected)
|
|
396
|
+
if 0.65 <= size_ratio <= 1.45:
|
|
397
|
+
ratio = SequenceMatcher(None, candidate_prefix, expected).ratio()
|
|
398
|
+
if ratio >= 0.84:
|
|
399
|
+
best = max(best, ratio)
|
|
400
|
+
return best
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _best_label_region(
|
|
404
|
+
regions: list[TextRegion], aliases: tuple[str, ...]
|
|
405
|
+
) -> Optional[TextRegion]:
|
|
406
|
+
scored = [
|
|
407
|
+
(_label_score(region.text, aliases), region.confidence, region)
|
|
408
|
+
for region in regions
|
|
409
|
+
]
|
|
410
|
+
scored = [entry for entry in scored if entry[0] >= 0.84]
|
|
411
|
+
if not scored:
|
|
412
|
+
return None
|
|
413
|
+
scored.sort(key=lambda entry: (entry[0], entry[1]), reverse=True)
|
|
414
|
+
return scored[0][2]
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _alias_token_count_at_start(text: str, aliases: tuple[str, ...]) -> int:
|
|
418
|
+
tokens = list(re.finditer(r"[^\W_]+", text, flags=re.UNICODE))
|
|
419
|
+
normalised_tokens = [_normalise(token.group(0), label=True) for token in tokens]
|
|
420
|
+
for alias in sorted(aliases, key=lambda value: len(_normalise(value)), reverse=True):
|
|
421
|
+
alias_tokens = _normalise(alias, label=True).split()
|
|
422
|
+
if normalised_tokens[: len(alias_tokens)] == alias_tokens:
|
|
423
|
+
return tokens[len(alias_tokens) - 1].end()
|
|
424
|
+
return 0
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _inline_value(text: str, aliases: tuple[str, ...]) -> Optional[str]:
|
|
428
|
+
for delimiter in (
|
|
429
|
+
re.compile(r"\s*[:=]\s*"),
|
|
430
|
+
re.compile(r"\s+[–—-]\s+"),
|
|
431
|
+
):
|
|
432
|
+
parts = delimiter.split(text, maxsplit=1)
|
|
433
|
+
if len(parts) == 2 and _label_score(parts[0], aliases) >= 0.84:
|
|
434
|
+
value = parts[1].strip(" :;|")
|
|
435
|
+
return value or None
|
|
436
|
+
|
|
437
|
+
end = _alias_token_count_at_start(text, aliases)
|
|
438
|
+
if end:
|
|
439
|
+
value = text[end:].strip(" :;|–—-")
|
|
440
|
+
return value or None
|
|
441
|
+
return None
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _looks_like_known_label(text: str) -> bool:
|
|
445
|
+
return any(_label_score(text, aliases) >= 0.92 for aliases in _ALL_FIELD_LABEL_GROUPS)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _value_for_labels(
|
|
449
|
+
regions: list[TextRegion], aliases: tuple[str, ...]
|
|
450
|
+
) -> Optional[str]:
|
|
451
|
+
label = _best_label_region(regions, aliases)
|
|
452
|
+
if label is None:
|
|
453
|
+
return None
|
|
454
|
+
|
|
455
|
+
inline = _inline_value(label.text, aliases)
|
|
456
|
+
if inline:
|
|
457
|
+
return inline
|
|
458
|
+
|
|
459
|
+
left, top, right, bottom = _bounds(label)
|
|
460
|
+
height = max(bottom - top, 1)
|
|
461
|
+
right_candidates: list[tuple[float, TextRegion]] = []
|
|
462
|
+
below_candidates: list[tuple[float, TextRegion]] = []
|
|
463
|
+
|
|
464
|
+
for region in regions:
|
|
465
|
+
if region is label or _looks_like_known_label(region.text):
|
|
466
|
+
continue
|
|
467
|
+
r_left, r_top, r_right, r_bottom = _bounds(region)
|
|
468
|
+
overlap = min(bottom, r_bottom) - max(top, r_top)
|
|
469
|
+
horizontal_gap = r_left - right
|
|
470
|
+
if overlap >= height * 0.4 and -5 <= horizontal_gap <= 500:
|
|
471
|
+
right_candidates.append((max(horizontal_gap, 0), region))
|
|
472
|
+
|
|
473
|
+
vertical_gap = r_top - bottom
|
|
474
|
+
x_overlap = min(right, r_right) - max(left, r_left)
|
|
475
|
+
aligned = abs(r_left - left) <= 220 or x_overlap > 0
|
|
476
|
+
if aligned and -4 <= vertical_gap <= 110:
|
|
477
|
+
below_candidates.append(
|
|
478
|
+
(max(vertical_gap, 0) + abs(r_left - left) * 0.15, region)
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
candidates = right_candidates or below_candidates
|
|
482
|
+
if not candidates:
|
|
483
|
+
return None
|
|
484
|
+
candidates.sort(key=lambda entry: (entry[0], -entry[1].confidence))
|
|
485
|
+
value = candidates[0][1].text.strip(" :;|")
|
|
486
|
+
return value or None
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _normalise_job_card_number(text: str) -> Optional[str]:
|
|
490
|
+
text = unicodedata.normalize("NFKC", text).upper()
|
|
491
|
+
matches: list[tuple[int, str]] = []
|
|
492
|
+
for match in _JOB_CARD_CANDIDATE_RE.finditer(text):
|
|
493
|
+
raw = re.sub(r"[–—]", "-", match.group(1))
|
|
494
|
+
parts = re.split(r"([-/])", raw)
|
|
495
|
+
state = parts[0].strip()
|
|
496
|
+
if not re.fullmatch(r"[A-Z]{2}", state):
|
|
497
|
+
continue
|
|
498
|
+
output = [state]
|
|
499
|
+
numeric_components = 0
|
|
500
|
+
for index in range(1, len(parts), 2):
|
|
501
|
+
separator = parts[index]
|
|
502
|
+
raw_component = parts[index + 1].strip()
|
|
503
|
+
component = raw_component.translate(_NUMERIC_OCR_TRANSLATION)
|
|
504
|
+
if not component.isdigit():
|
|
505
|
+
break
|
|
506
|
+
output.extend((separator, component))
|
|
507
|
+
numeric_components += 1
|
|
508
|
+
else:
|
|
509
|
+
if numeric_components >= 3:
|
|
510
|
+
value = "".join(output)
|
|
511
|
+
matches.append((numeric_components, value))
|
|
512
|
+
if not matches:
|
|
513
|
+
return None
|
|
514
|
+
matches.sort(key=lambda item: (item[0], len(item[1])), reverse=True)
|
|
515
|
+
return matches[0][1]
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _find_job_card_number(regions: list[TextRegion]) -> Optional[str]:
|
|
519
|
+
labelled = _value_for_labels(regions, _JOB_CARD_LABELS)
|
|
520
|
+
if labelled:
|
|
521
|
+
value = _normalise_job_card_number(labelled)
|
|
522
|
+
if value:
|
|
523
|
+
return value
|
|
524
|
+
|
|
525
|
+
# OCR can split a long number into adjacent regions, so inspect both
|
|
526
|
+
# individual regions and a reading-order join.
|
|
527
|
+
for region in regions:
|
|
528
|
+
value = _normalise_job_card_number(region.text)
|
|
529
|
+
if value:
|
|
530
|
+
return value
|
|
531
|
+
ordered = sorted(regions, key=lambda region: (_bounds(region)[1], _bounds(region)[0]))
|
|
532
|
+
return _normalise_job_card_number(" ".join(region.text for region in ordered))
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _extract_date(text: Optional[str]) -> Optional[str]:
|
|
536
|
+
if not text:
|
|
537
|
+
return None
|
|
538
|
+
match = _DATE_RE.search(text)
|
|
539
|
+
return match.group(0).strip() if match else None
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _normalise_gender(text: Optional[str]) -> Optional[str]:
|
|
543
|
+
if not text:
|
|
544
|
+
return None
|
|
545
|
+
value = _normalise(text)
|
|
546
|
+
if value in {_normalise(word) for word in _TRANSGENDER_WORDS}:
|
|
547
|
+
return "TRANSGENDER"
|
|
548
|
+
if value in {_normalise(word) for word in _FEMALE_WORDS}:
|
|
549
|
+
return "FEMALE"
|
|
550
|
+
if value in {_normalise(word) for word in _MALE_WORDS}:
|
|
551
|
+
return "MALE"
|
|
552
|
+
return None
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _normalise_category(text: Optional[str]) -> Optional[str]:
|
|
556
|
+
if not text:
|
|
557
|
+
return None
|
|
558
|
+
value = _normalise(text).upper()
|
|
559
|
+
compact = re.sub(r"[^A-Z]", "", value)
|
|
560
|
+
if compact in {"SC", "SCHEDULEDCASTE"} or "अनुसूचित जाति" in text:
|
|
561
|
+
return "SC"
|
|
562
|
+
if compact in {"ST", "SCHEDULEDTRIBE"} or "अनुसूचित जनजाति" in text:
|
|
563
|
+
return "ST"
|
|
564
|
+
if compact in {"OBC", "OTHERBACKWARDCLASS"} or "अन्य पिछड़ा वर्ग" in text:
|
|
565
|
+
return "OBC"
|
|
566
|
+
if compact in {"OTHER", "OTHERS", "GENERAL", "GEN"}:
|
|
567
|
+
return "OTHER"
|
|
568
|
+
return text.strip() or None
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _normalise_bpl(text: Optional[str]) -> Optional[bool]:
|
|
572
|
+
if not text:
|
|
573
|
+
return None
|
|
574
|
+
value = _normalise(text)
|
|
575
|
+
yes = {
|
|
576
|
+
_normalise(word)
|
|
577
|
+
for word in ("yes", "y", "true", "1", "हाँ", "हां", "জি", "হ্যাঁ", "అవును", "ஆம்")
|
|
578
|
+
}
|
|
579
|
+
no = {
|
|
580
|
+
_normalise(word)
|
|
581
|
+
for word in ("no", "n", "false", "0", "नहीं", "नही", "না", "కాదు", "இல்லை")
|
|
582
|
+
}
|
|
583
|
+
if value in yes or re.search(r"\byes\b", value):
|
|
584
|
+
return True
|
|
585
|
+
if value in no or re.search(r"\bno\b", value):
|
|
586
|
+
return False
|
|
587
|
+
return None
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def _parse_age(text: Optional[str]) -> Optional[int]:
|
|
591
|
+
if not text:
|
|
592
|
+
return None
|
|
593
|
+
match = re.search(r"\b(\d{1,3})\b", text)
|
|
594
|
+
if not match:
|
|
595
|
+
return None
|
|
596
|
+
age = int(match.group(1))
|
|
597
|
+
return age if 18 <= age <= 100 else None
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _header_kind(text: str) -> Optional[str]:
|
|
601
|
+
groups = (
|
|
602
|
+
("serial", _SERIAL_HEADER_LABELS),
|
|
603
|
+
("relation", _RELATION_HEADER_LABELS),
|
|
604
|
+
("gender", _GENDER_HEADER_LABELS),
|
|
605
|
+
("age", _AGE_HEADER_LABELS),
|
|
606
|
+
("name", _MEMBER_NAME_HEADER_LABELS),
|
|
607
|
+
)
|
|
608
|
+
matches = [
|
|
609
|
+
(kind, _label_score(text, aliases), max(map(len, aliases)))
|
|
610
|
+
for kind, aliases in groups
|
|
611
|
+
if _label_score(text, aliases) >= 0.84
|
|
612
|
+
]
|
|
613
|
+
if not matches:
|
|
614
|
+
return None
|
|
615
|
+
# Relation headings often contain "Name"; specific/strong matches win.
|
|
616
|
+
matches.sort(key=lambda entry: (entry[1], entry[2]), reverse=True)
|
|
617
|
+
return matches[0][0]
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _table_headers(regions: list[TextRegion]) -> dict[str, TextRegion]:
|
|
621
|
+
candidates = [
|
|
622
|
+
(region, _header_kind(region.text))
|
|
623
|
+
for region in regions
|
|
624
|
+
if _header_kind(region.text) is not None
|
|
625
|
+
]
|
|
626
|
+
best: dict[str, TextRegion] = {}
|
|
627
|
+
best_score = -1
|
|
628
|
+
|
|
629
|
+
for anchor, anchor_kind in candidates:
|
|
630
|
+
_, anchor_top, _, anchor_bottom = _bounds(anchor)
|
|
631
|
+
anchor_y = (anchor_top + anchor_bottom) / 2
|
|
632
|
+
clustered: dict[str, TextRegion] = {anchor_kind: anchor} # type: ignore[dict-item]
|
|
633
|
+
for region, kind in candidates:
|
|
634
|
+
_, top, _, bottom = _bounds(region)
|
|
635
|
+
if abs(((top + bottom) / 2) - anchor_y) <= 38:
|
|
636
|
+
current = clustered.get(kind) # type: ignore[arg-type]
|
|
637
|
+
if current is None or region.confidence > current.confidence:
|
|
638
|
+
clustered[kind] = region # type: ignore[index]
|
|
639
|
+
score = len(clustered) + int("name" in clustered) + int("age" in clustered)
|
|
640
|
+
if "name" in clustered and ("age" in clustered or "gender" in clustered):
|
|
641
|
+
if score > best_score:
|
|
642
|
+
best = clustered
|
|
643
|
+
best_score = score
|
|
644
|
+
return best
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def _parse_compact_member_row(text: str) -> Optional[NregaMember]:
|
|
648
|
+
"""Parse OCR rows kept as one region, using visible table separators."""
|
|
649
|
+
raw_parts = re.split(r"\s*[|│]\s*|\t+|\s{2,}", text.strip())
|
|
650
|
+
parts = [part.strip(" .:;-") for part in raw_parts if part.strip(" .:;-")]
|
|
651
|
+
if len(parts) < 3:
|
|
652
|
+
return None
|
|
653
|
+
|
|
654
|
+
serial: Optional[str] = None
|
|
655
|
+
serial_match = re.fullmatch(r"(\d{1,3})[.)]?", parts[0])
|
|
656
|
+
if serial_match:
|
|
657
|
+
serial = serial_match.group(1)
|
|
658
|
+
parts = parts[1:]
|
|
659
|
+
|
|
660
|
+
gender_index = next(
|
|
661
|
+
(index for index, part in enumerate(parts) if _normalise_gender(part)),
|
|
662
|
+
None,
|
|
663
|
+
)
|
|
664
|
+
age_index = next(
|
|
665
|
+
(
|
|
666
|
+
index
|
|
667
|
+
for index, part in enumerate(parts)
|
|
668
|
+
if re.fullmatch(r"(?:AGE\s*[:\-]?\s*)?\d{1,3}", part, re.IGNORECASE)
|
|
669
|
+
and _parse_age(part) is not None
|
|
670
|
+
),
|
|
671
|
+
None,
|
|
672
|
+
)
|
|
673
|
+
if gender_index is None and age_index is None:
|
|
674
|
+
return None
|
|
675
|
+
|
|
676
|
+
control_indexes = [index for index in (gender_index, age_index) if index is not None]
|
|
677
|
+
first_control = min(control_indexes)
|
|
678
|
+
identity_parts = parts[:first_control]
|
|
679
|
+
if not identity_parts:
|
|
680
|
+
return None
|
|
681
|
+
return NregaMember(
|
|
682
|
+
serial_number=serial,
|
|
683
|
+
name=identity_parts[0] or None,
|
|
684
|
+
father_or_husband_name=identity_parts[1] if len(identity_parts) > 1 else None,
|
|
685
|
+
gender=_normalise_gender(parts[gender_index]) if gender_index is not None else None,
|
|
686
|
+
age=_parse_age(parts[age_index]) if age_index is not None else None,
|
|
687
|
+
)
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _members_from_compact_rows(regions: list[TextRegion]) -> list[NregaMember]:
|
|
691
|
+
members = []
|
|
692
|
+
for region in regions:
|
|
693
|
+
member = _parse_compact_member_row(region.text)
|
|
694
|
+
if member and member.name:
|
|
695
|
+
members.append(member)
|
|
696
|
+
return members
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def _members_from_table(regions: list[TextRegion]) -> list[NregaMember]:
|
|
700
|
+
headers = _table_headers(regions)
|
|
701
|
+
if not headers:
|
|
702
|
+
return []
|
|
703
|
+
|
|
704
|
+
header_bottom = max(_bounds(region)[3] for region in headers.values())
|
|
705
|
+
header_centres = {
|
|
706
|
+
kind: (_bounds(region)[0] + _bounds(region)[2]) / 2
|
|
707
|
+
for kind, region in headers.items()
|
|
708
|
+
}
|
|
709
|
+
min_x = min(_bounds(region)[0] for region in headers.values()) - 80
|
|
710
|
+
max_x = max(_bounds(region)[2] for region in headers.values()) + 220
|
|
711
|
+
|
|
712
|
+
# Later household sections mark the end of the member table.
|
|
713
|
+
stop_y = header_bottom + 520
|
|
714
|
+
for region in regions:
|
|
715
|
+
_, top, _, _ = _bounds(region)
|
|
716
|
+
if top <= header_bottom:
|
|
717
|
+
continue
|
|
718
|
+
if any(
|
|
719
|
+
_label_score(region.text, aliases) >= 0.92
|
|
720
|
+
for aliases in (_ADDRESS_LABELS, _REGISTRATION_DATE_LABELS, _FAMILY_ID_LABELS)
|
|
721
|
+
):
|
|
722
|
+
stop_y = min(stop_y, top)
|
|
723
|
+
|
|
724
|
+
cells = []
|
|
725
|
+
for region in regions:
|
|
726
|
+
if region in headers.values():
|
|
727
|
+
continue
|
|
728
|
+
left, top, right, bottom = _bounds(region)
|
|
729
|
+
centre_x = (left + right) / 2
|
|
730
|
+
if top < header_bottom - 3 or top >= stop_y:
|
|
731
|
+
continue
|
|
732
|
+
if not (min_x <= centre_x <= max_x):
|
|
733
|
+
continue
|
|
734
|
+
if _header_kind(region.text) is not None:
|
|
735
|
+
continue
|
|
736
|
+
cells.append((region, (top + bottom) / 2))
|
|
737
|
+
|
|
738
|
+
rows: list[list[TextRegion]] = []
|
|
739
|
+
for region, centre_y in sorted(cells, key=lambda item: (item[1], _bounds(item[0])[0])):
|
|
740
|
+
if not rows:
|
|
741
|
+
rows.append([region])
|
|
742
|
+
continue
|
|
743
|
+
last_centres = [
|
|
744
|
+
(_bounds(item)[1] + _bounds(item)[3]) / 2 for item in rows[-1]
|
|
745
|
+
]
|
|
746
|
+
if abs(centre_y - sum(last_centres) / len(last_centres)) <= 18:
|
|
747
|
+
rows[-1].append(region)
|
|
748
|
+
else:
|
|
749
|
+
rows.append([region])
|
|
750
|
+
|
|
751
|
+
members: list[NregaMember] = []
|
|
752
|
+
for row in rows:
|
|
753
|
+
values: dict[str, str] = {}
|
|
754
|
+
for cell in row:
|
|
755
|
+
left, _, right, _ = _bounds(cell)
|
|
756
|
+
centre_x = (left + right) / 2
|
|
757
|
+
kind = min(header_centres, key=lambda key: abs(header_centres[key] - centre_x))
|
|
758
|
+
existing = values.get(kind)
|
|
759
|
+
values[kind] = (
|
|
760
|
+
f"{existing} {cell.text}".strip() if existing else cell.text.strip()
|
|
761
|
+
)
|
|
762
|
+
|
|
763
|
+
serial_match = re.search(r"\b\d{1,3}\b", values.get("serial", ""))
|
|
764
|
+
member = NregaMember(
|
|
765
|
+
serial_number=serial_match.group(0) if serial_match else None,
|
|
766
|
+
name=values.get("name") or None,
|
|
767
|
+
father_or_husband_name=values.get("relation") or None,
|
|
768
|
+
gender=_normalise_gender(values.get("gender")),
|
|
769
|
+
age=_parse_age(values.get("age")),
|
|
770
|
+
)
|
|
771
|
+
if member.name and (member.age is not None or member.gender is not None):
|
|
772
|
+
members.append(member)
|
|
773
|
+
return members
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
def _deduplicate_members(members: list[NregaMember]) -> list[NregaMember]:
|
|
777
|
+
output: list[NregaMember] = []
|
|
778
|
+
seen: set[tuple[Optional[str], str]] = set()
|
|
779
|
+
for member in members:
|
|
780
|
+
if not member.name:
|
|
781
|
+
continue
|
|
782
|
+
key = (member.serial_number, _normalise(member.name))
|
|
783
|
+
if key in seen:
|
|
784
|
+
continue
|
|
785
|
+
seen.add(key)
|
|
786
|
+
output.append(member)
|
|
787
|
+
return output
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _find_members(regions: list[TextRegion]) -> list[NregaMember]:
|
|
791
|
+
table_members = _members_from_table(regions)
|
|
792
|
+
compact_members = _members_from_compact_rows(regions)
|
|
793
|
+
return _deduplicate_members(table_members + compact_members)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _find_validity(regions: list[TextRegion]) -> tuple[Optional[str], Optional[str]]:
|
|
797
|
+
validity = _value_for_labels(regions, _VALIDITY_LABELS)
|
|
798
|
+
if validity:
|
|
799
|
+
dates = [match.group(0).strip() for match in _DATE_RE.finditer(validity)]
|
|
800
|
+
if len(dates) >= 2:
|
|
801
|
+
return dates[0], dates[1]
|
|
802
|
+
|
|
803
|
+
valid_from = _extract_date(_value_for_labels(regions, _VALID_FROM_LABELS))
|
|
804
|
+
valid_to = _extract_date(_value_for_labels(regions, _VALID_TO_LABELS))
|
|
805
|
+
return valid_from, valid_to
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def extract_nrega(regions: list[TextRegion]) -> NregaFields:
|
|
809
|
+
"""Extract household and adult-member identity fields from OCR regions."""
|
|
810
|
+
fields = NregaFields()
|
|
811
|
+
fields.job_card_number = _find_job_card_number(regions)
|
|
812
|
+
fields.head_of_household = _value_for_labels(regions, _HEAD_LABELS)
|
|
813
|
+
fields.category = _normalise_category(_value_for_labels(regions, _CATEGORY_LABELS))
|
|
814
|
+
fields.registration_date = _extract_date(
|
|
815
|
+
_value_for_labels(regions, _REGISTRATION_DATE_LABELS)
|
|
816
|
+
)
|
|
817
|
+
fields.validity_from, fields.validity_to = _find_validity(regions)
|
|
818
|
+
fields.address = _value_for_labels(regions, _ADDRESS_LABELS)
|
|
819
|
+
fields.village = _value_for_labels(regions, _VILLAGE_LABELS)
|
|
820
|
+
fields.gram_panchayat = _value_for_labels(regions, _PANCHAYAT_LABELS)
|
|
821
|
+
fields.block = _value_for_labels(regions, _BLOCK_LABELS)
|
|
822
|
+
fields.district = _value_for_labels(regions, _DISTRICT_LABELS)
|
|
823
|
+
fields.state = _value_for_labels(regions, _STATE_LABELS)
|
|
824
|
+
fields.bpl_status = _normalise_bpl(_value_for_labels(regions, _BPL_LABELS))
|
|
825
|
+
fields.family_id = _value_for_labels(regions, _FAMILY_ID_LABELS)
|
|
826
|
+
fields.members = _find_members(regions)
|
|
827
|
+
return fields
|