document-ocr 0.0.0-stage → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +260 -2
- package/dist/index.cjs +659 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +293 -0
- package/dist/index.d.ts +293 -0
- package/dist/index.js +616 -0
- package/dist/index.js.map +1 -0
- package/hosting/.dockerignore +17 -0
- package/hosting/.gcloudignore +14 -0
- package/hosting/FEATURES.md +172 -0
- package/hosting/HOSTING.md +268 -0
- package/hosting/LICENSE +21 -0
- package/hosting/README.md +369 -0
- package/hosting/core/__init__.py +0 -0
- package/hosting/core/aadhaar_extractor.py +298 -0
- package/hosting/core/back_page_extractor.py +492 -0
- package/hosting/core/barcodes.py +125 -0
- package/hosting/core/card_recovery.py +244 -0
- package/hosting/core/document_bundle.py +82 -0
- package/hosting/core/document_classifier.py +233 -0
- package/hosting/core/document_input.py +135 -0
- package/hosting/core/document_registry.py +143 -0
- package/hosting/core/driving_licence_extractor.py +510 -0
- package/hosting/core/ead_recovery.py +54 -0
- package/hosting/core/evidence.py +54 -0
- package/hosting/core/form_recovery.py +61 -0
- package/hosting/core/jobs.py +232 -0
- package/hosting/core/kyc_ocr.py +132 -0
- package/hosting/core/kyc_validation.py +411 -0
- package/hosting/core/model_setup.py +13 -0
- package/hosting/core/mrz_parser.py +336 -0
- package/hosting/core/mrz_recovery.py +193 -0
- package/hosting/core/npr_extractor.py +410 -0
- package/hosting/core/nrega_extractor.py +827 -0
- package/hosting/core/ocr_engine.py +207 -0
- package/hosting/core/page_classifier.py +137 -0
- package/hosting/core/pan_extractor.py +183 -0
- package/hosting/core/pipeline.py +712 -0
- package/hosting/core/preprocessor.py +297 -0
- package/hosting/core/recovery_budget.py +67 -0
- package/hosting/core/redaction.py +48 -0
- package/hosting/core/review.html +244 -0
- package/hosting/core/structured_documents.py +422 -0
- package/hosting/core/travel_mrz.py +164 -0
- package/hosting/core/travel_recovery.py +187 -0
- package/hosting/core/validator.py +396 -0
- package/hosting/core/validators.py +179 -0
- package/hosting/core/voter_id_extractor.py +363 -0
- package/hosting/deploy/cloudflare/deploy.sh +44 -0
- package/hosting/deploy/cloudflare/package-lock.json +3088 -0
- package/hosting/deploy/cloudflare/package.json +29 -0
- package/hosting/deploy/cloudflare/src/gateway.ts +42 -0
- package/hosting/deploy/cloudflare/src/index.ts +21 -0
- package/hosting/deploy/cloudflare/test/gateway.test.ts +59 -0
- package/hosting/deploy/cloudflare/tsconfig.json +12 -0
- package/hosting/deploy/cloudflare/worker-configuration.d.ts +16253 -0
- package/hosting/deploy/cloudflare/wrangler.jsonc +26 -0
- package/hosting/deploy/cloudrun/cloudbuild.yaml +18 -0
- package/hosting/deploy/cloudrun/deploy.sh +71 -0
- package/hosting/deploy/cloudrun/render_service.py +47 -0
- package/hosting/deploy/common.sh +14 -0
- package/hosting/deploy/docker/Dockerfile +16 -0
- package/hosting/deploy/docker/server.py +313 -0
- package/hosting/deploy/lambda/Dockerfile.lambda +11 -0
- package/hosting/deploy/lambda/deploy.sh +22 -0
- package/hosting/deploy/lambda/handler.py +132 -0
- package/hosting/deploy/lambda/template.yaml +59 -0
- package/hosting/deploy/local/dev.sh +18 -0
- package/hosting/deploy/server/Caddyfile +5 -0
- package/hosting/deploy/server/compose.jobs.yaml +36 -0
- package/hosting/deploy/server/compose.yaml +38 -0
- package/hosting/deploy/server/deploy.sh +23 -0
- package/hosting/infra/cloudrun/main.tf +104 -0
- package/hosting/infra/cloudrun/terraform.tfvars.example +7 -0
- package/hosting/infra/cloudrun/variables.tf +55 -0
- package/hosting/infra/lambda/main.tf +63 -0
- package/hosting/infra/lambda/terraform.tfvars.example +6 -0
- package/hosting/infra/lambda/variables.tf +37 -0
- package/hosting/pyproject.toml +65 -0
- package/hosting/requirements.lock +134 -0
- package/package.json +76 -4
- package/python/core/__init__.py +0 -0
- package/python/core/aadhaar_extractor.py +298 -0
- package/python/core/back_page_extractor.py +492 -0
- package/python/core/barcodes.py +125 -0
- package/python/core/card_recovery.py +244 -0
- package/python/core/document_bundle.py +82 -0
- package/python/core/document_classifier.py +233 -0
- package/python/core/document_input.py +135 -0
- package/python/core/document_registry.py +143 -0
- package/python/core/driving_licence_extractor.py +510 -0
- package/python/core/ead_recovery.py +54 -0
- package/python/core/evidence.py +54 -0
- package/python/core/form_recovery.py +61 -0
- package/python/core/jobs.py +232 -0
- package/python/core/kyc_ocr.py +132 -0
- package/python/core/kyc_validation.py +411 -0
- package/python/core/model_setup.py +13 -0
- package/python/core/mrz_parser.py +336 -0
- package/python/core/mrz_recovery.py +193 -0
- package/python/core/npr_extractor.py +410 -0
- package/python/core/nrega_extractor.py +827 -0
- package/python/core/ocr_engine.py +207 -0
- package/python/core/page_classifier.py +137 -0
- package/python/core/pan_extractor.py +183 -0
- package/python/core/pipeline.py +712 -0
- package/python/core/preprocessor.py +297 -0
- package/python/core/recovery_budget.py +67 -0
- package/python/core/redaction.py +48 -0
- package/python/core/review.html +244 -0
- package/python/core/structured_documents.py +422 -0
- package/python/core/travel_mrz.py +164 -0
- package/python/core/travel_recovery.py +187 -0
- package/python/core/validator.py +396 -0
- package/python/core/validators.py +179 -0
- package/python/core/voter_id_extractor.py +363 -0
- package/python/pyproject.toml +34 -0
- package/python/requirements.lock +134 -0
- package/python/server.py +313 -0
- package/scripts/dococr.mjs +163 -0
- package/scripts/postinstall.mjs +173 -0
- package/scripts/sync-hosting.mjs +35 -0
- package/scripts/sync-python.sh +20 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "./node_modules/wrangler/config-schema.json",
|
|
3
|
+
"name": "document-ocr",
|
|
4
|
+
"main": "src/index.ts",
|
|
5
|
+
"compatibility_date": "2026-09-07",
|
|
6
|
+
"compatibility_flags": ["nodejs_compat"],
|
|
7
|
+
"observability": { "enabled": true },
|
|
8
|
+
"secrets": { "required": ["API_TOKEN"] },
|
|
9
|
+
"vars": {
|
|
10
|
+
"DOCUMENT_OCR_KYC_LANGS": "en",
|
|
11
|
+
"CONTAINER_INSTANCES": "2",
|
|
12
|
+
"CONTAINER_IDLE_TIMEOUT": "10m"
|
|
13
|
+
},
|
|
14
|
+
"containers": [{
|
|
15
|
+
"class_name": "OcrContainer",
|
|
16
|
+
"image": "../docker/Dockerfile",
|
|
17
|
+
"image_build_context": "../..",
|
|
18
|
+
"image_vars": { "DOCUMENT_OCR_KYC_LANGS": "en" },
|
|
19
|
+
"instance_type": "standard-2",
|
|
20
|
+
"max_instances": 2
|
|
21
|
+
}],
|
|
22
|
+
"durable_objects": {
|
|
23
|
+
"bindings": [{ "name": "OCR", "class_name": "OcrContainer" }]
|
|
24
|
+
},
|
|
25
|
+
"migrations": [{ "tag": "v1", "new_sqlite_classes": ["OcrContainer"] }]
|
|
26
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Remote amd64 build. Deployment is owned by deploy.sh and render_service.py.
|
|
2
|
+
substitutions:
|
|
3
|
+
_IMAGE_URI: 'IMAGE_URI_REQUIRED'
|
|
4
|
+
_KYC_LANGS: en
|
|
5
|
+
steps:
|
|
6
|
+
- name: gcr.io/cloud-builders/docker
|
|
7
|
+
args:
|
|
8
|
+
- build
|
|
9
|
+
- --platform=linux/amd64
|
|
10
|
+
- --file=deploy/docker/Dockerfile
|
|
11
|
+
- '--build-arg=DOCUMENT_OCR_KYC_LANGS=${_KYC_LANGS}'
|
|
12
|
+
- '--tag=${_IMAGE_URI}'
|
|
13
|
+
- .
|
|
14
|
+
images:
|
|
15
|
+
- '${_IMAGE_URI}'
|
|
16
|
+
options:
|
|
17
|
+
machineType: E2_HIGHCPU_8
|
|
18
|
+
logging: CLOUD_LOGGING_ONLY
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
4
|
+
cd "$ROOT_DIR"
|
|
5
|
+
source deploy/common.sh
|
|
6
|
+
if [[ $# -gt 0 ]]; then
|
|
7
|
+
[[ "$1" == production || "$1" == development ]] || { printf 'Optional argument must be production or development\n' >&2; exit 1; }
|
|
8
|
+
set -a
|
|
9
|
+
source ".env.deploy.$1"
|
|
10
|
+
set +a
|
|
11
|
+
fi
|
|
12
|
+
hosting_defaults
|
|
13
|
+
require_command gcloud
|
|
14
|
+
require_command python3
|
|
15
|
+
: "${GCP_PROJECT:?Set GCP_PROJECT}"
|
|
16
|
+
export GCP_REGION="${GCP_REGION:-asia-south1}"
|
|
17
|
+
AR_REPO="${GCP_AR_REPO:-document-ocr}"
|
|
18
|
+
export DOCUMENT_OCR_SERVICE_ACCOUNT="${DOCUMENT_OCR_SERVICE_ACCOUNT:-${DOCUMENT_OCR_NAME:0:23}-ocr-sa@${GCP_PROJECT}.iam.gserviceaccount.com}"
|
|
19
|
+
TMP_DIR="$(mktemp -d)"
|
|
20
|
+
trap 'rm -rf "$TMP_DIR"' EXIT
|
|
21
|
+
|
|
22
|
+
gcloud services enable run.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com iam.googleapis.com secretmanager.googleapis.com --project "$GCP_PROJECT" --quiet
|
|
23
|
+
# List succeeds or the script fails; permission failures never mean "not found".
|
|
24
|
+
ACCOUNTS="$(gcloud iam service-accounts list --project "$GCP_PROJECT" --filter="email=${DOCUMENT_OCR_SERVICE_ACCOUNT}" --format='value(email)')"
|
|
25
|
+
if [[ -z "$ACCOUNTS" ]]; then
|
|
26
|
+
gcloud iam service-accounts create "${DOCUMENT_OCR_SERVICE_ACCOUNT%%@*}" --project "$GCP_PROJECT" --display-name 'Document OCR runtime' --quiet
|
|
27
|
+
fi
|
|
28
|
+
if [[ -n "${GCP_API_TOKEN_SECRET:-}" ]]; then
|
|
29
|
+
gcloud secrets add-iam-policy-binding "$GCP_API_TOKEN_SECRET" --project "$GCP_PROJECT" --member="serviceAccount:${DOCUMENT_OCR_SERVICE_ACCOUNT}" --role=roles/secretmanager.secretAccessor --quiet >/dev/null
|
|
30
|
+
fi
|
|
31
|
+
|
|
32
|
+
make_private() {
|
|
33
|
+
gcloud run services get-iam-policy "$DOCUMENT_OCR_NAME" --region "$GCP_REGION" --project "$GCP_PROJECT" --format=json > "$TMP_DIR/policy.json"
|
|
34
|
+
python3 - "$TMP_DIR/policy.json" <<'PY'
|
|
35
|
+
import json, sys
|
|
36
|
+
path = sys.argv[1]
|
|
37
|
+
with open(path) as source:
|
|
38
|
+
policy = json.load(source)
|
|
39
|
+
for binding in policy.get('bindings', []):
|
|
40
|
+
if binding.get('role') == 'roles/run.invoker':
|
|
41
|
+
binding['members'] = [member for member in binding.get('members', []) if member not in {'allUsers', 'allAuthenticatedUsers'}]
|
|
42
|
+
policy['bindings'] = [binding for binding in policy.get('bindings', []) if binding.get('members')]
|
|
43
|
+
with open(path, 'w') as target:
|
|
44
|
+
json.dump(policy, target)
|
|
45
|
+
PY
|
|
46
|
+
gcloud run services set-iam-policy "$DOCUMENT_OCR_NAME" "$TMP_DIR/policy.json" --region "$GCP_REGION" --project "$GCP_PROJECT" --quiet >/dev/null
|
|
47
|
+
}
|
|
48
|
+
EXISTING="$(gcloud run services list --region "$GCP_REGION" --project "$GCP_PROJECT" --filter="metadata.name=${DOCUMENT_OCR_NAME}" --format='value(metadata.name)')"
|
|
49
|
+
if [[ -n "$EXISTING" ]]; then
|
|
50
|
+
make_private
|
|
51
|
+
gcloud run services update "$DOCUMENT_OCR_NAME" --invoker-iam-check --region "$GCP_REGION" --project "$GCP_PROJECT" --quiet
|
|
52
|
+
fi
|
|
53
|
+
|
|
54
|
+
if [[ -z "${IMAGE_URI:-}" ]]; then
|
|
55
|
+
REPOSITORIES="$(gcloud artifacts repositories list --project "$GCP_PROJECT" --location "$GCP_REGION" --filter="name:${AR_REPO}" --format='value(name)')"
|
|
56
|
+
if [[ "$REPOSITORIES" != *"/repositories/${AR_REPO}"* ]]; then
|
|
57
|
+
gcloud artifacts repositories create "$AR_REPO" --repository-format=docker --location "$GCP_REGION" --project "$GCP_PROJECT" --quiet
|
|
58
|
+
fi
|
|
59
|
+
export IMAGE_URI="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT}/${AR_REPO}/${DOCUMENT_OCR_NAME}:$(date -u +%Y%m%d%H%M%S)"
|
|
60
|
+
gcloud builds submit . --project "$GCP_PROJECT" --region "$GCP_REGION" --config deploy/cloudrun/cloudbuild.yaml --substitutions="^|^_IMAGE_URI=${IMAGE_URI}|_KYC_LANGS=${DOCUMENT_OCR_KYC_LANGS}" --quiet
|
|
61
|
+
fi
|
|
62
|
+
export IMAGE_URI
|
|
63
|
+
python3 deploy/cloudrun/render_service.py > "$TMP_DIR/service.json"
|
|
64
|
+
gcloud run services replace "$TMP_DIR/service.json" --region "$GCP_REGION" --project "$GCP_PROJECT" --quiet
|
|
65
|
+
make_private
|
|
66
|
+
if [[ -n "${GCP_INVOKER_MEMBER:-}" ]]; then
|
|
67
|
+
[[ "$GCP_INVOKER_MEMBER" != allUsers && "$GCP_INVOKER_MEMBER" != allAuthenticatedUsers ]] || { printf 'A named IAM invoker is required\n' >&2; exit 1; }
|
|
68
|
+
gcloud run services add-iam-policy-binding "$DOCUMENT_OCR_NAME" --member "$GCP_INVOKER_MEMBER" --role roles/run.invoker --region "$GCP_REGION" --project "$GCP_PROJECT" --quiet >/dev/null
|
|
69
|
+
fi
|
|
70
|
+
URL="$(gcloud run services describe "$DOCUMENT_OCR_NAME" --region "$GCP_REGION" --project "$GCP_PROJECT" --format='value(status.url)')"
|
|
71
|
+
printf 'Deployed private Cloud Run service: %s\nUse an audience-matched Google ID token; see HOSTING.md.\n' "$URL"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Render the one Cloud Run service definition used by CLI and Cloud Build."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def service(env):
|
|
8
|
+
name = env.get('DOCUMENT_OCR_NAME', 'document-ocr')
|
|
9
|
+
profile = env.get('DOCUMENT_OCR_PROFILE', 'economy')
|
|
10
|
+
if profile not in {'economy', 'warm'}:
|
|
11
|
+
raise ValueError('Profile must be economy or warm')
|
|
12
|
+
variables = [{'name': 'DOCUMENT_OCR_KYC_LANGS', 'value': env.get('DOCUMENT_OCR_KYC_LANGS', 'en')}]
|
|
13
|
+
if env.get('GCP_API_TOKEN_SECRET'):
|
|
14
|
+
variables.append({'name': 'API_TOKEN', 'valueFrom': {'secretKeyRef': {
|
|
15
|
+
'name': env['GCP_API_TOKEN_SECRET'], 'key': env.get('GCP_API_TOKEN_VERSION', 'latest')}}})
|
|
16
|
+
return {
|
|
17
|
+
'apiVersion': 'serving.knative.dev/v1', 'kind': 'Service',
|
|
18
|
+
'metadata': {'name': name, 'annotations': {
|
|
19
|
+
'run.googleapis.com/ingress': 'all',
|
|
20
|
+
'run.googleapis.com/invoker-iam-disabled': 'false',
|
|
21
|
+
}},
|
|
22
|
+
'spec': {'template': {
|
|
23
|
+
'metadata': {'annotations': {
|
|
24
|
+
'autoscaling.knative.dev/minScale': '1' if profile == 'warm' else '0',
|
|
25
|
+
'autoscaling.knative.dev/maxScale': env.get('DOCUMENT_OCR_MAX_INSTANCES', '10'),
|
|
26
|
+
'run.googleapis.com/cpu-throttling': 'true',
|
|
27
|
+
'run.googleapis.com/startup-cpu-boost': 'true',
|
|
28
|
+
}},
|
|
29
|
+
'spec': {
|
|
30
|
+
'serviceAccountName': env['DOCUMENT_OCR_SERVICE_ACCOUNT'],
|
|
31
|
+
'containerConcurrency': 1, 'timeoutSeconds': 300,
|
|
32
|
+
'containers': [{
|
|
33
|
+
'image': env['IMAGE_URI'],
|
|
34
|
+
'ports': [{'name': 'http1', 'containerPort': 8000}],
|
|
35
|
+
'resources': {'limits': {'cpu': '2', 'memory': '2Gi'}},
|
|
36
|
+
'env': variables,
|
|
37
|
+
'startupProbe': {'httpGet': {'path': '/ready', 'port': 8000}, 'periodSeconds': 5, 'failureThreshold': 30, 'timeoutSeconds': 3},
|
|
38
|
+
'livenessProbe': {'httpGet': {'path': '/health', 'port': 8000}, 'periodSeconds': 30, 'timeoutSeconds': 3},
|
|
39
|
+
}],
|
|
40
|
+
},
|
|
41
|
+
}},
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
if __name__ == '__main__':
|
|
46
|
+
json.dump(service(os.environ), sys.stdout, indent=2)
|
|
47
|
+
print()
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Shared validation only. Sourcing this file performs no cloud operations.
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
require_command() {
|
|
5
|
+
command -v "$1" >/dev/null 2>&1 || { printf 'Required command is missing: %s\n' "$1" >&2; exit 1; }
|
|
6
|
+
}
|
|
7
|
+
hosting_defaults() {
|
|
8
|
+
export DOCUMENT_OCR_NAME="${DOCUMENT_OCR_NAME:-document-ocr}"
|
|
9
|
+
export DOCUMENT_OCR_PROFILE="${DOCUMENT_OCR_PROFILE:-economy}"
|
|
10
|
+
export DOCUMENT_OCR_KYC_LANGS="${DOCUMENT_OCR_KYC_LANGS:-en}"
|
|
11
|
+
[[ "$DOCUMENT_OCR_NAME" =~ ^[a-z][a-z0-9-]{0,39}$ ]] || { printf 'Invalid DOCUMENT_OCR_NAME\n' >&2; exit 1; }
|
|
12
|
+
case "$DOCUMENT_OCR_PROFILE" in economy|warm) ;; *) printf 'Profile must be economy or warm\n' >&2; exit 1;; esac
|
|
13
|
+
[[ "$DOCUMENT_OCR_KYC_LANGS" =~ ^(en|latin|devanagari|ka|ta|te)(,(en|latin|devanagari|ka|ta|te)){0,3}$ ]] || { printf 'Invalid DOCUMENT_OCR_KYC_LANGS (up to four supported languages)\n' >&2; exit 1; }
|
|
14
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
FROM python:3.14-slim
|
|
2
|
+
WORKDIR /app
|
|
3
|
+
ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1
|
|
4
|
+
RUN apt-get update && apt-get install -y --no-install-recommends libgl1 libglib2.0-0 libgomp1 && rm -rf /var/lib/apt/lists/*
|
|
5
|
+
COPY requirements.lock ./
|
|
6
|
+
RUN python -m pip install --no-cache-dir -r requirements.lock
|
|
7
|
+
COPY core/ core/
|
|
8
|
+
ARG DOCUMENT_OCR_KYC_LANGS=en
|
|
9
|
+
ENV DOCUMENT_OCR_KYC_LANGS=${DOCUMENT_OCR_KYC_LANGS}
|
|
10
|
+
RUN python -m core.model_setup
|
|
11
|
+
COPY deploy/docker/server.py server.py
|
|
12
|
+
RUN useradd --create-home --uid 10001 ocr && mkdir -p /var/lib/document-ocr/jobs && chown -R ocr:ocr /app /var/lib/document-ocr
|
|
13
|
+
USER ocr
|
|
14
|
+
EXPOSE 8000
|
|
15
|
+
HEALTHCHECK --interval=30s --timeout=3s --start-period=120s CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=2)"
|
|
16
|
+
CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-keep-alive", "30"]
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"""
|
|
2
|
+
FastAPI server wrapping the passport OCR pipeline.
|
|
3
|
+
|
|
4
|
+
Used by the document-ocr npm package to run OCR locally.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
from contextlib import asynccontextmanager
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import secrets
|
|
14
|
+
import uuid
|
|
15
|
+
from functools import partial
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from fastapi import FastAPI, File, Form, Header, UploadFile, HTTPException
|
|
19
|
+
from fastapi.responses import JSONResponse, HTMLResponse, Response
|
|
20
|
+
|
|
21
|
+
from core.kyc_ocr import KycOCRConfigError
|
|
22
|
+
from core.ocr_engine import OCRModelInitError
|
|
23
|
+
from core.pipeline import scan
|
|
24
|
+
from core.document_bundle import scan_batch, scan_document
|
|
25
|
+
from core.document_registry import list_document_profiles, validate_document_hint
|
|
26
|
+
from core.barcodes import barcode_decoder_available
|
|
27
|
+
from core.jobs import configured_store
|
|
28
|
+
from core.redaction import redact, render_preview
|
|
29
|
+
from core.preprocessor import ImageQualityError
|
|
30
|
+
|
|
31
|
+
# ---------------------------------------------------------------------------
|
|
32
|
+
# Config
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
MAX_UPLOAD_SIZE = 10 * 1024 * 1024 # 10 MB
|
|
36
|
+
SCAN_TIMEOUT_SECONDS = 60.0
|
|
37
|
+
logger = logging.getLogger("document-ocr")
|
|
38
|
+
# uvicorn configures only its own loggers, so without a handler these per-scan
|
|
39
|
+
# INFO lines (status, page type, confidence, timing; never document text) were
|
|
40
|
+
# dropped. Scoped to this logger so core modules keep their default level.
|
|
41
|
+
if not logger.handlers:
|
|
42
|
+
_handler = logging.StreamHandler()
|
|
43
|
+
_handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
|
|
44
|
+
logger.addHandler(_handler)
|
|
45
|
+
logger.setLevel(logging.INFO)
|
|
46
|
+
logger.propagate = False
|
|
47
|
+
# Matches the sibling pdf-worker service so a deployment sets one variable name
|
|
48
|
+
# everywhere. Unset means no auth is enforced, which is what local development wants.
|
|
49
|
+
API_TOKEN = os.getenv("API_TOKEN") or None
|
|
50
|
+
|
|
51
|
+
_ocr_semaphore = asyncio.Semaphore(1)
|
|
52
|
+
_models_ready = False
|
|
53
|
+
_model_init_error: str | None = None
|
|
54
|
+
|
|
55
|
+
# ---------------------------------------------------------------------------
|
|
56
|
+
# App lifecycle
|
|
57
|
+
# ---------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
@asynccontextmanager
|
|
60
|
+
async def _lifespan(app: FastAPI):
|
|
61
|
+
global _models_ready, _model_init_error
|
|
62
|
+
_models_ready = False
|
|
63
|
+
_model_init_error = None
|
|
64
|
+
logger.info("Loading OCR models...")
|
|
65
|
+
loop = asyncio.get_running_loop()
|
|
66
|
+
try:
|
|
67
|
+
await loop.run_in_executor(None, _warm_up_ocr)
|
|
68
|
+
except (OCRModelInitError, KycOCRConfigError) as exc:
|
|
69
|
+
# Stay up so /ready and /scan can report a clear error instead of the
|
|
70
|
+
# process crash-looping. Liveness (/health) remains green.
|
|
71
|
+
_model_init_error = str(exc)
|
|
72
|
+
logger.error("OCR model initialisation failed: %s", exc)
|
|
73
|
+
else:
|
|
74
|
+
_models_ready = True
|
|
75
|
+
logger.info("OCR models loaded.")
|
|
76
|
+
yield
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
app = FastAPI(title="Document OCR", version="3.1.2", lifespan=_lifespan)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@app.middleware("http")
|
|
83
|
+
async def _private_responses(request, call_next):
|
|
84
|
+
response = await call_next(request)
|
|
85
|
+
response.headers["Cache-Control"] = "no-store"
|
|
86
|
+
response.headers["X-Content-Type-Options"] = "nosniff"
|
|
87
|
+
return response
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _warm_up_ocr():
|
|
91
|
+
from core.kyc_ocr import configured_kyc_languages
|
|
92
|
+
from core.ocr_engine import _get_ocr
|
|
93
|
+
|
|
94
|
+
for language in dict.fromkeys(("en", *configured_kyc_languages())):
|
|
95
|
+
_get_ocr(language)
|
|
96
|
+
_get_ocr("en", band=True) # MRZ band re-reads; avoid loading on a live scan
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@app.get("/health")
|
|
100
|
+
async def health():
|
|
101
|
+
return {"status": "ok"}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@app.get("/ready")
|
|
105
|
+
async def ready():
|
|
106
|
+
if _model_init_error is not None:
|
|
107
|
+
return JSONResponse(
|
|
108
|
+
status_code=503,
|
|
109
|
+
content={"status": "model_init_failed", "error": _model_init_error},
|
|
110
|
+
)
|
|
111
|
+
if not _models_ready:
|
|
112
|
+
return JSONResponse(status_code=503, content={"status": "loading"})
|
|
113
|
+
return {"status": "ready"}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _scan_finished(future: asyncio.Future) -> None:
|
|
117
|
+
_ocr_semaphore.release()
|
|
118
|
+
# Consume late exceptions after a disconnected/timed-out caller has gone.
|
|
119
|
+
if not future.cancelled():
|
|
120
|
+
future.exception()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _authorize(authorization):
|
|
124
|
+
if API_TOKEN is not None:
|
|
125
|
+
expected = f"Bearer {API_TOKEN}"
|
|
126
|
+
if authorization is None or not secrets.compare_digest(authorization, expected):
|
|
127
|
+
raise HTTPException(status_code=401, detail="UNAUTHORIZED", headers={"WWW-Authenticate": "Bearer"})
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
async def _upload(image: UploadFile) -> bytes:
|
|
131
|
+
content_type = image.content_type or ""
|
|
132
|
+
if not content_type.startswith("image/") and content_type != "application/pdf":
|
|
133
|
+
raise HTTPException(status_code=400, detail="INVALID_CONTENT_TYPE")
|
|
134
|
+
data = await image.read(MAX_UPLOAD_SIZE + 1)
|
|
135
|
+
if len(data) > MAX_UPLOAD_SIZE:
|
|
136
|
+
raise HTTPException(status_code=400, detail="FILE_TOO_LARGE")
|
|
137
|
+
if not data:
|
|
138
|
+
raise HTTPException(status_code=400, detail="EMPTY_FILE")
|
|
139
|
+
return data
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
async def _uploads(images: list[UploadFile]) -> list[bytes]:
|
|
143
|
+
if not 1 <= len(images) <= 10:
|
|
144
|
+
raise HTTPException(status_code=400, detail="INVALID_BATCH_SIZE")
|
|
145
|
+
output, size = [], 0
|
|
146
|
+
for image in images:
|
|
147
|
+
data = await _upload(image)
|
|
148
|
+
size += len(data)
|
|
149
|
+
if size > MAX_UPLOAD_SIZE:
|
|
150
|
+
raise HTTPException(status_code=400, detail="BATCH_TOO_LARGE")
|
|
151
|
+
output.append(data)
|
|
152
|
+
return output
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _options(document_type, country, include_evidence):
|
|
156
|
+
try:
|
|
157
|
+
document_type, country = validate_document_hint(document_type, country)
|
|
158
|
+
except ValueError as exc:
|
|
159
|
+
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
160
|
+
# Omit defaults for compatibility with one-argument pipeline integrations.
|
|
161
|
+
return {key: value for key, value in {
|
|
162
|
+
'document_type': document_type, 'country': country,
|
|
163
|
+
'include_evidence': include_evidence,
|
|
164
|
+
}.items() if value}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
async def _execute(function, *args, **kwargs):
|
|
168
|
+
request_id = str(uuid.uuid4())[:8]
|
|
169
|
+
try:
|
|
170
|
+
deadline = asyncio.get_running_loop().time() + SCAN_TIMEOUT_SECONDS
|
|
171
|
+
await asyncio.wait_for(_ocr_semaphore.acquire(), timeout=SCAN_TIMEOUT_SECONDS)
|
|
172
|
+
try:
|
|
173
|
+
future = asyncio.get_running_loop().run_in_executor(None, partial(function, *args, **kwargs))
|
|
174
|
+
except BaseException:
|
|
175
|
+
_ocr_semaphore.release()
|
|
176
|
+
raise
|
|
177
|
+
future.add_done_callback(_scan_finished)
|
|
178
|
+
return await asyncio.wait_for(asyncio.shield(future), timeout=max(0, deadline - asyncio.get_running_loop().time()))
|
|
179
|
+
except asyncio.TimeoutError:
|
|
180
|
+
logger.warning("[%s] scan_timeout", request_id)
|
|
181
|
+
return JSONResponse(status_code=504, content={"error": "SCAN_TIMEOUT"})
|
|
182
|
+
except OCRModelInitError:
|
|
183
|
+
logger.error("[%s] model_init_failed", request_id)
|
|
184
|
+
return JSONResponse(status_code=503, content={"error": "MODEL_INIT_FAILED"})
|
|
185
|
+
except KycOCRConfigError:
|
|
186
|
+
logger.error("[%s] invalid_kyc_ocr_config", request_id)
|
|
187
|
+
return JSONResponse(status_code=503, content={"error": "INVALID_KYC_OCR_CONFIG"})
|
|
188
|
+
except (ImageQualityError, ValueError) as exc:
|
|
189
|
+
return JSONResponse(status_code=400, content={"error": str(exc)})
|
|
190
|
+
except Exception:
|
|
191
|
+
logger.exception("[%s] internal_error", request_id)
|
|
192
|
+
return JSONResponse(status_code=500, content={"error": "INTERNAL_ERROR"})
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@app.post("/scan")
|
|
196
|
+
async def scan_passport(
|
|
197
|
+
image: UploadFile = File(...), authorization: str | None = Header(default=None),
|
|
198
|
+
document_type: str | None = Form(default=None), country: str | None = Form(default=None),
|
|
199
|
+
include_evidence: bool = Form(default=False),
|
|
200
|
+
):
|
|
201
|
+
_authorize(authorization)
|
|
202
|
+
options = _options(document_type, country, include_evidence)
|
|
203
|
+
result = await _execute(scan, await _upload(image), **options)
|
|
204
|
+
if isinstance(result, Response):
|
|
205
|
+
return result
|
|
206
|
+
logger.info("status=%s page_type=%s confidence=%s processing_ms=%s", result.status,
|
|
207
|
+
result.page_type, result.confidence, result.processing_ms)
|
|
208
|
+
return JSONResponse(status_code=422 if result.status == 'failure' else 200, content=result.to_dict())
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@app.get("/documents")
|
|
212
|
+
async def documents(authorization: str | None = Header(default=None)):
|
|
213
|
+
_authorize(authorization)
|
|
214
|
+
return {"schemaVersion": 1, "documents": list_document_profiles(),
|
|
215
|
+
"capabilities": {"barcode": barcode_decoder_available(), "jobs": await asyncio.to_thread(_job_store, required=False) is not None,
|
|
216
|
+
"review": True, "multipage": True}}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@app.post("/scan/batch")
|
|
220
|
+
async def batch(
|
|
221
|
+
images: list[UploadFile] = File(...), authorization: str | None = Header(default=None),
|
|
222
|
+
document_type: str | None = Form(default=None), country: str | None = Form(default=None),
|
|
223
|
+
include_evidence: bool = Form(default=False),
|
|
224
|
+
):
|
|
225
|
+
_authorize(authorization)
|
|
226
|
+
options = _options(document_type, country, include_evidence)
|
|
227
|
+
return await _execute(scan_batch, await _uploads(images), **options)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@app.post("/scan/document")
|
|
231
|
+
async def grouped_document(
|
|
232
|
+
images: list[UploadFile] = File(...), authorization: str | None = Header(default=None),
|
|
233
|
+
document_type: str | None = Form(default=None), country: str | None = Form(default=None),
|
|
234
|
+
include_evidence: bool = Form(default=False),
|
|
235
|
+
):
|
|
236
|
+
_authorize(authorization)
|
|
237
|
+
options = _options(document_type, country, include_evidence)
|
|
238
|
+
return await _execute(scan_document, await _uploads(images), **options)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
@app.get("/review", response_class=HTMLResponse)
|
|
242
|
+
async def review():
|
|
243
|
+
import core
|
|
244
|
+
path = Path(core.__file__).with_name('review.html')
|
|
245
|
+
return HTMLResponse(path.read_text(), headers={'Cache-Control': 'no-store',
|
|
246
|
+
'Content-Security-Policy': "default-src 'self'; img-src 'self' blob: data:; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'"})
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@app.post("/preview")
|
|
250
|
+
async def preview(image: UploadFile = File(...), authorization: str | None = Header(default=None)):
|
|
251
|
+
_authorize(authorization)
|
|
252
|
+
result = await _execute(render_preview, await _upload(image))
|
|
253
|
+
if isinstance(result, Response):
|
|
254
|
+
return result
|
|
255
|
+
return Response(result, media_type='image/png', headers={'Cache-Control': 'no-store'})
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@app.post("/redact")
|
|
259
|
+
async def redacted(image: UploadFile = File(...), boxes: str = Form(...), authorization: str | None = Header(default=None)):
|
|
260
|
+
_authorize(authorization)
|
|
261
|
+
if len(boxes) > 50000:
|
|
262
|
+
raise HTTPException(status_code=400, detail='INVALID_REDACTION_BOXES')
|
|
263
|
+
result = await _execute(redact, await _upload(image), boxes)
|
|
264
|
+
if isinstance(result, Response):
|
|
265
|
+
return result
|
|
266
|
+
return Response(result, media_type='image/png', headers={'Cache-Control': 'no-store',
|
|
267
|
+
'Content-Disposition': 'attachment; filename="redacted.png"'})
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _job_store(*, required=True):
|
|
271
|
+
try:
|
|
272
|
+
store = configured_store()
|
|
273
|
+
except ValueError as exc:
|
|
274
|
+
raise HTTPException(status_code=503, detail='INVALID_JOB_CONFIGURATION') from exc
|
|
275
|
+
if required and store is None:
|
|
276
|
+
raise HTTPException(status_code=503, detail='JOBS_NOT_CONFIGURED')
|
|
277
|
+
return store
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
@app.post('/jobs', status_code=202)
|
|
281
|
+
async def enqueue_job(
|
|
282
|
+
images: list[UploadFile] = File(...), authorization: str | None = Header(default=None),
|
|
283
|
+
document_type: str | None = Form(default=None), country: str | None = Form(default=None),
|
|
284
|
+
include_evidence: bool = Form(default=False), grouped: bool = Form(default=False), notify: bool = Form(default=False),
|
|
285
|
+
):
|
|
286
|
+
_authorize(authorization)
|
|
287
|
+
options = _options(document_type, country, include_evidence)
|
|
288
|
+
store = await asyncio.to_thread(_job_store)
|
|
289
|
+
data = await _uploads(images)
|
|
290
|
+
try:
|
|
291
|
+
return await asyncio.to_thread(store.enqueue, data, options, grouped=grouped, notify=notify)
|
|
292
|
+
except ValueError as exc:
|
|
293
|
+
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@app.get('/jobs/{identifier}')
|
|
297
|
+
async def job(identifier: str, authorization: str | None = Header(default=None)):
|
|
298
|
+
_authorize(authorization)
|
|
299
|
+
store = await asyncio.to_thread(_job_store)
|
|
300
|
+
result = await asyncio.to_thread(store.get, identifier)
|
|
301
|
+
if result is None:
|
|
302
|
+
raise HTTPException(status_code=404, detail='JOB_NOT_FOUND')
|
|
303
|
+
return result
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@app.delete('/jobs/{identifier}')
|
|
307
|
+
async def delete_job(identifier: str, authorization: str | None = Header(default=None)):
|
|
308
|
+
_authorize(authorization)
|
|
309
|
+
store = await asyncio.to_thread(_job_store)
|
|
310
|
+
removed = await asyncio.to_thread(store.delete, identifier)
|
|
311
|
+
if not removed:
|
|
312
|
+
raise HTTPException(status_code=404, detail='JOB_NOT_FOUND')
|
|
313
|
+
return {'deleted': True}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
FROM public.ecr.aws/lambda/python:3.14
|
|
2
|
+
RUN microdnf install -y mesa-libGL glib2 libgomp && microdnf clean all
|
|
3
|
+
COPY requirements.lock ${LAMBDA_TASK_ROOT}/requirements.lock
|
|
4
|
+
RUN python -m pip install --no-cache-dir -r ${LAMBDA_TASK_ROOT}/requirements.lock
|
|
5
|
+
COPY core/ ${LAMBDA_TASK_ROOT}/core/
|
|
6
|
+
ARG DOCUMENT_OCR_KYC_LANGS=en
|
|
7
|
+
ENV DOCUMENT_OCR_KYC_LANGS=${DOCUMENT_OCR_KYC_LANGS} PYTHONDONTWRITEBYTECODE=1
|
|
8
|
+
# Lambda has a read-only root filesystem: package every selected model now.
|
|
9
|
+
RUN python -m core.model_setup
|
|
10
|
+
COPY deploy/lambda/handler.py ${LAMBDA_TASK_ROOT}/handler.py
|
|
11
|
+
CMD ["handler.handler"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
4
|
+
cd "$ROOT_DIR"
|
|
5
|
+
source deploy/common.sh
|
|
6
|
+
hosting_defaults
|
|
7
|
+
require_command sam
|
|
8
|
+
require_command aws
|
|
9
|
+
export AWS_REGION="${AWS_REGION:-ap-south-1}"
|
|
10
|
+
STACK="${AWS_STACK_NAME:-$DOCUMENT_OCR_NAME}"
|
|
11
|
+
PARAMETERS=("KycLanguages=$DOCUMENT_OCR_KYC_LANGS" "InputBucket=${DOCUMENT_OCR_S3_BUCKET:-}" "InputPrefix=${DOCUMENT_OCR_S3_PREFIX:-uploads/}")
|
|
12
|
+
TEMPLATE=deploy/lambda/template.yaml
|
|
13
|
+
if [[ -z "${IMAGE_URI:-}" ]]; then
|
|
14
|
+
require_command docker
|
|
15
|
+
export DOCKER_DEFAULT_PLATFORM=linux/amd64
|
|
16
|
+
sam build --template-file "$TEMPLATE" --parameter-overrides "${PARAMETERS[@]}"
|
|
17
|
+
TEMPLATE=.aws-sam/build/template.yaml
|
|
18
|
+
else
|
|
19
|
+
PARAMETERS+=("ImageUri=$IMAGE_URI")
|
|
20
|
+
fi
|
|
21
|
+
sam deploy --template-file "$TEMPLATE" --stack-name "$STACK" --region "$AWS_REGION" --resolve-image-repos --resolve-s3 --capabilities CAPABILITY_IAM --parameter-overrides "${PARAMETERS[@]}" --no-fail-on-empty-changeset
|
|
22
|
+
aws cloudformation describe-stacks --stack-name "$STACK" --region "$AWS_REGION" --query 'Stacks[0].Outputs' --output json
|