@1aboveio/skills 0.20.2 → 0.20.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -6
- package/package.json +1 -1
- package/runtime/skills/distribution/generated/recipes.json +220 -25
- package/runtime/skills/distribution/scripts/bundles.mjs +167 -48
- package/runtime/skills/engineering/engineering-runtime/scripts/workflow-coherence.mjs +4 -0
- package/runtime/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs +1 -1
- package/skills/compliance/compliance-audit/LICENSE +3 -0
- package/skills/compliance/compliance-audit/SKILL.md +466 -0
- package/skills/compliance/compliance-audit/references/findings-schema.md +290 -0
- package/skills/compliance/compliance-audit/references/vars-checklist.md +1068 -0
- package/skills/compliance/compliance-audit/references/vars-findings-schema.md +395 -0
- package/skills/compliance/compliance-audit/scripts/reconcile_tracker.py +454 -0
- package/skills/compliance/compliance-audit/scripts/test_validate_tracker.py +90 -0
- package/skills/compliance/compliance-audit/scripts/validate_tracker.py +752 -0
- package/skills/compliance/doc-authenticity/LICENSE +3 -0
- package/skills/compliance/doc-authenticity/SKILL.md +158 -0
- package/skills/compliance/doc-authenticity/references/agreement.md +66 -0
- package/skills/compliance/doc-authenticity/references/bank-statement.md +150 -0
- package/skills/compliance/doc-authenticity/references/forensic-signals.md +157 -0
- package/skills/compliance/doc-authenticity/references/image-signals.md +114 -0
- package/skills/compliance/doc-authenticity/references/invoice.md +66 -0
- package/skills/compliance/doc-authenticity/references/issuer-profiles.md +86 -0
- package/skills/compliance/doc-authenticity/references/verdict-and-report.md +95 -0
- package/skills/compliance/doc-authenticity/references/verdict.schema.json +81 -0
- package/skills/compliance/doc-authenticity/scripts/image_forensics.py +294 -0
- package/skills/compliance/doc-authenticity/scripts/pdf_forensics.py +401 -0
- package/skills/compliance/doc-authenticity/scripts/validate_codes.py +224 -0
- package/skills/compliance/doc-authenticity/scripts/validate_verdict.py +308 -0
- package/skills/compliance/report-writing/LICENSE +3 -0
- package/skills/compliance/report-writing/SKILL.md +206 -0
- package/skills/compliance/report-writing/references/audit-report-template.md +121 -0
- package/skills/compliance/report-writing/references/external-assessment-template.md +167 -0
- package/skills/compliance/report-writing/references/management-report-template.md +145 -0
- package/skills/compliance/report-writing/references/vars-report-template.md +224 -0
- package/skills/document-utils/doc-to-md/LICENSE +3 -0
- package/skills/document-utils/doc-to-md/SKILL.md +77 -0
- package/skills/document-utils/doc-to-md/scripts/convert.py +157 -0
- package/skills/document-utils/reg-parser/LICENSE +3 -0
- package/skills/document-utils/reg-parser/SKILL.md +201 -0
- package/skills/document-utils/reg-parser/references/checklist-schema.md +223 -0
- package/skills/document-utils/reg-parser/references/checklists/hk-amlo-checklist.json +1306 -0
- package/skills/document-utils/reg-parser/references/checklists/hk-amlo-checklist.md +469 -0
- package/skills/document-utils/reg-parser/references/checklists/hk-fi-corporate-onboarding-cdd-checklist.json +634 -0
- package/skills/document-utils/reg-parser/references/checklists/hk-fi-corporate-onboarding-cdd-checklist.md +233 -0
- package/skills/document-utils/reg-parser/references/checklists/hk-mso-guideline-checklist.json +2194 -0
- package/skills/document-utils/reg-parser/references/checklists/hk-mso-guideline-checklist.md +771 -0
- package/skills/document-utils/reg-parser/references/checklists/kyc-aml-checklist.md +258 -0
- package/skills/document-utils/reg-parser/references/checklists/visa-vars-checklist.md +1058 -0
- package/skills/document-utils/reg-parser/references/mandates/hk-amlo-cap615.md +389 -0
- package/skills/document-utils/reg-parser/references/mandates/visa-vars.md +205 -0
- package/skills/document-utils/reg-parser/scripts/compare_checklist.py +228 -0
- package/skills/document-utils/reg-parser/scripts/dump_checklist.py +348 -0
- package/skills/document-utils/reg-parser/scripts/preprocess-mandate.sh +171 -0
- package/skills/document-utils/reg-parser/scripts/test_quarantine.sh +102 -0
- package/skills/document-utils/reg-parser/scripts/validate_checklist.py +532 -0
- package/skills/document-utils/reg-parser/scripts/validate_mandate.py +242 -0
- package/skills/engineering/engineering-runtime/coherence/workflow.json +16 -16
- package/skills/engineering/engineering-runtime/scripts/workflow-coherence.mjs +4 -0
- package/skills/engineering/engineering-runtime/scripts/workflow-policy.mjs +1 -1
- package/skills/payment/similar-domain/LICENSE +3 -0
- package/skills/payment/similar-domain/SKILL.md +178 -0
- package/skills/payment/similar-domain/references/entities.md +58 -0
- package/skills/payment/similar-domain/references/fingerprints.md +82 -0
- package/skills/payment/similar-domain/references/pivots.md +66 -0
- package/skills/payment/similar-domain/references/report-template.md +76 -0
- package/skills/payment/similar-domain/references/scoring.md +61 -0
- package/skills/payment/similar-domain/references/terminology.md +37 -0
- package/skills/payment/similar-domain/references/workflow.md +79 -0
- package/skills/payment/tld-detection/SKILL.md +76 -21
- package/skills/payment/tld-detection/references/clean.md +1 -1
- package/skills/payment/tld-detection/references/clustering.md +21 -6
- package/skills/payment/tld-detection/references/decision.md +7 -2
- package/skills/payment/tld-detection/references/fx-conversion.md +229 -0
- package/skills/payment/tld-detection/references/report-template.md +63 -27
- package/skills/payment/tld-detection/references/streams.md +90 -0
- package/skills/payment/tld-detection/references/terminology.md +15 -7
- package/skills/payment/tld-detection/references/workflow.md +1 -1
- package/skills/payment/tld-detection/scripts/detect_streams.py +542 -0
- package/skills/engineering/ensure-coverage/evals/evals.json +0 -125
- package/skills/engineering/rush-issues/evals/evals.json +0 -65
- package/skills/engineering/rush-release/evals/evals.json +0 -58
- package/skills/fullstack/shadcn/evals/evals.json +0 -90
- package/skills/payment/fraud-analysis/evals/evals.json +0 -40
- package/skills/payment/tld-detection/evals/evals.json +0 -107
- package/skills/payment/tld-detection/tests/test_detect_ladders.py +0 -132
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
# /// script
|
|
2
|
+
# requires-python = ">=3.10"
|
|
3
|
+
# dependencies = ["pikepdf>=8", "pdfplumber>=0.11"]
|
|
4
|
+
# ///
|
|
5
|
+
"""Extract forensic signals from a PDF for the doc-authenticity skill.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
uv run pdf_forensics.py input.pdf --out outdir/
|
|
9
|
+
|
|
10
|
+
Writes:
|
|
11
|
+
outdir/forensics.json structural + metadata signals (F1-F9 inputs)
|
|
12
|
+
outdir/pages/page-N.txt extracted text per page
|
|
13
|
+
outdir/pages/page-N.fonts.tsv word-level font map (word, font, size, x0, top)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import hashlib
|
|
20
|
+
import json
|
|
21
|
+
import re
|
|
22
|
+
import sys
|
|
23
|
+
from collections import Counter
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
import pdfplumber
|
|
27
|
+
import pikepdf
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_pdf_date(raw: str | None) -> str | None:
|
|
31
|
+
"""D:20260115093000+00'00' -> 2026-01-15T09:30:00+00:00 (best effort)."""
|
|
32
|
+
if not raw:
|
|
33
|
+
return None
|
|
34
|
+
s = str(raw)
|
|
35
|
+
m = re.match(r"D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?([Zz+\-].*)?", s)
|
|
36
|
+
if not m:
|
|
37
|
+
return s
|
|
38
|
+
y, mo, d, h, mi, sec, tz = m.groups()
|
|
39
|
+
iso = f"{y}-{mo or '01'}-{d or '01'}T{h or '00'}:{mi or '00'}:{sec or '00'}"
|
|
40
|
+
if tz and tz not in ("Z", "z"):
|
|
41
|
+
tz = tz.replace("'", ":").rstrip(":")
|
|
42
|
+
iso += tz if re.match(r"[+\-]\d{2}:\d{2}$", tz) else ""
|
|
43
|
+
elif tz:
|
|
44
|
+
iso += "+00:00"
|
|
45
|
+
return iso
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def scan_raw_structure(data: bytes) -> dict:
|
|
49
|
+
"""Byte-level scan: save generations and objects redefined across them."""
|
|
50
|
+
eof_positions = [m.start() for m in re.finditer(rb"%%EOF", data)]
|
|
51
|
+
startxrefs = len(re.findall(rb"startxref", data))
|
|
52
|
+
prev_pointers = len(re.findall(rb"/Prev\s+\d+", data))
|
|
53
|
+
|
|
54
|
+
# Objects defined more than once = redefined in a later generation
|
|
55
|
+
# (approximation: also matched inside object streams is not possible since
|
|
56
|
+
# those are compressed; raw matches are top-level definitions).
|
|
57
|
+
defs: dict[int, list[int]] = {}
|
|
58
|
+
for m in re.finditer(rb"(?m)^\s*(\d+)\s+\d+\s+obj\b", data):
|
|
59
|
+
defs.setdefault(int(m.group(1)), []).append(m.start())
|
|
60
|
+
updated = sorted(n for n, offs in defs.items() if len(offs) > 1)
|
|
61
|
+
|
|
62
|
+
trailing = data[eof_positions[-1] + 5 :] if eof_positions else b""
|
|
63
|
+
return {
|
|
64
|
+
"file_size": len(data),
|
|
65
|
+
"eof_count": len(eof_positions),
|
|
66
|
+
"startxref_count": startxrefs,
|
|
67
|
+
"prev_pointers": prev_pointers,
|
|
68
|
+
"incremental_updates": max(0, len(eof_positions) - 1),
|
|
69
|
+
"updated_objects": updated[:50],
|
|
70
|
+
"updated_object_count": len(updated),
|
|
71
|
+
"bytes_after_last_eof": len(trailing.strip()),
|
|
72
|
+
"header": data[:16].decode("latin-1", "replace").strip(),
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_metadata(pdf: pikepdf.Pdf) -> dict:
|
|
77
|
+
info = {}
|
|
78
|
+
try:
|
|
79
|
+
for k, v in (pdf.docinfo or {}).items():
|
|
80
|
+
info[str(k).lstrip("/")] = str(v)
|
|
81
|
+
except Exception as e: # noqa: BLE001
|
|
82
|
+
info["_error"] = str(e)
|
|
83
|
+
return {
|
|
84
|
+
"producer": info.get("Producer"),
|
|
85
|
+
"creator": info.get("Creator"),
|
|
86
|
+
"author": info.get("Author"),
|
|
87
|
+
"title": info.get("Title"),
|
|
88
|
+
"creation_date_raw": info.get("CreationDate"),
|
|
89
|
+
"mod_date_raw": info.get("ModDate"),
|
|
90
|
+
"creation_date": parse_pdf_date(info.get("CreationDate")),
|
|
91
|
+
"mod_date": parse_pdf_date(info.get("ModDate")),
|
|
92
|
+
"all_docinfo": info,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def extract_xmp(pdf: pikepdf.Pdf) -> dict:
|
|
97
|
+
out: dict = {"present": False}
|
|
98
|
+
try:
|
|
99
|
+
meta = pdf.open_metadata()
|
|
100
|
+
if not str(meta):
|
|
101
|
+
return out
|
|
102
|
+
out["present"] = True
|
|
103
|
+
keys = [
|
|
104
|
+
("xmp:CreatorTool", "creator_tool"),
|
|
105
|
+
("pdf:Producer", "producer"),
|
|
106
|
+
("xmp:CreateDate", "create_date"),
|
|
107
|
+
("xmp:ModifyDate", "modify_date"),
|
|
108
|
+
("xmp:MetadataDate", "metadata_date"),
|
|
109
|
+
("xmpMM:DocumentID", "document_id"),
|
|
110
|
+
("xmpMM:InstanceID", "instance_id"),
|
|
111
|
+
]
|
|
112
|
+
for xmp_key, name in keys:
|
|
113
|
+
try:
|
|
114
|
+
if xmp_key in meta:
|
|
115
|
+
out[name] = str(meta[xmp_key])
|
|
116
|
+
except Exception: # noqa: BLE001, S110
|
|
117
|
+
pass
|
|
118
|
+
raw = str(meta)
|
|
119
|
+
out["xmp_toolkit"] = None
|
|
120
|
+
tk = re.search(r'x:xmptk="([^"]+)"', raw)
|
|
121
|
+
if tk:
|
|
122
|
+
out["xmp_toolkit"] = tk.group(1)
|
|
123
|
+
out["has_history"] = "xmpMM:History" in raw
|
|
124
|
+
out["has_derived_from"] = "xmpMM:DerivedFrom" in raw
|
|
125
|
+
agents = sorted(set(re.findall(r"softwareAgent>([^<]+)<", raw)))
|
|
126
|
+
agents += sorted(set(re.findall(r'softwareAgent="([^"]+)"', raw)))
|
|
127
|
+
out["history_software_agents"] = sorted(set(agents))
|
|
128
|
+
except Exception as e: # noqa: BLE001
|
|
129
|
+
out["_error"] = str(e)
|
|
130
|
+
return out
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def extract_signatures(pdf: pikepdf.Pdf, file_size: int) -> list[dict]:
|
|
134
|
+
sigs = []
|
|
135
|
+
try:
|
|
136
|
+
root = pdf.Root
|
|
137
|
+
acro = root.get("/AcroForm")
|
|
138
|
+
if acro is None:
|
|
139
|
+
return sigs
|
|
140
|
+
for field in acro.get("/Fields", []):
|
|
141
|
+
try:
|
|
142
|
+
if str(field.get("/FT", "")) != "/Sig":
|
|
143
|
+
continue
|
|
144
|
+
v = field.get("/V")
|
|
145
|
+
entry: dict = {"field_name": str(field.get("/T", ""))}
|
|
146
|
+
if v is None:
|
|
147
|
+
entry["signed"] = False
|
|
148
|
+
sigs.append(entry)
|
|
149
|
+
continue
|
|
150
|
+
entry["signed"] = True
|
|
151
|
+
entry["cryptographic_validation"] = "not_performed"
|
|
152
|
+
entry["sub_filter"] = str(v.get("/SubFilter", "")) or None
|
|
153
|
+
entry["signer_name"] = str(v.get("/Name", "")) or None
|
|
154
|
+
entry["sign_date"] = parse_pdf_date(
|
|
155
|
+
str(v.get("/M")) if v.get("/M") is not None else None
|
|
156
|
+
)
|
|
157
|
+
br = v.get("/ByteRange")
|
|
158
|
+
if br is not None:
|
|
159
|
+
br = [int(x) for x in br]
|
|
160
|
+
entry["byte_range"] = br
|
|
161
|
+
# This checks structural coverage only; it does not verify
|
|
162
|
+
# the signature value, certificate chain, or trust anchor.
|
|
163
|
+
entry["byte_range_covers_file"] = (
|
|
164
|
+
len(br) == 4
|
|
165
|
+
and br[0] == 0
|
|
166
|
+
and br[1] >= 0
|
|
167
|
+
and br[2] >= br[1]
|
|
168
|
+
and br[3] >= 0
|
|
169
|
+
and br[2] + br[3] >= file_size - 4
|
|
170
|
+
)
|
|
171
|
+
sigs.append(entry)
|
|
172
|
+
except Exception as e: # noqa: BLE001
|
|
173
|
+
sigs.append({"_error": str(e)})
|
|
174
|
+
except Exception as e: # noqa: BLE001
|
|
175
|
+
sigs.append({"_error": str(e)})
|
|
176
|
+
return sigs
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def extract_fonts_and_annots(pdf: pikepdf.Pdf) -> tuple[list[dict], list[dict], dict]:
|
|
180
|
+
fonts: dict[tuple, dict] = {}
|
|
181
|
+
annots: list[dict] = []
|
|
182
|
+
doc_flags = {
|
|
183
|
+
"has_javascript": False,
|
|
184
|
+
"has_embedded_files": False,
|
|
185
|
+
"has_open_action": False,
|
|
186
|
+
}
|
|
187
|
+
try:
|
|
188
|
+
names = pdf.Root.get("/Names")
|
|
189
|
+
if names is not None:
|
|
190
|
+
if names.get("/JavaScript") is not None:
|
|
191
|
+
doc_flags["has_javascript"] = True
|
|
192
|
+
if names.get("/EmbeddedFiles") is not None:
|
|
193
|
+
doc_flags["has_embedded_files"] = True
|
|
194
|
+
if pdf.Root.get("/OpenAction") is not None:
|
|
195
|
+
doc_flags["has_open_action"] = True
|
|
196
|
+
except Exception: # noqa: BLE001, S110
|
|
197
|
+
pass
|
|
198
|
+
|
|
199
|
+
for pageno, page in enumerate(pdf.pages, 1):
|
|
200
|
+
try:
|
|
201
|
+
res = page.get("/Resources", {})
|
|
202
|
+
fdict = res.get("/Font", {}) if res is not None else {}
|
|
203
|
+
for _, fobj in fdict.items() if fdict is not None else []:
|
|
204
|
+
try:
|
|
205
|
+
base = str(fobj.get("/BaseFont", "unknown")).lstrip("/")
|
|
206
|
+
desc = fobj.get("/FontDescriptor")
|
|
207
|
+
if desc is None and fobj.get("/DescendantFonts") is not None:
|
|
208
|
+
desc = fobj.get("/DescendantFonts")[0].get("/FontDescriptor")
|
|
209
|
+
embedded = bool(
|
|
210
|
+
desc is not None
|
|
211
|
+
and any(
|
|
212
|
+
desc.get(k) is not None
|
|
213
|
+
for k in ("/FontFile", "/FontFile2", "/FontFile3")
|
|
214
|
+
)
|
|
215
|
+
)
|
|
216
|
+
m = re.match(r"^([A-Z]{6})\+(.+)$", base)
|
|
217
|
+
key = (base, embedded)
|
|
218
|
+
rec = fonts.setdefault(
|
|
219
|
+
key,
|
|
220
|
+
{
|
|
221
|
+
"base_font": base,
|
|
222
|
+
"family": m.group(2) if m else base,
|
|
223
|
+
"subset_prefix": m.group(1) if m else None,
|
|
224
|
+
"embedded": embedded,
|
|
225
|
+
"subtype": str(fobj.get("/Subtype", "")).lstrip("/"),
|
|
226
|
+
"pages": [],
|
|
227
|
+
},
|
|
228
|
+
)
|
|
229
|
+
if pageno not in rec["pages"]:
|
|
230
|
+
rec["pages"].append(pageno)
|
|
231
|
+
except Exception: # noqa: BLE001, S112
|
|
232
|
+
continue
|
|
233
|
+
except Exception: # noqa: BLE001, S110
|
|
234
|
+
pass
|
|
235
|
+
try:
|
|
236
|
+
for a in page.get("/Annots", []) or []:
|
|
237
|
+
sub = str(a.get("/Subtype", "")).lstrip("/")
|
|
238
|
+
annots.append(
|
|
239
|
+
{
|
|
240
|
+
"page": pageno,
|
|
241
|
+
"subtype": sub,
|
|
242
|
+
"contents": str(a.get("/Contents", ""))[:200] or None,
|
|
243
|
+
"rect": [float(x) for x in a.get("/Rect", [])] or None,
|
|
244
|
+
}
|
|
245
|
+
)
|
|
246
|
+
except Exception: # noqa: BLE001, S110
|
|
247
|
+
pass
|
|
248
|
+
|
|
249
|
+
font_list = sorted(fonts.values(), key=lambda f: f["base_font"])
|
|
250
|
+
# duplicate subset embeddings of the same family = edit trace (F5)
|
|
251
|
+
fam_prefixes: dict[str, set] = {}
|
|
252
|
+
for f in font_list:
|
|
253
|
+
if f["subset_prefix"]:
|
|
254
|
+
fam_prefixes.setdefault(f["family"], set()).add(f["subset_prefix"])
|
|
255
|
+
doc_flags["families_with_multiple_subsets"] = sorted(
|
|
256
|
+
fam for fam, p in fam_prefixes.items() if len(p) > 1
|
|
257
|
+
)
|
|
258
|
+
return font_list, annots, doc_flags
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def analyze_text(pdf_path: Path, pages_dir: Path) -> tuple[list[dict], list[dict]]:
|
|
262
|
+
"""Per-page text + font map; returns (page_stats, font_anomalies)."""
|
|
263
|
+
pages_dir.mkdir(parents=True, exist_ok=True)
|
|
264
|
+
page_stats: list[dict] = []
|
|
265
|
+
anomalies: list[dict] = []
|
|
266
|
+
with pdfplumber.open(pdf_path) as pdf:
|
|
267
|
+
for i, page in enumerate(pdf.pages, 1):
|
|
268
|
+
text = page.extract_text() or ""
|
|
269
|
+
(pages_dir / f"page-{i}.txt").write_text(text, encoding="utf-8")
|
|
270
|
+
words = page.extract_words(extra_attrs=["fontname", "size"]) or []
|
|
271
|
+
with (pages_dir / f"page-{i}.fonts.tsv").open("w", encoding="utf-8") as fh:
|
|
272
|
+
fh.write("word\tfont\tsize\tx0\ttop\n")
|
|
273
|
+
for w in words:
|
|
274
|
+
fh.write(
|
|
275
|
+
f"{w['text']}\t{w.get('fontname')}\t"
|
|
276
|
+
f"{round(float(w.get('size', 0)), 2)}\t"
|
|
277
|
+
f"{round(w['x0'], 1)}\t{round(w['top'], 1)}\n"
|
|
278
|
+
)
|
|
279
|
+
page_area = float(page.width) * float(page.height) or 1.0
|
|
280
|
+
img_area = sum(
|
|
281
|
+
max(0.0, (im["x1"] - im["x0"]) * (im["bottom"] - im["top"]))
|
|
282
|
+
for im in page.images
|
|
283
|
+
)
|
|
284
|
+
page_stats.append(
|
|
285
|
+
{
|
|
286
|
+
"page": i,
|
|
287
|
+
"char_count": len(text),
|
|
288
|
+
"word_count": len(words),
|
|
289
|
+
"image_count": len(page.images),
|
|
290
|
+
"image_coverage": round(min(1.0, img_area / page_area), 3),
|
|
291
|
+
}
|
|
292
|
+
)
|
|
293
|
+
# group words into lines by 'top', find words whose font deviates
|
|
294
|
+
# from the line's modal font (F6 splice candidates)
|
|
295
|
+
lines: dict[int, list[dict]] = {}
|
|
296
|
+
for w in words:
|
|
297
|
+
lines.setdefault(round(w["top"] / 3.0), []).append(w)
|
|
298
|
+
for line_words in lines.values():
|
|
299
|
+
if len(line_words) < 3:
|
|
300
|
+
continue
|
|
301
|
+
fnames = Counter(w.get("fontname") for w in line_words)
|
|
302
|
+
modal, modal_n = fnames.most_common(1)[0]
|
|
303
|
+
if len(fnames) == 1 or modal_n < max(3, len(line_words) - 2):
|
|
304
|
+
continue
|
|
305
|
+
for w in line_words:
|
|
306
|
+
if w.get("fontname") != modal:
|
|
307
|
+
anomalies.append(
|
|
308
|
+
{
|
|
309
|
+
"page": i,
|
|
310
|
+
"word": w["text"],
|
|
311
|
+
"font": w.get("fontname"),
|
|
312
|
+
"line_modal_font": modal,
|
|
313
|
+
"size": round(float(w.get("size", 0)), 2),
|
|
314
|
+
"x0": round(w["x0"], 1),
|
|
315
|
+
"top": round(w["top"], 1),
|
|
316
|
+
"line_text": " ".join(
|
|
317
|
+
x["text"]
|
|
318
|
+
for x in sorted(line_words, key=lambda x: x["x0"])
|
|
319
|
+
)[:200],
|
|
320
|
+
}
|
|
321
|
+
)
|
|
322
|
+
return page_stats, anomalies
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def main() -> int:
|
|
326
|
+
ap = argparse.ArgumentParser(description=__doc__)
|
|
327
|
+
ap.add_argument("pdf", type=Path)
|
|
328
|
+
ap.add_argument(
|
|
329
|
+
"--out",
|
|
330
|
+
type=Path,
|
|
331
|
+
default=None,
|
|
332
|
+
help="output dir (default: ./doc-authenticity-out/<stem>)",
|
|
333
|
+
)
|
|
334
|
+
args = ap.parse_args()
|
|
335
|
+
|
|
336
|
+
if not args.pdf.is_file():
|
|
337
|
+
print(f"error: {args.pdf} not found", file=sys.stderr)
|
|
338
|
+
return 2
|
|
339
|
+
outdir = args.out or Path("doc-authenticity-out") / args.pdf.stem
|
|
340
|
+
outdir.mkdir(parents=True, exist_ok=True)
|
|
341
|
+
|
|
342
|
+
data = args.pdf.read_bytes()
|
|
343
|
+
result: dict = {
|
|
344
|
+
"input": str(args.pdf.resolve()),
|
|
345
|
+
"media_type": "application/pdf",
|
|
346
|
+
"size_bytes": len(data),
|
|
347
|
+
"sha256": hashlib.sha256(data).hexdigest(),
|
|
348
|
+
"extension_magic_match": args.pdf.suffix.lower() == ".pdf",
|
|
349
|
+
"structure": scan_raw_structure(data),
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
repairs: list[str] = []
|
|
353
|
+
try:
|
|
354
|
+
import warnings
|
|
355
|
+
|
|
356
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
357
|
+
warnings.simplefilter("always")
|
|
358
|
+
pdf = pikepdf.open(args.pdf)
|
|
359
|
+
repairs = [str(w.message) for w in caught]
|
|
360
|
+
except pikepdf.PasswordError:
|
|
361
|
+
result["error"] = "encrypted: password required"
|
|
362
|
+
(outdir / "forensics.json").write_text(json.dumps(result, indent=2))
|
|
363
|
+
print(json.dumps(result, indent=2))
|
|
364
|
+
return 1
|
|
365
|
+
except Exception as exc: # noqa: BLE001
|
|
366
|
+
result["error"] = f"PDF parsing failed: {exc}"
|
|
367
|
+
(outdir / "forensics.json").write_text(json.dumps(result, indent=2))
|
|
368
|
+
print(json.dumps(result, indent=2))
|
|
369
|
+
return 1
|
|
370
|
+
|
|
371
|
+
with pdf:
|
|
372
|
+
result["pdf_version"] = str(pdf.pdf_version)
|
|
373
|
+
result["encrypted"] = pdf.is_encrypted
|
|
374
|
+
result["linearized"] = pdf.is_linearized
|
|
375
|
+
result["page_count"] = len(pdf.pages)
|
|
376
|
+
result["metadata"] = extract_metadata(pdf)
|
|
377
|
+
result["xmp"] = extract_xmp(pdf)
|
|
378
|
+
result["signatures"] = extract_signatures(pdf, result["structure"]["file_size"])
|
|
379
|
+
fonts, annots, doc_flags = extract_fonts_and_annots(pdf)
|
|
380
|
+
result["fonts"] = fonts
|
|
381
|
+
result["annotations"] = annots
|
|
382
|
+
result["structure"].update(doc_flags)
|
|
383
|
+
result["structure"]["repairs"] = repairs
|
|
384
|
+
|
|
385
|
+
try:
|
|
386
|
+
page_stats, anomalies = analyze_text(args.pdf, outdir / "pages")
|
|
387
|
+
result["pages"] = page_stats
|
|
388
|
+
result["text_font_anomalies"] = anomalies
|
|
389
|
+
except Exception as e: # noqa: BLE001
|
|
390
|
+
result["pages_error"] = str(e)
|
|
391
|
+
|
|
392
|
+
out_path = outdir / "forensics.json"
|
|
393
|
+
out_path.write_text(json.dumps(result, indent=2), encoding="utf-8")
|
|
394
|
+
print(json.dumps(result, indent=2))
|
|
395
|
+
print(f"\n[written] {out_path}", file=sys.stderr)
|
|
396
|
+
print(f"[written] {outdir}/pages/page-N.txt + .fonts.tsv", file=sys.stderr)
|
|
397
|
+
return 0
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
if __name__ == "__main__":
|
|
401
|
+
sys.exit(main())
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# /// script
|
|
2
|
+
# requires-python = ">=3.10"
|
|
3
|
+
# dependencies = []
|
|
4
|
+
# ///
|
|
5
|
+
"""Mechanical validation of banking identifiers (doc-authenticity D1/D2).
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
uv run validate_codes.py --iban GB29NWBK60161331926819 --bic NWBKGB2L \
|
|
9
|
+
--sort-code 60-16-13 --account 31926819 --aba 021000021
|
|
10
|
+
|
|
11
|
+
Pass only the identifiers found on the document. Prints one JSON object with
|
|
12
|
+
a result per identifier: valid (bool), plus detail. Pure stdlib.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import json
|
|
19
|
+
import re
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
# ISO 13616 IBAN lengths per country (common subset)
|
|
23
|
+
IBAN_LENGTHS = {
|
|
24
|
+
"AD": 24, "AE": 23, "AL": 28, "AT": 20, "AZ": 28, "BA": 20, "BE": 16,
|
|
25
|
+
"BG": 22, "BH": 22, "BR": 29, "CH": 21, "CR": 22, "CY": 28, "CZ": 24,
|
|
26
|
+
"DE": 22, "DK": 18, "DO": 28, "EE": 20, "EG": 29, "ES": 24, "FI": 18,
|
|
27
|
+
"FO": 18, "FR": 27, "GB": 22, "GE": 22, "GI": 23, "GL": 18, "GR": 27,
|
|
28
|
+
"GT": 28, "HR": 21, "HU": 28, "IE": 22, "IL": 23, "IS": 26, "IT": 27,
|
|
29
|
+
"JO": 30, "KW": 30, "KZ": 20, "LB": 28, "LI": 21, "LT": 20, "LU": 20,
|
|
30
|
+
"LV": 21, "MC": 27, "MD": 24, "ME": 22, "MK": 19, "MR": 27, "MT": 31,
|
|
31
|
+
"MU": 30, "NL": 18, "NO": 15, "PK": 24, "PL": 28, "PS": 29, "PT": 25,
|
|
32
|
+
"QA": 29, "RO": 24, "RS": 22, "SA": 24, "SE": 24, "SI": 19, "SK": 24,
|
|
33
|
+
"SM": 27, "TN": 24, "TR": 26, "UA": 29, "VA": 22, "VG": 24, "XK": 20,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
# UK sort-code leading-pair → bank group hints (indicative, not exhaustive)
|
|
37
|
+
UK_SORT_HINTS = {
|
|
38
|
+
range(1, 10): "generic clearing (01-09: various incl. NatWest 01, "
|
|
39
|
+
"Santander 09, Nationwide 07, fintechs 04)",
|
|
40
|
+
range(10, 20): "Bank of Scotland / Halifax (10-19: 12 BoS, 11 Halifax)",
|
|
41
|
+
range(20, 30): "Barclays (20-29)",
|
|
42
|
+
range(30, 40): "Lloyds Banking Group (30-39: 30-38 Lloyds/TSB, 40s excl.)",
|
|
43
|
+
range(40, 50): "HSBC group (40-49: 40 HSBC, 40-47 first direct 40-47-xx)",
|
|
44
|
+
range(50, 60): "NatWest group (50-59: 50-55 NatWest, 56-58 Ulster)",
|
|
45
|
+
range(60, 70): "NatWest group (60-66) / others",
|
|
46
|
+
range(70, 80): "misc (77 Lloyds/TSB Scotland ranges etc.)",
|
|
47
|
+
range(80, 90): "Bank of Scotland (80-89)",
|
|
48
|
+
range(90, 100): "Bank of Ireland / NI (90-99)",
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def check_iban(iban: str) -> dict:
|
|
53
|
+
s = re.sub(r"[\s\-]", "", iban).upper()
|
|
54
|
+
out: dict = {"input": iban, "normalized": s}
|
|
55
|
+
if not re.fullmatch(r"[A-Z]{2}\d{2}[A-Z0-9]{10,30}", s):
|
|
56
|
+
out.update(valid=False, reason="malformed: expected CCkk + 10-30 alphanumerics")
|
|
57
|
+
return out
|
|
58
|
+
cc = s[:2]
|
|
59
|
+
expected = IBAN_LENGTHS.get(cc)
|
|
60
|
+
if expected and len(s) != expected:
|
|
61
|
+
out.update(
|
|
62
|
+
valid=False,
|
|
63
|
+
reason=f"wrong length for {cc}: got {len(s)}, expected {expected}",
|
|
64
|
+
)
|
|
65
|
+
return out
|
|
66
|
+
if not expected:
|
|
67
|
+
out["note"] = f"country {cc} not in built-in length table; checksum only"
|
|
68
|
+
rearranged = s[4:] + s[:4]
|
|
69
|
+
digits = "".join(str(int(ch, 36)) for ch in rearranged)
|
|
70
|
+
ok = int(digits) % 97 == 1
|
|
71
|
+
out.update(
|
|
72
|
+
valid=ok,
|
|
73
|
+
country=cc,
|
|
74
|
+
check_digits=s[2:4],
|
|
75
|
+
reason=None if ok else "mod-97 checksum failed (banks do not misprint IBANs)",
|
|
76
|
+
)
|
|
77
|
+
if cc == "GB" and ok:
|
|
78
|
+
out["gb_bank_code"] = s[4:8]
|
|
79
|
+
out["gb_sort_code"] = f"{s[8:10]}-{s[10:12]}-{s[12:14]}"
|
|
80
|
+
out["gb_account"] = s[14:]
|
|
81
|
+
return out
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def check_bic(bic: str) -> dict:
|
|
85
|
+
s = re.sub(r"\s", "", bic).upper()
|
|
86
|
+
out: dict = {"input": bic, "normalized": s}
|
|
87
|
+
m = re.fullmatch(r"([A-Z]{4})([A-Z]{2})([A-Z0-9]{2})([A-Z0-9]{3})?", s)
|
|
88
|
+
if not m:
|
|
89
|
+
out.update(valid=False, reason="malformed: expected AAAA CC LL (BBB)")
|
|
90
|
+
return out
|
|
91
|
+
bank, country, location, branch = m.groups()
|
|
92
|
+
out.update(
|
|
93
|
+
valid=True,
|
|
94
|
+
bank_code=bank,
|
|
95
|
+
country=country,
|
|
96
|
+
location_code=location,
|
|
97
|
+
branch_code=branch,
|
|
98
|
+
is_test_bic=location.endswith("0"),
|
|
99
|
+
note="format-valid; confirm bank_code matches the named bank and "
|
|
100
|
+
"country matches the branch address (D2/D3)",
|
|
101
|
+
)
|
|
102
|
+
return out
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def check_sort_code(sc: str) -> dict:
|
|
106
|
+
s = re.sub(r"[\s\-]", "", sc)
|
|
107
|
+
out: dict = {"input": sc, "normalized": s}
|
|
108
|
+
if not re.fullmatch(r"\d{6}", s):
|
|
109
|
+
out.update(valid=False, reason="malformed: expected 6 digits (XX-XX-XX)")
|
|
110
|
+
return out
|
|
111
|
+
lead = int(s[:2])
|
|
112
|
+
hint = next((v for k, v in UK_SORT_HINTS.items() if lead in k), "unknown range")
|
|
113
|
+
out.update(
|
|
114
|
+
valid=True,
|
|
115
|
+
formatted=f"{s[0:2]}-{s[2:4]}-{s[4:6]}",
|
|
116
|
+
leading_pair=s[:2],
|
|
117
|
+
bank_group_hint=hint,
|
|
118
|
+
note="format-valid only; the EISCD modulus check is not public. "
|
|
119
|
+
"Verify sort code ↔ named bank (D3), by web search if available.",
|
|
120
|
+
)
|
|
121
|
+
return out
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def check_aba(aba: str) -> dict:
|
|
125
|
+
s = re.sub(r"[\s\-]", "", aba)
|
|
126
|
+
out: dict = {"input": aba, "normalized": s}
|
|
127
|
+
if not re.fullmatch(r"\d{9}", s):
|
|
128
|
+
out.update(valid=False, reason="malformed: expected 9 digits")
|
|
129
|
+
return out
|
|
130
|
+
d = [int(c) for c in s]
|
|
131
|
+
total = 3 * (d[0] + d[3] + d[6]) + 7 * (d[1] + d[4] + d[7]) + (d[2] + d[5] + d[8])
|
|
132
|
+
ok = total % 10 == 0
|
|
133
|
+
ff = int(s[:2])
|
|
134
|
+
district_ok = ff <= 12 or 21 <= ff <= 32 or 61 <= ff <= 72 or ff == 80
|
|
135
|
+
out.update(
|
|
136
|
+
valid=ok and district_ok,
|
|
137
|
+
checksum_ok=ok,
|
|
138
|
+
federal_district_prefix_ok=district_ok,
|
|
139
|
+
reason=None if ok and district_ok else (
|
|
140
|
+
"checksum failed" if not ok else "invalid Federal Reserve district prefix"
|
|
141
|
+
),
|
|
142
|
+
)
|
|
143
|
+
return out
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def check_account(acct: str, country: str) -> dict:
|
|
147
|
+
s = re.sub(r"[\s\-]", "", acct)
|
|
148
|
+
out: dict = {"input": acct, "normalized": s, "country_assumed": country}
|
|
149
|
+
if country.upper() == "GB":
|
|
150
|
+
ok = bool(re.fullmatch(r"\d{8}", s))
|
|
151
|
+
out.update(
|
|
152
|
+
valid=ok,
|
|
153
|
+
reason=None if ok else "UK account numbers are 8 digits",
|
|
154
|
+
)
|
|
155
|
+
else:
|
|
156
|
+
ok = bool(re.fullmatch(r"\d{4,17}", s))
|
|
157
|
+
out.update(
|
|
158
|
+
valid=ok,
|
|
159
|
+
note="generic length check only (4-17 digits); no per-bank format applied",
|
|
160
|
+
reason=None if ok else "not a plausible account number",
|
|
161
|
+
)
|
|
162
|
+
return out
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def main() -> int:
|
|
166
|
+
ap = argparse.ArgumentParser(description=__doc__)
|
|
167
|
+
ap.add_argument("--iban")
|
|
168
|
+
ap.add_argument("--bic")
|
|
169
|
+
ap.add_argument("--sort-code")
|
|
170
|
+
ap.add_argument("--aba")
|
|
171
|
+
ap.add_argument("--account")
|
|
172
|
+
ap.add_argument("--country", default="GB",
|
|
173
|
+
help="ISO country for --account format rules (default GB)")
|
|
174
|
+
args = ap.parse_args()
|
|
175
|
+
|
|
176
|
+
results: dict = {}
|
|
177
|
+
if args.iban:
|
|
178
|
+
results["iban"] = check_iban(args.iban)
|
|
179
|
+
if args.bic:
|
|
180
|
+
results["bic"] = check_bic(args.bic)
|
|
181
|
+
if args.sort_code:
|
|
182
|
+
results["sort_code"] = check_sort_code(args.sort_code)
|
|
183
|
+
if args.aba:
|
|
184
|
+
results["aba"] = check_aba(args.aba)
|
|
185
|
+
if args.account:
|
|
186
|
+
results["account"] = check_account(args.account, args.country)
|
|
187
|
+
if not results:
|
|
188
|
+
print("nothing to check; pass at least one identifier", file=sys.stderr)
|
|
189
|
+
return 2
|
|
190
|
+
|
|
191
|
+
# cross-checks when multiple identifiers were supplied
|
|
192
|
+
cross = []
|
|
193
|
+
if "iban" in results and "sort_code" in results:
|
|
194
|
+
gb_sc = results["iban"].get("gb_sort_code")
|
|
195
|
+
if gb_sc and gb_sc != results["sort_code"].get("formatted"):
|
|
196
|
+
cross.append(
|
|
197
|
+
f"MISMATCH: sort code embedded in IBAN ({gb_sc}) != printed "
|
|
198
|
+
f"sort code ({results['sort_code'].get('formatted')})"
|
|
199
|
+
)
|
|
200
|
+
if "iban" in results and "account" in results:
|
|
201
|
+
gb_acct = results["iban"].get("gb_account")
|
|
202
|
+
if gb_acct and gb_acct != results["account"]["normalized"]:
|
|
203
|
+
cross.append(
|
|
204
|
+
f"MISMATCH: account embedded in IBAN ({gb_acct}) != printed "
|
|
205
|
+
f"account ({results['account']['normalized']})"
|
|
206
|
+
)
|
|
207
|
+
if "iban" in results and "bic" in results:
|
|
208
|
+
if (
|
|
209
|
+
results["bic"].get("valid")
|
|
210
|
+
and results["iban"].get("valid")
|
|
211
|
+
and results["bic"]["country"] != results["iban"]["country"]
|
|
212
|
+
):
|
|
213
|
+
cross.append(
|
|
214
|
+
f"MISMATCH: BIC country {results['bic']['country']} != IBAN "
|
|
215
|
+
f"country {results['iban']['country']}"
|
|
216
|
+
)
|
|
217
|
+
results["cross_checks"] = cross or ["no mismatches among supplied identifiers"]
|
|
218
|
+
|
|
219
|
+
print(json.dumps(results, indent=2))
|
|
220
|
+
return 0
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
if __name__ == "__main__":
|
|
224
|
+
sys.exit(main())
|