@youweichen/pi-harness 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/server/document-conversion/extension.js +164 -0
- package/dist/server/document-conversion/python/bridge.py +424 -0
- package/dist/server/document-conversion/python/fixtures.py +201 -0
- package/dist/server/document-conversion/python/native_code.py +149 -0
- package/dist/server/document-conversion/runtime.js +321 -0
- package/dist/server/document-conversion/service.js +340 -0
- package/dist/server/document-conversion/settings.js +63 -0
- package/dist/server/index.js +2 -0
- package/dist/server/native-tools.js +4 -0
- package/dist/server/okf/extension.js +118 -0
- package/dist/server/okf/markdown-sources.js +58 -0
- package/dist/server/okf/service.js +713 -0
- package/dist/server/okf/storage.js +165 -0
- package/dist/server/okf/types.js +1 -0
- package/dist/server/wiki-links.js +24 -0
- package/dist/server/wiki-service.js +37 -5
- package/package.json +2 -2
- package/web/dist/assets/{NodeWorkbench-BiP2ktQP.js → NodeWorkbench-DM6YTS5e.js} +1 -1
- package/web/dist/assets/{TerminalPanel-CKaSUsN-.js → TerminalPanel-aFJylyxg.js} +1 -1
- package/web/dist/assets/{abnfDiagram-VCTEODGH-CnwRJDRL.js → abnfDiagram-VCTEODGH-CcM7f7oX.js} +1 -1
- package/web/dist/assets/{arc-CG4Q7-W_.js → arc-Bv7vS5l5.js} +1 -1
- package/web/dist/assets/{architectureDiagram-5GKGNRK7-BrGqX_Rg.js → architectureDiagram-5GKGNRK7-Cl8QcB3w.js} +1 -1
- package/web/dist/assets/{blockDiagram-I7D4REHJ-C2t0PjZT.js → blockDiagram-I7D4REHJ-DarS2-4G.js} +1 -1
- package/web/dist/assets/{c4Diagram-7LVT6UL2-CadrcgGL.js → c4Diagram-7LVT6UL2-BY0_ttBt.js} +1 -1
- package/web/dist/assets/channel-DJGMgVTh.js +1 -0
- package/web/dist/assets/{chunk-2Q5K7J3B-CC3V2ITZ.js → chunk-2Q5K7J3B-Cz-zS7_X.js} +1 -1
- package/web/dist/assets/{chunk-5VM5RSS4-DqKSbGmE.js → chunk-5VM5RSS4-B7AyqkmI.js} +1 -1
- package/web/dist/assets/{chunk-F27PBJKO-_CDr3JbY.js → chunk-F27PBJKO-B7hVduzL.js} +1 -1
- package/web/dist/assets/{chunk-IMKFNOWR-BIR4H4fX.js → chunk-IMKFNOWR-DNGJqYn5.js} +1 -1
- package/web/dist/assets/{chunk-JWPE2WC7-CrsdBGXT.js → chunk-JWPE2WC7-B5iJgXvl.js} +1 -1
- package/web/dist/assets/{chunk-POPQ4Y6H-C6q9kQmf.js → chunk-POPQ4Y6H-BvWCkriR.js} +1 -1
- package/web/dist/assets/{chunk-SVP7TREG-l2XXTr2v.js → chunk-SVP7TREG-BcdlBuOQ.js} +1 -1
- package/web/dist/assets/{chunk-TICWLB2K-29I_mZgB.js → chunk-TICWLB2K-B1igqjQd.js} +1 -1
- package/web/dist/assets/{chunk-XXDRQBXY-htV9FBqi.js → chunk-XXDRQBXY-DKGDquaw.js} +1 -1
- package/web/dist/assets/classDiagram-ZZMXUADV-D9nxgF0M.js +1 -0
- package/web/dist/assets/classDiagram-v2-VYDZK3BY-D9nxgF0M.js +1 -0
- package/web/dist/assets/{cose-bilkent-JH36ORCC-Cop-ihyx.js → cose-bilkent-JH36ORCC-BU_9nu88.js} +1 -1
- package/web/dist/assets/{cynefin-OW5HDTMX-IEk8qBRq.js → cynefin-OW5HDTMX-DY0BDLjc.js} +1 -1
- package/web/dist/assets/{cynefinDiagram-5FMLGOSQ-Bmzt6Qb5.js → cynefinDiagram-5FMLGOSQ-B-zM25Fk.js} +1 -1
- package/web/dist/assets/{dagre-GXQ25YYZ-DiXTJYYU.js → dagre-GXQ25YYZ-BxooDyBc.js} +1 -1
- package/web/dist/assets/{diagram-S7CK7UJ4-BT7CLuNk.js → diagram-S7CK7UJ4-CCO-8de9.js} +1 -1
- package/web/dist/assets/{diagram-UQ7AKVKN-BZZ3nTCO.js → diagram-UQ7AKVKN-CmGsH525.js} +1 -1
- package/web/dist/assets/{diagram-VSXAHHWV-Dal8LeZh.js → diagram-VSXAHHWV-CSQJYo3m.js} +1 -1
- package/web/dist/assets/{diagram-VX7I27RA-D6V9FfNA.js → diagram-VX7I27RA-CwuJsF66.js} +1 -1
- package/web/dist/assets/{diagram-Z3DM3KII-KUAIcFiD.js → diagram-Z3DM3KII-BF13T_lj.js} +1 -1
- package/web/dist/assets/{ebnfDiagram-PWID7BFC-s0FEPZrV.js → ebnfDiagram-PWID7BFC-ZtoDLro6.js} +1 -1
- package/web/dist/assets/{erDiagram-RLTQ6QDP-CiIQPFCD.js → erDiagram-RLTQ6QDP-BBo-qmNH.js} +1 -1
- package/web/dist/assets/{flowDiagram-HODETNUW-CvG4MYWn.js → flowDiagram-HODETNUW-DAq9qdP8.js} +1 -1
- package/web/dist/assets/{ganttDiagram-EL5Y4UJY-CUf115ii.js → ganttDiagram-EL5Y4UJY-C53eRlMu.js} +1 -1
- package/web/dist/assets/{gitGraphDiagram-WWUBYQGX-o3K1r6fL.js → gitGraphDiagram-WWUBYQGX-BviBMJxj.js} +1 -1
- package/web/dist/assets/{index-C5or_GAq.js → index-Bab44wX8.js} +81 -81
- package/web/dist/assets/{infoDiagram-27XIBGKW-BxiBeQ8E.js → infoDiagram-27XIBGKW-DI95SO3j.js} +1 -1
- package/web/dist/assets/{ishikawaDiagram-5VMMS53U-B1ONuCkH.js → ishikawaDiagram-5VMMS53U-Cl8IDXwf.js} +1 -1
- package/web/dist/assets/{journeyDiagram-3NMN7TZE-DwmuoXcX.js → journeyDiagram-3NMN7TZE-Co6aJeTZ.js} +1 -1
- package/web/dist/assets/{kanban-definition-UXKFOSKX-pMKHubxH.js → kanban-definition-UXKFOSKX-CknQQpwW.js} +1 -1
- package/web/dist/assets/{layout-CEwfjQR0.js → layout-CFtUmtTW.js} +1 -1
- package/web/dist/assets/{linear-D22Gdbt1.js → linear-mRlGUAKy.js} +1 -1
- package/web/dist/assets/{mermaid.core-CO2nrD04.js → mermaid.core-CvQEO3ue.js} +5 -5
- package/web/dist/assets/{mindmap-definition-YA3MSWOX-9MQn9ZAp.js → mindmap-definition-YA3MSWOX-DpqhsxUl.js} +1 -1
- package/web/dist/assets/{pegDiagram-XKGWAZYB-Dy6fLjM-.js → pegDiagram-XKGWAZYB-C9eawB5u.js} +1 -1
- package/web/dist/assets/{pieDiagram-E7YTZNPT-EDLez6nA.js → pieDiagram-E7YTZNPT-8vaqXVD6.js} +1 -1
- package/web/dist/assets/{quadrantDiagram-AXDQQJYC-Cgcd9mvy.js → quadrantDiagram-AXDQQJYC-Diwr8ynm.js} +1 -1
- package/web/dist/assets/{railroadDiagram-O6MQD6OU-B4Td0tjj.js → railroadDiagram-O6MQD6OU-krW50Cgi.js} +1 -1
- package/web/dist/assets/{requirementDiagram-BXWQKSXE-aF2QNZ78.js → requirementDiagram-BXWQKSXE-BPLI6eMG.js} +1 -1
- package/web/dist/assets/{sankeyDiagram-P5KCCOFB-DgTYwYMa.js → sankeyDiagram-P5KCCOFB-Bix_elmH.js} +1 -1
- package/web/dist/assets/{sequenceDiagram-WJ2MYXX4-EUcddWUc.js → sequenceDiagram-WJ2MYXX4-0m0xaUKi.js} +1 -1
- package/web/dist/assets/{sizeCapture-INFHLROL-CXHOv9Sx.js → sizeCapture-INFHLROL-BtzJ_oOR.js} +1 -1
- package/web/dist/assets/{stateDiagram-D77RDMKH-B-xlgfaK.js → stateDiagram-D77RDMKH-Clq5bN0v.js} +1 -1
- package/web/dist/assets/stateDiagram-v2-MP3YSRHH-Bn1oW4gp.js +1 -0
- package/web/dist/assets/{swimlanes-42K2YHIH-BtFBOCDh.js → swimlanes-42K2YHIH-11MDGnYA.js} +1 -1
- package/web/dist/assets/swimlanesDiagram-VR7AAH4N-BW96Edgo.js +8 -0
- package/web/dist/assets/{timeline-definition-24CTP7MA-CZlQY6pb.js → timeline-definition-24CTP7MA-BeQfWP95.js} +1 -1
- package/web/dist/assets/{vennDiagram-4TSXK5OY-Cn61dJvL.js → vennDiagram-4TSXK5OY-BMYkX0KL.js} +1 -1
- package/web/dist/assets/{wardleyDiagram-VM6X3IG4-fiMNrWpr.js → wardleyDiagram-VM6X3IG4-BV3fTUkU.js} +1 -1
- package/web/dist/assets/{xychartDiagram-S5SC5T6Z-CuzM_sof.js → xychartDiagram-S5SC5T6Z-Ct7ezLnc.js} +1 -1
- package/web/dist/index.html +1 -1
- package/web/dist/assets/channel-C27Jizb6.js +0 -1
- package/web/dist/assets/classDiagram-ZZMXUADV-DYTBrAEP.js +0 -1
- package/web/dist/assets/classDiagram-v2-VYDZK3BY-DYTBrAEP.js +0 -1
- package/web/dist/assets/stateDiagram-v2-MP3YSRHH-kwAwAlWX.js +0 -1
- package/web/dist/assets/swimlanesDiagram-VR7AAH4N-D5j5Cj3S.js +0 -8
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Small synthetic fixtures; these verify engine wiring, not real-document fidelity."""
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from base64 import b64decode
|
|
4
|
+
from io import BytesIO
|
|
5
|
+
|
|
6
|
+
# Self-authored raster caption keeps the Chinese OCR check independent of OS fonts.
|
|
7
|
+
CHINESE_CAPTION_PNG = (
|
|
8
|
+
"iVBORw0KGgoAAAANSUhEUgAAAa4AAAA8CAIAAACxTskxAAAegUlEQVR42u1dZ1gUV9uepYgsiIii2AVsoIIVFSwgIMUCEV+NEWKi"
|
|
9
|
+
"gmBDMKBIFLGESwFRQBGNSoIGX5RLUQELFlBiFBSVKgLSO7vALkvb8v04n+eazO7Ozi6L0Tfn/sE1zJw5c+bszD3PeSpNIBBgCAgI"
|
|
10
|
+
"CP9uKKApQEBAQFBCU4CA8OVj69atysrKbm5uU6ZMQbOBpMJ/Ej/99JOjo2NsbGx+fj6Px5Njz/v27Zs5c+by5cudnJx+/fVXqc69"
|
|
11
|
+
"cuXKuHHjFixYYGtru3r16qqqKnmNau/evQkJCQ0NDT09PZ9hegUCgb6+vpmZ2dKlS+3t7W/evCmyWVJSEo1Go9FoampqWVlZ+EOL"
|
|
12
|
+
"Fi0Ch1asWCHfsf3888/6+voLFixYtmyZr6+vvLotKChQVVXV0tJSU1MbNWpUZ2enuJY8Hi8+Pj48PHzq1KkjR468evUqeh/75BFE"
|
|
13
|
+
"kIi2trYBAwbASfvll1/k2HlERAToduTIkc3NzVKdm5OTA0cVHBwsx1F5eHjAnv/zn/98hkmeNGkSuNycOXNaWlpEtnn9+jVoY2pq"
|
|
14
|
+
"Sji0bt06cCg5OVm+AwsNDYVT8fr1a3l1i+c+8t/u3r17sKWFhUV7ezt6JeUO6aRCNptNk4Tly5fLzMuOjo40eYPJZJJc8cGDB5aW"
|
|
15
|
+
"lmlpaeQDO336NIvFAtuRkZF+fn5y/Bppa2uDDTMzMy0tLXHNcnNz16xZ09XVhd85atQouL148WI5jmrIkCFgQ09P77fffvsMX+XB"
|
|
16
|
+
"gweDjR07dgwcOFBkG01NTbBBo9EIhzQ0NMQd6iUGDRoENuh0+vTp0+XVrYqKirq6OtjW0dEhafnHH3/AkcTFxdHpdCTDfQUL5D//"
|
|
17
|
+
"/PPLsUorKirC51gYNTU1zs7Ojx49Mjc3Nzc3F0eIZWVlhw8fBtt79uzZtm2byGZ8Pr+np6ejo4PFYjGZzIaGBnEs3NLSEhYWBoUC"
|
|
18
|
+
"+GQrKyuDDQ6HM3PmzLNnz5aUlIA9Fy9eNDExuXbt2r59+/BdDRw4EL758qUASEZTpkxRVVUlHK2oqHj8+HFhYWFNTU1bW1tXVxeP"
|
|
19
|
+
"x4O/u0Ag6O7uZrFYdXV1BQUFqampFy5cIIyc5Ip4AZwANTU1cYeEBykvQMJSVVWV7ySTPJwQTCYzPj4ebB88eHDYsGGItv6BBXJ7"
|
|
20
|
+
"e7uFhUVcXFxXV5dAIICSETnevXsnm4zq4OAg37uj0+kkl3N1dSW0t7e3LywsxLfh8XjW1tayXf3MmTPCF/X29gbv89WrV8GeBw8e"
|
|
21
|
+
"gPbr168He/BasMuXLwsEgpMnT/7/t0tB4fnz5/gOITtkZmbKcb1w+vRp0K2Dg4Pw0djYWGlnQ09Pj/yKq1atAi1v3LgBdzY0NKSn"
|
|
22
|
+
"pxcVFTU1NXE4nObmZihBE07fs2cPOJSSksLj8fBcnJGRQbIe7yP4+/tT1wnExsaKaxMSEkL9ovr6+nw+H612ZYAEKoTaa3V1dWtr"
|
|
23
|
+
"awaDIa4ln88fMWIEaHzgwIF/9q78/f2hEEHSjMvlRkdHE9YmdDq9trYWtpF5LdyvX7+mpibhi65evRrSLtjz9OlTAhWGh4eDPebm"
|
|
24
|
+
"5nB6LSwsoJjW09MDO4TClHyp8MKFC/KlwtmzZ5Nfcf369cJUmJKS0ntWGjt27JdJhdOmTSOnQj6fP378eOoXDQ0NRaTWJ7pCSIVd"
|
|
25
|
+
"XV3btm0jkedpNJq9vT3Y/losXIqKim5ubh8+fPD29oYLn4CAAEiOERERQUFBJIs1AwMDXV3dESNGaGlp0el0RUVFeNTOzg4qv/DY"
|
|
26
|
+
"vHkz2Lh//35TU5PInh8+fAjNuHB6o6Ki+vXrh2FYXl4eFNkwDFNS6hOPKLl3K1HDBfUD/x5InOTExMTi4mKKvQ0YMGDTpk1opSt/"
|
|
27
|
+
"XSGLxbp27RrY9vPzk7h6/eabb8BGUVHR3bt3v5YpUFdXDw0NTU5OHjp0qJ2dnY+PD9h/8uTJnTt3Yp/sldBsV1hYCHULT58+LS0t"
|
|
28
|
+
"ra6ubm5ubm9vf/78OezWxcVF5OWsra2BrYPL5YoUedhs9v379zEMGz16NH5tPmnSJDiewMBABoMhx0l4/fr127dvq6qqWltbu7u7"
|
|
29
|
+
"+Xw+XikGFpuNjY0lJSUZGRlsNtvCwuLp06cFBQWVlZWNjY0tLS0cDqerq6unp4fL5fJxsLGxgWKyxC/T5/zdo6Ojnzx5kpWVVVBQ"
|
|
30
|
+
"AH/EtrY2cCNcLpfH44FbuHHjBjTsgD08Hq+np6e7u5vD4bBYLAaDUV9fX1FRUVRUlJOT8/z58wcPHgg/AF1dXUlJSW/evKmrq2Oz"
|
|
31
|
+
"2VwuF04yn89vbW2trKzMzs6+c+cOl8sFTwh+UfLy5UuR4szatWtBA1dXV3HmJoRe6QrPnj0L2owYMYLNZkuUMLlc7siRI7FP1swv"
|
|
32
|
+
"f4FMQG1tbWNjI9gODg6GUzR9+nQmkwmb7dixAx7av38/vgdbW1vsk6Gzs7NT3IUgo61Zs0Z4gRwTEwP+9fHxIZzIYDCgYP7TTz+B"
|
|
33
|
+
"nVD27M0CGRIWRs0hToaebWxsCIfS09Nfv35dWVnZ0tLS3d3t5uYGWl6/fp3BYBQXFz98+LC6ujojI6OoqKiurg7Yo8TpCuGPnpyc"
|
|
34
|
+
"3NPTAxSLpaWlmZmZ9+/f783jhKfC3vTT2NhIcYZZLJZAIIiKisLv/P7774X7LCgoUFBQACIhfHoRZIBY+ZzL5UI68PPzI7Hc4b/q"
|
|
35
|
+
"mzdvDgwMxDAsLS3t5s2bjo6OVH749vb2pKQkTU1NTU1NdXV1dXV1Op3er18/ZWVlZWVlBQUF8GPjbbXAXAue+La2tubm5vr6emtr"
|
|
36
|
+
"ayrjFAe80tDLy2vatGk7d+5UVVVNTU2FPhwfPnyIjo4Gyz0OhxMcHLxu3ToDAwNwF5MnT1ZWVi4pKbGyslJRURF3IScnp/DwcHV1"
|
|
37
|
+
"deiwggekwqVLlwobHHfu3BkYGEin0+FX5+uFl5fXq1evhPdDdSogBVNTU7zILLFbGo2mpKSkpKQEHJh1dXVJGpeUlOTn52tqampo"
|
|
38
|
+
"aIBnr3///uDBU1RUBP5YIgUIIBt2d3eDh5DNZrNYrJaWltbWVicnp95PDpvNPnjwIH7P1atXDx06NHbsWPxOT09PPp+PYdju3btF"
|
|
39
|
+
"Pk4IvZUKL168CEXCjo4Oisza3NwMPSFGjRrV1tZG5ayPHz/K5V4+fvzYG6kQuIAEBQUdPHgQ/Mtms/E+z11dXbNnzwY9JyQkAH9A"
|
|
40
|
+
"IyMjvMxIBTwe7/r16xwOR5zZpKys7Pz587ABQbJwcnKCd/pVS4WzZs2iKB9BQB8GEqkwJSWF+vAIklefvlNSSYXQxZ1Op0OP0R9+"
|
|
41
|
+
"+AHfIfT3HD16NGGiEORjQWYymdB96dSpU1L1+PPPP8NfdOXKlVRM+18IFT569AhY9IRfM7D8h197W1tbfBCCiYlJfX29zL+BMBVK"
|
|
42
|
+
"BblQYWZmZnZ2dkVFBZPJ7Ozs5HK50Ebs4ODQ2dnJZDJramoKCwvT0tKkeusQFeLDS27evPnXX38VFxc3Nzd3dnYaGxuDU3777Tc2"
|
|
43
|
+
"m11TU/Pu3bt79+7h4w5jY2PfvXsHhFMajfbs2TPQW35+PnB4pNFoqampiMv6hAqh1kZXV1dYJKypqSF3RcTL8N7e3l8+Fb548cLK"
|
|
44
|
+
"ygq/wiovLyd8G6AeUEdHp6qqCpCjiYkJ3JmYmEhFar5z505mZmZFRUVLSwvwTMZTITBQ1NfXl5aWvnr1KiUlBSqAeDyeSIdNuVAh"
|
|
45
|
+
"ibuMSGcauVDhvXv3nj179uHDh8bGRg6Hs2XLFtDy2rVrDQ0N+fn5SUlJeLchILaHhYWFhYXFx8cLax6DgoKCgoJKS0u/TCokoLq6"
|
|
46
|
+
"Gmp+8M40ubm5MHLG3d0d7ISm4YkTJ7a2tjY1NUFnIE9PT0RkfUKF0GoMFNiEoz09PYMHD161alVWVpa4TpOSkvBPhqurK5fLJRkE"
|
|
47
|
+
"m82+fv36/fv3X7x4kZeXV1ZWBo2SeIskUNDw+Xwul9vd3d3e3s5gMCorKwsLC58/f56YmAhtOxSpkM/n37p1y9zcHD/aiRMnEoJY"
|
|
48
|
+
"k5OTIblraGi8ePEC/zSPGTMGnjt37tyYmBhx8bNA+JL2vXr69Ck4cfbs2erq6gSO/qqpkABIhXi/QiogBPlUVVW9f/8+MzPzwYMH"
|
|
49
|
+
"cgkZlpfZhIDjx4/jRT+4v6OjIyEhwcLCYtq0aVAQYTAY8CG0s7ODipolS5aA8AcEOVNhTk4O/CKJtALjaW7WrFkiAyoEAsHu3bvx"
|
|
50
|
+
"77OFhYXwO/wPWpDLysoCAwMJCnU1NbVjx47hH6zMzEw7Ozu8XUXYoaG0tJSQN0lZWXnWrFnu7u4nTpwgRIbIRoWdnZ3Dhw8XR0z/"
|
|
51
|
+
"81T4/v17Jycnc3PzGTNm6OvrDx8+XFNTExg3CPY0TFTWny+TCvl8/oQJE/BUWFlZuX37dhC3ChdY+FNevnxJsAoaGxu3trYiFpM/"
|
|
52
|
+
"FVZUVEC7ZL9+/XJycoRPgM6DYCEJLQzCH2pLS0vs79HyERERn+cLJpEKYTIYiFWrVlVUVMAGaWlpCxYsIPgDitMMtLW1ubq6Cpsa"
|
|
53
|
+
"VVVVCXMos1SIT4jw8OHDf4oKwXL+M1NhQUGBzGvVY8eOiVRT3Lhx4/Hjx69evSoqKqqqqmpqampra+vo6ABLECpUCL0LOzs72Ww2"
|
|
54
|
+
"EEjLy8sLCwuzs7OfPn2anJxMorUErqN4oznwuyT3QsMr4jEMW7duXUNDA2IxOVNhcXExXko6cuSIcOvKykp8SEBISIiANLEVFOMh"
|
|
55
|
+
"xowZk5aW9o9TIZ/Ph0w3bty4pKQkQgMOhwPTLkyePPm///2vxItmZGTA2DiAw4cPE9q0tLSkpqYCAwWDweBwOOXl5TDkAOgK2Ww2"
|
|
56
|
+
"0BVmZ2c/fPgQmLD5fL6RkRH2KYfVP0KFd+/eNTIywjOamZkZdVaSlgpbW1sPHjzY2dnZ0NAgMxWGhYXJRU0hGyZNmiTuZvG6aQzD"
|
|
57
|
+
"IiIioMfozZs3hduXlpbipRC8hOHj4yOVTR+BjApfvnwJl2AYhllaWoJEIwTASAwMw+bNmyfROtzU1DRnzhz8L7d27dru7m5x7YFP"
|
|
58
|
+
"ogyIiYmR1sX62bNnysrKvr6+JNnfoqOj7927J3IqxCErK8vNzW3IkCF0Op0kZBsiICAA3gW5BRnKJiAe63NSYXJy8vz588G/y5cv"
|
|
59
|
+
"72sqjIqK8vf3HzRokIaGBvimimMWGLZM4IKwsDDsU0a1L5AKX7x4gf09GU9sbOzRo0ehuICPaMjPz3d1dcW7qYp0dTQ2Nt61a9eN"
|
|
60
|
+
"GzdKS0tRRgYZqfDWrVv4iR43bpxI1xAmkwl/ORqN9tdff1G5AIvFgh/AqVOnki+Qpc3hjOHSDsoQbVJWVga3YeqX3gDSBI/HKy4u"
|
|
61
|
+
"pqLvhzksIBXy+fzjx49XVlYKhLwRYWT+/Pnz+5oKf//9d5HxcFZWVn1NhXin/c7OTpiiUQYqjIqK+gKpEFjqjI2NoTtRbGxsfX09"
|
|
62
|
+
"8Ot2d3dvaWnp6OiIj4+3tbUlEN+UKVMyMjJu3rwpMsIduiKKTMaDICBPx2BraxsYGAjSvWlqaiYlJQ0dOlR4fk+cONHa2gq2nZ2d"
|
|
63
|
+
"586dSzHCNyUlxdPTE8OwM2fOkAeikiSqIwc09WBSJizBhPKn9gYw8aqCgoK+vr7E9teuXaupqcH+nspw5cqVvr6+0J8JQkFBAUwj"
|
|
64
|
+
"hmGtra1QVpIvPn78GBERYWNjAzOYgeoF/fr1c3R0jIuLS0xMxGvi0tPTc3JySktLa2pqmpubWSwWIYaX3Hmby+W+evUqNDTU3t6e"
|
|
65
|
+
"kO1m5syZe/fu5XK5EoOXMXnnyBF88qiHAT9KSkqPHz8GmhPqbxcMV8cjLi7uyZMn4CXC7x86dOjRo0cfPnxobm7u6empo6OzZs2a"
|
|
66
|
+
"u3fvwiyQAwcOPHHixJs3b0xNTR0cHHJzc2H0sXDc1Lx581D8iIw+UC9evBg+fDjB4onXNEO6GTBgALlroUCMI7HENrdv36buD4hP"
|
|
67
|
+
"nkiwTsjgYn39+vXeT6abm5tAmsgWQJdQtlq/fj14SeA7IxDyOhozZkxwcDBeyYCXCpubmxMTEz08PCi+7fjoF3d3d5H5oIYPH37q"
|
|
68
|
+
"1CmRCcd6aTbZvHmzyHQ1Li4uhKyR8pUKKfIgPmw0Kiqqrq7O09Nz+PDh4tgwJiZGoh68ubkZyBnKysq1tbV4qVDw9/h0PPr37+/t"
|
|
69
|
+
"7S3yJ3jy5MnChQsJ7eVbcOLfaEEmybng5eWFSarDwGAwEhISejMamKlFWiokvAkyUCGJw0RLS0tycjJJrDvM2bNlyxbqNwuTEp4/"
|
|
70
|
+
"fx6/QP7xxx9hyKNw5CJBw9De3g7V7aNHj8ZntCbRyQqDJE993znTiCuZJOxX+JmpsLW1dcmSJXA8R48e7enpgc4V58+fF4jK7AkC"
|
|
71
|
+
"n48fP06iqoOS4Lp16wS4eBtIhd3d3fj1lpqampeXV3V1NfmA09PTv/322+nTpy9btkxFRYWKcgaBLF+huFwGZWVlZ86cgeZUuEyD"
|
|
72
|
+
"6OzsDA4O1tfXX716NZTs/meQl5dnb2+vra09aNCgWbNmlZaW9r5PBoMBigTMnz9/8uTJ+EMhISFgoV1TU3Po0CHCiRwOJy0tLSws"
|
|
73
|
+
"zMXFZcqUKRoaGjBZC1AvQmG/vr6e+nhgFkWgKwgICMDn5ukjLFq0CIarBwcHf/fdd1/Cz52fnz9v3rxHjx6Bf4OCgvbt26ekpLRr"
|
|
74
|
+
"1y7sU/Y2kAEB4tdff129ejWXy+Vyub6+vkuWLBFZy+H8+fOXL18G2yKlPyAtxsXFaWhozJgxw8DAoL29PSwsbOTIkeQFfBYtWnT1"
|
|
75
|
+
"6tU5c+bcuXOHyWRSUc4gYDLUQfb29oaq68jISOEUmx8/fty/fz9o88MPP2RnZ+NjML52VFdXQ0Weg4ODnp5e7/vcsmULCM4X/q5o"
|
|
76
|
+
"aWkFBAR4enrq6urOmjUrKysLZMHLzc199+5dRUUFRloexMDAQE9Pb+zYsSSpcTBRWRTBpf38/LZu3Uqn0+FL23cYN27c2LFj/fz8"
|
|
77
|
+
"Nm7cqKys7O7uTt7+/fv34kqLgORAvVcWnTt3ztvbm8PhAKNNZGQkHNWOHTsiIyPLy8vz8vLi4uKAQNrW1rZz507o9TlmzJjt27fP"
|
|
78
|
+
"nz+/f//+hM7fvn0L07uZm5uT6PJ0dXXfv3+vo6Pj6uoqlU8lWB/0XY2XfzsVpqamwsWjs7MzwXEaPoiBgYEg5TKDwVi3bl16evpn"
|
|
79
|
+
"zsTZd/jw4QP2yWXa29u79x3GxMQA1eT48eNXr16NT/gKiZLFYnl5eW3atIk8JXj//v1BplUMw+7cubNs2TLZhjRx4kQfHx8/Pz8q"
|
|
80
|
+
"hYfkaGD98OHDF5K8+s2bN9u3b8/IyIDS2a1bt2DgOYZhKioqhw8f/v777zEM27t3r6OjY2pq6rZt28CXUkdH58CBA5s2bRJn5Bk1"
|
|
81
|
+
"atSgQYPq6uowDNu/fz9GIV+ctKT2L0wDjn22inccDmfr1q3wm4MvCIsJVUyHuQn+/PNPkjz4GLUMhhKrespsbpYW0Ats6dKlsPiZ"
|
|
82
|
+
"zMjJyYGLI19fX5EfDBUVFX9/fzqdTkhOBzBhwgRnZ+fIyMjMzMy2tjaYS7GXtdCOHz/+OXkQmGWlentl0BVSQXZ29po1a2bOnAl5"
|
|
83
|
+
"EMMwDQ0NPA9CUQD4wVRVVU2bNs3R0bG6ulpNTS0gIKC4uNjDw4PE2D148GCQC9nCwgKviCR7OT+FFZI7nAo7ISHIXyrcs2cPlImC"
|
|
84
|
+
"g4NFOtlgn1zAoqKi5syZAySUwMBAW1tb4WgTcvT09Mi8tOnlRBC0Pxgul/2zZ88woZSiIgH8TkhQXl5ua2sLDD6GhobQQoKRuvvQ"
|
|
85
|
+
"aDRjY2Nzc/NFixYtXLiQkKETrx9EDzR1NDY2JiYmxsTE4BlQXV2dJDssjUa7dOnStGnT2Gz2x48faTSai4tLUFAQ3jmUBA4ODosX"
|
|
86
|
+
"L8ZnYUD4OqTCx48fw3JCNjY2EivIzJw5E36guFyui4sLLPVLER0dHbLdRnd3dy8nglBqHcNZM0EVEXV19ZUrVwKNQU5OjlSdQFHX"
|
|
87
|
+
"xsYGOhKeOHFCYpWfhQsXXr58ubGxMTs7Oyws7JtvvhHOVAzqYJCwOQImVJR1xowZQBmH58FVq1a9fv0ak6TfhGsjZWXlFStWUORB"
|
|
88
|
+
"7JMnqbTyAcI/TIUsFuvHH38EgoaGhgZ0+CAQUFNTE4iWTUtLu337tqGhIVRsFxYWHjhwQKrRQBduaZ1pxBVfl+HSBP0AVOt4eHho"
|
|
89
|
+
"aGhUVFQ4OTkZGRlZWFjcuHGDIAaK7ATDGeihXOnk5EQlcfTUqVPXr19PEleAF0UlyqTyQlFRUWpqKrDn1NbWgtAIkIftq3jonZyc"
|
|
90
|
+
"Wltb8V+OSZMmJSUlJSQk4MNPMfHeo+Cj2N3dvWbNmn379lFfzcjFkx/hcyyQi4uLGxoaWlpaYmJiysvLsU8Oa/7+/iDCAQBskwtB"
|
|
91
|
+
"QPBxcnKiGJeCYZi4YpgSQYjZwGQqWwycvKDqjcfjbd68GegHtLW1Qfz15s2bQZjHkydPnjx5Mnbs2G3bti1YsGDcuHFAi0d+lSNH"
|
|
92
|
+
"jjQ0NCQmJsoxYyhkQCge9jXi4+MlKv7lqMToDcWLvOiwYcOio6NBJMmIESP8/f1dXV0lai3LysrOnTtnaWlpaWkZFxdnaWkJYk+D"
|
|
93
|
+
"goJu37598uRJkRZFihohpNz44qgwMjLy1KlTmJBjXV5enmwP8aZNm7Kzsylqx6XKQaKqqgpdsg0NDakvVEVi48aNGzduxHB+G25u"
|
|
94
|
+
"bunp6UBDdOHCBfA9/+WXX4YNGxYfHw+W5OXl5b6+vnQ6fcOGDV5eXhKpEMOw8PDwDRs24KUD+KrLtsLtIyqE3YrzX5H5tf9sJ5JM"
|
|
95
|
+
"qbW19XfffWdmZrZx40ZhxxfCPCQnJ589e/bu3buwQDudTk9KSlq0aBF4L3Jzc62srBYvXrxr1y57e3vqkYLw1sjp/sqVK1euXEFs"
|
|
96
|
+
"9VmpcOfOneHh4VS+UUpKSkOHDtXW1oZ/tbS0tLS0Bg8eXFtbC51O8vLyQkJC8OVcMQoefBi1gu7CNj7w9b516xbF2rvCyMzMjIiI"
|
|
97
|
+
"iIuLA8yioKAQHR29YsUK7FP4amxsbEhIyKlTp06fPg0kRA6HExUVBZr5+PiQJyno378/oQGkMBneeYFAAE8H3nC9oSpFRUVotXz5"
|
|
98
|
+
"8iX2yZwthcJFQUFBQQGY+EEMMnyWyO+Oz+dDAx2BfOF9yeBXSHJRieRSW1t74cKFc+fOVVZWYrhkkeCTqaWllZGR4ezsfOfOHXAo"
|
|
99
|
+
"LS0tLS1t4MCBVlZWpqamBgYGJiYm5MoNODyRlC2tqNh7jTmGYpDxwKfVotPphoaG9vb27u7uR44cuXTp0r1793JychoaGsgTAeEr"
|
|
100
|
+
"+GhoaJDkuBfIIyGwmZnZ0KFDR48eTdDFGBgYUOmhrKwsISHB09OTEISrra1NklyeyWQeOnQIpmDAcLYOQkkAipHX+PxXFIGnvz/+"
|
|
101
|
+
"+KM301hdXa2lpaWtra2np4d/gbdt20ZomZeXl5KSkpWVVVJSUl9fD0oviKvZAFWiJiYmhEMdHR1Tp07V09MzNDTEX5Ewe1BRIwNE"
|
|
102
|
+
"pt2kooBWUFAgGLU0NTU9PDwIdSz4fP7BgwdFmr+WLl0qsXo4NLmcPn1a+CjMmEkR27dvR/Fz8qxtEh8fHxoampqaCooZyQaY3cDa"
|
|
103
|
+
"2jovL6+vb0NcMoUdO3YISEv8GBkZifRPVFFRcXV1pZKDgMlk7t27l+AN++2331If/KVLlzAxVdwEFOpGwYuePXu2l9NIKLwL8Pvv"
|
|
104
|
+
"v8slBllfX19kCgNMVOgbvs3bt29lpsLdu3dTHGdJSUl4eLiwvxSNRrOysoqLiyMpgZuXlwfCdTBc9kCJPCgQCOCzJzLbNtTYUPQr"
|
|
105
|
+
"dHFxQYwm54p3vQeXyzU3NxeZj7cvwOFwhL/MkyZNqqurE5CmCxR2dp0wYUJAQAD5iSJFqg0bNoDl2/jx4ykWgAY4duyYVDKsQEyx"
|
|
106
|
+
"wKNHj/ZyGoWj+tauXUteoos6FQ4cOFBkeh68IK+kpLR3715Cm9TUVIlJoQmALtbAC4IK3r59S3iEBg8e7OPjU1JSQrGHpKQkkMZc"
|
|
107
|
+
"UVHx7du3AgrZKuG1hO9aIBCsWrVKKirEp5JEkBZKfbTuVlRUfPz48Wdb5quqqhoYGOTk5CgpKeno6EydOnXZsmUbN24UmQMK/+Jd"
|
|
108
|
+
"vHjRxMRkzJgxxsbGc+fONTc3p2L6EMaIESNiYmJcXV1379595swZqSJhhgwZAvhC2GcQo1AmuPf2d4jRo0cbGhoqKioaGBjMmDHD"
|
|
109
|
+
"zs4OFuqVGRMnTgQDA9pDQnQNcM179+7dnDlzTE1NbWxshN1NQLCaVBgwYABIJCOxDhSEkZGRh4cHKHpjaGjo7e29fv16cosKAfb2"
|
|
110
|
+
"9vb29rm5uenp6bD6AkaakgNu46V7iKqqKirXnTJlCnh+RMYmIVAE7X/Jii8QCORl7kT4F6K4uNjZ2Xnfvn0rVqxADxKiQgQEBAQM"
|
|
111
|
+
"RZsgICAgICpEQEBAQFSIgICAgKgQAQEBAVEhAgICAqJCBAQEBESFCAgICIgKERAQEBAVIiAgICAqREBAQEBUiICAgICoEAEBAQFR"
|
|
112
|
+
"IQICAgKiQgQEBAREhQgICAiIChEQEBAQFSIgICAgKkRTgICAgICoEAEBAQH7PzsHvqY55zH8AAAAAElFTkSuQmCC"
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def make_fixtures(root):
|
|
117
|
+
from PIL import Image, ImageDraw
|
|
118
|
+
from docx import Document
|
|
119
|
+
from openpyxl import Workbook
|
|
120
|
+
from pptx import Presentation
|
|
121
|
+
from pptx.util import Inches
|
|
122
|
+
files = []
|
|
123
|
+
# Raster PDF exercises the actual OCR path without a hidden text layer.
|
|
124
|
+
image = Image.new("RGB", (1000, 600), "white")
|
|
125
|
+
draw = ImageDraw.Draw(image)
|
|
126
|
+
draw.text((80, 70), "TOTAL 123", fill="black", font_size=60)
|
|
127
|
+
draw.text((80, 180), "Document conversion fixture", fill="black", font_size=32)
|
|
128
|
+
image.paste(Image.open(BytesIO(b64decode(CHINESE_CAPTION_PNG))), (80, 260))
|
|
129
|
+
draw.rectangle((75, 350, 700, 520), outline="black", width=3)
|
|
130
|
+
draw.line((75, 430, 700, 430), fill="black", width=3)
|
|
131
|
+
draw.line((400, 350, 400, 520), fill="black", width=3)
|
|
132
|
+
draw.text((95, 375), "Name", fill="black", font_size=32)
|
|
133
|
+
draw.text((425, 375), "Value", fill="black", font_size=32)
|
|
134
|
+
draw.text((95, 455), "Total", fill="black", font_size=32)
|
|
135
|
+
draw.text((425, 455), "123", fill="black", font_size=32)
|
|
136
|
+
path = root / "scanned.pdf"
|
|
137
|
+
image.save(path, "PDF", resolution=120)
|
|
138
|
+
files.append((path, "123"))
|
|
139
|
+
# A born-digital PDF preserves text independently of OCR.
|
|
140
|
+
objects = [b"<< /Type /Catalog /Pages 2 0 R >>", b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R /F2 6 0 R /F3 7 0 R >> >> /Contents 5 0 R >>", b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"]
|
|
141
|
+
chinese = "公司技术资料".encode("utf-16-be").hex().encode()
|
|
142
|
+
stream = b"BT /F1 18 Tf 72 720 Td (Document conversion fixture 456) Tj ET\n"
|
|
143
|
+
stream += b"BT /F3 16 Tf 72 680 Td <" + chinese + b"> Tj ET\n"
|
|
144
|
+
stream += b"72 500 400 100 re 72 550 m 472 550 l 272 500 m 272 600 l S\n"
|
|
145
|
+
stream += b"BT /F1 14 Tf 85 570 Td (Name) Tj 200 0 Td (Value) Tj ET\nBT /F1 14 Tf 85 520 Td (Total) Tj 200 0 Td (456) Tj ET\n"
|
|
146
|
+
header_stream = stream
|
|
147
|
+
code_lines = ["def calculate_total(values):", " return sum(values)", "", "def print_report(values):", " total = calculate_total(values)", " print('Total:', total)", "", "print_report([1, 2, 3])"]
|
|
148
|
+
stream += b"0.95 g 62 284 470 164 re f 0 g\nBT /F2 12 Tf 72 430 Td\n"
|
|
149
|
+
for index, line in enumerate(code_lines):
|
|
150
|
+
if index:
|
|
151
|
+
stream += b"0 -18 Td\n"
|
|
152
|
+
escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
|
|
153
|
+
stream += b"(" + escaped.encode("ascii") + b") Tj\n"
|
|
154
|
+
stream += b"ET\n"
|
|
155
|
+
objects.append(b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream")
|
|
156
|
+
objects.extend([b"<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>", b"<< /Type /Font /Subtype /Type0 /BaseFont /STSong-Light /Encoding /UniGB-UCS2-H /DescendantFonts [8 0 R] >>", b"<< /Type /Font /Subtype /CIDFontType0 /BaseFont /STSong-Light /CIDSystemInfo << /Registry (Adobe) /Ordering (GB1) /Supplement 4 >> /DW 1000 >>"])
|
|
157
|
+
def write_pdf(name, content):
|
|
158
|
+
objects[4] = b"<< /Length " + str(len(content)).encode() + b" >>\nstream\n" + content + b"\nendstream"
|
|
159
|
+
data, offsets = bytearray(b"%PDF-1.4\n"), [0]
|
|
160
|
+
for index, value in enumerate(objects, 1):
|
|
161
|
+
offsets.append(len(data))
|
|
162
|
+
data.extend(str(index).encode() + b" 0 obj\n" + value + b"\nendobj\n")
|
|
163
|
+
xref = len(data)
|
|
164
|
+
data.extend(("xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1)).encode())
|
|
165
|
+
for offset in offsets[1:]:
|
|
166
|
+
data.extend(("%010d 00000 n \n" % offset).encode())
|
|
167
|
+
data.extend(("trailer\n<< /Root 1 0 R /Size %d >>\nstartxref\n%d\n%%%%EOF\n" % (len(objects) + 1, xref)).encode())
|
|
168
|
+
path = root / name
|
|
169
|
+
path.write_bytes(data)
|
|
170
|
+
files.append((path, "456"))
|
|
171
|
+
objects[5] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Courier /Encoding /WinAnsiEncoding >>"
|
|
172
|
+
write_pdf("digital.pdf", stream)
|
|
173
|
+
# StandardEncoding maps this font's quote byte to U+2019. Retain the
|
|
174
|
+
# ambiguous original case: a recognizer must not silently turn it into ASCII.
|
|
175
|
+
objects[5] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>"
|
|
176
|
+
write_pdf("digital-ambiguous.pdf", stream)
|
|
177
|
+
# Keep the short, unboxed snippet which the layout model misclassified in the
|
|
178
|
+
# first real qualification: this must be preserved or explicitly report partial.
|
|
179
|
+
write_pdf("digital-short.pdf", header_stream + b"BT /F2 12 Tf 72 430 Td (def calculate_total\\(values\\):) Tj 0 -18 Td ( return sum\\(values\\)) Tj ET\n")
|
|
180
|
+
document = Document()
|
|
181
|
+
document.add_heading("公司资料 Fixture document", 1)
|
|
182
|
+
document.add_paragraph("Office text 789")
|
|
183
|
+
table = document.add_table(rows=2, cols=2)
|
|
184
|
+
for cell, value in zip([cell for row in table.rows for cell in row.cells], ["Name", "Value", "Total", "789"]):
|
|
185
|
+
cell.text = value
|
|
186
|
+
path = root / "office.docx"
|
|
187
|
+
document.save(path)
|
|
188
|
+
files.append((path, "789"))
|
|
189
|
+
book = Workbook()
|
|
190
|
+
book.active.append(["Name", "Value"])
|
|
191
|
+
book.active.append(["Total", 321])
|
|
192
|
+
path = root / "sheet.xlsx"
|
|
193
|
+
book.save(path)
|
|
194
|
+
files.append((path, "321"))
|
|
195
|
+
presentation = Presentation()
|
|
196
|
+
slide = presentation.slides.add_slide(presentation.slide_layouts[6])
|
|
197
|
+
slide.shapes.add_textbox(Inches(1), Inches(1), Inches(7), Inches(2)).text = "Presentation fixture 654"
|
|
198
|
+
path = root / "slides.pptx"
|
|
199
|
+
presentation.save(path)
|
|
200
|
+
files.append((path, "654"))
|
|
201
|
+
return files
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Recover visible PDF code spacing only when native glyphs prove a fixed grid.
|
|
2
|
+
|
|
3
|
+
PDF text APIs can collapse explicit spaces. We retain native Unicode characters
|
|
4
|
+
and reconstruct their visible columns from glyph origins, never from code syntax.
|
|
5
|
+
Ambiguous geometry remains unchanged and is reported for review.
|
|
6
|
+
"""
|
|
7
|
+
import ctypes
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_SIGNATURE = re.compile(r"^\s*(?:(?:async\s+)?def\s+[A-Za-z_]\w*\s*\(|(?:export\s+)?(?:async\s+)?function(?:\s+[A-Za-z_$][\w$]*)?\s*\(|(?:public\s+|private\s+|static\s+)*(?:void|int|bool|float|double|char|long)\s+[A-Za-z_]\w*\s*\()")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _compact(value):
|
|
15
|
+
return re.sub(r"\s+", "", value)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _native_region(page, textpage, bbox):
|
|
19
|
+
import pypdfium2 as pdfium
|
|
20
|
+
if page.get_rotation() != 0:
|
|
21
|
+
raise ValueError("rotated page")
|
|
22
|
+
if textpage.count_chars() > 100_000:
|
|
23
|
+
raise ValueError("native page exceeds the character inspection limit")
|
|
24
|
+
bounds = bbox.to_bottom_left_origin(page.get_height())
|
|
25
|
+
chars, pitches, font_heights = [], [], []
|
|
26
|
+
font_cache = {}
|
|
27
|
+
for index in range(textpage.count_chars()):
|
|
28
|
+
codepoint = pdfium.raw.FPDFText_GetUnicode(textpage.raw, index)
|
|
29
|
+
if codepoint in (10, 13):
|
|
30
|
+
continue
|
|
31
|
+
x, y = ctypes.c_double(), ctypes.c_double()
|
|
32
|
+
if not pdfium.raw.FPDFText_GetCharOrigin(textpage.raw, index, ctypes.byref(x), ctypes.byref(y)):
|
|
33
|
+
continue
|
|
34
|
+
left, bottom, right, top = textpage.get_charbox(index)
|
|
35
|
+
if not (bounds.l - 0.25 <= (left + right) / 2 <= bounds.r + 0.25 and bounds.b - 0.25 <= y.value <= bounds.t + 0.25):
|
|
36
|
+
continue
|
|
37
|
+
if pdfium.raw.FPDFText_IsGenerated(textpage.raw, index):
|
|
38
|
+
if codepoint == 32:
|
|
39
|
+
continue
|
|
40
|
+
raise ValueError("generated non-whitespace character")
|
|
41
|
+
if codepoint < 32 or codepoint > 0x10ffff or 0xd800 <= codepoint <= 0xdfff or pdfium.raw.FPDFText_HasUnicodeMapError(textpage.raw, index):
|
|
42
|
+
raise ValueError("unmapped or control character")
|
|
43
|
+
obj = textpage.get_textobj(index)
|
|
44
|
+
if obj is None:
|
|
45
|
+
raise ValueError("character has no native text object")
|
|
46
|
+
matrix = obj.get_matrix()
|
|
47
|
+
if abs(matrix.b) > 1e-5 or abs(matrix.c) > 1e-5 or matrix.a <= 0 or matrix.d <= 0 or abs(pdfium.raw.FPDFText_GetCharAngle(textpage.raw, index)) > 1e-5:
|
|
48
|
+
raise ValueError("rotated or skewed text")
|
|
49
|
+
if pdfium.raw.FPDFTextObj_GetTextRenderMode(obj.raw) in (3, 7):
|
|
50
|
+
raise ValueError("invisible text layer")
|
|
51
|
+
font, size = obj.get_font(), obj.get_font_size()
|
|
52
|
+
font_key = (ctypes.cast(font.raw, ctypes.c_void_p).value, size, matrix.a)
|
|
53
|
+
def width(character):
|
|
54
|
+
value = ctypes.c_float()
|
|
55
|
+
if not pdfium.raw.FPDFFont_GetGlyphWidth(font.raw, ord(character), size, ctypes.byref(value)):
|
|
56
|
+
raise ValueError("unavailable glyph advance")
|
|
57
|
+
return value.value * matrix.a
|
|
58
|
+
if font_key not in font_cache:
|
|
59
|
+
advances = [width(character) for character in " iMW0"]
|
|
60
|
+
if min(advances) <= 0 or max(advances) - min(advances) > max(0.01, advances[0] * 0.002):
|
|
61
|
+
raise ValueError("proportional font")
|
|
62
|
+
font_cache[font_key] = advances[0]
|
|
63
|
+
pitch = font_cache[font_key]
|
|
64
|
+
advance = width(chr(codepoint))
|
|
65
|
+
cells = round(advance / pitch)
|
|
66
|
+
if cells not in (1, 2) or abs(advance / pitch - cells) > 0.02:
|
|
67
|
+
raise ValueError("character advance is not an integral grid width")
|
|
68
|
+
chars.append({"index": index, "character": chr(codepoint), "x": x.value, "y": y.value, "cells": cells})
|
|
69
|
+
pitches.append(pitch)
|
|
70
|
+
font_heights.append(size * matrix.d)
|
|
71
|
+
if not chars:
|
|
72
|
+
raise ValueError("no visible native text in this region")
|
|
73
|
+
pitch = sum(pitches) / len(pitches)
|
|
74
|
+
if max(pitches) - min(pitches) > max(0.01, pitch * 0.002):
|
|
75
|
+
raise ValueError("mixed glyph advances")
|
|
76
|
+
rows = []
|
|
77
|
+
for char in sorted(chars, key=lambda value: (-value["y"], value["x"])):
|
|
78
|
+
if not rows or abs(rows[-1][0]["y"] - char["y"]) > 0.15:
|
|
79
|
+
rows.append([])
|
|
80
|
+
rows[-1].append(char)
|
|
81
|
+
gaps = [rows[i][0]["y"] - rows[i + 1][0]["y"] for i in range(len(rows) - 1)]
|
|
82
|
+
line_advance = min(gaps) if gaps else None
|
|
83
|
+
gap_cells = []
|
|
84
|
+
if line_advance is not None:
|
|
85
|
+
if line_advance < pitch or line_advance > 2 * min(font_heights):
|
|
86
|
+
raise ValueError("baseline spacing cannot establish a continuous line grid")
|
|
87
|
+
for gap in gaps:
|
|
88
|
+
cells = round(gap / line_advance)
|
|
89
|
+
if cells < 1 or cells > 8 or abs(gap / line_advance - cells) > 0.025:
|
|
90
|
+
raise ValueError("non-integral baseline gaps may contain significant blank lines")
|
|
91
|
+
gap_cells.append(cells)
|
|
92
|
+
origin = min(char["x"] for char in chars)
|
|
93
|
+
lines = []
|
|
94
|
+
for row_index, row in enumerate(rows):
|
|
95
|
+
if row_index:
|
|
96
|
+
# Empty rows preserve measured visual gaps, without interpreting syntax.
|
|
97
|
+
lines.extend([""] * (gap_cells[row_index - 1] - 1))
|
|
98
|
+
line, cursor = "", 0
|
|
99
|
+
for char in sorted(row, key=lambda value: value["x"]):
|
|
100
|
+
column = (char["x"] - origin) / pitch
|
|
101
|
+
cell = round(column)
|
|
102
|
+
if cell < cursor or cell > 2000 or abs(column - cell) > 0.025:
|
|
103
|
+
raise ValueError("overlapping or non-integral character positions")
|
|
104
|
+
line += " " * (cell - cursor) + char["character"]
|
|
105
|
+
cursor = cell + char["cells"]
|
|
106
|
+
lines.append(line.rstrip())
|
|
107
|
+
return "\n".join(lines), {"advance": pitch, "lineAdvance": line_advance, "rowPitch": line_advance, "baselineGaps": gaps, "gapRows": gap_cells, "originX": origin, "baselines": [row[0]["y"] for row in rows], "characters": chars}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def recover_native_code(document, source):
|
|
111
|
+
"""Mutate qualified Docling text/code items; return auditable recovery metadata."""
|
|
112
|
+
import pypdfium2 as pdfium
|
|
113
|
+
from docling_core.types.doc import CodeItem
|
|
114
|
+
items = [item for item, _level in document.iterate_items() if str(getattr(item, "label", "")) in ("code", "text") and (str(item.label) == "code" or _SIGNATURE.match(getattr(item, "text", "")))]
|
|
115
|
+
result = {"recovered": 0, "warnings": [], "needs_review": False, "recoveries": []}
|
|
116
|
+
if not items:
|
|
117
|
+
return result
|
|
118
|
+
with pdfium.PdfDocument(source) as pdf:
|
|
119
|
+
for item in items:
|
|
120
|
+
try:
|
|
121
|
+
if len(item.prov) != 1:
|
|
122
|
+
raise ValueError("code spans multiple or unspecified regions")
|
|
123
|
+
prov = item.prov[0]
|
|
124
|
+
page = pdf[prov.page_no - 1]
|
|
125
|
+
try:
|
|
126
|
+
textpage = page.get_textpage()
|
|
127
|
+
try:
|
|
128
|
+
text, evidence = _native_region(page, textpage, prov.bbox)
|
|
129
|
+
finally:
|
|
130
|
+
textpage.close()
|
|
131
|
+
finally:
|
|
132
|
+
page.close()
|
|
133
|
+
if _compact(text) not in {_compact(item.text), _compact(item.orig)}:
|
|
134
|
+
raise ValueError("native characters differ from the recognized region")
|
|
135
|
+
if any(delimiter in text or delimiter in item.text for delimiter in ('"""', "'''", "`")):
|
|
136
|
+
raise ValueError("multiline string whitespace requires explicit review")
|
|
137
|
+
if str(item.label) != "code" and "\n" not in text:
|
|
138
|
+
continue
|
|
139
|
+
data = item.model_dump()
|
|
140
|
+
data.update(label="code", text=text, orig=text)
|
|
141
|
+
new_item = CodeItem(**data)
|
|
142
|
+
new_item.prov[0].charspan = (0, len(text))
|
|
143
|
+
document.replace_item(old_item=item, new_item=new_item)
|
|
144
|
+
result["recovered"] += 1
|
|
145
|
+
result["recoveries"].append({"id": new_item.self_ref, "page": prov.page_no, "bbox": prov.bbox.model_dump(mode="json"), "originalText": item.text, "text": text, **evidence})
|
|
146
|
+
except Exception as error:
|
|
147
|
+
result["needs_review"] = True
|
|
148
|
+
result["warnings"].append(f"Native code layout could not be verified for {item.self_ref}: {error}; recognized text was preserved for review.")
|
|
149
|
+
return result
|