@aryacode/dsh-engineering 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/LICENSE +21 -0
- package/README.md +228 -2
- package/cordis.patch.yml +122 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +77 -3
- package/vendor/ftview/README.md +55 -0
- package/vendor/ftview/SKILL.md +74 -0
- package/vendor/ftview/lib/checks.d.ts +27 -0
- package/vendor/ftview/lib/checks.js +39 -0
- package/vendor/ftview/lib/core.d.ts +208 -0
- package/vendor/ftview/lib/core.js +160 -0
- package/vendor/ftview/lib/edit/drafts.d.ts +49 -0
- package/vendor/ftview/lib/edit/drafts.js +84 -0
- package/vendor/ftview/lib/edit/operations.d.ts +62 -0
- package/vendor/ftview/lib/edit/operations.js +126 -0
- package/vendor/ftview/lib/edit/prepare.d.ts +38 -0
- package/vendor/ftview/lib/edit/prepare.js +133 -0
- package/vendor/ftview/lib/edit/tools.d.ts +25 -0
- package/vendor/ftview/lib/edit/tools.js +198 -0
- package/vendor/ftview/lib/edit.d.ts +30 -0
- package/vendor/ftview/lib/edit.js +38 -0
- package/vendor/ftview/lib/index.d.ts +31 -0
- package/vendor/ftview/lib/index.js +36 -0
- package/vendor/ftview/lib/model.d.ts +80 -0
- package/vendor/ftview/lib/model.js +267 -0
- package/vendor/ftview/lib/session.d.ts +48 -0
- package/vendor/ftview/lib/session.js +98 -0
- package/vendor/ftview/lib/skill.d.ts +29 -0
- package/vendor/ftview/lib/skill.js +43 -0
- package/vendor/ftview/lib/studio/discover.d.ts +36 -0
- package/vendor/ftview/lib/studio/discover.js +87 -0
- package/vendor/ftview/lib/studio/preflight.d.ts +51 -0
- package/vendor/ftview/lib/studio/preflight.js +64 -0
- package/vendor/ftview/lib/studio/tools.d.ts +15 -0
- package/vendor/ftview/lib/studio/tools.js +83 -0
- package/vendor/ftview/lib/studio.d.ts +30 -0
- package/vendor/ftview/lib/studio.js +37 -0
- package/vendor/ftview/lib/tools.d.ts +14 -0
- package/vendor/ftview/lib/tools.js +202 -0
- package/vendor/l5x/README.md +62 -0
- package/vendor/l5x/SKILL.md +81 -0
- package/vendor/l5x/lib/approvals.d.ts +50 -0
- package/vendor/l5x/lib/approvals.js +93 -0
- package/vendor/l5x/lib/client.js +412 -0
- package/vendor/l5x/lib/client.js.map +1 -0
- package/vendor/l5x/lib/core/errors.d.ts +38 -0
- package/vendor/l5x/lib/core/errors.js +48 -0
- package/vendor/l5x/lib/core/params.d.ts +49 -0
- package/vendor/l5x/lib/core/params.js +39 -0
- package/vendor/l5x/lib/core/result.d.ts +46 -0
- package/vendor/l5x/lib/core/result.js +21 -0
- package/vendor/l5x/lib/core/tool.d.ts +102 -0
- package/vendor/l5x/lib/core/tool.js +19 -0
- package/vendor/l5x/lib/drafts.d.ts +66 -0
- package/vendor/l5x/lib/drafts.js +98 -0
- package/vendor/l5x/lib/editing.d.ts +30 -0
- package/vendor/l5x/lib/editing.js +42 -0
- package/vendor/l5x/lib/engine/acd-edit.d.ts +13 -0
- package/vendor/l5x/lib/engine/acd-edit.js +115 -0
- package/vendor/l5x/lib/engine/analysis.d.ts +22 -0
- package/vendor/l5x/lib/engine/analysis.js +258 -0
- package/vendor/l5x/lib/engine/capabilities.d.ts +9 -0
- package/vendor/l5x/lib/engine/capabilities.js +40 -0
- package/vendor/l5x/lib/engine/change-set.d.ts +122 -0
- package/vendor/l5x/lib/engine/change-set.js +446 -0
- package/vendor/l5x/lib/engine/configuration.d.ts +21 -0
- package/vendor/l5x/lib/engine/configuration.js +42 -0
- package/vendor/l5x/lib/engine/details.d.ts +47 -0
- package/vendor/l5x/lib/engine/details.js +173 -0
- package/vendor/l5x/lib/engine/direct-polling.d.ts +91 -0
- package/vendor/l5x/lib/engine/direct-polling.js +193 -0
- package/vendor/l5x/lib/engine/download-review.d.ts +15 -0
- package/vendor/l5x/lib/engine/download-review.js +63 -0
- package/vendor/l5x/lib/engine/logic-syntax.d.ts +60 -0
- package/vendor/l5x/lib/engine/logic-syntax.js +237 -0
- package/vendor/l5x/lib/engine/observations.d.ts +41 -0
- package/vendor/l5x/lib/engine/observations.js +67 -0
- package/vendor/l5x/lib/engine/project-analysis.d.ts +131 -0
- package/vendor/l5x/lib/engine/project-analysis.js +314 -0
- package/vendor/l5x/lib/engine/review-evidence.d.ts +8 -0
- package/vendor/l5x/lib/engine/review-evidence.js +49 -0
- package/vendor/l5x/lib/engine/review.d.ts +18 -0
- package/vendor/l5x/lib/engine/review.js +123 -0
- package/vendor/l5x/lib/engine/studio-sdk.d.ts +101 -0
- package/vendor/l5x/lib/engine/studio-sdk.js +571 -0
- package/vendor/l5x/lib/engine/tag-resolution.d.ts +20 -0
- package/vendor/l5x/lib/engine/tag-resolution.js +169 -0
- package/vendor/l5x/lib/engine/usage.d.ts +16 -0
- package/vendor/l5x/lib/engine/usage.js +92 -0
- package/vendor/l5x/lib/engine/xml-regions.d.ts +23 -0
- package/vendor/l5x/lib/engine/xml-regions.js +77 -0
- package/vendor/l5x/lib/fs.d.ts +17 -0
- package/vendor/l5x/lib/fs.js +36 -0
- package/vendor/l5x/lib/index.d.ts +36 -0
- package/vendor/l5x/lib/index.js +42 -0
- package/vendor/l5x/lib/live.d.ts +48 -0
- package/vendor/l5x/lib/live.js +48 -0
- package/vendor/l5x/lib/model/aois.d.ts +39 -0
- package/vendor/l5x/lib/model/aois.js +112 -0
- package/vendor/l5x/lib/model/errors.d.ts +15 -0
- package/vendor/l5x/lib/model/errors.js +18 -0
- package/vendor/l5x/lib/model/index.d.ts +20 -0
- package/vendor/l5x/lib/model/index.js +19 -0
- package/vendor/l5x/lib/model/module.d.ts +37 -0
- package/vendor/l5x/lib/model/module.js +54 -0
- package/vendor/l5x/lib/model/modules.d.ts +24 -0
- package/vendor/l5x/lib/model/modules.js +55 -0
- package/vendor/l5x/lib/model/parse.d.ts +32 -0
- package/vendor/l5x/lib/model/parse.js +116 -0
- package/vendor/l5x/lib/model/project.d.ts +15 -0
- package/vendor/l5x/lib/model/project.js +31 -0
- package/vendor/l5x/lib/model/routines.d.ts +71 -0
- package/vendor/l5x/lib/model/routines.js +183 -0
- package/vendor/l5x/lib/model/tags.d.ts +51 -0
- package/vendor/l5x/lib/model/tags.js +119 -0
- package/vendor/l5x/lib/model/types.d.ts +304 -0
- package/vendor/l5x/lib/model/types.js +12 -0
- package/vendor/l5x/lib/model/udts.d.ts +34 -0
- package/vendor/l5x/lib/model/udts.js +81 -0
- package/vendor/l5x/lib/model/xml.d.ts +139 -0
- package/vendor/l5x/lib/model/xml.js +291 -0
- package/vendor/l5x/lib/model/xref.d.ts +28 -0
- package/vendor/l5x/lib/model/xref.js +154 -0
- package/vendor/l5x/lib/session.d.ts +119 -0
- package/vendor/l5x/lib/session.js +236 -0
- package/vendor/l5x/lib/skill.d.ts +30 -0
- package/vendor/l5x/lib/skill.js +44 -0
- package/vendor/l5x/lib/studio-state.d.ts +77 -0
- package/vendor/l5x/lib/studio-state.js +79 -0
- package/vendor/l5x/lib/studio.d.ts +29 -0
- package/vendor/l5x/lib/studio.js +40 -0
- package/vendor/l5x/lib/tools/checks.d.ts +19 -0
- package/vendor/l5x/lib/tools/checks.js +140 -0
- package/vendor/l5x/lib/tools/common.d.ts +41 -0
- package/vendor/l5x/lib/tools/common.js +84 -0
- package/vendor/l5x/lib/tools/controller.d.ts +21 -0
- package/vendor/l5x/lib/tools/controller.js +170 -0
- package/vendor/l5x/lib/tools/drafts.d.ts +23 -0
- package/vendor/l5x/lib/tools/drafts.js +270 -0
- package/vendor/l5x/lib/tools/live.d.ts +21 -0
- package/vendor/l5x/lib/tools/live.js +175 -0
- package/vendor/l5x/lib/tools/read.d.ts +19 -0
- package/vendor/l5x/lib/tools/read.js +245 -0
- package/vendor/l5x/lib/tools/references.d.ts +20 -0
- package/vendor/l5x/lib/tools/references.js +269 -0
- package/vendor/l5x/lib/tools/studio.d.ts +21 -0
- package/vendor/l5x/lib/tools/studio.js +193 -0
- package/vendor/l5x/poll-worker.mjs +77 -0
- package/vendor/l5x/studio-bridge/NuGet.Config +1 -0
- package/vendor/l5x/studio-bridge/Program.cs +304 -0
- package/vendor/l5x/studio-bridge/StudioBridge.csproj +11 -0
- package/vendor/mwi/README.md +60 -0
- package/vendor/mwi/SKILL.md +87 -0
- package/vendor/mwi/bridge/README.md +8 -0
- package/vendor/mwi/bridge/mw_bridge.ps1 +2763 -0
- package/vendor/mwi/bridge/sdk-interop-importer.cs +30 -0
- package/vendor/mwi/bridge/sdk-variable-reader.cs +22 -0
- package/vendor/mwi/bridge/start_bridge.cmd +21 -0
- package/vendor/mwi/bridge/watch_dialogs.cmd +13 -0
- package/vendor/mwi/bridge/watch_dialogs.ps1 +153 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__init__.py +14 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/__init__.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/block_interfaces.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/cfb.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/datatypes.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/declarations.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/errors.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/graphical_listing.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/grid.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/knowledge.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/mwt_bind.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/navigation.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/program_checks.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/project.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/staging.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/structure.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/task_bindings.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/tasks.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/tree.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/validation.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/variable_edit.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/variables.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/__pycache__/workflow.cpython-312.pyc +0 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/block_interfaces.py +275 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/build.py +339 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/capture.py +448 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/cfb.py +765 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/com.py +268 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/datatypes.py +204 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/declarations.py +399 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/descriptions.py +30 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/eip.py +292 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/errors.py +36 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/graphical_listing.py +208 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/grid.py +227 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/ide.py +317 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/iec.py +317 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/installed_help.py +151 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/knowledge.py +218 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/libraries.py +250 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/manuals.py +520 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/mwt_bind.py +133 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/navigation.py +44 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/pou_writer.py +1547 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/program_checks.py +360 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/project.py +704 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/reference_data/catalog.json +672 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/reference_data/patterns.json +86 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/restore.py +208 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/server.py +1913 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/snapshot.py +138 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/st_comments.py +25 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/staging.py +179 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/stlint.py +676 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/structure.py +98 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/task_bindings.py +68 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/tasks.py +162 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/transaction.py +157 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/tree.py +563 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/tree_writer.py +790 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/uia.py +465 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/validation.py +41 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/variable_edit.py +183 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/variables.py +472 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/workflow.py +130 -0
- package/vendor/mwi/code/engine/motionworks_iec_mcp/writer.py +666 -0
- package/vendor/mwi/code/mw_code.py +1258 -0
- package/vendor/mwi/legacy/edit-session.js +50 -0
- package/vendor/mwi/legacy/fb-insertion.js +74 -0
- package/vendor/mwi/legacy/native-code.js +83 -0
- package/vendor/mwi/legacy/native-groups.js +75 -0
- package/vendor/mwi/legacy/native-structure.js +143 -0
- package/vendor/mwi/legacy/native-variables.js +77 -0
- package/vendor/mwi/legacy/pou-conversion.js +47 -0
- package/vendor/mwi/legacy/pou-package.js +66 -0
- package/vendor/mwi/lib/core/bridge.d.ts +71 -0
- package/vendor/mwi/lib/core/bridge.js +53 -0
- package/vendor/mwi/lib/core/code-engine.d.ts +47 -0
- package/vendor/mwi/lib/core/code-engine.js +109 -0
- package/vendor/mwi/lib/core/errors.d.ts +37 -0
- package/vendor/mwi/lib/core/errors.js +47 -0
- package/vendor/mwi/lib/core/params.d.ts +52 -0
- package/vendor/mwi/lib/core/params.js +46 -0
- package/vendor/mwi/lib/core/ps-bridge.d.ts +62 -0
- package/vendor/mwi/lib/core/ps-bridge.js +127 -0
- package/vendor/mwi/lib/core/result.d.ts +45 -0
- package/vendor/mwi/lib/core/result.js +20 -0
- package/vendor/mwi/lib/core/tool.d.ts +80 -0
- package/vendor/mwi/lib/core/tool.js +55 -0
- package/vendor/mwi/lib/edit/compare.d.ts +33 -0
- package/vendor/mwi/lib/edit/compare.js +70 -0
- package/vendor/mwi/lib/edit.d.ts +20 -0
- package/vendor/mwi/lib/edit.js +32 -0
- package/vendor/mwi/lib/index.d.ts +21 -0
- package/vendor/mwi/lib/index.js +33 -0
- package/vendor/mwi/lib/read/service.d.ts +117 -0
- package/vendor/mwi/lib/read/service.js +229 -0
- package/vendor/mwi/lib/read.d.ts +21 -0
- package/vendor/mwi/lib/read.js +33 -0
- package/vendor/mwi/lib/session/project.d.ts +34 -0
- package/vendor/mwi/lib/session/project.js +54 -0
- package/vendor/mwi/lib/session/service.d.ts +167 -0
- package/vendor/mwi/lib/session/service.js +566 -0
- package/vendor/mwi/lib/session/stage.d.ts +84 -0
- package/vendor/mwi/lib/session/stage.js +187 -0
- package/vendor/mwi/lib/session/workspace.d.ts +52 -0
- package/vendor/mwi/lib/session/workspace.js +90 -0
- package/vendor/mwi/lib/skill.d.ts +36 -0
- package/vendor/mwi/lib/skill.js +56 -0
- package/vendor/mwi/lib/tools/edit.d.ts +20 -0
- package/vendor/mwi/lib/tools/edit.js +286 -0
- package/vendor/mwi/lib/tools/read.d.ts +19 -0
- package/vendor/mwi/lib/tools/read.js +161 -0
- package/vendor/mwi/lib/tools/session.d.ts +20 -0
- package/vendor/mwi/lib/tools/session.js +166 -0
- package/vendor/mwi/lib/tools/verify.d.ts +19 -0
- package/vendor/mwi/lib/tools/verify.js +85 -0
- package/vendor/mwi/lib/verify.d.ts +17 -0
- package/vendor/mwi/lib/verify.js +29 -0
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
"""Search the MotionWorks manuals that ship with the IDE.
|
|
2
|
+
|
|
3
|
+
The goal for this plugin says to learn about MotionWorks programming, and it turns out the IDE
|
|
4
|
+
installs its own documentation - three PDF manuals plus 259 .chm help files - so an agent writing
|
|
5
|
+
MotionWorks code can be given the vendor's own words instead of guessing at an API.
|
|
6
|
+
|
|
7
|
+
WHAT IS READABLE AND WHAT IS NOT.
|
|
8
|
+
|
|
9
|
+
The PDFs are ordinary PDFs and are read here with zlib alone: inflate every stream, keep the ones
|
|
10
|
+
containing text operators, and pull the drawn strings out. Measured on the Toolbox Manual,
|
|
11
|
+
217 KB of searchable text from 285 streams.
|
|
12
|
+
|
|
13
|
+
Part of that text is drawn with a font whose encoding is the ASCII codepoint shifted by 29, in
|
|
14
|
+
EITHER direction:
|
|
15
|
+
|
|
16
|
+
'5HSODFHB0H' +29 -> 'Replace_Me'
|
|
17
|
+
'ptboolo|pqor' -29 -> 'SWERROR_STRUCT'
|
|
18
|
+
'7KLV' +29 -> 'This'
|
|
19
|
+
'Introduction' neither - already plain
|
|
20
|
+
|
|
21
|
+
so a token is decoded only when it carries a character that does not occur inside English words -
|
|
22
|
+
a digit, '|', a backtick - and then both directions are tried and the one that scores as more
|
|
23
|
+
English wins. Applying one direction to everything makes plain words worse, which is how a first
|
|
24
|
+
attempt turned "Introduction" into "fntroduction".
|
|
25
|
+
|
|
26
|
+
The .chm files are NOT read. They are ITSF containers with LZX-compressed content; the file list
|
|
27
|
+
is visible and the text is not, and hh.exe -decompile produced nothing here. Their topics are
|
|
28
|
+
listed in the manifest, so a caller can at least be told that a topic exists and which help file
|
|
29
|
+
covers it.
|
|
30
|
+
|
|
31
|
+
The extracted text is cached beside the plugin, because inflating a 6.7 MB manual takes a moment
|
|
32
|
+
and the manuals do not change.
|
|
33
|
+
"""
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import re
|
|
37
|
+
import zlib
|
|
38
|
+
from dataclasses import dataclass
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
|
|
41
|
+
from .errors import NotFound
|
|
42
|
+
|
|
43
|
+
#: Where the IDE installs its documentation. The manuals directory is the readable part.
|
|
44
|
+
IDE_ROOTS = (
|
|
45
|
+
Path(r"C:\Program Files (x86)\Yaskawa\MotionWorks IEC 3 Pro"),
|
|
46
|
+
Path(r"C:\Program Files\Yaskawa\MotionWorks IEC 3 Pro"),
|
|
47
|
+
)
|
|
48
|
+
MANUAL_DIR = "Manuals"
|
|
49
|
+
HELP_DIR = "Help"
|
|
50
|
+
|
|
51
|
+
STREAM = re.compile(rb"stream\r?\n(.*?)endstream", re.DOTALL)
|
|
52
|
+
LITERAL = re.compile(rb"\((?:\\.|[^\\()])*\)")
|
|
53
|
+
HEXSTR = re.compile(rb"<([0-9A-Fa-f\s]+)>")
|
|
54
|
+
|
|
55
|
+
_SHIFT = 0x1D
|
|
56
|
+
_FINGERPRINT = set("0123456789|`~\\^}{[]#@$%&*+=<>/")
|
|
57
|
+
_VOWELS = set("aeiouAEIOU")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class Manual:
|
|
62
|
+
"""One document, and whether its text can be read."""
|
|
63
|
+
|
|
64
|
+
name: str
|
|
65
|
+
path: Path
|
|
66
|
+
bytes: int
|
|
67
|
+
readable: bool
|
|
68
|
+
note: str = ""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def manuals_root() -> Path:
|
|
72
|
+
for root in IDE_ROOTS:
|
|
73
|
+
candidate = root / MANUAL_DIR
|
|
74
|
+
if candidate.is_dir():
|
|
75
|
+
return candidate
|
|
76
|
+
raise NotFound(
|
|
77
|
+
"no MotionWorks Manuals directory found; looked in "
|
|
78
|
+
+ " and ".join(str(r / MANUAL_DIR) for r in IDE_ROOTS)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def list_manuals() -> list[Manual]:
|
|
83
|
+
root = manuals_root()
|
|
84
|
+
found = []
|
|
85
|
+
for path in sorted(root.glob("*.pdf")):
|
|
86
|
+
found.append(Manual(name=path.stem, path=path, bytes=path.stat().st_size, readable=True))
|
|
87
|
+
for path in sorted(root.glob("*.chm")):
|
|
88
|
+
found.append(Manual(
|
|
89
|
+
name=path.stem, path=path, bytes=path.stat().st_size, readable=False,
|
|
90
|
+
note="compiled help (ITSF/LZX); the topic list is available, the text is not",
|
|
91
|
+
))
|
|
92
|
+
return found
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def help_topics() -> list[str]:
|
|
96
|
+
"""The .chm help files, which name their subject in the filename."""
|
|
97
|
+
for root in IDE_ROOTS:
|
|
98
|
+
help_dir = root / HELP_DIR
|
|
99
|
+
if help_dir.is_dir():
|
|
100
|
+
names = {p.stem for p in help_dir.glob("*.chm")}
|
|
101
|
+
# Drop the locale-suffixed duplicates (edit033 vs edit001) and locale folders.
|
|
102
|
+
english = sorted(
|
|
103
|
+
n for n in names
|
|
104
|
+
if re.fullmatch(r"[A-Za-z_]+0*1?", n) or not re.search(r"\d{3}$", n)
|
|
105
|
+
)
|
|
106
|
+
return english
|
|
107
|
+
raise NotFound("no MotionWorks Help directory found")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# ── PDF text ─────────────────────────────────────────────────────────────────────
|
|
111
|
+
|
|
112
|
+
def _unescape(raw: bytes) -> str:
|
|
113
|
+
out = bytearray()
|
|
114
|
+
i = 0
|
|
115
|
+
while i < len(raw):
|
|
116
|
+
c = raw[i]
|
|
117
|
+
if c == 0x5C and i + 1 < len(raw):
|
|
118
|
+
n = raw[i + 1]
|
|
119
|
+
simple = {0x6E: 10, 0x72: 13, 0x74: 9, 0x62: 8, 0x66: 12}
|
|
120
|
+
if n in simple:
|
|
121
|
+
out.append(simple[n]); i += 2; continue
|
|
122
|
+
if 0x30 <= n <= 0x37:
|
|
123
|
+
j, digits = i + 1, b""
|
|
124
|
+
while j < len(raw) and len(digits) < 3 and 0x30 <= raw[j] <= 0x37:
|
|
125
|
+
digits += bytes([raw[j]]); j += 1
|
|
126
|
+
out.append(int(digits, 8) & 0xFF); i = j; continue
|
|
127
|
+
out.append(n); i += 2; continue
|
|
128
|
+
out.append(c); i += 1
|
|
129
|
+
return out.decode("latin1")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _score(text: str) -> float:
|
|
133
|
+
if not text or "\ufffd" in text:
|
|
134
|
+
return -1.0
|
|
135
|
+
letters = sum(1 for c in text if c.isalpha())
|
|
136
|
+
if letters == 0:
|
|
137
|
+
return -1.0
|
|
138
|
+
digits = sum(1 for c in text if c.isdigit())
|
|
139
|
+
symbols = sum(1 for c in text if not c.isalnum() and c not in " _")
|
|
140
|
+
vowels = sum(1 for c in text if c in _VOWELS) / letters
|
|
141
|
+
return vowels * 2.0 - digits * 0.5 - symbols * 0.3
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _decode_token(token: str) -> str:
|
|
145
|
+
if not any(c in _FINGERPRINT for c in token):
|
|
146
|
+
return token
|
|
147
|
+
|
|
148
|
+
def shift(delta: int) -> str:
|
|
149
|
+
out = []
|
|
150
|
+
for c in token:
|
|
151
|
+
n = ord(c) + delta
|
|
152
|
+
out.append(chr(n) if 0x20 <= n <= 0x7E else "\ufffd")
|
|
153
|
+
return "".join(out)
|
|
154
|
+
|
|
155
|
+
return max((token, shift(_SHIFT), shift(-_SHIFT)), key=_score)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ── PDF text, by the document's own character maps ──────────────────────────────
|
|
159
|
+
#
|
|
160
|
+
# The heuristic below guesses at a font encoding. This does not: a PDF that uses a non-ASCII font
|
|
161
|
+
# ships a /ToUnicode CMap saying what each byte means, and reading that is exact. The guess stays as
|
|
162
|
+
# a fallback for fonts that carry no map.
|
|
163
|
+
|
|
164
|
+
_OBJ = re.compile(rb"(?<![0-9])(\d+)\s+(\d+)\s+obj\b")
|
|
165
|
+
_STREAM = re.compile(rb"stream\r?\n", re.DOTALL)
|
|
166
|
+
_FONT_REF = re.compile(rb"/([A-Za-z0-9]+)\s+(\d+)\s+0\s+R")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _inflate(data: bytes) -> bytes | None:
|
|
170
|
+
"""FlateDecode, tolerating the variants that appear in real files."""
|
|
171
|
+
for attempt in (data, data[1:], data[2:]):
|
|
172
|
+
try:
|
|
173
|
+
return zlib.decompress(attempt)
|
|
174
|
+
except zlib.error:
|
|
175
|
+
continue
|
|
176
|
+
try:
|
|
177
|
+
return zlib.decompressobj(-15).decompress(data)
|
|
178
|
+
except zlib.error:
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _objects(data: bytes) -> dict[int, bytes]:
|
|
183
|
+
"""Every `N G obj ... endobj` body, keyed by object number."""
|
|
184
|
+
found: dict[int, bytes] = {}
|
|
185
|
+
for m in _OBJ.finditer(data):
|
|
186
|
+
end = data.find(b"endobj", m.end())
|
|
187
|
+
found[int(m.group(1))] = data[m.end():end if end >= 0 else len(data)]
|
|
188
|
+
return found
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _object_stream(body: bytes) -> bytes | None:
|
|
192
|
+
m = _STREAM.search(body)
|
|
193
|
+
if not m:
|
|
194
|
+
return None
|
|
195
|
+
raw = body[m.end():body.find(b"endstream", m.end())]
|
|
196
|
+
if b"FlateDecode" in body[:m.start()]:
|
|
197
|
+
return _inflate(raw)
|
|
198
|
+
return raw
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _parse_cmap(text: bytes) -> dict[int, str]:
|
|
202
|
+
"""code -> character, from the bfchar and bfrange sections of a ToUnicode CMap."""
|
|
203
|
+
mapping: dict[int, str] = {}
|
|
204
|
+
for block in re.findall(rb"beginbfchar(.*?)endbfchar", text, re.DOTALL):
|
|
205
|
+
for src, dst in re.findall(rb"<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>", block):
|
|
206
|
+
try:
|
|
207
|
+
mapping[int(src, 16)] = bytes.fromhex(dst.decode()).decode("utf-16-be", "replace")
|
|
208
|
+
except Exception:
|
|
209
|
+
continue
|
|
210
|
+
for block in re.findall(rb"beginbfrange(.*?)endbfrange", text, re.DOTALL):
|
|
211
|
+
for lo, hi, dst in re.findall(
|
|
212
|
+
rb"<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>", block):
|
|
213
|
+
try:
|
|
214
|
+
start, stop, base = int(lo, 16), int(hi, 16), int(dst, 16)
|
|
215
|
+
except Exception:
|
|
216
|
+
continue
|
|
217
|
+
if stop - start > 65535:
|
|
218
|
+
continue
|
|
219
|
+
for offset in range(stop - start + 1):
|
|
220
|
+
mapping[start + offset] = chr(base + offset)
|
|
221
|
+
return mapping
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _font_maps(objects: dict[int, bytes]) -> tuple[dict[str, dict[int, str]], dict[str, bool]]:
|
|
225
|
+
"""Resource name -> its CMap, and resource name -> whether the font is two-byte.
|
|
226
|
+
|
|
227
|
+
The name is assigned by whoever REFERENCES the font, not by the font itself, so the lookup runs
|
|
228
|
+
in that direction: find the font object, then find every object naming it.
|
|
229
|
+
"""
|
|
230
|
+
cmaps: dict[int, dict[int, str]] = {}
|
|
231
|
+
for number, body in objects.items():
|
|
232
|
+
if b"beginbfchar" in body or b"beginbfrange" in body:
|
|
233
|
+
stream = _object_stream(body)
|
|
234
|
+
if stream:
|
|
235
|
+
cmaps[number] = _parse_cmap(stream)
|
|
236
|
+
if not cmaps:
|
|
237
|
+
return {}, {}
|
|
238
|
+
|
|
239
|
+
# font object number -> the CMap it points at
|
|
240
|
+
font_to_cmap: dict[int, dict[int, str]] = {}
|
|
241
|
+
two_byte_objects: set[int] = set()
|
|
242
|
+
for number, body in objects.items():
|
|
243
|
+
ref = re.search(rb"/ToUnicode\s+(\d+)\s+0\s+R", body)
|
|
244
|
+
if ref and int(ref.group(1)) in cmaps:
|
|
245
|
+
font_to_cmap[number] = cmaps[int(ref.group(1))]
|
|
246
|
+
if b"/Type0" in body:
|
|
247
|
+
two_byte_objects.add(number)
|
|
248
|
+
|
|
249
|
+
# every `/Name N 0 R` anywhere, where N is a font object
|
|
250
|
+
by_name: dict[str, dict[int, str]] = {}
|
|
251
|
+
two_byte: dict[str, bool] = {}
|
|
252
|
+
for body in objects.values():
|
|
253
|
+
for name, target in _FONT_REF.findall(body):
|
|
254
|
+
target = int(target)
|
|
255
|
+
if target in font_to_cmap:
|
|
256
|
+
key = name.decode()
|
|
257
|
+
by_name[key] = font_to_cmap[target]
|
|
258
|
+
if target in two_byte_objects:
|
|
259
|
+
two_byte[key] = True
|
|
260
|
+
return by_name, two_byte
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _apply_cmap(raw: bytes, cmap: dict[int, str] | None, two_byte: bool) -> str:
|
|
264
|
+
if not cmap:
|
|
265
|
+
return raw.decode("latin1", "replace")
|
|
266
|
+
if two_byte:
|
|
267
|
+
return "".join(cmap.get((raw[i] << 8) | raw[i + 1], "")
|
|
268
|
+
for i in range(0, len(raw) - 1, 2))
|
|
269
|
+
return "".join(cmap.get(c, chr(c)) for c in raw)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
_TEXT_OP = re.compile(
|
|
273
|
+
rb"/([A-Za-z0-9]+)\s+[\d.]+\s+Tf" # 1 font selection
|
|
274
|
+
rb"|\[(.*?)\]\s*TJ" # 2 array show
|
|
275
|
+
rb"|\((.*?)(?<!\\)\)\s*Tj" # 3 string show
|
|
276
|
+
rb"|(T\*|Td|TD)", # 4 positioning
|
|
277
|
+
re.DOTALL,
|
|
278
|
+
)
|
|
279
|
+
_IN_ARRAY = re.compile(rb"\((.*?)(?<!\\)\)", re.DOTALL)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def extract_pdf_by_cmap(path: Path) -> str | None:
|
|
283
|
+
"""Text read through the document's own ToUnicode maps, or None if it has none."""
|
|
284
|
+
data = path.read_bytes()
|
|
285
|
+
objects = _objects(data)
|
|
286
|
+
by_name, two_byte = _font_maps(objects)
|
|
287
|
+
if not by_name:
|
|
288
|
+
return None
|
|
289
|
+
|
|
290
|
+
pieces: list[str] = []
|
|
291
|
+
for body in objects.values():
|
|
292
|
+
if b"Tj" not in body and b"TJ" not in body:
|
|
293
|
+
continue
|
|
294
|
+
stream = _object_stream(body)
|
|
295
|
+
if not stream or (b"Tj" not in stream and b"TJ" not in stream):
|
|
296
|
+
continue
|
|
297
|
+
cmap, wide = None, False
|
|
298
|
+
for m in _TEXT_OP.finditer(stream):
|
|
299
|
+
if m.group(1):
|
|
300
|
+
name = m.group(1).decode()
|
|
301
|
+
cmap, wide = by_name.get(name), two_byte.get(name, False)
|
|
302
|
+
elif m.group(2) is not None:
|
|
303
|
+
for s in _IN_ARRAY.finditer(m.group(2)):
|
|
304
|
+
pieces.append(_apply_cmap(_unescape_bytes(s.group(1)), cmap, wide))
|
|
305
|
+
elif m.group(3) is not None:
|
|
306
|
+
pieces.append(_apply_cmap(_unescape_bytes(m.group(3)), cmap, wide))
|
|
307
|
+
elif m.group(4):
|
|
308
|
+
pieces.append("\n")
|
|
309
|
+
|
|
310
|
+
if not pieces:
|
|
311
|
+
return None
|
|
312
|
+
text = "".join(pieces)
|
|
313
|
+
text = re.sub(r"[ \t]+", " ", text)
|
|
314
|
+
return re.sub(r"\n{3,}", "\n\n", text)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _unescape_bytes(raw: bytes) -> bytes:
|
|
318
|
+
"""PDF string escapes, as bytes, so a multi-byte font encoding survives intact."""
|
|
319
|
+
out, i = bytearray(), 0
|
|
320
|
+
simple = {0x6E: 10, 0x72: 13, 0x74: 9, 0x62: 8, 0x66: 12,
|
|
321
|
+
0x28: 0x28, 0x29: 0x29, 0x5C: 0x5C}
|
|
322
|
+
while i < len(raw):
|
|
323
|
+
if raw[i] == 0x5C and i + 1 < len(raw):
|
|
324
|
+
nxt = raw[i + 1]
|
|
325
|
+
if nxt in simple:
|
|
326
|
+
out.append(simple[nxt]); i += 2; continue
|
|
327
|
+
if 0x30 <= nxt <= 0x37:
|
|
328
|
+
j, digits = i + 1, b""
|
|
329
|
+
while j < len(raw) and len(digits) < 3 and 0x30 <= raw[j] <= 0x37:
|
|
330
|
+
digits += bytes([raw[j]]); j += 1
|
|
331
|
+
out.append(int(digits, 8) & 0xFF); i = j; continue
|
|
332
|
+
i += 2; continue
|
|
333
|
+
out.append(raw[i]); i += 1
|
|
334
|
+
return bytes(out)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _extract_pdf_by_shift(path: Path) -> str:
|
|
338
|
+
data = path.read_bytes()
|
|
339
|
+
chunks: list[str] = []
|
|
340
|
+
for raw in STREAM.findall(data):
|
|
341
|
+
body = raw
|
|
342
|
+
if body[:2] in (b"\x78\x9c", b"\x78\x01", b"\x78\xda"):
|
|
343
|
+
try:
|
|
344
|
+
body = zlib.decompress(body)
|
|
345
|
+
except Exception:
|
|
346
|
+
continue
|
|
347
|
+
if b"Tj" not in body and b"TJ" not in body:
|
|
348
|
+
continue
|
|
349
|
+
for line in body.split(b"\n"):
|
|
350
|
+
if b"Tj" not in line and b"TJ" not in line:
|
|
351
|
+
continue
|
|
352
|
+
piece = [_unescape(m.group(0)[1:-1]) for m in LITERAL.finditer(line)]
|
|
353
|
+
for m in HEXSTR.finditer(line):
|
|
354
|
+
try:
|
|
355
|
+
piece.append(bytes.fromhex(m.group(1).decode("ascii")).decode("utf-16-be", "replace"))
|
|
356
|
+
except Exception:
|
|
357
|
+
pass
|
|
358
|
+
if piece:
|
|
359
|
+
chunks.append("".join(piece))
|
|
360
|
+
text = " ".join(chunks)
|
|
361
|
+
text = re.sub(r"[\x00-\x1f]+", " ", text)
|
|
362
|
+
text = re.sub(r"\s{2,}", " ", text)
|
|
363
|
+
return re.sub(r"[!-~]{4,}", lambda m: _decode_token(m.group(0)), text)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def manual_text(name: str, cache_dir: Path | None = None) -> str:
|
|
367
|
+
"""The decoded text of one manual, cached on disk."""
|
|
368
|
+
root = manuals_root()
|
|
369
|
+
matches = [p for p in root.glob("*.pdf") if name.lower() in p.stem.lower()]
|
|
370
|
+
if not matches:
|
|
371
|
+
available = ", ".join(p.stem for p in root.glob("*.pdf"))
|
|
372
|
+
raise NotFound(f"no manual matching {name!r}. Available: {available}")
|
|
373
|
+
path = matches[0]
|
|
374
|
+
|
|
375
|
+
from .knowledge import cache_root
|
|
376
|
+
import hashlib
|
|
377
|
+
root_cache = cache_root()
|
|
378
|
+
cache_base = (cache_dir or root_cache / 'legacy').resolve()
|
|
379
|
+
if not cache_base.is_relative_to(root_cache):
|
|
380
|
+
raise ValueError('Manual extraction cache must stay inside workspace reference storage')
|
|
381
|
+
key = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
382
|
+
cache = cache_base / (key + '.txt')
|
|
383
|
+
if not cache.resolve().is_relative_to(root_cache):
|
|
384
|
+
raise ValueError('Linked manual extraction cache escapes reference storage')
|
|
385
|
+
try:
|
|
386
|
+
if cache.is_file() and cache.stat().st_mtime >= path.stat().st_mtime:
|
|
387
|
+
return cache.read_text(encoding="utf-8")
|
|
388
|
+
except OSError:
|
|
389
|
+
pass
|
|
390
|
+
|
|
391
|
+
text = extract_pdf(path)
|
|
392
|
+
try:
|
|
393
|
+
cache.parent.mkdir(parents=True, exist_ok=True)
|
|
394
|
+
cache.write_text(text, encoding="utf-8")
|
|
395
|
+
except OSError:
|
|
396
|
+
pass
|
|
397
|
+
return text
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def search(term: str, name: str | None = None, limit: int = 5,
|
|
401
|
+
cache_dir: Path | None = None) -> list[dict]:
|
|
402
|
+
"""Find a term across the manuals, with surrounding context.
|
|
403
|
+
|
|
404
|
+
A term's words are allowed to sit apart in the text: the PDFs draw each word as its own
|
|
405
|
+
operation, so a phrase has to be matched with whitespace between its parts or it never hits.
|
|
406
|
+
"""
|
|
407
|
+
targets = list_manuals()
|
|
408
|
+
if name:
|
|
409
|
+
targets = [m for m in targets if name.lower() in m.name.lower() and m.readable]
|
|
410
|
+
else:
|
|
411
|
+
targets = [m for m in targets if m.readable]
|
|
412
|
+
|
|
413
|
+
pattern = re.compile(r"\W+".join(re.escape(w) for w in term.split()), re.IGNORECASE)
|
|
414
|
+
out: list[dict] = []
|
|
415
|
+
for manual in targets:
|
|
416
|
+
try:
|
|
417
|
+
text = manual_text(manual.name, cache_dir=cache_dir)
|
|
418
|
+
except Exception:
|
|
419
|
+
continue
|
|
420
|
+
hits = list(pattern.finditer(text))
|
|
421
|
+
if not hits:
|
|
422
|
+
continue
|
|
423
|
+
out.append({
|
|
424
|
+
"manual": manual.name,
|
|
425
|
+
"hits": len(hits),
|
|
426
|
+
"passages": [
|
|
427
|
+
text[max(0, h.start() - 200):h.end() + 400].strip()
|
|
428
|
+
for h in hits[:limit]
|
|
429
|
+
],
|
|
430
|
+
})
|
|
431
|
+
return out
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def extract_pdf(path: Path) -> str:
|
|
435
|
+
"""Text from a PDF, preferring the document's own character maps.
|
|
436
|
+
|
|
437
|
+
The CMap path is exact and is tried first. The older heuristic - inflate, pull the drawn
|
|
438
|
+
strings, and guess at a +/-29 font shift - remains as a fallback, because some PDFs really do
|
|
439
|
+
use a shifted encoding and ship no map to read.
|
|
440
|
+
"""
|
|
441
|
+
candidates: list[tuple[str, str]] = []
|
|
442
|
+
|
|
443
|
+
try:
|
|
444
|
+
by_cmap = extract_pdf_by_cmap(path)
|
|
445
|
+
if by_cmap and by_cmap.strip():
|
|
446
|
+
candidates.append(("cmap", by_cmap))
|
|
447
|
+
except Exception:
|
|
448
|
+
pass
|
|
449
|
+
|
|
450
|
+
try:
|
|
451
|
+
plain = _extract_pdf_plain(path)
|
|
452
|
+
if plain.strip():
|
|
453
|
+
candidates.append(("plain", plain))
|
|
454
|
+
except Exception:
|
|
455
|
+
pass
|
|
456
|
+
|
|
457
|
+
try:
|
|
458
|
+
shifted = _extract_pdf_by_shift(path)
|
|
459
|
+
if shifted.strip():
|
|
460
|
+
candidates.append(("shift", shifted))
|
|
461
|
+
except Exception:
|
|
462
|
+
pass
|
|
463
|
+
|
|
464
|
+
if not candidates:
|
|
465
|
+
return ""
|
|
466
|
+
|
|
467
|
+
# Judge by English: count the ordinary words in a sample. The three decoders differ enormously
|
|
468
|
+
# when the right one is chosen and barely at all when it is not, so this only has to be roughly
|
|
469
|
+
# right - but it does have to be about words, which the vowel-based _score is not.
|
|
470
|
+
return max(candidates, key=lambda pair: _english(pair[1]))[1]
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
#: Ordinary words, used only to decide which decoder produced readable text.
|
|
474
|
+
_ENGLISH = frozenset("""
|
|
475
|
+
the and for with this that are not from which function block variable type value input output
|
|
476
|
+
is of to in on be by or an as it at we you they can may must will would should has have been
|
|
477
|
+
when where what how all any each other more most such only same than then there their them
|
|
478
|
+
""".split())
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _english(text: str) -> float:
|
|
482
|
+
"""Fraction of the words in a sample that are ordinary English words."""
|
|
483
|
+
words = re.findall(r"[A-Za-z]{3,}", text[:20000])
|
|
484
|
+
if not words:
|
|
485
|
+
return 0.0
|
|
486
|
+
return sum(1 for w in words if w.lower() in _ENGLISH) / len(words)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _extract_pdf_plain(path: Path) -> str:
|
|
490
|
+
"""The drawn strings decoded as latin-1, with no re-encoding guess applied.
|
|
491
|
+
|
|
492
|
+
This is what the shift heuristic is a correction FOR. Some documents are already plain and the
|
|
493
|
+
correction damages them - AN.MPIEC.01 reads at 24% this way and 0% with the shift - so the two
|
|
494
|
+
are offered as alternatives and the scorer picks.
|
|
495
|
+
"""
|
|
496
|
+
data = path.read_bytes()
|
|
497
|
+
chunks: list[str] = []
|
|
498
|
+
for raw in STREAM.findall(data):
|
|
499
|
+
body = raw
|
|
500
|
+
if body[:2] in (b"\x78\x9c", b"\x78\x01", b"\x78\xda"):
|
|
501
|
+
try:
|
|
502
|
+
body = zlib.decompress(body)
|
|
503
|
+
except Exception:
|
|
504
|
+
continue
|
|
505
|
+
if b"Tj" not in body and b"TJ" not in body:
|
|
506
|
+
continue
|
|
507
|
+
# Walk the whole stream, not line by line: an operator and its operand are often on
|
|
508
|
+
# different lines, and a drawn string can contain a newline of its own.
|
|
509
|
+
for m in _TEXT_RUN.finditer(body):
|
|
510
|
+
piece = m.group(1) if m.group(1) is not None else m.group(2)
|
|
511
|
+
for s in _STRING_IN.finditer(piece):
|
|
512
|
+
chunks.append(_unescape(s.group(1)))
|
|
513
|
+
chunks.append("\n")
|
|
514
|
+
text = "".join(chunks)
|
|
515
|
+
text = re.sub(r"[ \t]{2,}", " ", text)
|
|
516
|
+
return re.sub(r"\n{3,}", "\n\n", text)
|
|
517
|
+
|
|
518
|
+
#: A drawn-string run: either the operand of Tj, or the array operand of TJ.
|
|
519
|
+
_TEXT_RUN = re.compile(rb"\((.*?)(?<!\\)\)\s*Tj|\[(.*?)\]\s*TJ", re.DOTALL)
|
|
520
|
+
_STRING_IN = re.compile(rb"\((.*?)(?<!\\)\)", re.DOTALL)
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Point a staged ``.mwt`` at the staged project, not at wherever it used to live.
|
|
2
|
+
|
|
3
|
+
A MotionWorks ``.mwt`` is a small OLE property set. One of its strings is an
|
|
4
|
+
absolute directory — measured on a real project as the last place that project
|
|
5
|
+
lived, which was not the copy under ``stage/``. Opening that wrapper in the IDE
|
|
6
|
+
loads the directory named inside it. The open call then sees *a* project and
|
|
7
|
+
reports success, while the window is showing a project outside the workspace.
|
|
8
|
+
|
|
9
|
+
``retarget`` rewrites every Windows path in that property set so it names the
|
|
10
|
+
staged directory. The source project is never opened here; callers pass the
|
|
11
|
+
staged copy only.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
import struct
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from .cfb import CompoundFile
|
|
21
|
+
|
|
22
|
+
_PATH = re.compile(r"^(?:[A-Za-z]:[\\/]|\\\\)")
|
|
23
|
+
_VT_BSTR = 8
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _padded(n: int) -> int:
|
|
27
|
+
return (n + 3) & ~3
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _section(data: bytes) -> tuple[int, int, list[tuple[int, int]]]:
|
|
31
|
+
"""Return ``(section_offset, section_size, [(pid, relative_offset), ...])``."""
|
|
32
|
+
if len(data) < 48 or data[:2] != b"\xfe\xff":
|
|
33
|
+
raise ValueError("the .mwt property stream is not an OLE property set")
|
|
34
|
+
nsets = struct.unpack_from("<I", data, 24)[0]
|
|
35
|
+
if nsets != 1:
|
|
36
|
+
raise ValueError(f"expected one property set, found {nsets}")
|
|
37
|
+
section = struct.unpack_from("<I", data, 44)[0]
|
|
38
|
+
if section + 8 > len(data):
|
|
39
|
+
raise ValueError("property section points past the end of the stream")
|
|
40
|
+
size, count = struct.unpack_from("<II", data, section)
|
|
41
|
+
props: list[tuple[int, int]] = []
|
|
42
|
+
for i in range(count):
|
|
43
|
+
pid, off = struct.unpack_from("<II", data, section + 8 + i * 8)
|
|
44
|
+
props.append((pid, off))
|
|
45
|
+
return section, size, props
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def embedded_paths(mwt: Path | str) -> list[str]:
|
|
49
|
+
"""Absolute Windows paths stored in the wrapper. Empty when it has none."""
|
|
50
|
+
cfb = CompoundFile(mwt)
|
|
51
|
+
found: list[str] = []
|
|
52
|
+
for name in cfb.stream_names():
|
|
53
|
+
data = cfb.read_stream(name)
|
|
54
|
+
try:
|
|
55
|
+
section, size, props = _section(data)
|
|
56
|
+
except ValueError:
|
|
57
|
+
continue
|
|
58
|
+
for _pid, off in props:
|
|
59
|
+
text = _bstr_at(data, section + off, section + size)
|
|
60
|
+
if text and _PATH.match(text):
|
|
61
|
+
found.append(text)
|
|
62
|
+
return found
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _bstr_at(data: bytes, abs_off: int, section_end: int) -> str | None:
|
|
66
|
+
if abs_off + 8 > len(data) or abs_off >= section_end:
|
|
67
|
+
return None
|
|
68
|
+
if struct.unpack_from("<I", data, abs_off)[0] != _VT_BSTR:
|
|
69
|
+
return None
|
|
70
|
+
nbytes = struct.unpack_from("<I", data, abs_off + 4)[0]
|
|
71
|
+
end = abs_off + 8 + nbytes
|
|
72
|
+
if nbytes < 2 or end > len(data):
|
|
73
|
+
return None
|
|
74
|
+
return data[abs_off + 8 : end].decode("utf-16le", "replace").rstrip("\x00")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def retarget(mwt: Path | str, directory: Path | str) -> dict[str, object]:
|
|
78
|
+
"""Rewrite path strings in ``mwt`` so they name ``directory``.
|
|
79
|
+
|
|
80
|
+
Returns which paths changed. A wrapper with no path is left byte-for-byte
|
|
81
|
+
alone and reported as unchanged — some projects simply do not store one.
|
|
82
|
+
"""
|
|
83
|
+
mwt_path = Path(mwt)
|
|
84
|
+
directory = Path(directory).resolve()
|
|
85
|
+
new = str(directory)
|
|
86
|
+
cfb = CompoundFile(mwt_path)
|
|
87
|
+
changed: list[dict[str, str]] = []
|
|
88
|
+
|
|
89
|
+
for name in cfb.stream_names():
|
|
90
|
+
raw = cfb.read_stream(name)
|
|
91
|
+
try:
|
|
92
|
+
section, size, props = _section(raw)
|
|
93
|
+
except ValueError:
|
|
94
|
+
continue
|
|
95
|
+
data = bytearray(raw)
|
|
96
|
+
# Later properties first, so a splice does not move a property still queued.
|
|
97
|
+
located: list[tuple[int, int, str]] = []
|
|
98
|
+
abs_offs = sorted(section + off for _pid, off in props)
|
|
99
|
+
for _pid, off in props:
|
|
100
|
+
abs_off = section + off
|
|
101
|
+
text = _bstr_at(data, abs_off, section + size)
|
|
102
|
+
if text and _PATH.match(text) and Path(text) != directory:
|
|
103
|
+
later = [a for a in abs_offs if a > abs_off]
|
|
104
|
+
span_end = min(later) if later else section + size
|
|
105
|
+
located.append((abs_off, span_end - abs_off, text))
|
|
106
|
+
if not located:
|
|
107
|
+
continue
|
|
108
|
+
for abs_off, span, text in sorted(located, reverse=True):
|
|
109
|
+
encoded = (new + "\x00").encode("utf-16le")
|
|
110
|
+
body = struct.pack("<II", _VT_BSTR, len(encoded)) + encoded
|
|
111
|
+
body += b"\x00" * (_padded(len(body)) - len(body))
|
|
112
|
+
delta = len(body) - span
|
|
113
|
+
data[abs_off : abs_off + span] = body
|
|
114
|
+
rel = abs_off - section
|
|
115
|
+
props = [(pid, off + delta if off > rel else off) for pid, off in props]
|
|
116
|
+
for i, (pid, off) in enumerate(props):
|
|
117
|
+
struct.pack_into("<II", data, section + 8 + i * 8, pid, off)
|
|
118
|
+
size += delta
|
|
119
|
+
struct.pack_into("<I", data, section, size)
|
|
120
|
+
changed.append({"from": text, "to": new, "stream": name})
|
|
121
|
+
if changed:
|
|
122
|
+
cfb = CompoundFile(mwt_path) # re-read; replace uses the current file
|
|
123
|
+
# The loop may see several streams; write this one now, then rescan
|
|
124
|
+
# would double-apply. Write once per stream below.
|
|
125
|
+
cfb.replace_streams({name: bytes(data)}, mwt_path)
|
|
126
|
+
# Refresh so a later stream is read from the file we just wrote.
|
|
127
|
+
cfb = CompoundFile(mwt_path)
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
"changed": bool(changed),
|
|
131
|
+
"bound_to": new,
|
|
132
|
+
"paths": changed,
|
|
133
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Resolve native document URNs from the saved project tree, without writes."""
|
|
2
|
+
from .cfb import CompoundFile
|
|
3
|
+
from .tree import parse_tree
|
|
4
|
+
|
|
5
|
+
def worksheet_target(project, kind, pou=None):
|
|
6
|
+
if kind not in ('variables', 'code'):
|
|
7
|
+
raise ValueError('Unknown worksheet kind')
|
|
8
|
+
roots, warnings = parse_tree(CompoundFile(project.root / 'src.st1').read_stream('PROJECT.TRE').decode('latin1'))
|
|
9
|
+
if warnings:
|
|
10
|
+
raise ValueError('Project tree has parse warnings; navigation refused')
|
|
11
|
+
nodes = [node for root in roots for _, node in root.walk()]
|
|
12
|
+
def parts(node):
|
|
13
|
+
return node.path.split('\t', 1)[0].replace('\\', '/').split('/')
|
|
14
|
+
if pou:
|
|
15
|
+
parents = [n for n in nodes if len(parts(n)) == 2 and parts(n)[0].upper() == 'POE' and n.name.casefold() == pou.casefold()]
|
|
16
|
+
if len(parents) != 1:
|
|
17
|
+
raise ValueError('Exact POU tree identity is absent or ambiguous')
|
|
18
|
+
parent = parents[0]
|
|
19
|
+
extensions = ('.VGR',) if kind == 'variables' else ('.STB', '.AB', '.GB')
|
|
20
|
+
matches = [n for n in parent.children if parts(n)[-1].upper().endswith(extensions)]
|
|
21
|
+
if len(matches) != 1:
|
|
22
|
+
raise ValueError('Exact worksheet is absent or ambiguous')
|
|
23
|
+
node = matches[0]
|
|
24
|
+
segments = [parent.name, node.name]
|
|
25
|
+
logical = '/Pous/' + '/'.join(segments)
|
|
26
|
+
urn = '@POUS.' + '.'.join(segments)
|
|
27
|
+
else:
|
|
28
|
+
if kind != 'variables':
|
|
29
|
+
raise ValueError('Code navigation requires a POU')
|
|
30
|
+
matches = [n for n in nodes if len(parts(n)) == 5 and parts(n)[0].upper() == 'C' and parts(n)[2].upper() == 'R' and parts(n)[-1].upper().endswith('.VGR')]
|
|
31
|
+
if len(matches) != 1:
|
|
32
|
+
raise ValueError('Exact resource globals worksheet is absent or ambiguous')
|
|
33
|
+
node = matches[0]
|
|
34
|
+
segments = [parts(node)[1], parts(node)[3], node.name]
|
|
35
|
+
logical = '/Hardware/' + '/'.join(segments)
|
|
36
|
+
urn = '@HW.' + '.'.join(segments)
|
|
37
|
+
if any(not s or any(c in s for c in './\\\r\n') for s in segments):
|
|
38
|
+
raise ValueError('Unsupported native worksheet identity')
|
|
39
|
+
document_logical = logical
|
|
40
|
+
if pou and kind == 'code':
|
|
41
|
+
# The active code editor reports the POU identity, not its body child.
|
|
42
|
+
logical = '/Pous/' + segments[0]
|
|
43
|
+
return dict(kind=kind, pou=segments[0] if pou else None, logical_name=logical,
|
|
44
|
+
document_logical_name=document_logical, urn=urn)
|