@dforge-core/dforge-mcp 0.2.44 → 0.2.46-next
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +3 -2
- package/dist/server.js +109 -84
- package/package.json +48 -46
- package/resources/docs/conventions.md +17 -14
- package/resources/docs/dsl-reference.md +15 -5
- package/resources/schemas/entity.schema.json +1 -1
- package/resources/schemas/registers.schema.json +8 -3
- package/scripts/install-skills.mjs +1 -0
- package/skills/dforge-1c-import/SKILL.md +374 -0
- package/skills/dforge-1c-import/references/environment.md +97 -0
- package/skills/dforge-1c-import/references/loading.md +161 -0
- package/skills/dforge-1c-import/references/measurements.md +210 -0
- package/skills/dforge-1c-import/references/print-forms.md +148 -0
- package/skills/dforge-1c-import/scripts/1c.py +178 -0
- package/skills/dforge-1c-import/scripts/README.md +110 -0
- package/skills/dforge-1c-import/scripts/lib/analyze.py +83 -0
- package/skills/dforge-1c-import/scripts/lib/attrs.py +278 -0
- package/skills/dforge-1c-import/scripts/lib/autospec.py +964 -0
- package/skills/dforge-1c-import/scripts/lib/binding.py +257 -0
- package/skills/dforge-1c-import/scripts/lib/bsl.py +811 -0
- package/skills/dforge-1c-import/scripts/lib/callgraph.py +717 -0
- package/skills/dforge-1c-import/scripts/lib/cli.py +53 -0
- package/skills/dforge-1c-import/scripts/lib/columnmap.py +346 -0
- package/skills/dforge-1c-import/scripts/lib/coverage.py +242 -0
- package/skills/dforge-1c-import/scripts/lib/custom.py +83 -0
- package/skills/dforge-1c-import/scripts/lib/delta.py +741 -0
- package/skills/dforge-1c-import/scripts/lib/deps.py +136 -0
- package/skills/dforge-1c-import/scripts/lib/docusage.py +105 -0
- package/skills/dforge-1c-import/scripts/lib/dump1c.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/explore.py +55 -0
- package/skills/dforge-1c-import/scripts/lib/external.py +201 -0
- package/skills/dforge-1c-import/scripts/lib/extract_code.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/forms.py +239 -0
- package/skills/dforge-1c-import/scripts/lib/genmodule.py +1432 -0
- package/skills/dforge-1c-import/scripts/lib/handlers.py +202 -0
- package/skills/dforge-1c-import/scripts/lib/handmap.py +142 -0
- package/skills/dforge-1c-import/scripts/lib/i18n_coverage.py +87 -0
- package/skills/dforge-1c-import/scripts/lib/idioms.py +273 -0
- package/skills/dforge-1c-import/scripts/lib/integrity.py +264 -0
- package/skills/dforge-1c-import/scripts/lib/inventory.py +535 -0
- package/skills/dforge-1c-import/scripts/lib/layers.py +106 -0
- package/skills/dforge-1c-import/scripts/lib/meta.py +315 -0
- package/skills/dforge-1c-import/scripts/lib/migrate.py +1299 -0
- package/skills/dforge-1c-import/scripts/lib/mxl.py +368 -0
- package/skills/dforge-1c-import/scripts/lib/naming.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/paths.py +90 -0
- package/skills/dforge-1c-import/scripts/lib/peek.py +70 -0
- package/skills/dforge-1c-import/scripts/lib/posted.py +218 -0
- package/skills/dforge-1c-import/scripts/lib/posting.py +655 -0
- package/skills/dforge-1c-import/scripts/lib/printgen.py +239 -0
- package/skills/dforge-1c-import/scripts/lib/rebuild.py +194 -0
- package/skills/dforge-1c-import/scripts/lib/recorded.py +170 -0
- package/skills/dforge-1c-import/scripts/lib/recorders.py +94 -0
- package/skills/dforge-1c-import/scripts/lib/reftargets.py +95 -0
- package/skills/dforge-1c-import/scripts/lib/reportgen.py +157 -0
- package/skills/dforge-1c-import/scripts/lib/roles.py +33 -0
- package/skills/dforge-1c-import/scripts/lib/seedstate.py +488 -0
- package/skills/dforge-1c-import/scripts/lib/sizing.py +184 -0
- package/skills/dforge-1c-import/scripts/lib/spec.py +400 -0
- package/skills/dforge-1c-import/scripts/lib/statements.py +774 -0
- package/skills/dforge-1c-import/scripts/lib/subsystems.py +480 -0
- package/skills/dforge-1c-import/scripts/lib/terms.py +451 -0
- package/skills/dforge-1c-import/scripts/lib/terms_uk.py +555 -0
- package/skills/dforge-1c-import/scripts/lib/totals.py +387 -0
- package/skills/dforge-1c-import/scripts/lib/typemap.py +73 -0
- package/skills/dforge-1c-import/scripts/lib/unpack.py +136 -0
- package/skills/dforge-1c-import/scripts/lib/validate.py +508 -0
- package/skills/dforge-1c-import/scripts/project-template/contours.py +47 -0
- package/skills/dforge-1c-import/scripts/project-template/sources.py +14 -0
- package/skills/dforge-mcp-author/SKILL.md +8 -2
- package/skills/dforge-mcp-author/examples/column-security/entities/product.json +1 -1
- package/skills/dforge-mcp-author/examples/column-security/manifest.json +0 -1
- package/skills/dforge-mcp-author/examples/column-security/security/roles.json +2 -2
- package/skills/dforge-mcp-author/references/action-dsl.md +20 -7
- package/skills/dforge-mcp-author/references/conventions.md +7 -1
- package/skills/dforge-mcp-author/references/field-types.md +3 -2
- package/skills/dforge-mcp-author/references/formulas.md +5 -1
- package/skills/dforge-mcp-author/references/print-templates.md +5 -2
- package/skills/dforge-mcp-author/references/queries.md +1 -1
- package/skills/dforge-mcp-author/references/registers.md +18 -3
- package/skills/dforge-mcp-author/references/reports.md +4 -1
- package/skills/dforge-mcp-author/references/schema-import.md +1 -1
- package/skills/dforge-mcp-author/references/security.md +1 -1
- package/skills/dforge-mcp-author/references/translations.md +38 -15
- package/skills/dforge-mcp-author/references/validation-checklist.md +2 -3
- package/skills/dforge-module-build/SKILL.md +10 -1
- package/skills/dforge-module-ship/SKILL.md +31 -3
- package/skills/dforge-mcp-author/examples/column-security/translations/en-US.json +0 -35
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
One way in, so the split between library and project stays a fact rather than a habit.
|
|
5
|
+
|
|
6
|
+
python tools/1c.py <command> [args...]
|
|
7
|
+
python tools/1c.py --list # what there is
|
|
8
|
+
python tools/1c.py --boundary # does the library still stand on its own?
|
|
9
|
+
|
|
10
|
+
`tools/lib` is 1C and dForge: how a configuration is stored, how its metadata parses,
|
|
11
|
+
what the physical tables mean, and how rows are copied and checked. It is meant to be
|
|
12
|
+
lifted out and pointed at a different infobase.
|
|
13
|
+
|
|
14
|
+
`tools/project` is this infobase: which object belongs to which module, which 1C column
|
|
15
|
+
feeds each ported one, which regulated statements it holds, this company's own proper
|
|
16
|
+
nouns, and the modules written by hand. `DF1C_PROJECT` points the library at a different
|
|
17
|
+
one; `binding.py` lists the whole of what a project has to provide.
|
|
18
|
+
|
|
19
|
+
Both directories go on the path, so the modules import each other by plain name and a
|
|
20
|
+
move between them costs no edits. That convenience is also the hazard, so `--boundary`
|
|
21
|
+
checks two things rather than trusting either: that no library module IMPORTS a project
|
|
22
|
+
one, and that no library module NAMES this project's world - its server, its infobase,
|
|
23
|
+
its tenant, its company's abbreviations. A literal is the quiet way to bind a library to
|
|
24
|
+
one configuration, and it is how `DF1C_DB` came to default to `utp_biscuit` for months.
|
|
25
|
+
|
|
26
|
+
Because both directories are on one path, a project module may not take a name the
|
|
27
|
+
library already uses: `regulated.py` rather than `forms.py`, `vocabulary.py` rather than
|
|
28
|
+
`terms.py`. `--boundary` catches the shadowing as an import crossing.
|
|
29
|
+
"""
|
|
30
|
+
from __future__ import print_function
|
|
31
|
+
|
|
32
|
+
import io
|
|
33
|
+
import os
|
|
34
|
+
import sys
|
|
35
|
+
|
|
36
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
37
|
+
LIB = os.path.join(HERE, "lib")
|
|
38
|
+
PROJECT = os.path.abspath(os.environ.get("DF1C_PROJECT") or os.path.join(HERE, "project"))
|
|
39
|
+
|
|
40
|
+
for entry in (PROJECT, LIB):
|
|
41
|
+
if entry not in sys.path:
|
|
42
|
+
sys.path.insert(0, entry)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def commands():
|
|
46
|
+
"""Every runnable module, library or project, mapped to (path, first docstring line)."""
|
|
47
|
+
found = {}
|
|
48
|
+
for directory in (LIB, PROJECT):
|
|
49
|
+
for name in sorted(os.listdir(directory)):
|
|
50
|
+
if not name.endswith(".py") or name.startswith("_") or name[:-3] in found:
|
|
51
|
+
continue
|
|
52
|
+
path = os.path.join(directory, name)
|
|
53
|
+
summary = _runnable(path)
|
|
54
|
+
if summary is not None:
|
|
55
|
+
found[name[:-3]] = (path, summary)
|
|
56
|
+
return found
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _runnable(path):
|
|
60
|
+
"""The module's first docstring line if it can be run as a script, else None."""
|
|
61
|
+
with io.open(path, encoding="utf-8") as f:
|
|
62
|
+
text = f.read()
|
|
63
|
+
# Anything runnable: most define main(), a few only a __main__ block.
|
|
64
|
+
if "def main(" not in text and "__name__" not in text:
|
|
65
|
+
return None
|
|
66
|
+
if '"""' not in text:
|
|
67
|
+
return ""
|
|
68
|
+
body = text.split('"""', 2)[1].strip().splitlines()
|
|
69
|
+
return (body[0] if body else "").strip()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def boundary():
|
|
73
|
+
"""Two ways the library can stop standing on its own, both read off the source.
|
|
74
|
+
|
|
75
|
+
IMPORTS. The path puts both directories in reach, so nothing stops a library module
|
|
76
|
+
importing `contours` directly and nothing would complain until someone tried to lift
|
|
77
|
+
the library out.
|
|
78
|
+
|
|
79
|
+
NAMES. An import is the loud way to bind to one infobase; a literal is the quiet one.
|
|
80
|
+
`DF1C_DB` defaulting to `utp_biscuit` and `DF_TENANT` to `hbf1` sat in the library for
|
|
81
|
+
a long while, harmless with one infobase and a trap with two - point the tools at a
|
|
82
|
+
second configuration, forget one variable, and they read the first one with
|
|
83
|
+
right-looking output. So the names of THIS project's world are refused in the library,
|
|
84
|
+
including in its comments, since a stale comment is how the next reader learns the
|
|
85
|
+
wrong thing.
|
|
86
|
+
|
|
87
|
+
The list is read from the project rather than written here, so a project that renames
|
|
88
|
+
itself does not have to remember this file.
|
|
89
|
+
"""
|
|
90
|
+
import ast
|
|
91
|
+
import re
|
|
92
|
+
|
|
93
|
+
project_modules = set(n[:-3] for n in os.listdir(PROJECT) if n.endswith(".py"))
|
|
94
|
+
|
|
95
|
+
# What this project calls itself: the infobase, the tenant and its own proper nouns.
|
|
96
|
+
reserved = set()
|
|
97
|
+
for variable in ("DF1C_DB", "DF1C_SERVER", "DF_TENANT"):
|
|
98
|
+
value = os.environ.get(variable)
|
|
99
|
+
if value:
|
|
100
|
+
reserved.add(value.lower())
|
|
101
|
+
env_example = os.path.join(os.path.dirname(os.path.abspath(PROJECT)), os.pardir,
|
|
102
|
+
".env.example")
|
|
103
|
+
if os.path.exists(env_example):
|
|
104
|
+
with io.open(env_example, encoding="utf-8") as f:
|
|
105
|
+
for line in f:
|
|
106
|
+
match = re.match(r"\s*(DF1C_DB|DF1C_SERVER|DF_TENANT)\s*=\s*(\S+)", line)
|
|
107
|
+
if match:
|
|
108
|
+
reserved.add(match.group(2).lower())
|
|
109
|
+
terms = os.path.join(PROJECT, "terms.py")
|
|
110
|
+
if os.path.exists(terms):
|
|
111
|
+
with io.open(terms, encoding="utf-8") as f:
|
|
112
|
+
for value in re.findall(r'u"([^"]+)"\s*:', f.read()):
|
|
113
|
+
if len(value) > 3: # two- and three-letter forms collide with real words
|
|
114
|
+
reserved.add(value.lower())
|
|
115
|
+
|
|
116
|
+
problems = []
|
|
117
|
+
for name in sorted(os.listdir(LIB)):
|
|
118
|
+
if not name.endswith(".py"):
|
|
119
|
+
continue
|
|
120
|
+
with io.open(os.path.join(LIB, name), encoding="utf-8") as f:
|
|
121
|
+
text = f.read()
|
|
122
|
+
tree = ast.parse(text)
|
|
123
|
+
for node in ast.walk(tree):
|
|
124
|
+
named = []
|
|
125
|
+
if isinstance(node, ast.Import):
|
|
126
|
+
named = [a.name.split(".")[0] for a in node.names]
|
|
127
|
+
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
128
|
+
named = [node.module.split(".")[0]]
|
|
129
|
+
for target in named:
|
|
130
|
+
if target in project_modules:
|
|
131
|
+
problems.append("lib/%s imports %s - go through binding.py instead"
|
|
132
|
+
% (name, target))
|
|
133
|
+
# paths.py explains why these names are refused, so it is allowed to name them.
|
|
134
|
+
if name == "paths.py":
|
|
135
|
+
continue
|
|
136
|
+
lowered = text.lower()
|
|
137
|
+
for word in sorted(reserved):
|
|
138
|
+
if word in lowered:
|
|
139
|
+
problems.append("lib/%s names %r - that is the project's, not the library's"
|
|
140
|
+
% (name, word))
|
|
141
|
+
return problems
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def main():
|
|
145
|
+
args = sys.argv[1:]
|
|
146
|
+
if not args or args[0] in ("-h", "--help", "--list"):
|
|
147
|
+
import paths
|
|
148
|
+
paths.show()
|
|
149
|
+
print()
|
|
150
|
+
for name, (_, summary) in sorted(commands().items()):
|
|
151
|
+
print(" %-16s %s" % (name, summary[:60]))
|
|
152
|
+
return 0
|
|
153
|
+
|
|
154
|
+
if args[0] == "--boundary":
|
|
155
|
+
problems = boundary()
|
|
156
|
+
for line in problems:
|
|
157
|
+
print(line)
|
|
158
|
+
print("%d crossing(s) of the library/project boundary" % len(problems))
|
|
159
|
+
return 1 if problems else 0
|
|
160
|
+
|
|
161
|
+
name, rest = args[0], args[1:]
|
|
162
|
+
found = commands()
|
|
163
|
+
if name not in found:
|
|
164
|
+
print("no such command: %s (try --list)" % name)
|
|
165
|
+
return 2
|
|
166
|
+
# run_path rather than calling main(): a few tools only have a __main__
|
|
167
|
+
# block, and this runs every one of them exactly as a script would.
|
|
168
|
+
import runpy
|
|
169
|
+
sys.argv = [name] + rest
|
|
170
|
+
try:
|
|
171
|
+
runpy.run_path(found[name][0], run_name="__main__")
|
|
172
|
+
except SystemExit as stop:
|
|
173
|
+
return stop.code or 0
|
|
174
|
+
return 0
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
if __name__ == "__main__":
|
|
178
|
+
sys.exit(main() or 0)
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# The 1C porting toolkit
|
|
2
|
+
|
|
3
|
+
A Python library that reads a 1C:Enterprise 8 infobase hosted on MS SQL Server. It needs
|
|
4
|
+
no 1C software and no licence. From the infobase it:
|
|
5
|
+
|
|
6
|
+
- generates dForge modules;
|
|
7
|
+
- copies the rows;
|
|
8
|
+
- checks the result.
|
|
9
|
+
|
|
10
|
+
It was built on a real port: an УТП-based configuration, 16 contours, 30 M+ rows,
|
|
11
|
+
reconciled to the row and the kopeck. It is meant to be pointed at the next infobase
|
|
12
|
+
unchanged.
|
|
13
|
+
|
|
14
|
+
```
|
|
15
|
+
scripts/
|
|
16
|
+
1c.py the one entry point: python 1c.py <command> [args]
|
|
17
|
+
lib/ the library: 1C storage, metadata, generation, loading, checks
|
|
18
|
+
project-template/ what a new port starts its project package from
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Library and project
|
|
22
|
+
|
|
23
|
+
- **`lib/` knows 1C and dForge**: how a configuration is stored, how its metadata
|
|
24
|
+
parses, what the physical tables mean, and how rows are copied and checked.
|
|
25
|
+
- **The project knows this infobase**:
|
|
26
|
+
- which object goes to which module (`contours.py`);
|
|
27
|
+
- which 1C column feeds an authored column (`sources.py`);
|
|
28
|
+
- the integrator's prefixes;
|
|
29
|
+
- the hand-written modules (`authored/<module>/`).
|
|
30
|
+
|
|
31
|
+
`lib/binding.py` lists everything a project may provide. Only `contours` and `sources`
|
|
32
|
+
are required.
|
|
33
|
+
|
|
34
|
+
**Never put project facts in `lib/`.** That covers infobase names, tenant names, the
|
|
35
|
+
company's abbreviations and prefixes. `python 1c.py --boundary` checks that no library
|
|
36
|
+
module imports a project module or names the project's world.
|
|
37
|
+
|
|
38
|
+
**Never hand-edit generated output.** Put the change in the project, either as a spec
|
|
39
|
+
override or as an authored file under `authored/<module>/` that `genmodule` merges.
|
|
40
|
+
Then regenerate. If a port keeps needing hand edits, the generator is wrong.
|
|
41
|
+
|
|
42
|
+
## Setup
|
|
43
|
+
|
|
44
|
+
- Python 3.7+.
|
|
45
|
+
- `pyodbc` and an ODBC driver for SQL Server (driver 17 or 18), to read the infobase.
|
|
46
|
+
- `psycopg2`, to load and check the tenant.
|
|
47
|
+
- `jsonschema`, for `validate`.
|
|
48
|
+
- Node with `liquidjs` in `PRINT_NODE_DIR`, only for `printgen preview`.
|
|
49
|
+
|
|
50
|
+
On Windows, set `PYTHONIOENCODING=utf-8`.
|
|
51
|
+
|
|
52
|
+
Lay out a port directory and point the toolkit at it:
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
myport/
|
|
56
|
+
.env credentials (never committed)
|
|
57
|
+
project/ copied from project-template/, then filled in
|
|
58
|
+
out/ what the extraction writes (DF1C_OUT, default <root>/out)
|
|
59
|
+
modules/ generated modules (DF1C_MODULES, default <root>/modules)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
DF1C_ROOT=/path/to/myport
|
|
64
|
+
DF1C_PROJECT=/path/to/myport/project
|
|
65
|
+
DF1C_SERVER=<sql server> DF1C_DB=<infobase db> DF1C_USER= DF1C_PASS= # read-only login; Windows auth when USER is empty
|
|
66
|
+
DF_TENANT=<tenant code> DF_PG_HOST= DF_PG_PORT= DF_PG_USER= DF_PG_PASS= # the tenant's Postgres, for load and checks
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
None of these has a default. A default that named one infobase would make the toolkit
|
|
70
|
+
quietly read or write that infobase when a variable was forgotten.
|
|
71
|
+
|
|
72
|
+
## The pipeline
|
|
73
|
+
|
|
74
|
+
Each step reads what the previous one wrote to `out/`. Every command has `--help` or a
|
|
75
|
+
docstring with its options, and `python 1c.py --list` lists them all.
|
|
76
|
+
|
|
77
|
+
| # | step | command |
|
|
78
|
+
|---|---|---|
|
|
79
|
+
| 1 | Read the configuration and DBNames from the SQL tables | `dump1c probe` · `dump1c dbnames` · `dump1c schema` · `dump1c extract` |
|
|
80
|
+
| 2 | Parse the metadata, resolve type GUIDs, join the physical schema | `meta allobjects` · `typemap` · `analyze` · `attrs <Object>` · `explore <Object>` |
|
|
81
|
+
| 3 | Extract the module texts (posting handlers, print procedures) | `extract_code` · `unpack <Object> --code` |
|
|
82
|
+
| 4 | Inventory: what exists, what is used, what is the client's | `inventory` · `docusage` · `custom` · `sizing` · `subsystems` |
|
|
83
|
+
| 5 | Read the posting handlers: which registers each document writes and from where | `handlers [--client]` · `callgraph` · `recorders` · `recorded` · `idioms` · `spec` |
|
|
84
|
+
| 6 | Decide the module boundaries | `subsystems --contours --acyclic`, copied into `project/contours.py` and corrected |
|
|
85
|
+
| 7 | Generate and validate the modules | `autospec [module]` · `genmodule` · `validate` · `deps` · `layers` |
|
|
86
|
+
| 8 | Install them | `npx -y @dforge-core/dforge-cli@latest module install ...`, or `rebuild --code <tenant> --install` |
|
|
87
|
+
| 9 | Copy the rows and reconcile | `migrate plan\|check\|run\|verify\|fill <module>` · `integrity` · `columnmap <module>` |
|
|
88
|
+
| 10 | Post state and registers | `posted [--mark]` · `posting` · `seedstate` · `totals` · `coverage` |
|
|
89
|
+
| 11 | Catch up with what 1C changed | `delta --baseline` · `delta [--apply]` |
|
|
90
|
+
| 12 | Print forms and reports | `mxl list [--client]` · `mxl dump <Object>` · `printgen <spec>` · `printgen preview <spec> <form> <key>` · `reportgen <spec>` |
|
|
91
|
+
| 13 | External reports and processings stored in the infobase | `external` |
|
|
92
|
+
|
|
93
|
+
`rebuild --code <tenant>` runs steps 8–10 from nothing, which makes a tenant
|
|
94
|
+
reproducible.
|
|
95
|
+
|
|
96
|
+
## Rules the toolkit assumes
|
|
97
|
+
|
|
98
|
+
- **The infobase is read-only.** Use a SELECT-only login.
|
|
99
|
+
- **Open ad-hoc database connections in autocommit.** A read transaction left open
|
|
100
|
+
blocks the next install's `ALTER`.
|
|
101
|
+
- **Resolve a physical column (`_FldNNN`) only through DBNames** (`attrs`, `columnmap`).
|
|
102
|
+
Never resolve it by cardinality.
|
|
103
|
+
- **Stop on a 401 from the tenant.** Ask for a fresh login, then re-run
|
|
104
|
+
`posted --mark` for documents loaded in the meantime.
|
|
105
|
+
|
|
106
|
+
## Source
|
|
107
|
+
|
|
108
|
+
This copy is kept in step with the port it was developed on (`tools/lib`, `tools/1c.py`
|
|
109
|
+
there). Fixes made in a port go back to the library here. A project's own files never
|
|
110
|
+
come here.
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Join the physical schema against DBNames to see which parts of the configuration
|
|
5
|
+
the client actually uses. A stock UTP ships hundreds of catalogs and documents;
|
|
6
|
+
only the populated ones are worth porting.
|
|
7
|
+
"""
|
|
8
|
+
import codecs
|
|
9
|
+
import collections
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
import paths
|
|
16
|
+
|
|
17
|
+
OUT = paths.OUT
|
|
18
|
+
|
|
19
|
+
# Physical table families that carry business data, in the order we care about.
|
|
20
|
+
DATA_KINDS = ("Reference", "Document", "InfoRg", "AccumRg", "AccRg", "Chrc", "Acc",
|
|
21
|
+
"Enum", "DocumentJournal", "Node", "BPr", "Task", "Const")
|
|
22
|
+
|
|
23
|
+
SPLIT = re.compile(r"^_([A-Za-z]+?)(\d+)(?:_(.+))?$")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def load(name):
|
|
27
|
+
with codecs.open(os.path.join(OUT, name), "r", "utf-8") as f:
|
|
28
|
+
return json.load(f)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def classify(table_name):
|
|
32
|
+
"""_Document123_VT456 -> ('Document', 123, 'VT456')"""
|
|
33
|
+
m = SPLIT.match(table_name)
|
|
34
|
+
if not m:
|
|
35
|
+
return None, None, None
|
|
36
|
+
return m.group(1), int(m.group(2)), m.group(3)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def main():
|
|
40
|
+
schema = load("schema.json")
|
|
41
|
+
tables = {t["name"]: t for t in schema["tables"]}
|
|
42
|
+
|
|
43
|
+
summary = collections.OrderedDict((k, {"tables": 0, "used": 0, "rows": 0}) for k in DATA_KINDS)
|
|
44
|
+
detail = collections.defaultdict(list)
|
|
45
|
+
|
|
46
|
+
for name, t in tables.items():
|
|
47
|
+
kind, number, suffix = classify(name)
|
|
48
|
+
if kind not in summary or suffix: # skip tabular sections, roll them up separately
|
|
49
|
+
continue
|
|
50
|
+
rows = t["rows"] or 0
|
|
51
|
+
# a document's tabular sections belong to the document's footprint
|
|
52
|
+
child_rows = sum((tables[n]["rows"] or 0) for n in tables
|
|
53
|
+
if n.startswith("_%s%d_" % (kind, number)))
|
|
54
|
+
summary[kind]["tables"] += 1
|
|
55
|
+
summary[kind]["rows"] += rows
|
|
56
|
+
if rows:
|
|
57
|
+
summary[kind]["used"] += 1
|
|
58
|
+
detail[kind].append({"table": name, "number": number, "rows": rows,
|
|
59
|
+
"childRows": child_rows, "columns": len(t["columns"])})
|
|
60
|
+
|
|
61
|
+
print("%-18s %8s %8s %14s" % ("kind", "tables", "used", "rows"))
|
|
62
|
+
print("-" * 52)
|
|
63
|
+
for kind, s in summary.items():
|
|
64
|
+
if s["tables"]:
|
|
65
|
+
print("%-18s %8d %8d %14s" % (kind, s["tables"], s["used"], "{:,}".format(s["rows"])))
|
|
66
|
+
|
|
67
|
+
top = int(sys.argv[1]) if len(sys.argv) > 1 else 25
|
|
68
|
+
for kind in ("Document", "Reference", "AccumRg", "InfoRg"):
|
|
69
|
+
rows = sorted(detail[kind], key=lambda d: -(d["rows"] + d["childRows"]))[:top]
|
|
70
|
+
print("\n== %s: top %d by volume ==" % (kind, top))
|
|
71
|
+
for d in rows:
|
|
72
|
+
if not (d["rows"] or d["childRows"]):
|
|
73
|
+
continue
|
|
74
|
+
print(" %-22s rows=%-12s child=%-12s cols=%d"
|
|
75
|
+
% (d["table"], "{:,}".format(d["rows"]),
|
|
76
|
+
"{:,}".format(d["childRows"]), d["columns"]))
|
|
77
|
+
|
|
78
|
+
with codecs.open(os.path.join(OUT, "usage.json"), "w", "utf-8") as f:
|
|
79
|
+
json.dump({"summary": summary, "detail": detail}, f, ensure_ascii=False, indent="\t")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
main()
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Extract the field list of a metadata object, with references resolved.
|
|
5
|
+
|
|
6
|
+
Every typed thing in a 1C object file - a catalog attribute, a register dimension
|
|
7
|
+
or resource, a tabular-section column - is encoded the same way:
|
|
8
|
+
|
|
9
|
+
["2", ["1", ["0","0",<own guid>], "<Name>", ["1","ru","<Synonym>"], ...],
|
|
10
|
+
["Pattern", <type...>]]
|
|
11
|
+
|
|
12
|
+
so the collector looks for that shape anywhere in the tree rather than hard-coding
|
|
13
|
+
per-kind offsets, and takes the nearest named ancestor as the tabular section.
|
|
14
|
+
|
|
15
|
+
python attrs.py <ObjectName> one object, to the console
|
|
16
|
+
python attrs.py --model <file> <names...> several objects, to a markdown file
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import print_function
|
|
19
|
+
|
|
20
|
+
import codecs
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import paths
|
|
24
|
+
import sys
|
|
25
|
+
|
|
26
|
+
import meta
|
|
27
|
+
|
|
28
|
+
OUT = meta.OUT
|
|
29
|
+
|
|
30
|
+
PRIMITIVE = {"S": "string", "N": "number", "B": "bool", "D": "date"}
|
|
31
|
+
|
|
32
|
+
# Collection class GUIDs -> what the fields in them mean, confirmed by checking a
|
|
33
|
+
# register whose meaning is unambiguous (ТоварыНаСкладах: Количество is the resource,
|
|
34
|
+
# Склад/Номенклатура the dimensions). Unknown GUIDs print truncated so they show up.
|
|
35
|
+
ROLE = {
|
|
36
|
+
"cf4abea7-37b2-11d4-940f-008048da11f9": "attr", # catalog attribute
|
|
37
|
+
"45e46cbc-3e24-4165-8b7b-cc98a6f80211": "attr", # document attribute
|
|
38
|
+
"888744e1-b616-11d4-9436-004095e12fc7": "column", # tabular section column
|
|
39
|
+
"b64d9a41-1642-11d6-a3c7-0050bae0a776": "resource", # accumulation register
|
|
40
|
+
"b64d9a42-1642-11d6-a3c7-0050bae0a776": "attr",
|
|
41
|
+
"b64d9a43-1642-11d6-a3c7-0050bae0a776": "dimension",
|
|
42
|
+
"13134202-f60b-11d5-a3c7-0050bae0a776": "resource", # information register
|
|
43
|
+
"13134203-f60b-11d5-a3c7-0050bae0a776": "dimension",
|
|
44
|
+
"13134204-f60b-11d5-a3c7-0050bae0a776": "attr",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
ROLE_ORDER = ("dimension", "resource", "attr", "column")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_identity(node):
|
|
51
|
+
"""["1", ["0","0",<guid>], "<Name>", ["1","ru","<Synonym>"], "", "0", "0"]"""
|
|
52
|
+
if not (isinstance(node, list) and len(node) >= 3 and isinstance(node[1], list)):
|
|
53
|
+
return False
|
|
54
|
+
if len(node[1]) != 3 or not isinstance(node[1][2], str) or not meta.GUID.match(node[1][2]):
|
|
55
|
+
return False
|
|
56
|
+
return isinstance(node[2], str) and bool(node[2]) and not node[2].isdigit()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def find_identity(node):
|
|
60
|
+
"""First identity block in a header subtree, depth first."""
|
|
61
|
+
if not isinstance(node, list):
|
|
62
|
+
return None
|
|
63
|
+
if is_identity(node):
|
|
64
|
+
return node
|
|
65
|
+
for item in node:
|
|
66
|
+
found = find_identity(item)
|
|
67
|
+
if found is not None:
|
|
68
|
+
return found
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def is_field(node):
|
|
73
|
+
"""A field is an identity block immediately followed by its type pattern."""
|
|
74
|
+
return (isinstance(node, list) and len(node) >= 3 and is_identity(node[1])
|
|
75
|
+
and isinstance(node[2], list) and node[2] and node[2][0] == "Pattern")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def name_of(node):
|
|
79
|
+
return node[1][2]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def guid_of(node):
|
|
83
|
+
"""The attribute's own GUID, from its identity block.
|
|
84
|
+
|
|
85
|
+
This is what DBNames keys its `Fld` entries by, so it is how a ported column is
|
|
86
|
+
traced back to the `_FldNNN` the data actually sits in.
|
|
87
|
+
"""
|
|
88
|
+
ident = node[1]
|
|
89
|
+
head = ident[1] if len(ident) > 1 else None
|
|
90
|
+
return head[2] if isinstance(head, list) and len(head) > 2 else None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def synonym_of(node):
|
|
94
|
+
syn = node[1][3] if len(node[1]) > 3 else None
|
|
95
|
+
return syn[2] if isinstance(syn, list) and len(syn) >= 3 else None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def is_collection(node):
|
|
99
|
+
"""[<class guid>, <count>, member, member, ...] - attributes, dimensions, resources."""
|
|
100
|
+
return (isinstance(node, list) and len(node) > 2 and isinstance(node[0], str)
|
|
101
|
+
and meta.GUID.match(node[0]) and isinstance(node[1], str) and node[1].isdigit())
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def decode_type(pattern, types):
|
|
105
|
+
"""['Pattern', ['S','30','1'], guid, ...] -> readable type."""
|
|
106
|
+
parts, refs = [], []
|
|
107
|
+
for item in pattern[1:]:
|
|
108
|
+
if isinstance(item, str) and meta.GUID.match(item):
|
|
109
|
+
refs.append(item)
|
|
110
|
+
elif isinstance(item, list) and item:
|
|
111
|
+
head = item[0]
|
|
112
|
+
if head == "S":
|
|
113
|
+
parts.append("string(%s)" % (item[1] if len(item) > 1 else "?"))
|
|
114
|
+
elif head == "N":
|
|
115
|
+
parts.append("number(%s,%s)" % (item[1] if len(item) > 1 else "?",
|
|
116
|
+
item[2] if len(item) > 2 else "0"))
|
|
117
|
+
elif head == "B":
|
|
118
|
+
parts.append("bool")
|
|
119
|
+
elif head == "D":
|
|
120
|
+
parts.append("date(%s)" % (item[1] if len(item) > 1 else "DateTime"))
|
|
121
|
+
else:
|
|
122
|
+
refs.extend(s for s in item if isinstance(s, str) and meta.GUID.match(s))
|
|
123
|
+
for guid in refs:
|
|
124
|
+
target = types.get(guid)
|
|
125
|
+
parts.append("-> %s.%s" % (target["family"], target["name"]) if target
|
|
126
|
+
else "-> <%s>" % guid[:8])
|
|
127
|
+
return ", ".join(parts) or "?"
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def has_pattern(node):
|
|
131
|
+
if not isinstance(node, list):
|
|
132
|
+
return False
|
|
133
|
+
if node and node[0] == "Pattern":
|
|
134
|
+
return True
|
|
135
|
+
return any(has_pattern(item) for item in node)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def tabular_section_info(node):
|
|
139
|
+
"""A tabular section is [<header>, ..., <columns collection>]: a named header that
|
|
140
|
+
carries no type of its own, followed by a collection of typed columns.
|
|
141
|
+
|
|
142
|
+
Returns (name, guid). The GUID is what DBNames keys the section's `_VT` table and
|
|
143
|
+
its `_LineNo` column by.
|
|
144
|
+
"""
|
|
145
|
+
if len(node) < 3 or not isinstance(node[0], list) or not is_collection(node[2]):
|
|
146
|
+
return None
|
|
147
|
+
if has_pattern(node[0]):
|
|
148
|
+
return None
|
|
149
|
+
ident = find_identity(node[0])
|
|
150
|
+
if ident is None:
|
|
151
|
+
return None
|
|
152
|
+
head = ident[1] if len(ident) > 1 else None
|
|
153
|
+
guid = head[2] if isinstance(head, list) and len(head) > 2 else None
|
|
154
|
+
return ident[2], guid
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def tabular_section_name(node):
|
|
158
|
+
found = tabular_section_info(node)
|
|
159
|
+
return found[0] if found else None
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def collect(node, section, collection, types, fields, section_guid=None):
|
|
163
|
+
if not isinstance(node, list):
|
|
164
|
+
return
|
|
165
|
+
if is_field(node):
|
|
166
|
+
fields.append({
|
|
167
|
+
"section": section,
|
|
168
|
+
"sectionGuid": section_guid,
|
|
169
|
+
"collection": collection,
|
|
170
|
+
"name": name_of(node),
|
|
171
|
+
"guid": guid_of(node),
|
|
172
|
+
"synonym": synonym_of(node),
|
|
173
|
+
"type": decode_type(node[2], types),
|
|
174
|
+
})
|
|
175
|
+
return
|
|
176
|
+
found = tabular_section_info(node)
|
|
177
|
+
if found:
|
|
178
|
+
section, section_guid = found
|
|
179
|
+
if is_collection(node):
|
|
180
|
+
collection = node[0]
|
|
181
|
+
node = node[2:]
|
|
182
|
+
for item in node:
|
|
183
|
+
collect(item, section, collection, types, fields, section_guid)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def enum_values(tree, object_name):
|
|
187
|
+
"""Enum members carry no type pattern, so they are plain identity blocks."""
|
|
188
|
+
values = []
|
|
189
|
+
def walk(node):
|
|
190
|
+
if not isinstance(node, list):
|
|
191
|
+
return
|
|
192
|
+
if is_identity(node):
|
|
193
|
+
if node[2] != object_name and node[2] not in values:
|
|
194
|
+
values.append(node[2])
|
|
195
|
+
return
|
|
196
|
+
for item in node:
|
|
197
|
+
walk(item)
|
|
198
|
+
walk(tree)
|
|
199
|
+
return values
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def load_json(name):
|
|
203
|
+
with codecs.open(os.path.join(OUT, name), "r", "utf-8") as f:
|
|
204
|
+
return json.load(f)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def fields_of(name, objects, types):
|
|
208
|
+
guid = next((o["guid"] for o in objects if o.get("name") == name), None)
|
|
209
|
+
if not guid:
|
|
210
|
+
return None, None
|
|
211
|
+
text = meta.load(guid)
|
|
212
|
+
if text is None:
|
|
213
|
+
return guid, None
|
|
214
|
+
fields = []
|
|
215
|
+
collect(meta.parse(text), None, None, types, fields)
|
|
216
|
+
return guid, fields
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def main():
|
|
220
|
+
objects = load_json("objects.json")
|
|
221
|
+
types = load_json("typemap.json")
|
|
222
|
+
report = {r["name"]: r for r in load_json("report.json")}
|
|
223
|
+
|
|
224
|
+
if sys.argv[1] == "--model":
|
|
225
|
+
out_path, names = sys.argv[2], sys.argv[3:]
|
|
226
|
+
lines = ["# Transport & shipping contour - extracted model", "",
|
|
227
|
+
"Generated from the `%s` MS SQL infobase without 1C software."
|
|
228
|
+
% os.environ.get("DF1C_DB", "source"), ""]
|
|
229
|
+
for name in names:
|
|
230
|
+
guid, fields = fields_of(name, objects, types)
|
|
231
|
+
row = report.get(name, {})
|
|
232
|
+
if fields is None:
|
|
233
|
+
lines += ["## %s *(not found)*" % name, ""]
|
|
234
|
+
continue
|
|
235
|
+
lines += ["## %s.%s - %s" % (row.get("family", "?"), name, row.get("synonym") or ""),
|
|
236
|
+
"",
|
|
237
|
+
"`%s` - %s rows, %s child rows" % (
|
|
238
|
+
row.get("physical"),
|
|
239
|
+
"{:,}".format(row.get("rows") or 0),
|
|
240
|
+
"{:,}".format(row.get("childRows") or 0)), ""]
|
|
241
|
+
if not fields:
|
|
242
|
+
values = enum_values(meta.parse(meta.load(guid)), name)
|
|
243
|
+
if values:
|
|
244
|
+
lines += ["Values: " + ", ".join(values), ""]
|
|
245
|
+
continue
|
|
246
|
+
sections = [None] + sorted({f["section"] for f in fields if f["section"]})
|
|
247
|
+
for section in sections:
|
|
248
|
+
chunk = [f for f in fields if f["section"] == section]
|
|
249
|
+
if not chunk:
|
|
250
|
+
continue
|
|
251
|
+
chunk.sort(key=lambda f: ROLE_ORDER.index(ROLE[f["collection"]])
|
|
252
|
+
if ROLE.get(f["collection"]) in ROLE_ORDER else 9)
|
|
253
|
+
lines += ["**%s**" % (section or "header"), "",
|
|
254
|
+
"| role | field | synonym | type |", "|---|---|---|---|"]
|
|
255
|
+
for f in chunk:
|
|
256
|
+
lines.append("| %s | %s | %s | %s |" % (
|
|
257
|
+
ROLE.get(f["collection"], "?"), f["name"],
|
|
258
|
+
f["synonym"] or "", f["type"]))
|
|
259
|
+
lines.append("")
|
|
260
|
+
with codecs.open(out_path, "w", "utf-8") as f:
|
|
261
|
+
f.write("\n".join(lines))
|
|
262
|
+
print("wrote %s (%d objects)" % (out_path, len(names)))
|
|
263
|
+
return
|
|
264
|
+
|
|
265
|
+
name = sys.argv[1]
|
|
266
|
+
guid, fields = fields_of(name, objects, types)
|
|
267
|
+
if fields is None:
|
|
268
|
+
print("no such object: %s" % name)
|
|
269
|
+
return
|
|
270
|
+
print("%s (%s) fields=%d\n" % (name, guid, len(fields)))
|
|
271
|
+
for f in fields:
|
|
272
|
+
print(" %-10s %-22s %-34s %-28s %s"
|
|
273
|
+
% (ROLE.get(f["collection"], (f["collection"] or "")[:8]),
|
|
274
|
+
f["section"] or "", f["name"], f["synonym"] or "", f["type"]))
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
if __name__ == "__main__":
|
|
278
|
+
main()
|