@dforge-core/dforge-mcp 0.2.44 → 0.2.46-next

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +73 -0
  2. package/README.md +3 -2
  3. package/dist/server.js +109 -84
  4. package/package.json +48 -46
  5. package/resources/docs/conventions.md +17 -14
  6. package/resources/docs/dsl-reference.md +15 -5
  7. package/resources/schemas/entity.schema.json +1 -1
  8. package/resources/schemas/registers.schema.json +8 -3
  9. package/scripts/install-skills.mjs +1 -0
  10. package/skills/dforge-1c-import/SKILL.md +374 -0
  11. package/skills/dforge-1c-import/references/environment.md +97 -0
  12. package/skills/dforge-1c-import/references/loading.md +161 -0
  13. package/skills/dforge-1c-import/references/measurements.md +210 -0
  14. package/skills/dforge-1c-import/references/print-forms.md +148 -0
  15. package/skills/dforge-1c-import/scripts/1c.py +178 -0
  16. package/skills/dforge-1c-import/scripts/README.md +110 -0
  17. package/skills/dforge-1c-import/scripts/lib/analyze.py +83 -0
  18. package/skills/dforge-1c-import/scripts/lib/attrs.py +278 -0
  19. package/skills/dforge-1c-import/scripts/lib/autospec.py +964 -0
  20. package/skills/dforge-1c-import/scripts/lib/binding.py +257 -0
  21. package/skills/dforge-1c-import/scripts/lib/bsl.py +811 -0
  22. package/skills/dforge-1c-import/scripts/lib/callgraph.py +717 -0
  23. package/skills/dforge-1c-import/scripts/lib/cli.py +53 -0
  24. package/skills/dforge-1c-import/scripts/lib/columnmap.py +346 -0
  25. package/skills/dforge-1c-import/scripts/lib/coverage.py +242 -0
  26. package/skills/dforge-1c-import/scripts/lib/custom.py +83 -0
  27. package/skills/dforge-1c-import/scripts/lib/delta.py +741 -0
  28. package/skills/dforge-1c-import/scripts/lib/deps.py +136 -0
  29. package/skills/dforge-1c-import/scripts/lib/docusage.py +105 -0
  30. package/skills/dforge-1c-import/scripts/lib/dump1c.py +251 -0
  31. package/skills/dforge-1c-import/scripts/lib/explore.py +55 -0
  32. package/skills/dforge-1c-import/scripts/lib/external.py +201 -0
  33. package/skills/dforge-1c-import/scripts/lib/extract_code.py +251 -0
  34. package/skills/dforge-1c-import/scripts/lib/forms.py +239 -0
  35. package/skills/dforge-1c-import/scripts/lib/genmodule.py +1432 -0
  36. package/skills/dforge-1c-import/scripts/lib/handlers.py +202 -0
  37. package/skills/dforge-1c-import/scripts/lib/handmap.py +142 -0
  38. package/skills/dforge-1c-import/scripts/lib/i18n_coverage.py +87 -0
  39. package/skills/dforge-1c-import/scripts/lib/idioms.py +273 -0
  40. package/skills/dforge-1c-import/scripts/lib/integrity.py +264 -0
  41. package/skills/dforge-1c-import/scripts/lib/inventory.py +535 -0
  42. package/skills/dforge-1c-import/scripts/lib/layers.py +106 -0
  43. package/skills/dforge-1c-import/scripts/lib/meta.py +315 -0
  44. package/skills/dforge-1c-import/scripts/lib/migrate.py +1299 -0
  45. package/skills/dforge-1c-import/scripts/lib/mxl.py +368 -0
  46. package/skills/dforge-1c-import/scripts/lib/naming.py +251 -0
  47. package/skills/dforge-1c-import/scripts/lib/paths.py +90 -0
  48. package/skills/dforge-1c-import/scripts/lib/peek.py +70 -0
  49. package/skills/dforge-1c-import/scripts/lib/posted.py +218 -0
  50. package/skills/dforge-1c-import/scripts/lib/posting.py +655 -0
  51. package/skills/dforge-1c-import/scripts/lib/printgen.py +239 -0
  52. package/skills/dforge-1c-import/scripts/lib/rebuild.py +194 -0
  53. package/skills/dforge-1c-import/scripts/lib/recorded.py +170 -0
  54. package/skills/dforge-1c-import/scripts/lib/recorders.py +94 -0
  55. package/skills/dforge-1c-import/scripts/lib/reftargets.py +95 -0
  56. package/skills/dforge-1c-import/scripts/lib/reportgen.py +157 -0
  57. package/skills/dforge-1c-import/scripts/lib/roles.py +33 -0
  58. package/skills/dforge-1c-import/scripts/lib/seedstate.py +488 -0
  59. package/skills/dforge-1c-import/scripts/lib/sizing.py +184 -0
  60. package/skills/dforge-1c-import/scripts/lib/spec.py +400 -0
  61. package/skills/dforge-1c-import/scripts/lib/statements.py +774 -0
  62. package/skills/dforge-1c-import/scripts/lib/subsystems.py +480 -0
  63. package/skills/dforge-1c-import/scripts/lib/terms.py +451 -0
  64. package/skills/dforge-1c-import/scripts/lib/terms_uk.py +555 -0
  65. package/skills/dforge-1c-import/scripts/lib/totals.py +387 -0
  66. package/skills/dforge-1c-import/scripts/lib/typemap.py +73 -0
  67. package/skills/dforge-1c-import/scripts/lib/unpack.py +136 -0
  68. package/skills/dforge-1c-import/scripts/lib/validate.py +508 -0
  69. package/skills/dforge-1c-import/scripts/project-template/contours.py +47 -0
  70. package/skills/dforge-1c-import/scripts/project-template/sources.py +14 -0
  71. package/skills/dforge-mcp-author/SKILL.md +8 -2
  72. package/skills/dforge-mcp-author/examples/column-security/entities/product.json +1 -1
  73. package/skills/dforge-mcp-author/examples/column-security/manifest.json +0 -1
  74. package/skills/dforge-mcp-author/examples/column-security/security/roles.json +2 -2
  75. package/skills/dforge-mcp-author/references/action-dsl.md +20 -7
  76. package/skills/dforge-mcp-author/references/conventions.md +7 -1
  77. package/skills/dforge-mcp-author/references/field-types.md +3 -2
  78. package/skills/dforge-mcp-author/references/formulas.md +5 -1
  79. package/skills/dforge-mcp-author/references/print-templates.md +5 -2
  80. package/skills/dforge-mcp-author/references/queries.md +1 -1
  81. package/skills/dforge-mcp-author/references/registers.md +18 -3
  82. package/skills/dforge-mcp-author/references/reports.md +4 -1
  83. package/skills/dforge-mcp-author/references/schema-import.md +1 -1
  84. package/skills/dforge-mcp-author/references/security.md +1 -1
  85. package/skills/dforge-mcp-author/references/translations.md +38 -15
  86. package/skills/dforge-mcp-author/references/validation-checklist.md +2 -3
  87. package/skills/dforge-module-build/SKILL.md +10 -1
  88. package/skills/dforge-module-ship/SKILL.md +31 -3
  89. package/skills/dforge-mcp-author/examples/column-security/translations/en-US.json +0 -35
@@ -0,0 +1,178 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ One way in, so the split between library and project stays a fact rather than a habit.
5
+
6
+ python tools/1c.py <command> [args...]
7
+ python tools/1c.py --list # what there is
8
+ python tools/1c.py --boundary # does the library still stand on its own?
9
+
10
+ `tools/lib` is 1C and dForge: how a configuration is stored, how its metadata parses,
11
+ what the physical tables mean, and how rows are copied and checked. It is meant to be
12
+ lifted out and pointed at a different infobase.
13
+
14
+ `tools/project` is this infobase: which object belongs to which module, which 1C column
15
+ feeds each ported one, which regulated statements it holds, this company's own proper
16
+ nouns, and the modules written by hand. `DF1C_PROJECT` points the library at a different
17
+ one; `binding.py` lists the whole of what a project has to provide.
18
+
19
+ Both directories go on the path, so the modules import each other by plain name and a
20
+ move between them costs no edits. That convenience is also the hazard, so `--boundary`
21
+ checks two things rather than trusting either: that no library module IMPORTS a project
22
+ one, and that no library module NAMES this project's world - its server, its infobase,
23
+ its tenant, its company's abbreviations. A literal is the quiet way to bind a library to
24
+ one configuration, and it is how `DF1C_DB` came to default to `utp_biscuit` for months.
25
+
26
+ Because both directories are on one path, a project module may not take a name the
27
+ library already uses: `regulated.py` rather than `forms.py`, `vocabulary.py` rather than
28
+ `terms.py`. `--boundary` catches the shadowing as an import crossing.
29
+ """
30
+ from __future__ import print_function
31
+
32
+ import io
33
+ import os
34
+ import sys
35
+
36
+ HERE = os.path.dirname(os.path.abspath(__file__))
37
+ LIB = os.path.join(HERE, "lib")
38
+ PROJECT = os.path.abspath(os.environ.get("DF1C_PROJECT") or os.path.join(HERE, "project"))
39
+
40
+ for entry in (PROJECT, LIB):
41
+ if entry not in sys.path:
42
+ sys.path.insert(0, entry)
43
+
44
+
45
+ def commands():
46
+ """Every runnable module, library or project, mapped to (path, first docstring line)."""
47
+ found = {}
48
+ for directory in (LIB, PROJECT):
49
+ for name in sorted(os.listdir(directory)):
50
+ if not name.endswith(".py") or name.startswith("_") or name[:-3] in found:
51
+ continue
52
+ path = os.path.join(directory, name)
53
+ summary = _runnable(path)
54
+ if summary is not None:
55
+ found[name[:-3]] = (path, summary)
56
+ return found
57
+
58
+
59
+ def _runnable(path):
60
+ """The module's first docstring line if it can be run as a script, else None."""
61
+ with io.open(path, encoding="utf-8") as f:
62
+ text = f.read()
63
+ # Anything runnable: most define main(), a few only a __main__ block.
64
+ if "def main(" not in text and "__name__" not in text:
65
+ return None
66
+ if '"""' not in text:
67
+ return ""
68
+ body = text.split('"""', 2)[1].strip().splitlines()
69
+ return (body[0] if body else "").strip()
70
+
71
+
72
+ def boundary():
73
+ """Two ways the library can stop standing on its own, both read off the source.
74
+
75
+ IMPORTS. The path puts both directories in reach, so nothing stops a library module
76
+ importing `contours` directly and nothing would complain until someone tried to lift
77
+ the library out.
78
+
79
+ NAMES. An import is the loud way to bind to one infobase; a literal is the quiet one.
80
+ `DF1C_DB` defaulting to `utp_biscuit` and `DF_TENANT` to `hbf1` sat in the library for
81
+ a long while, harmless with one infobase and a trap with two - point the tools at a
82
+ second configuration, forget one variable, and they read the first one with
83
+ right-looking output. So the names of THIS project's world are refused in the library,
84
+ including in its comments, since a stale comment is how the next reader learns the
85
+ wrong thing.
86
+
87
+ The list is read from the project rather than written here, so a project that renames
88
+ itself does not have to remember this file.
89
+ """
90
+ import ast
91
+ import re
92
+
93
+ project_modules = set(n[:-3] for n in os.listdir(PROJECT) if n.endswith(".py"))
94
+
95
+ # What this project calls itself: the infobase, the tenant and its own proper nouns.
96
+ reserved = set()
97
+ for variable in ("DF1C_DB", "DF1C_SERVER", "DF_TENANT"):
98
+ value = os.environ.get(variable)
99
+ if value:
100
+ reserved.add(value.lower())
101
+ env_example = os.path.join(os.path.dirname(os.path.abspath(PROJECT)), os.pardir,
102
+ ".env.example")
103
+ if os.path.exists(env_example):
104
+ with io.open(env_example, encoding="utf-8") as f:
105
+ for line in f:
106
+ match = re.match(r"\s*(DF1C_DB|DF1C_SERVER|DF_TENANT)\s*=\s*(\S+)", line)
107
+ if match:
108
+ reserved.add(match.group(2).lower())
109
+ terms = os.path.join(PROJECT, "terms.py")
110
+ if os.path.exists(terms):
111
+ with io.open(terms, encoding="utf-8") as f:
112
+ for value in re.findall(r'u"([^"]+)"\s*:', f.read()):
113
+ if len(value) > 3: # two- and three-letter forms collide with real words
114
+ reserved.add(value.lower())
115
+
116
+ problems = []
117
+ for name in sorted(os.listdir(LIB)):
118
+ if not name.endswith(".py"):
119
+ continue
120
+ with io.open(os.path.join(LIB, name), encoding="utf-8") as f:
121
+ text = f.read()
122
+ tree = ast.parse(text)
123
+ for node in ast.walk(tree):
124
+ named = []
125
+ if isinstance(node, ast.Import):
126
+ named = [a.name.split(".")[0] for a in node.names]
127
+ elif isinstance(node, ast.ImportFrom) and node.module:
128
+ named = [node.module.split(".")[0]]
129
+ for target in named:
130
+ if target in project_modules:
131
+ problems.append("lib/%s imports %s - go through binding.py instead"
132
+ % (name, target))
133
+ # paths.py explains why these names are refused, so it is allowed to name them.
134
+ if name == "paths.py":
135
+ continue
136
+ lowered = text.lower()
137
+ for word in sorted(reserved):
138
+ if word in lowered:
139
+ problems.append("lib/%s names %r - that is the project's, not the library's"
140
+ % (name, word))
141
+ return problems
142
+
143
+
144
+ def main():
145
+ args = sys.argv[1:]
146
+ if not args or args[0] in ("-h", "--help", "--list"):
147
+ import paths
148
+ paths.show()
149
+ print()
150
+ for name, (_, summary) in sorted(commands().items()):
151
+ print(" %-16s %s" % (name, summary[:60]))
152
+ return 0
153
+
154
+ if args[0] == "--boundary":
155
+ problems = boundary()
156
+ for line in problems:
157
+ print(line)
158
+ print("%d crossing(s) of the library/project boundary" % len(problems))
159
+ return 1 if problems else 0
160
+
161
+ name, rest = args[0], args[1:]
162
+ found = commands()
163
+ if name not in found:
164
+ print("no such command: %s (try --list)" % name)
165
+ return 2
166
+ # run_path rather than calling main(): a few tools only have a __main__
167
+ # block, and this runs every one of them exactly as a script would.
168
+ import runpy
169
+ sys.argv = [name] + rest
170
+ try:
171
+ runpy.run_path(found[name][0], run_name="__main__")
172
+ except SystemExit as stop:
173
+ return stop.code or 0
174
+ return 0
175
+
176
+
177
+ if __name__ == "__main__":
178
+ sys.exit(main() or 0)
@@ -0,0 +1,110 @@
1
+ # The 1C porting toolkit
2
+
3
+ A Python library that reads a 1C:Enterprise 8 infobase hosted on MS SQL Server. It needs
4
+ no 1C software and no licence. From the infobase it:
5
+
6
+ - generates dForge modules;
7
+ - copies the rows;
8
+ - checks the result.
9
+
10
+ It was built on a real port: an УТП-based configuration, 16 contours, 30 M+ rows,
11
+ reconciled to the row and the kopeck. It is meant to be pointed at the next infobase
12
+ unchanged.
13
+
14
+ ```
15
+ scripts/
16
+ 1c.py the one entry point: python 1c.py <command> [args]
17
+ lib/ the library: 1C storage, metadata, generation, loading, checks
18
+ project-template/ what a new port starts its project package from
19
+ ```
20
+
21
+ ## Library and project
22
+
23
+ - **`lib/` knows 1C and dForge**: how a configuration is stored, how its metadata
24
+ parses, what the physical tables mean, and how rows are copied and checked.
25
+ - **The project knows this infobase**:
26
+ - which object goes to which module (`contours.py`);
27
+ - which 1C column feeds an authored column (`sources.py`);
28
+ - the integrator's prefixes;
29
+ - the hand-written modules (`authored/<module>/`).
30
+
31
+ `lib/binding.py` lists everything a project may provide. Only `contours` and `sources`
32
+ are required.
33
+
34
+ **Never put project facts in `lib/`.** That covers infobase names, tenant names, the
35
+ company's abbreviations and prefixes. `python 1c.py --boundary` checks that no library
36
+ module imports a project module or names the project's world.
37
+
38
+ **Never hand-edit generated output.** Put the change in the project, either as a spec
39
+ override or as an authored file under `authored/<module>/` that `genmodule` merges.
40
+ Then regenerate. If a port keeps needing hand edits, the generator is wrong.
41
+
42
+ ## Setup
43
+
44
+ - Python 3.7+.
45
+ - `pyodbc` and an ODBC driver for SQL Server (driver 17 or 18), to read the infobase.
46
+ - `psycopg2`, to load and check the tenant.
47
+ - `jsonschema`, for `validate`.
48
+ - Node with `liquidjs` in `PRINT_NODE_DIR`, only for `printgen preview`.
49
+
50
+ On Windows, set `PYTHONIOENCODING=utf-8`.
51
+
52
+ Lay out a port directory and point the toolkit at it:
53
+
54
+ ```
55
+ myport/
56
+ .env credentials (never committed)
57
+ project/ copied from project-template/, then filled in
58
+ out/ what the extraction writes (DF1C_OUT, default <root>/out)
59
+ modules/ generated modules (DF1C_MODULES, default <root>/modules)
60
+ ```
61
+
62
+ ```
63
+ DF1C_ROOT=/path/to/myport
64
+ DF1C_PROJECT=/path/to/myport/project
65
+ DF1C_SERVER=<sql server> DF1C_DB=<infobase db> DF1C_USER= DF1C_PASS= # read-only login; Windows auth when USER is empty
66
+ DF_TENANT=<tenant code> DF_PG_HOST= DF_PG_PORT= DF_PG_USER= DF_PG_PASS= # the tenant's Postgres, for load and checks
67
+ ```
68
+
69
+ None of these has a default. A default that named one infobase would make the toolkit
70
+ quietly read or write that infobase when a variable was forgotten.
71
+
72
+ ## The pipeline
73
+
74
+ Each step reads what the previous one wrote to `out/`. Every command has `--help` or a
75
+ docstring with its options, and `python 1c.py --list` lists them all.
76
+
77
+ | # | step | command |
78
+ |---|---|---|
79
+ | 1 | Read the configuration and DBNames from the SQL tables | `dump1c probe` · `dump1c dbnames` · `dump1c schema` · `dump1c extract` |
80
+ | 2 | Parse the metadata, resolve type GUIDs, join the physical schema | `meta allobjects` · `typemap` · `analyze` · `attrs <Object>` · `explore <Object>` |
81
+ | 3 | Extract the module texts (posting handlers, print procedures) | `extract_code` · `unpack <Object> --code` |
82
+ | 4 | Inventory: what exists, what is used, what is the client's | `inventory` · `docusage` · `custom` · `sizing` · `subsystems` |
83
+ | 5 | Read the posting handlers: which registers each document writes and from where | `handlers [--client]` · `callgraph` · `recorders` · `recorded` · `idioms` · `spec` |
84
+ | 6 | Decide the module boundaries | `subsystems --contours --acyclic`, copied into `project/contours.py` and corrected |
85
+ | 7 | Generate and validate the modules | `autospec [module]` · `genmodule` · `validate` · `deps` · `layers` |
86
+ | 8 | Install them | `npx -y @dforge-core/dforge-cli@latest module install ...`, or `rebuild --code <tenant> --install` |
87
+ | 9 | Copy the rows and reconcile | `migrate plan\|check\|run\|verify\|fill <module>` · `integrity` · `columnmap <module>` |
88
+ | 10 | Post state and registers | `posted [--mark]` · `posting` · `seedstate` · `totals` · `coverage` |
89
+ | 11 | Catch up with what 1C changed | `delta --baseline` · `delta [--apply]` |
90
+ | 12 | Print forms and reports | `mxl list [--client]` · `mxl dump <Object>` · `printgen <spec>` · `printgen preview <spec> <form> <key>` · `reportgen <spec>` |
91
+ | 13 | External reports and processings stored in the infobase | `external` |
92
+
93
+ `rebuild --code <tenant>` runs steps 8–10 from nothing, which makes a tenant
94
+ reproducible.
95
+
96
+ ## Rules the toolkit assumes
97
+
98
+ - **The infobase is read-only.** Use a SELECT-only login.
99
+ - **Open ad-hoc database connections in autocommit.** A read transaction left open
100
+ blocks the next install's `ALTER`.
101
+ - **Resolve a physical column (`_FldNNN`) only through DBNames** (`attrs`, `columnmap`).
102
+ Never resolve it by cardinality.
103
+ - **Stop on a 401 from the tenant.** Ask for a fresh login, then re-run
104
+ `posted --mark` for documents loaded in the meantime.
105
+
106
+ ## Source
107
+
108
+ This copy is kept in step with the port it was developed on (`tools/lib`, `tools/1c.py`
109
+ there). Fixes made in a port go back to the library here. A project's own files never
110
+ come here.
@@ -0,0 +1,83 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Join the physical schema against DBNames to see which parts of the configuration
5
+ the client actually uses. A stock UTP ships hundreds of catalogs and documents;
6
+ only the populated ones are worth porting.
7
+ """
8
+ import codecs
9
+ import collections
10
+ import json
11
+ import os
12
+ import re
13
+ import sys
14
+
15
+ import paths
16
+
17
+ OUT = paths.OUT
18
+
19
+ # Physical table families that carry business data, in the order we care about.
20
+ DATA_KINDS = ("Reference", "Document", "InfoRg", "AccumRg", "AccRg", "Chrc", "Acc",
21
+ "Enum", "DocumentJournal", "Node", "BPr", "Task", "Const")
22
+
23
+ SPLIT = re.compile(r"^_([A-Za-z]+?)(\d+)(?:_(.+))?$")
24
+
25
+
26
+ def load(name):
27
+ with codecs.open(os.path.join(OUT, name), "r", "utf-8") as f:
28
+ return json.load(f)
29
+
30
+
31
+ def classify(table_name):
32
+ """_Document123_VT456 -> ('Document', 123, 'VT456')"""
33
+ m = SPLIT.match(table_name)
34
+ if not m:
35
+ return None, None, None
36
+ return m.group(1), int(m.group(2)), m.group(3)
37
+
38
+
39
+ def main():
40
+ schema = load("schema.json")
41
+ tables = {t["name"]: t for t in schema["tables"]}
42
+
43
+ summary = collections.OrderedDict((k, {"tables": 0, "used": 0, "rows": 0}) for k in DATA_KINDS)
44
+ detail = collections.defaultdict(list)
45
+
46
+ for name, t in tables.items():
47
+ kind, number, suffix = classify(name)
48
+ if kind not in summary or suffix: # skip tabular sections, roll them up separately
49
+ continue
50
+ rows = t["rows"] or 0
51
+ # a document's tabular sections belong to the document's footprint
52
+ child_rows = sum((tables[n]["rows"] or 0) for n in tables
53
+ if n.startswith("_%s%d_" % (kind, number)))
54
+ summary[kind]["tables"] += 1
55
+ summary[kind]["rows"] += rows
56
+ if rows:
57
+ summary[kind]["used"] += 1
58
+ detail[kind].append({"table": name, "number": number, "rows": rows,
59
+ "childRows": child_rows, "columns": len(t["columns"])})
60
+
61
+ print("%-18s %8s %8s %14s" % ("kind", "tables", "used", "rows"))
62
+ print("-" * 52)
63
+ for kind, s in summary.items():
64
+ if s["tables"]:
65
+ print("%-18s %8d %8d %14s" % (kind, s["tables"], s["used"], "{:,}".format(s["rows"])))
66
+
67
+ top = int(sys.argv[1]) if len(sys.argv) > 1 else 25
68
+ for kind in ("Document", "Reference", "AccumRg", "InfoRg"):
69
+ rows = sorted(detail[kind], key=lambda d: -(d["rows"] + d["childRows"]))[:top]
70
+ print("\n== %s: top %d by volume ==" % (kind, top))
71
+ for d in rows:
72
+ if not (d["rows"] or d["childRows"]):
73
+ continue
74
+ print(" %-22s rows=%-12s child=%-12s cols=%d"
75
+ % (d["table"], "{:,}".format(d["rows"]),
76
+ "{:,}".format(d["childRows"]), d["columns"]))
77
+
78
+ with codecs.open(os.path.join(OUT, "usage.json"), "w", "utf-8") as f:
79
+ json.dump({"summary": summary, "detail": detail}, f, ensure_ascii=False, indent="\t")
80
+
81
+
82
+ if __name__ == "__main__":
83
+ main()
@@ -0,0 +1,278 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Extract the field list of a metadata object, with references resolved.
5
+
6
+ Every typed thing in a 1C object file - a catalog attribute, a register dimension
7
+ or resource, a tabular-section column - is encoded the same way:
8
+
9
+ ["2", ["1", ["0","0",<own guid>], "<Name>", ["1","ru","<Synonym>"], ...],
10
+ ["Pattern", <type...>]]
11
+
12
+ so the collector looks for that shape anywhere in the tree rather than hard-coding
13
+ per-kind offsets, and takes the nearest named ancestor as the tabular section.
14
+
15
+ python attrs.py <ObjectName> one object, to the console
16
+ python attrs.py --model <file> <names...> several objects, to a markdown file
17
+ """
18
+ from __future__ import print_function
19
+
20
+ import codecs
21
+ import json
22
+ import os
23
+ import paths
24
+ import sys
25
+
26
+ import meta
27
+
28
+ OUT = meta.OUT
29
+
30
+ PRIMITIVE = {"S": "string", "N": "number", "B": "bool", "D": "date"}
31
+
32
+ # Collection class GUIDs -> what the fields in them mean, confirmed by checking a
33
+ # register whose meaning is unambiguous (ТоварыНаСкладах: Количество is the resource,
34
+ # Склад/Номенклатура the dimensions). Unknown GUIDs print truncated so they show up.
35
+ ROLE = {
36
+ "cf4abea7-37b2-11d4-940f-008048da11f9": "attr", # catalog attribute
37
+ "45e46cbc-3e24-4165-8b7b-cc98a6f80211": "attr", # document attribute
38
+ "888744e1-b616-11d4-9436-004095e12fc7": "column", # tabular section column
39
+ "b64d9a41-1642-11d6-a3c7-0050bae0a776": "resource", # accumulation register
40
+ "b64d9a42-1642-11d6-a3c7-0050bae0a776": "attr",
41
+ "b64d9a43-1642-11d6-a3c7-0050bae0a776": "dimension",
42
+ "13134202-f60b-11d5-a3c7-0050bae0a776": "resource", # information register
43
+ "13134203-f60b-11d5-a3c7-0050bae0a776": "dimension",
44
+ "13134204-f60b-11d5-a3c7-0050bae0a776": "attr",
45
+ }
46
+
47
+ ROLE_ORDER = ("dimension", "resource", "attr", "column")
48
+
49
+
50
+ def is_identity(node):
51
+ """["1", ["0","0",<guid>], "<Name>", ["1","ru","<Synonym>"], "", "0", "0"]"""
52
+ if not (isinstance(node, list) and len(node) >= 3 and isinstance(node[1], list)):
53
+ return False
54
+ if len(node[1]) != 3 or not isinstance(node[1][2], str) or not meta.GUID.match(node[1][2]):
55
+ return False
56
+ return isinstance(node[2], str) and bool(node[2]) and not node[2].isdigit()
57
+
58
+
59
+ def find_identity(node):
60
+ """First identity block in a header subtree, depth first."""
61
+ if not isinstance(node, list):
62
+ return None
63
+ if is_identity(node):
64
+ return node
65
+ for item in node:
66
+ found = find_identity(item)
67
+ if found is not None:
68
+ return found
69
+ return None
70
+
71
+
72
+ def is_field(node):
73
+ """A field is an identity block immediately followed by its type pattern."""
74
+ return (isinstance(node, list) and len(node) >= 3 and is_identity(node[1])
75
+ and isinstance(node[2], list) and node[2] and node[2][0] == "Pattern")
76
+
77
+
78
+ def name_of(node):
79
+ return node[1][2]
80
+
81
+
82
+ def guid_of(node):
83
+ """The attribute's own GUID, from its identity block.
84
+
85
+ This is what DBNames keys its `Fld` entries by, so it is how a ported column is
86
+ traced back to the `_FldNNN` the data actually sits in.
87
+ """
88
+ ident = node[1]
89
+ head = ident[1] if len(ident) > 1 else None
90
+ return head[2] if isinstance(head, list) and len(head) > 2 else None
91
+
92
+
93
+ def synonym_of(node):
94
+ syn = node[1][3] if len(node[1]) > 3 else None
95
+ return syn[2] if isinstance(syn, list) and len(syn) >= 3 else None
96
+
97
+
98
+ def is_collection(node):
99
+ """[<class guid>, <count>, member, member, ...] - attributes, dimensions, resources."""
100
+ return (isinstance(node, list) and len(node) > 2 and isinstance(node[0], str)
101
+ and meta.GUID.match(node[0]) and isinstance(node[1], str) and node[1].isdigit())
102
+
103
+
104
+ def decode_type(pattern, types):
105
+ """['Pattern', ['S','30','1'], guid, ...] -> readable type."""
106
+ parts, refs = [], []
107
+ for item in pattern[1:]:
108
+ if isinstance(item, str) and meta.GUID.match(item):
109
+ refs.append(item)
110
+ elif isinstance(item, list) and item:
111
+ head = item[0]
112
+ if head == "S":
113
+ parts.append("string(%s)" % (item[1] if len(item) > 1 else "?"))
114
+ elif head == "N":
115
+ parts.append("number(%s,%s)" % (item[1] if len(item) > 1 else "?",
116
+ item[2] if len(item) > 2 else "0"))
117
+ elif head == "B":
118
+ parts.append("bool")
119
+ elif head == "D":
120
+ parts.append("date(%s)" % (item[1] if len(item) > 1 else "DateTime"))
121
+ else:
122
+ refs.extend(s for s in item if isinstance(s, str) and meta.GUID.match(s))
123
+ for guid in refs:
124
+ target = types.get(guid)
125
+ parts.append("-> %s.%s" % (target["family"], target["name"]) if target
126
+ else "-> <%s>" % guid[:8])
127
+ return ", ".join(parts) or "?"
128
+
129
+
130
+ def has_pattern(node):
131
+ if not isinstance(node, list):
132
+ return False
133
+ if node and node[0] == "Pattern":
134
+ return True
135
+ return any(has_pattern(item) for item in node)
136
+
137
+
138
+ def tabular_section_info(node):
139
+ """A tabular section is [<header>, ..., <columns collection>]: a named header that
140
+ carries no type of its own, followed by a collection of typed columns.
141
+
142
+ Returns (name, guid). The GUID is what DBNames keys the section's `_VT` table and
143
+ its `_LineNo` column by.
144
+ """
145
+ if len(node) < 3 or not isinstance(node[0], list) or not is_collection(node[2]):
146
+ return None
147
+ if has_pattern(node[0]):
148
+ return None
149
+ ident = find_identity(node[0])
150
+ if ident is None:
151
+ return None
152
+ head = ident[1] if len(ident) > 1 else None
153
+ guid = head[2] if isinstance(head, list) and len(head) > 2 else None
154
+ return ident[2], guid
155
+
156
+
157
+ def tabular_section_name(node):
158
+ found = tabular_section_info(node)
159
+ return found[0] if found else None
160
+
161
+
162
+ def collect(node, section, collection, types, fields, section_guid=None):
163
+ if not isinstance(node, list):
164
+ return
165
+ if is_field(node):
166
+ fields.append({
167
+ "section": section,
168
+ "sectionGuid": section_guid,
169
+ "collection": collection,
170
+ "name": name_of(node),
171
+ "guid": guid_of(node),
172
+ "synonym": synonym_of(node),
173
+ "type": decode_type(node[2], types),
174
+ })
175
+ return
176
+ found = tabular_section_info(node)
177
+ if found:
178
+ section, section_guid = found
179
+ if is_collection(node):
180
+ collection = node[0]
181
+ node = node[2:]
182
+ for item in node:
183
+ collect(item, section, collection, types, fields, section_guid)
184
+
185
+
186
+ def enum_values(tree, object_name):
187
+ """Enum members carry no type pattern, so they are plain identity blocks."""
188
+ values = []
189
+ def walk(node):
190
+ if not isinstance(node, list):
191
+ return
192
+ if is_identity(node):
193
+ if node[2] != object_name and node[2] not in values:
194
+ values.append(node[2])
195
+ return
196
+ for item in node:
197
+ walk(item)
198
+ walk(tree)
199
+ return values
200
+
201
+
202
+ def load_json(name):
203
+ with codecs.open(os.path.join(OUT, name), "r", "utf-8") as f:
204
+ return json.load(f)
205
+
206
+
207
+ def fields_of(name, objects, types):
208
+ guid = next((o["guid"] for o in objects if o.get("name") == name), None)
209
+ if not guid:
210
+ return None, None
211
+ text = meta.load(guid)
212
+ if text is None:
213
+ return guid, None
214
+ fields = []
215
+ collect(meta.parse(text), None, None, types, fields)
216
+ return guid, fields
217
+
218
+
219
+ def main():
220
+ objects = load_json("objects.json")
221
+ types = load_json("typemap.json")
222
+ report = {r["name"]: r for r in load_json("report.json")}
223
+
224
+ if sys.argv[1] == "--model":
225
+ out_path, names = sys.argv[2], sys.argv[3:]
226
+ lines = ["# Transport & shipping contour - extracted model", "",
227
+ "Generated from the `%s` MS SQL infobase without 1C software."
228
+ % os.environ.get("DF1C_DB", "source"), ""]
229
+ for name in names:
230
+ guid, fields = fields_of(name, objects, types)
231
+ row = report.get(name, {})
232
+ if fields is None:
233
+ lines += ["## %s *(not found)*" % name, ""]
234
+ continue
235
+ lines += ["## %s.%s - %s" % (row.get("family", "?"), name, row.get("synonym") or ""),
236
+ "",
237
+ "`%s` - %s rows, %s child rows" % (
238
+ row.get("physical"),
239
+ "{:,}".format(row.get("rows") or 0),
240
+ "{:,}".format(row.get("childRows") or 0)), ""]
241
+ if not fields:
242
+ values = enum_values(meta.parse(meta.load(guid)), name)
243
+ if values:
244
+ lines += ["Values: " + ", ".join(values), ""]
245
+ continue
246
+ sections = [None] + sorted({f["section"] for f in fields if f["section"]})
247
+ for section in sections:
248
+ chunk = [f for f in fields if f["section"] == section]
249
+ if not chunk:
250
+ continue
251
+ chunk.sort(key=lambda f: ROLE_ORDER.index(ROLE[f["collection"]])
252
+ if ROLE.get(f["collection"]) in ROLE_ORDER else 9)
253
+ lines += ["**%s**" % (section or "header"), "",
254
+ "| role | field | synonym | type |", "|---|---|---|---|"]
255
+ for f in chunk:
256
+ lines.append("| %s | %s | %s | %s |" % (
257
+ ROLE.get(f["collection"], "?"), f["name"],
258
+ f["synonym"] or "", f["type"]))
259
+ lines.append("")
260
+ with codecs.open(out_path, "w", "utf-8") as f:
261
+ f.write("\n".join(lines))
262
+ print("wrote %s (%d objects)" % (out_path, len(names)))
263
+ return
264
+
265
+ name = sys.argv[1]
266
+ guid, fields = fields_of(name, objects, types)
267
+ if fields is None:
268
+ print("no such object: %s" % name)
269
+ return
270
+ print("%s (%s) fields=%d\n" % (name, guid, len(fields)))
271
+ for f in fields:
272
+ print(" %-10s %-22s %-34s %-28s %s"
273
+ % (ROLE.get(f["collection"], (f["collection"] or "")[:8]),
274
+ f["section"] or "", f["name"], f["synonym"] or "", f["type"]))
275
+
276
+
277
+ if __name__ == "__main__":
278
+ main()