@dforge-core/dforge-mcp 0.2.44 → 0.2.46-next

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +73 -0
  2. package/README.md +3 -2
  3. package/dist/server.js +109 -84
  4. package/package.json +48 -46
  5. package/resources/docs/conventions.md +17 -14
  6. package/resources/docs/dsl-reference.md +15 -5
  7. package/resources/schemas/entity.schema.json +1 -1
  8. package/resources/schemas/registers.schema.json +8 -3
  9. package/scripts/install-skills.mjs +1 -0
  10. package/skills/dforge-1c-import/SKILL.md +374 -0
  11. package/skills/dforge-1c-import/references/environment.md +97 -0
  12. package/skills/dforge-1c-import/references/loading.md +161 -0
  13. package/skills/dforge-1c-import/references/measurements.md +210 -0
  14. package/skills/dforge-1c-import/references/print-forms.md +148 -0
  15. package/skills/dforge-1c-import/scripts/1c.py +178 -0
  16. package/skills/dforge-1c-import/scripts/README.md +110 -0
  17. package/skills/dforge-1c-import/scripts/lib/analyze.py +83 -0
  18. package/skills/dforge-1c-import/scripts/lib/attrs.py +278 -0
  19. package/skills/dforge-1c-import/scripts/lib/autospec.py +964 -0
  20. package/skills/dforge-1c-import/scripts/lib/binding.py +257 -0
  21. package/skills/dforge-1c-import/scripts/lib/bsl.py +811 -0
  22. package/skills/dforge-1c-import/scripts/lib/callgraph.py +717 -0
  23. package/skills/dforge-1c-import/scripts/lib/cli.py +53 -0
  24. package/skills/dforge-1c-import/scripts/lib/columnmap.py +346 -0
  25. package/skills/dforge-1c-import/scripts/lib/coverage.py +242 -0
  26. package/skills/dforge-1c-import/scripts/lib/custom.py +83 -0
  27. package/skills/dforge-1c-import/scripts/lib/delta.py +741 -0
  28. package/skills/dforge-1c-import/scripts/lib/deps.py +136 -0
  29. package/skills/dforge-1c-import/scripts/lib/docusage.py +105 -0
  30. package/skills/dforge-1c-import/scripts/lib/dump1c.py +251 -0
  31. package/skills/dforge-1c-import/scripts/lib/explore.py +55 -0
  32. package/skills/dforge-1c-import/scripts/lib/external.py +201 -0
  33. package/skills/dforge-1c-import/scripts/lib/extract_code.py +251 -0
  34. package/skills/dforge-1c-import/scripts/lib/forms.py +239 -0
  35. package/skills/dforge-1c-import/scripts/lib/genmodule.py +1432 -0
  36. package/skills/dforge-1c-import/scripts/lib/handlers.py +202 -0
  37. package/skills/dforge-1c-import/scripts/lib/handmap.py +142 -0
  38. package/skills/dforge-1c-import/scripts/lib/i18n_coverage.py +87 -0
  39. package/skills/dforge-1c-import/scripts/lib/idioms.py +273 -0
  40. package/skills/dforge-1c-import/scripts/lib/integrity.py +264 -0
  41. package/skills/dforge-1c-import/scripts/lib/inventory.py +535 -0
  42. package/skills/dforge-1c-import/scripts/lib/layers.py +106 -0
  43. package/skills/dforge-1c-import/scripts/lib/meta.py +315 -0
  44. package/skills/dforge-1c-import/scripts/lib/migrate.py +1299 -0
  45. package/skills/dforge-1c-import/scripts/lib/mxl.py +368 -0
  46. package/skills/dforge-1c-import/scripts/lib/naming.py +251 -0
  47. package/skills/dforge-1c-import/scripts/lib/paths.py +90 -0
  48. package/skills/dforge-1c-import/scripts/lib/peek.py +70 -0
  49. package/skills/dforge-1c-import/scripts/lib/posted.py +218 -0
  50. package/skills/dforge-1c-import/scripts/lib/posting.py +655 -0
  51. package/skills/dforge-1c-import/scripts/lib/printgen.py +239 -0
  52. package/skills/dforge-1c-import/scripts/lib/rebuild.py +194 -0
  53. package/skills/dforge-1c-import/scripts/lib/recorded.py +170 -0
  54. package/skills/dforge-1c-import/scripts/lib/recorders.py +94 -0
  55. package/skills/dforge-1c-import/scripts/lib/reftargets.py +95 -0
  56. package/skills/dforge-1c-import/scripts/lib/reportgen.py +157 -0
  57. package/skills/dforge-1c-import/scripts/lib/roles.py +33 -0
  58. package/skills/dforge-1c-import/scripts/lib/seedstate.py +488 -0
  59. package/skills/dforge-1c-import/scripts/lib/sizing.py +184 -0
  60. package/skills/dforge-1c-import/scripts/lib/spec.py +400 -0
  61. package/skills/dforge-1c-import/scripts/lib/statements.py +774 -0
  62. package/skills/dforge-1c-import/scripts/lib/subsystems.py +480 -0
  63. package/skills/dforge-1c-import/scripts/lib/terms.py +451 -0
  64. package/skills/dforge-1c-import/scripts/lib/terms_uk.py +555 -0
  65. package/skills/dforge-1c-import/scripts/lib/totals.py +387 -0
  66. package/skills/dforge-1c-import/scripts/lib/typemap.py +73 -0
  67. package/skills/dforge-1c-import/scripts/lib/unpack.py +136 -0
  68. package/skills/dforge-1c-import/scripts/lib/validate.py +508 -0
  69. package/skills/dforge-1c-import/scripts/project-template/contours.py +47 -0
  70. package/skills/dforge-1c-import/scripts/project-template/sources.py +14 -0
  71. package/skills/dforge-mcp-author/SKILL.md +8 -2
  72. package/skills/dforge-mcp-author/examples/column-security/entities/product.json +1 -1
  73. package/skills/dforge-mcp-author/examples/column-security/manifest.json +0 -1
  74. package/skills/dforge-mcp-author/examples/column-security/security/roles.json +2 -2
  75. package/skills/dforge-mcp-author/references/action-dsl.md +20 -7
  76. package/skills/dforge-mcp-author/references/conventions.md +7 -1
  77. package/skills/dforge-mcp-author/references/field-types.md +3 -2
  78. package/skills/dforge-mcp-author/references/formulas.md +5 -1
  79. package/skills/dforge-mcp-author/references/print-templates.md +5 -2
  80. package/skills/dforge-mcp-author/references/queries.md +1 -1
  81. package/skills/dforge-mcp-author/references/registers.md +18 -3
  82. package/skills/dforge-mcp-author/references/reports.md +4 -1
  83. package/skills/dforge-mcp-author/references/schema-import.md +1 -1
  84. package/skills/dforge-mcp-author/references/security.md +1 -1
  85. package/skills/dforge-mcp-author/references/translations.md +38 -15
  86. package/skills/dforge-mcp-author/references/validation-checklist.md +2 -3
  87. package/skills/dforge-module-build/SKILL.md +10 -1
  88. package/skills/dforge-module-ship/SKILL.md +31 -3
  89. package/skills/dforge-mcp-author/examples/column-security/translations/en-US.json +0 -35
@@ -0,0 +1,53 @@
1
+ # -*- coding: utf-8 -*-
2
+ """The platform's own CLI, run on this machine: `npx -y @dforge-core/dforge-cli@latest`.
3
+
4
+ The same command the editor extension runs, so what it installs and what it reports are
5
+ what the editor would install and report - no copy of the CLI baked into a container image
6
+ that can lag the published one, and no staging of modules into the platform checkout.
7
+
8
+ `DFORGE_CLI` overrides the command (a whitespace-separated command plus flags). A local
9
+ install opens the platform's databases itself, so it needs the auth database's connection
10
+ string; `ConnectionStrings__AuthDatabase` is taken as set, or built from the same
11
+ `DF_PG_*` the loader uses, against the platform's `dforge` database (`DF_AUTH_DB`
12
+ overrides the name).
13
+ """
14
+ from __future__ import print_function
15
+
16
+ import os
17
+ import shutil
18
+ import subprocess
19
+
20
+ DEFAULT = "npx -y @dforge-core/dforge-cli@latest"
21
+
22
+
23
+ def command():
24
+ """The CLI as an argv prefix. On Windows `npx` is a .cmd, so it is resolved by PATH."""
25
+ parts = (os.environ.get("DFORGE_CLI") or DEFAULT).split()
26
+ found = shutil.which(parts[0])
27
+ if not found:
28
+ raise SystemExit("cli: %s is not on PATH (install Node.js, or set DFORGE_CLI)" % parts[0])
29
+ return [found] + parts[1:]
30
+
31
+
32
+ def environment():
33
+ """os.environ plus the auth database's connection string, for a local install."""
34
+ import migrate
35
+ migrate.load_env()
36
+ env = dict(os.environ)
37
+ if not env.get("ConnectionStrings__AuthDatabase"):
38
+ env["ConnectionStrings__AuthDatabase"] = (
39
+ "Host=%s;Port=%s;Database=%s;Username=%s;Password=%s" % (
40
+ env.get("DF_PG_HOST", "localhost"), env.get("DF_PG_PORT", "5432"),
41
+ env.get("DF_AUTH_DB", "dforge"), env.get("DF_PG_USER", "dforge"),
42
+ env.get("DF_PG_PASS", "")))
43
+ return env
44
+
45
+
46
+ def run(args, cwd=None):
47
+ """(exit code, combined output) of one CLI command."""
48
+ proc = subprocess.Popen(command() + list(args), cwd=cwd, env=environment(),
49
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
50
+ out = proc.communicate()[0].decode("utf-8", "replace")
51
+ # npx announces itself on every run; none of it is the CLI's answer.
52
+ out = "\n".join(l for l in out.splitlines() if not l.startswith("npm notice"))
53
+ return proc.returncode, out
@@ -0,0 +1,346 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Which 1C column each ported column is copied from.
5
+
6
+ This is what a direct data copy stands on, and the one mapping that cannot be guessed
7
+ from names: 1C stores an attribute in `_Fld2036`, and only DBNames knows which attribute
8
+ that is - it keys a `Fld` entry by the attribute's own GUID. The same holds for tabular
9
+ sections (`VT`) and their line numbers (`LineNo`).
10
+
11
+ The map is built by `autospec` while it generates the entities, not derived afterwards,
12
+ so it cannot drift from the schema it is a map into.
13
+
14
+ python tools/columnmap.py # coverage per module -> out/columnmap.json
15
+ python tools/columnmap.py <module> # one module, column by column
16
+ """
17
+ from __future__ import print_function
18
+
19
+ import codecs
20
+ import collections
21
+ import json
22
+ import os
23
+ import sys
24
+
25
+ import attrs
26
+ import paths
27
+ import autospec
28
+ import handmap
29
+ import naming
30
+ import binding
31
+
32
+ sources = binding.sources()
33
+
34
+
35
+ def hand_written(model, mapped):
36
+ """Fold the authored modules in, resolving `sources.py` to physical columns."""
37
+ resolved = collections.OrderedDict()
38
+ for module, spec in handmap.SPECS.items():
39
+ entities = collections.OrderedDict()
40
+ for entity, entity_spec in spec["entities"].items():
41
+ declared = sources.SOURCES.get(module, {}).get(entity, {})
42
+ object_name = declared.get("object")
43
+ if not object_name:
44
+ continue
45
+ family = declared.get("family")
46
+ section = declared.get("section")
47
+ table, columns = model.physical(object_name, family)
48
+ if declared.get("table"):
49
+ # DBNames resolves an object to its own table and a section to a `_VT`
50
+ # one. A chart of accounts keeps what each account declares in neither -
51
+ # `_Acc34_ExtDim780` has no metadata name to resolve from - so the
52
+ # project names the table and the columns are read off it.
53
+ table = declared["table"]
54
+ columns = model.table_columns(table)
55
+
56
+ fields = model.fields(object_name, family)
57
+ by_name = dict((f["name"], f) for f in fields
58
+ if (f.get("section") or None) == section)
59
+ if section:
60
+ guid = next((f.get("sectionGuid") for f in fields
61
+ if f.get("section") == section), None)
62
+ suffix = model.vt.get(guid)
63
+ table = (table + suffix) if (table and suffix) else None
64
+ columns = model.table_columns(table)
65
+
66
+ found = handmap.propose(module, entity, entity_spec, model)
67
+ # A reference is stored in the hidden FK beside it, so that is the column
68
+ # the copy fills - `head_party` is declared, `head_party_id` is written.
69
+ # a field's type is a short string ("ref:...", "drop1c:...") or a column spec dict
70
+ kinds = dict((f[0], f[1] if isinstance(f[1], str) else "") for f in entity_spec["fields"])
71
+
72
+ def target(column):
73
+ return (column + "_id"
74
+ if kinds.get(column, "").startswith("ref:") else column)
75
+
76
+ # `drop1c:` names the 1C enum a dropdown's options were read from, and
77
+ # that is what the migration needs to turn a GUID back into a code.
78
+ enums = collections.OrderedDict()
79
+ for item in entity_spec["fields"]:
80
+ if isinstance(item[1], str) and item[1].startswith("drop1c:"):
81
+ declared_enum = autospec.enum_spec(
82
+ model, item[1][len("drop1c:"):])
83
+ if declared_enum and declared_enum["source"]:
84
+ enums[item[0]] = declared_enum["source"]
85
+
86
+ # Where the port chose its own codes rather than taking the enum's, the
87
+ # mapping is declared; the option list stays positional, because
88
+ # `_EnumOrder` is what the decoder has to work with.
89
+ for (owner, name, column), (enum, values) in sources.ENUMS.items():
90
+ if owner != module or name != entity:
91
+ continue
92
+ declared_enum = autospec.enum_spec(model, enum)
93
+ if declared_enum and declared_enum["source"]:
94
+ held = declared_enum["source"]
95
+ # The enum's options come pinned to what the map already holds (autospec's
96
+ # PINNED_ENUMS), and that is this mapping's output - the port's own codes -
97
+ # from the second run on; a code that is already one of ours stays as it is.
98
+ ours = set(values.values())
99
+ enums[column] = {
100
+ "table": held["table"],
101
+ "options": [code if code in ours else values.get(code)
102
+ for code in held["options"]],
103
+ }
104
+
105
+ tablerefs = dict(
106
+ (column, dict((str(number), code) for number, code in table.items()))
107
+ for (owner, name, column), table in sources.TABLEREFS.items()
108
+ if owner == module and name == entity)
109
+
110
+ pairs, joined = {}, {}
111
+ if section:
112
+ line_number = model.line_no.get(guid)
113
+ if line_number in columns:
114
+ pairs["line_no"] = line_number
115
+ else:
116
+ pairs[entity + "_id"] = "_IDRRef"
117
+ for column, source, _how in (found[0] if found else []):
118
+ if source == "(standard)":
119
+ for candidate in autospec.STANDARD_PHYSICAL.get(column, []):
120
+ if candidate in columns:
121
+ pairs[target(column)] = candidate
122
+ break
123
+ continue
124
+ if source.startswith("_"):
125
+ # `_Table._Column` is a column the entity's own table does not
126
+ # have: the query in QUERIES joins it in. The loader selects it
127
+ # by its bare name, but it is the other table that says what type
128
+ # it is, so which table that is travels with the map.
129
+ held, _, single = source.rpartition(".")
130
+ if held:
131
+ if single in model.table_columns(held):
132
+ pairs[target(column)] = single
133
+ joined[single] = held
134
+ continue
135
+ if source in columns or sources.QUERIES.get((module, entity)):
136
+ # A declared query may produce a column the table has not got -
137
+ # the accounting pivot folds 1C's analytic ROWS into eight slot
138
+ # COLUMNS. The query names them, so they belong in the map; the
139
+ # schema dump describes the table and knows nothing of them.
140
+ pairs[target(column)] = source
141
+ continue
142
+ field = by_name.get(source)
143
+ physical = autospec.physical_of(field, model, columns) if field else None
144
+ if physical:
145
+ pairs[target(column)] = physical
146
+
147
+ entities[entity] = {"object": object_name, "family": family or "",
148
+ "table": table, "columns": pairs,
149
+ "joined": joined,
150
+ "enums": enums, "tablerefs": tablerefs}
151
+ declared_query = sources.QUERIES.get((module, entity))
152
+ if declared_query:
153
+ entities[entity]["query"] = " ".join(declared_query.split())
154
+ resolved[module] = entities
155
+ return resolved
156
+
157
+
158
+ def register_entities(model, mapped):
159
+ """Where an overlay ENTITY's rows come from, for a generated module.
160
+
161
+ A balance register's state has no 1C original: the infobase holds the movements and
162
+ sums them on every read, so there is no table to derive the entity from and the
163
+ overlay introduces it. `sources.SOURCES` then says which 1C object its rows are
164
+ GROUPED from, and `sources.QUERIES` how.
165
+
166
+ Resolved the way an authored module's entities are - by 1C attribute name through
167
+ DBNames - so that no `_Fld` is written down in the project. Without this the entity
168
+ has no mapped source at all: the map is built from what autospec extracted, and
169
+ autospec cannot extract a table the infobase does not have.
170
+
171
+ The state of a register that declares a `journal` has no 1C source at all and is
172
+ skipped: it is rebuilt from the journal in the tenant (`seedstate`), so the two can
173
+ never disagree about what was summed.
174
+ """
175
+ for module, overlay in binding.registers().items():
176
+ seeded = set(d.get("entity") for d in (overlay.get("defs") or {}).values()
177
+ if d.get("kind") == "balance" and d.get("journal"))
178
+ for entity, spec in (overlay.get("entities") or {}).items():
179
+ # Neither is an entity with a 1C source: a seeded state is rebuilt from its
180
+ # journal, and a `computed` one is written by the platform - a document's
181
+ # ledger postings, filled when it is posted. Nothing is copied into either.
182
+ if entity in seeded or spec.get("computed"):
183
+ continue
184
+ declared = sources.SOURCES.get(module, {}).get(entity, {})
185
+ object_name = declared.get("object")
186
+ if not object_name:
187
+ raise KeyError("sources.py: %s.%s is introduced by the register overlay "
188
+ "and has no SOURCES entry saying what it is grouped from"
189
+ % (module, entity))
190
+ family = declared.get("family")
191
+ table, columns = model.physical(object_name, family)
192
+ by_name = dict((f["name"], f) for f in model.fields(object_name, family))
193
+
194
+ # A reference is declared by its visible name and stored in the hidden FK
195
+ # beside it, exactly as on the authored path: `product` is declared,
196
+ # `product_id` is filled.
197
+ kinds = dict((f[0], f[1]) for f in spec["fields"])
198
+
199
+ resolved = collections.OrderedDict()
200
+ resolved[entity + "_id"] = "_IDRRef"
201
+ for ours, theirs in sorted((declared.get("columns") or {}).items()):
202
+ field = by_name.get(theirs)
203
+ physical = (autospec.physical_of(field, model, columns) if field
204
+ else (theirs if theirs in columns else None))
205
+ if not physical:
206
+ raise KeyError("sources.py: %s.%s reads %r, which %s does not have"
207
+ % (module, entity, theirs, object_name))
208
+ target = (ours + "_id"
209
+ if str(kinds.get(ours, "")).startswith("ref:") else ours)
210
+ resolved[target] = physical
211
+
212
+ found = collections.OrderedDict([
213
+ ("object", object_name), ("family", family or ""), ("table", table),
214
+ ("columns", resolved), ("joined", {}), ("enums", {}), ("tablerefs", {}),
215
+ ])
216
+ query = sources.QUERIES.get((module, entity))
217
+ if query:
218
+ found["query"] = " ".join(query.split())
219
+ mapped.setdefault(module, collections.OrderedDict())[entity] = found
220
+ return mapped
221
+
222
+
223
+ def register_load(mapped):
224
+ """Fold the register overlay's `load` in: a second ported column off one 1C column.
225
+
226
+ A turnover register wants a magnitude and a direction where the infobase keeps a signed
227
+ value, so one `_Fld` feeds two of our columns and each is converted differently. The
228
+ overlay names the ported column whose source to reuse rather than the physical column
229
+ itself, which keeps DBNames the only thing that resolves an `_Fld` - naming
230
+ `_Fld17248` here would be the cardinality mistake in another costume.
231
+ """
232
+ for module, overlay in binding.registers().items():
233
+ for entity, columns in (overlay.get("load") or {}).items():
234
+ found = mapped.get(module, {}).get(entity)
235
+ if found is None:
236
+ raise KeyError("registers.py: %s has no mapped entity %r to load into"
237
+ % (module, entity))
238
+ sources_of = found.setdefault("columns", {})
239
+ transforms = found.setdefault("transforms", {})
240
+ for column, how in sorted(columns.items()):
241
+ borrowed = how["from"]
242
+ if borrowed not in sources_of:
243
+ raise KeyError("registers.py: %s.%s reads %r, which has no 1C source"
244
+ % (module, entity, borrowed))
245
+ sources_of[column] = sources_of[borrowed]
246
+ transforms[column] = how["transform"]
247
+ return mapped
248
+
249
+
250
+ def build():
251
+ """(module, entity) -> {table, columns}, alongside the generated entities."""
252
+ generated, model = autospec.build()
253
+ out = collections.OrderedDict()
254
+ for (module, entity), found in autospec.COLUMNS.items():
255
+ # A declared query counts for a generated entity too. It was only attached on the
256
+ # hand-written path, so the chart of accounts - generated, and needing a query to
257
+ # decode `_Kind` into a code - silently went on being read straight from its table.
258
+ declared_query = sources.QUERIES.get((module, entity))
259
+ if declared_query:
260
+ found = dict(found, query=" ".join(declared_query.split()))
261
+ out.setdefault(module, collections.OrderedDict())[entity] = found
262
+ for module, entities in hand_written(model, out).items():
263
+ out.setdefault(module, collections.OrderedDict()).update(entities)
264
+ # Before `register_load`, which needs the entity to be mapped before it can borrow a
265
+ # source for a second column of it.
266
+ register_entities(model, out)
267
+ return generated, register_load(out)
268
+
269
+
270
+ def coverage(generated, mapped):
271
+ """How many of a module's columns have somewhere to be copied from.
272
+
273
+ A column with no source is not a defect by itself - the primary key of a register is
274
+ generated here, and a set column has no storage at all - but the count is what says
275
+ whether a contour is ready to be migrated.
276
+ """
277
+ # The authored modules are not in `generated` - autospec never built them - so their
278
+ # entities come from the specs themselves.
279
+ everything = collections.OrderedDict()
280
+ for module, entities in generated.items():
281
+ everything.setdefault(module, collections.OrderedDict()).update(entities)
282
+ for module, spec in handmap.SPECS.items():
283
+ everything.setdefault(module, collections.OrderedDict()).update(spec["entities"])
284
+
285
+ rows = []
286
+ for module, entities in everything.items():
287
+ total = matched = 0
288
+ missing = []
289
+ for entity, spec in entities.items():
290
+ found = mapped.get(module, {}).get(entity, {})
291
+ columns = found.get("columns", {})
292
+ # Only physical columns can have a source. A Reference column is virtual -
293
+ # the FK beside it is what holds the value - and so is a set column.
294
+ # A type spec may be a dict handed straight through (genmodule.parse_type).
295
+ names = set(f[0] + "_id" if not isinstance(f[1], dict) and f[1].startswith("ref:")
296
+ else f[0]
297
+ for f in spec["fields"])
298
+ names.add(entity + "_id" if not spec.get("lineOf") else "line_no")
299
+ total += len(names)
300
+ matched += len(names & set(columns))
301
+ missing.extend((entity, n) for n in sorted(names - set(columns)))
302
+ rows.append((module, matched, total, missing))
303
+ return rows
304
+
305
+
306
+ def main():
307
+ generated, mapped = build()
308
+ wanted = sys.argv[1] if len(sys.argv) > 1 else None
309
+
310
+ if wanted:
311
+ for entity, found in mapped.get(wanted, {}).items():
312
+ print("\n%s <- %s (%s)"
313
+ % (entity, found.get("table") or "-", found.get("object")))
314
+ for column, source in sorted(found.get("columns", {}).items()):
315
+ print(" %-44s %s" % (column, source))
316
+ return
317
+
318
+ path = paths.derived("columnmap.json")
319
+ with codecs.open(path, "w", "utf-8") as f:
320
+ f.write(json.dumps(mapped, ensure_ascii=False, indent="\t", sort_keys=False))
321
+ f.write("\n")
322
+
323
+ print("%-12s %10s %10s %8s" % ("module", "mapped", "columns", "covered"))
324
+ print("-" * 44)
325
+ grand_matched = grand_total = 0
326
+ unmapped = []
327
+ for module, matched, total, missing in coverage(generated, mapped):
328
+ grand_matched += matched
329
+ grand_total += total
330
+ unmapped.extend((module,) + item for item in missing)
331
+ print("%-12s %10d %10d %7d%%"
332
+ % (module, matched, total, 100 * matched // max(total, 1)))
333
+ print("-" * 44)
334
+ print("%-12s %10d %10d %7d%%"
335
+ % ("total", grand_matched, grand_total,
336
+ 100 * grand_matched // max(grand_total, 1)))
337
+
338
+ if unmapped:
339
+ print("\n%d column(s) with no source, first 20:" % len(unmapped))
340
+ for module, entity, column in unmapped[:20]:
341
+ print(" %-10s %-40s %s" % (module, entity, column))
342
+ print("\n-> %s" % path)
343
+
344
+
345
+ if __name__ == "__main__":
346
+ main()
@@ -0,0 +1,242 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ How much of what 1C wrote the engine can write: per register, per document type that wrote it.
5
+
6
+ The port is measured one document at a time - a model against 1C's rows, then the engine
7
+ against them - and each measurement lands in a commit message or a paragraph. This is the
8
+ sum of them: for every register the tenant declares, the rows the infobase holds in a window,
9
+ split by the document type that recorded them, and for each type whether a binding writes
10
+ that register (an `A` or `L` column naming it) and what the last engine check said.
11
+
12
+ python tools/1c.py coverage # every register, the last 365 days
13
+ python tools/1c.py coverage --since 2025-10-10 # from a date
14
+ python tools/1c.py coverage accounting.self_supporting --all # one register, every type
15
+ python tools/1c.py coverage --save # and keep a snapshot to compare with
16
+
17
+ Everything is read from the tenant's own metadata ("dForge".register and the binding
18
+ columns), so a register or binding installed tomorrow is counted without editing this file.
19
+ A row's document type is the document table its recorder is a key of; a recorder that is a
20
+ key of none is reported as such rather than dropped. A type counts as bound when any entity
21
+ over its table - the document's own module or an extension in another - has a binding column
22
+ for the register. Bound is not the same as ported in full: a binding may still refuse some of
23
+ its documents, which is what the engine column is for.
24
+
25
+ The engine column is filled by the tools that put documents through the running engine and
26
+ compare with 1C's rows (`record_engine`): the last result per document type and check, with
27
+ its date. `--save` writes the table to `<out>/coverage/<date>.json`; the next run prints what
28
+ moved since the latest snapshot.
29
+ """
30
+ from __future__ import print_function
31
+
32
+ import argparse
33
+ import datetime
34
+ import io
35
+ import json
36
+ import os
37
+ import sys
38
+
39
+ import migrate
40
+ import paths
41
+
42
+ FOLDER = os.path.join(paths.OUT, "coverage")
43
+ ENGINE = os.path.join(FOLDER, "engine.json")
44
+
45
+
46
+ def record_engine(table, check, checked, equal, when=None):
47
+ """Keep the result of an engine check of `table` (schema.table) under the name `check`."""
48
+ if not os.path.isdir(FOLDER):
49
+ os.makedirs(FOLDER)
50
+ found = _read(ENGINE) or {}
51
+ found.setdefault(table, {})[check] = {
52
+ "date": when or datetime.date.today().isoformat(), "checked": checked, "equal": equal}
53
+ _write(ENGINE, found)
54
+
55
+
56
+ def _read(path):
57
+ if not os.path.isfile(path):
58
+ return None
59
+ with io.open(path, encoding="utf-8") as f:
60
+ return json.load(f)
61
+
62
+
63
+ def _write(path, value):
64
+ with io.open(path, "w", encoding="utf-8", newline="\n") as f:
65
+ f.write(json.dumps(value, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
66
+
67
+
68
+ def _column(cur, schema, table, *names):
69
+ """The first of `names` the table has, or None."""
70
+ for name in names:
71
+ if not name:
72
+ continue
73
+ cur.execute("""SELECT 1 FROM information_schema.columns
74
+ WHERE table_schema = %s AND table_name = %s AND column_name = %s""",
75
+ (schema, table, name))
76
+ if cur.fetchone():
77
+ return name
78
+ return None
79
+
80
+
81
+ def registers(cur, only=None):
82
+ """Every register with the table its rows sit in, the column naming the recorder, and the
83
+ column dating a row - (code, kind, schema, table, recorder, date). A register whose rows
84
+ name no recorder is left out: nothing can be said about who wrote them."""
85
+ cur.execute("""
86
+ SELECT m.module_cd || '.' || r.register_cd, r.kind,
87
+ coalesce(j.schema_name, e.schema_name), coalesce(j.db_object, e.db_object),
88
+ r.recorder_field, coalesce(r.journal_date, r.period_field)
89
+ FROM "dForge".register r
90
+ JOIN "dForge".module m ON m.module_id = r.module_id
91
+ LEFT JOIN "dForge".entity e ON e.entity_id = r.entity_id
92
+ LEFT JOIN "dForge".entity j ON j.entity_id = r.journal_entity_id
93
+ ORDER BY 1""")
94
+ out = []
95
+ for code, kind, schema, table, recorder, date in cur.fetchall():
96
+ if only and code not in only:
97
+ continue
98
+ if not schema or not table:
99
+ continue
100
+ # 1C's recorder where the rows were loaded; the binding's own field (`source`, which the
101
+ # engine fills) only after it
102
+ recorder = _column(cur, schema, table, "recorder", "recorder_ref", recorder)
103
+ if recorder is None:
104
+ continue
105
+ cur.execute("""SELECT data_type FROM information_schema.columns
106
+ WHERE table_schema = %s AND table_name = %s AND column_name = %s""", (schema, table, recorder))
107
+ if cur.fetchone()[0] == 'jsonb':
108
+ recorder = "\"%s\"->>'value'" % recorder
109
+ else:
110
+ recorder = '"%s"::text' % recorder
111
+ out.append((code, kind, schema, table, recorder, _column(cur, schema, table, date)))
112
+ return out
113
+
114
+
115
+ def documents(cur):
116
+ """Every document table in the tenant - one with a `posted` flag and a key named after it -
117
+ as (schema, table, key)."""
118
+ cur.execute("""
119
+ SELECT c.table_schema, c.table_name, c.table_name || '_id'
120
+ FROM information_schema.columns c
121
+ WHERE c.column_name = 'posted'
122
+ AND EXISTS (SELECT 1 FROM information_schema.columns k
123
+ WHERE k.table_schema = c.table_schema AND k.table_name = c.table_name
124
+ AND k.column_name = c.table_name || '_id')
125
+ ORDER BY 1, 2""")
126
+ return cur.fetchall()
127
+
128
+
129
+ def bindings(cur):
130
+ """register code -> the document tables (schema.table) a binding column writes it from."""
131
+ cur.execute("""
132
+ SELECT c.params->>'register', e.schema_name || '.' || e.db_object
133
+ FROM "dForge".entity_column c
134
+ JOIN "dForge".entity e ON e.entity_id = c.entity_id
135
+ WHERE c.column_type IN ('A', 'L') AND c.params->>'register' IS NOT NULL""")
136
+ out = {}
137
+ for register, table in cur.fetchall():
138
+ out.setdefault(register, set()).add(table)
139
+ return out
140
+
141
+
142
+ def measure(pg, since, only=None):
143
+ """{register: {"kind", "rows", "types": {table: {"docs", "rows"}}}} for rows dated from
144
+ `since` (all of them when the register keeps no date)."""
145
+ cur = pg.cursor()
146
+ docs = documents(cur)
147
+ out = {}
148
+ for code, kind, schema, table, recorder, date in registers(cur, only):
149
+ window = ' WHERE "%s" >= %%(since)s' % date if date else ""
150
+ cur.execute('DROP TABLE IF EXISTS coverage_rec')
151
+ cur.execute('CREATE TEMP TABLE coverage_rec AS SELECT %s AS recorder, count(*) AS n FROM "%s"."%s"%s GROUP BY 1'
152
+ % (recorder, schema, table, window), {"since": since})
153
+ cur.execute("SELECT count(*), coalesce(sum(n), 0) FROM coverage_rec")
154
+ recorders, rows = cur.fetchone()
155
+ types = {}
156
+ if rows:
157
+ cur.execute("CREATE INDEX ON coverage_rec (recorder)")
158
+ for dschema, dtable, key in docs:
159
+ cur.execute('SELECT count(*), coalesce(sum(r.n), 0) FROM coverage_rec r JOIN "%s"."%s" d ON d."%s"::text = r.recorder'
160
+ % (dschema, dtable, key))
161
+ n, total = cur.fetchone()
162
+ if n:
163
+ types["%s.%s" % (dschema, dtable)] = {"docs": int(n), "rows": int(total)}
164
+ known = sum(t["rows"] for t in types.values())
165
+ if rows - known:
166
+ types["(no document)"] = {"docs": None, "rows": int(rows - known)}
167
+ out[code] = {"kind": kind, "rows": int(rows), "types": types}
168
+ return out
169
+
170
+
171
+ def report(measured, bound, engine, show_all=False, previous=None):
172
+ for code in sorted(measured):
173
+ reg = measured[code]
174
+ if not reg["rows"]:
175
+ continue
176
+ tables = bound.get(code, set())
177
+ covered = sum(t["rows"] for name, t in reg["types"].items() if name in tables)
178
+ print("\n%s (%s) - %s rows, %s by bound types (%.1f%%)" % (
179
+ code, reg["kind"], _n(reg["rows"]), _n(covered), 100.0 * covered / reg["rows"]))
180
+ before = (previous or {}).get(code, {}).get("types", {})
181
+ listed = sorted(reg["types"].items(), key=lambda kv: -kv[1]["rows"])
182
+ for name, t in listed:
183
+ if not show_all and name in tables and len(listed) > 12:
184
+ continue
185
+ mark = "bound" if name in tables else "-"
186
+ checks = engine.get(name, {})
187
+ check = " ".join("%s %d/%d %s" % (k, v["equal"], v["checked"], v["date"]) for k, v in sorted(checks.items()))
188
+ moved = ""
189
+ if name in before and before[name]["rows"] != t["rows"]:
190
+ moved = " (was %s)" % _n(before[name]["rows"])
191
+ print(" %-58s %-6s docs %8s rows %11s%s %s" % (
192
+ name, mark, _n(t["docs"]) if t["docs"] is not None else "", _n(t["rows"]), moved, check))
193
+ if not show_all and len(listed) > 12:
194
+ hidden = [n for n, _ in listed if n in tables]
195
+ if hidden:
196
+ print(" ... and %d bound types (--all lists them)" % len(hidden))
197
+
198
+
199
+ def _n(value):
200
+ return "{:,}".format(value)
201
+
202
+
203
+ def main():
204
+ parser = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0])
205
+ parser.add_argument("registers", nargs="*", help="register codes (module.register); every one when omitted")
206
+ parser.add_argument("--since", help="the window's first date (default: 365 days back)")
207
+ parser.add_argument("--all", action="store_true", help="list bound types too where a register has many")
208
+ parser.add_argument("--save", action="store_true", help="keep this run as a snapshot")
209
+ args = parser.parse_args()
210
+ since = args.since or (datetime.date.today() - datetime.timedelta(days=365)).isoformat()
211
+
212
+ migrate.load_env()
213
+ pg = migrate.pg_connect()
214
+ pg.autocommit = True
215
+ print("window from %s" % since)
216
+ measured = measure(pg, since, set(args.registers) or None)
217
+ bound = bindings(pg.cursor())
218
+ engine = _read(ENGINE) or {}
219
+
220
+ previous = None
221
+ if os.path.isdir(FOLDER):
222
+ snaps = sorted(n for n in os.listdir(FOLDER) if n[:4].isdigit() and n.endswith(".json"))
223
+ for name in reversed(snaps):
224
+ snap = _read(os.path.join(FOLDER, name))
225
+ if snap and snap.get("since") == since:
226
+ previous = snap["registers"]
227
+ print("compared with the snapshot %s" % name)
228
+ break
229
+ report(measured, bound, engine, args.all, previous)
230
+
231
+ if args.save:
232
+ if not os.path.isdir(FOLDER):
233
+ os.makedirs(FOLDER)
234
+ path = os.path.join(FOLDER, datetime.date.today().isoformat() + ".json")
235
+ _write(path, {"since": since, "registers": measured,
236
+ "bound": dict((k, sorted(v)) for k, v in bound.items())})
237
+ print("\nsaved %s" % path)
238
+ return 0
239
+
240
+
241
+ if __name__ == "__main__":
242
+ sys.exit(main())