@dforge-core/dforge-mcp 0.2.45 → 0.2.47

package/skills/dforge-1c-import/scripts/lib/columnmap.py

@@ -0,0 +1,346 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Which 1C column each ported column is copied from.
+
+This is what a direct data copy stands on, and the one mapping that cannot be guessed
+from names: 1C stores an attribute in `_Fld2036`, and only DBNames knows which attribute
+that is - it keys a `Fld` entry by the attribute's own GUID. The same holds for tabular
+sections (`VT`) and their line numbers (`LineNo`).
+
+The map is built by `autospec` while it generates the entities, not derived afterwards,
+so it cannot drift from the schema it is a map into.
+
+    python tools/columnmap.py             # coverage per module   -> out/columnmap.json
+    python tools/columnmap.py <module>    # one module, column by column
+"""
+from __future__ import print_function
+
+import codecs
+import collections
+import json
+import os
+import sys
+
+import attrs
+import paths
+import autospec
+import handmap
+import naming
+import binding
+
+sources = binding.sources()
+
+
+def hand_written(model, mapped):
+	"""Fold the authored modules in, resolving `sources.py` to physical columns."""
+	resolved = collections.OrderedDict()
+	for module, spec in handmap.SPECS.items():
+		entities = collections.OrderedDict()
+		for entity, entity_spec in spec["entities"].items():
+			declared = sources.SOURCES.get(module, {}).get(entity, {})
+			object_name = declared.get("object")
+			if not object_name:
+				continue
+			family = declared.get("family")
+			section = declared.get("section")
+			table, columns = model.physical(object_name, family)
+			if declared.get("table"):
+				# DBNames resolves an object to its own table and a section to a `_VT`
+				# one. A chart of accounts keeps what each account declares in neither -
+				# `_Acc34_ExtDim780` has no metadata name to resolve from - so the
+				# project names the table and the columns are read off it.
+				table = declared["table"]
+				columns = model.table_columns(table)
+
+			fields = model.fields(object_name, family)
+			by_name = dict((f["name"], f) for f in fields
+			               if (f.get("section") or None) == section)
+			if section:
+				guid = next((f.get("sectionGuid") for f in fields
+				             if f.get("section") == section), None)
+				suffix = model.vt.get(guid)
+				table = (table + suffix) if (table and suffix) else None
+				columns = model.table_columns(table)
+
+			found = handmap.propose(module, entity, entity_spec, model)
+			# A reference is stored in the hidden FK beside it, so that is the column
+			# the copy fills - `head_party` is declared, `head_party_id` is written.
+			# a field's type is a short string ("ref:...", "drop1c:...") or a column spec dict
+			kinds = dict((f[0], f[1] if isinstance(f[1], str) else "") for f in entity_spec["fields"])
+
+			def target(column):
+				return (column + "_id"
+				        if kinds.get(column, "").startswith("ref:") else column)
+
+			# `drop1c:` names the 1C enum a dropdown's options were read from, and
+			# that is what the migration needs to turn a GUID back into a code.
+			enums = collections.OrderedDict()
+			for item in entity_spec["fields"]:
+				if isinstance(item[1], str) and item[1].startswith("drop1c:"):
+					declared_enum = autospec.enum_spec(
+						model, item[1][len("drop1c:"):])
+					if declared_enum and declared_enum["source"]:
+						enums[item[0]] = declared_enum["source"]
+
+			# Where the port chose its own codes rather than taking the enum's, the
+			# mapping is declared; the option list stays positional, because
+			# `_EnumOrder` is what the decoder has to work with.
+			for (owner, name, column), (enum, values) in sources.ENUMS.items():
+				if owner != module or name != entity:
+					continue
+				declared_enum = autospec.enum_spec(model, enum)
+				if declared_enum and declared_enum["source"]:
+					held = declared_enum["source"]
+					# The enum's options come pinned to what the map already holds (autospec's
+					# PINNED_ENUMS), and that is this mapping's output - the port's own codes -
+					# from the second run on; a code that is already one of ours stays as it is.
+					ours = set(values.values())
+					enums[column] = {
+						"table": held["table"],
+						"options": [code if code in ours else values.get(code)
+						            for code in held["options"]],
+					}
+
+			tablerefs = dict(
+				(column, dict((str(number), code) for number, code in table.items()))
+				for (owner, name, column), table in sources.TABLEREFS.items()
+				if owner == module and name == entity)
+
+			pairs, joined = {}, {}
+			if section:
+				line_number = model.line_no.get(guid)
+				if line_number in columns:
+					pairs["line_no"] = line_number
+			else:
+				pairs[entity + "_id"] = "_IDRRef"
+			for column, source, _how in (found[0] if found else []):
+				if source == "(standard)":
+					for candidate in autospec.STANDARD_PHYSICAL.get(column, []):
+						if candidate in columns:
+							pairs[target(column)] = candidate
+							break
+					continue
+				if source.startswith("_"):
+					# `_Table._Column` is a column the entity's own table does not
+					# have: the query in QUERIES joins it in. The loader selects it
+					# by its bare name, but it is the other table that says what type
+					# it is, so which table that is travels with the map.
+					held, _, single = source.rpartition(".")
+					if held:
+						if single in model.table_columns(held):
+							pairs[target(column)] = single
+							joined[single] = held
+						continue
+					if source in columns or sources.QUERIES.get((module, entity)):
+						# A declared query may produce a column the table has not got -
+						# the accounting pivot folds 1C's analytic ROWS into eight slot
+						# COLUMNS. The query names them, so they belong in the map; the
+						# schema dump describes the table and knows nothing of them.
+						pairs[target(column)] = source
+					continue
+				field = by_name.get(source)
+				physical = autospec.physical_of(field, model, columns) if field else None
+				if physical:
+					pairs[target(column)] = physical
+
+			entities[entity] = {"object": object_name, "family": family or "",
+			                    "table": table, "columns": pairs,
+			                    "joined": joined,
+			                    "enums": enums, "tablerefs": tablerefs}
+			declared_query = sources.QUERIES.get((module, entity))
+			if declared_query:
+				entities[entity]["query"] = " ".join(declared_query.split())
+		resolved[module] = entities
+	return resolved
+
+
+def register_entities(model, mapped):
+	"""Where an overlay ENTITY's rows come from, for a generated module.
+
+	A balance register's state has no 1C original: the infobase holds the movements and
+	sums them on every read, so there is no table to derive the entity from and the
+	overlay introduces it. `sources.SOURCES` then says which 1C object its rows are
+	GROUPED from, and `sources.QUERIES` how.
+
+	Resolved the way an authored module's entities are - by 1C attribute name through
+	DBNames - so that no `_Fld` is written down in the project. Without this the entity
+	has no mapped source at all: the map is built from what autospec extracted, and
+	autospec cannot extract a table the infobase does not have.
+
+	The state of a register that declares a `journal` has no 1C source at all and is
+	skipped: it is rebuilt from the journal in the tenant (`seedstate`), so the two can
+	never disagree about what was summed.
+	"""
+	for module, overlay in binding.registers().items():
+		seeded = set(d.get("entity") for d in (overlay.get("defs") or {}).values()
+		             if d.get("kind") == "balance" and d.get("journal"))
+		for entity, spec in (overlay.get("entities") or {}).items():
+			# Neither is an entity with a 1C source: a seeded state is rebuilt from its
+			# journal, and a `computed` one is written by the platform - a document's
+			# ledger postings, filled when it is posted. Nothing is copied into either.
+			if entity in seeded or spec.get("computed"):
+				continue
+			declared = sources.SOURCES.get(module, {}).get(entity, {})
+			object_name = declared.get("object")
+			if not object_name:
+				raise KeyError("sources.py: %s.%s is introduced by the register overlay "
+				               "and has no SOURCES entry saying what it is grouped from"
+				               % (module, entity))
+			family = declared.get("family")
+			table, columns = model.physical(object_name, family)
+			by_name = dict((f["name"], f) for f in model.fields(object_name, family))
+
+			# A reference is declared by its visible name and stored in the hidden FK
+			# beside it, exactly as on the authored path: `product` is declared,
+			# `product_id` is filled.
+			kinds = dict((f[0], f[1]) for f in spec["fields"])
+
+			resolved = collections.OrderedDict()
+			resolved[entity + "_id"] = "_IDRRef"
+			for ours, theirs in sorted((declared.get("columns") or {}).items()):
+				field = by_name.get(theirs)
+				physical = (autospec.physical_of(field, model, columns) if field
+				            else (theirs if theirs in columns else None))
+				if not physical:
+					raise KeyError("sources.py: %s.%s reads %r, which %s does not have"
+					               % (module, entity, theirs, object_name))
+				target = (ours + "_id"
+				          if str(kinds.get(ours, "")).startswith("ref:") else ours)
+				resolved[target] = physical
+
+			found = collections.OrderedDict([
+				("object", object_name), ("family", family or ""), ("table", table),
+				("columns", resolved), ("joined", {}), ("enums", {}), ("tablerefs", {}),
+			])
+			query = sources.QUERIES.get((module, entity))
+			if query:
+				found["query"] = " ".join(query.split())
+			mapped.setdefault(module, collections.OrderedDict())[entity] = found
+	return mapped
+
+
+def register_load(mapped):
+	"""Fold the register overlay's `load` in: a second ported column off one 1C column.
+
+	A turnover register wants a magnitude and a direction where the infobase keeps a signed
+	value, so one `_Fld` feeds two of our columns and each is converted differently. The
+	overlay names the ported column whose source to reuse rather than the physical column
+	itself, which keeps DBNames the only thing that resolves an `_Fld` - naming
+	`_Fld17248` here would be the cardinality mistake in another costume.
+	"""
+	for module, overlay in binding.registers().items():
+		for entity, columns in (overlay.get("load") or {}).items():
+			found = mapped.get(module, {}).get(entity)
+			if found is None:
+				raise KeyError("registers.py: %s has no mapped entity %r to load into"
+				               % (module, entity))
+			sources_of = found.setdefault("columns", {})
+			transforms = found.setdefault("transforms", {})
+			for column, how in sorted(columns.items()):
+				borrowed = how["from"]
+				if borrowed not in sources_of:
+					raise KeyError("registers.py: %s.%s reads %r, which has no 1C source"
+					               % (module, entity, borrowed))
+				sources_of[column] = sources_of[borrowed]
+				transforms[column] = how["transform"]
+	return mapped
+
+
+def build():
+	"""(module, entity) -> {table, columns}, alongside the generated entities."""
+	generated, model = autospec.build()
+	out = collections.OrderedDict()
+	for (module, entity), found in autospec.COLUMNS.items():
+		# A declared query counts for a generated entity too. It was only attached on the
+		# hand-written path, so the chart of accounts - generated, and needing a query to
+		# decode `_Kind` into a code - silently went on being read straight from its table.
+		declared_query = sources.QUERIES.get((module, entity))
+		if declared_query:
+			found = dict(found, query=" ".join(declared_query.split()))
+		out.setdefault(module, collections.OrderedDict())[entity] = found
+	for module, entities in hand_written(model, out).items():
+		out.setdefault(module, collections.OrderedDict()).update(entities)
+	# Before `register_load`, which needs the entity to be mapped before it can borrow a
+	# source for a second column of it.
+	register_entities(model, out)
+	return generated, register_load(out)
+
+
+def coverage(generated, mapped):
+	"""How many of a module's columns have somewhere to be copied from.
+
+	A column with no source is not a defect by itself - the primary key of a register is
+	generated here, and a set column has no storage at all - but the count is what says
+	whether a contour is ready to be migrated.
+	"""
+	# The authored modules are not in `generated` - autospec never built them - so their
+	# entities come from the specs themselves.
+	everything = collections.OrderedDict()
+	for module, entities in generated.items():
+		everything.setdefault(module, collections.OrderedDict()).update(entities)
+	for module, spec in handmap.SPECS.items():
+		everything.setdefault(module, collections.OrderedDict()).update(spec["entities"])
+
+	rows = []
+	for module, entities in everything.items():
+		total = matched = 0
+		missing = []
+		for entity, spec in entities.items():
+			found = mapped.get(module, {}).get(entity, {})
+			columns = found.get("columns", {})
+			# Only physical columns can have a source. A Reference column is virtual -
+			# the FK beside it is what holds the value - and so is a set column.
+			# A type spec may be a dict handed straight through (genmodule.parse_type).
+			names = set(f[0] + "_id" if not isinstance(f[1], dict) and f[1].startswith("ref:")
+			            else f[0]
+			            for f in spec["fields"])
+			names.add(entity + "_id" if not spec.get("lineOf") else "line_no")
+			total += len(names)
+			matched += len(names & set(columns))
+			missing.extend((entity, n) for n in sorted(names - set(columns)))
+		rows.append((module, matched, total, missing))
+	return rows
+
+
+def main():
+	generated, mapped = build()
+	wanted = sys.argv[1] if len(sys.argv) > 1 else None
+
+	if wanted:
+		for entity, found in mapped.get(wanted, {}).items():
+			print("\n%s  <-  %s  (%s)"
+			      % (entity, found.get("table") or "-", found.get("object")))
+			for column, source in sorted(found.get("columns", {}).items()):
+				print("   %-44s %s" % (column, source))
+		return
+
+	path = paths.derived("columnmap.json")
+	with codecs.open(path, "w", "utf-8") as f:
+		f.write(json.dumps(mapped, ensure_ascii=False, indent="\t", sort_keys=False))
+		f.write("\n")
+
+	print("%-12s %10s %10s %8s" % ("module", "mapped", "columns", "covered"))
+	print("-" * 44)
+	grand_matched = grand_total = 0
+	unmapped = []
+	for module, matched, total, missing in coverage(generated, mapped):
+		grand_matched += matched
+		grand_total += total
+		unmapped.extend((module,) + item for item in missing)
+		print("%-12s %10d %10d %7d%%"
+		      % (module, matched, total, 100 * matched // max(total, 1)))
+	print("-" * 44)
+	print("%-12s %10d %10d %7d%%"
+	      % ("total", grand_matched, grand_total,
+	         100 * grand_matched // max(grand_total, 1)))
+
+	if unmapped:
+		print("\n%d column(s) with no source, first 20:" % len(unmapped))
+		for module, entity, column in unmapped[:20]:
+			print("   %-10s %-40s %s" % (module, entity, column))
+	print("\n-> %s" % path)
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/coverage.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+How much of what 1C wrote the engine can write: per register, per document type that wrote it.
+
+The port is measured one document at a time - a model against 1C's rows, then the engine
+against them - and each measurement lands in a commit message or a paragraph. This is the
+sum of them: for every register the tenant declares, the rows the infobase holds in a window,
+split by the document type that recorded them, and for each type whether a binding writes
+that register (an `A` or `L` column naming it) and what the last engine check said.
+
+    python tools/1c.py coverage                       # every register, the last 365 days
+    python tools/1c.py coverage --since 2025-10-10    # from a date
+    python tools/1c.py coverage accounting.self_supporting --all   # one register, every type
+    python tools/1c.py coverage --save                # and keep a snapshot to compare with
+
+Everything is read from the tenant's own metadata ("dForge".register and the binding
+columns), so a register or binding installed tomorrow is counted without editing this file.
+A row's document type is the document table its recorder is a key of; a recorder that is a
+key of none is reported as such rather than dropped. A type counts as bound when any entity
+over its table - the document's own module or an extension in another - has a binding column
+for the register. Bound is not the same as ported in full: a binding may still refuse some of
+its documents, which is what the engine column is for.
+
+The engine column is filled by the tools that put documents through the running engine and
+compare with 1C's rows (`record_engine`): the last result per document type and check, with
+its date. `--save` writes the table to `<out>/coverage/<date>.json`; the next run prints what
+moved since the latest snapshot.
+"""
+from __future__ import print_function
+
+import argparse
+import datetime
+import io
+import json
+import os
+import sys
+
+import migrate
+import paths
+
+FOLDER = os.path.join(paths.OUT, "coverage")
+ENGINE = os.path.join(FOLDER, "engine.json")
+
+
+def record_engine(table, check, checked, equal, when=None):
+	"""Keep the result of an engine check of `table` (schema.table) under the name `check`."""
+	if not os.path.isdir(FOLDER):
+		os.makedirs(FOLDER)
+	found = _read(ENGINE) or {}
+	found.setdefault(table, {})[check] = {
+		"date": when or datetime.date.today().isoformat(), "checked": checked, "equal": equal}
+	_write(ENGINE, found)
+
+
+def _read(path):
+	if not os.path.isfile(path):
+		return None
+	with io.open(path, encoding="utf-8") as f:
+		return json.load(f)
+
+
+def _write(path, value):
+	with io.open(path, "w", encoding="utf-8", newline="\n") as f:
+		f.write(json.dumps(value, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
+
+
+def _column(cur, schema, table, *names):
+	"""The first of `names` the table has, or None."""
+	for name in names:
+		if not name:
+			continue
+		cur.execute("""SELECT 1 FROM information_schema.columns
+		               WHERE table_schema = %s AND table_name = %s AND column_name = %s""",
+		            (schema, table, name))
+		if cur.fetchone():
+			return name
+	return None
+
+
+def registers(cur, only=None):
+	"""Every register with the table its rows sit in, the column naming the recorder, and the
+	column dating a row - (code, kind, schema, table, recorder, date). A register whose rows
+	name no recorder is left out: nothing can be said about who wrote them."""
+	cur.execute("""
+		SELECT m.module_cd || '.' || r.register_cd, r.kind,
+		       coalesce(j.schema_name, e.schema_name), coalesce(j.db_object, e.db_object),
+		       r.recorder_field, coalesce(r.journal_date, r.period_field)
+		FROM "dForge".register r
+		JOIN "dForge".module m ON m.module_id = r.module_id
+		LEFT JOIN "dForge".entity e ON e.entity_id = r.entity_id
+		LEFT JOIN "dForge".entity j ON j.entity_id = r.journal_entity_id
+		ORDER BY 1""")
+	out = []
+	for code, kind, schema, table, recorder, date in cur.fetchall():
+		if only and code not in only:
+			continue
+		if not schema or not table:
+			continue
+		# 1C's recorder where the rows were loaded; the binding's own field (`source`, which the
+		# engine fills) only after it
+		recorder = _column(cur, schema, table, "recorder", "recorder_ref", recorder)
+		if recorder is None:
+			continue
+		cur.execute("""SELECT data_type FROM information_schema.columns
+		               WHERE table_schema = %s AND table_name = %s AND column_name = %s""", (schema, table, recorder))
+		if cur.fetchone()[0] == 'jsonb':
+			recorder = "\"%s\"->>'value'" % recorder
+		else:
+			recorder = '"%s"::text' % recorder
+		out.append((code, kind, schema, table, recorder, _column(cur, schema, table, date)))
+	return out
+
+
+def documents(cur):
+	"""Every document table in the tenant - one with a `posted` flag and a key named after it -
+	as (schema, table, key)."""
+	cur.execute("""
+		SELECT c.table_schema, c.table_name, c.table_name || '_id'
+		FROM information_schema.columns c
+		WHERE c.column_name = 'posted'
+		  AND EXISTS (SELECT 1 FROM information_schema.columns k
+		              WHERE k.table_schema = c.table_schema AND k.table_name = c.table_name
+		                AND k.column_name = c.table_name || '_id')
+		ORDER BY 1, 2""")
+	return cur.fetchall()
+
+
+def bindings(cur):
+	"""register code -> the document tables (schema.table) a binding column writes it from."""
+	cur.execute("""
+		SELECT c.params->>'register', e.schema_name || '.' || e.db_object
+		FROM "dForge".entity_column c
+		JOIN "dForge".entity e ON e.entity_id = c.entity_id
+		WHERE c.column_type IN ('A', 'L') AND c.params->>'register' IS NOT NULL""")
+	out = {}
+	for register, table in cur.fetchall():
+		out.setdefault(register, set()).add(table)
+	return out
+
+
+def measure(pg, since, only=None):
+	"""{register: {"kind", "rows", "types": {table: {"docs", "rows"}}}} for rows dated from
+	`since` (all of them when the register keeps no date)."""
+	cur = pg.cursor()
+	docs = documents(cur)
+	out = {}
+	for code, kind, schema, table, recorder, date in registers(cur, only):
+		window = ' WHERE "%s" >= %%(since)s' % date if date else ""
+		cur.execute('DROP TABLE IF EXISTS coverage_rec')
+		cur.execute('CREATE TEMP TABLE coverage_rec AS SELECT %s AS recorder, count(*) AS n FROM "%s"."%s"%s GROUP BY 1'
+		            % (recorder, schema, table, window), {"since": since})
+		cur.execute("SELECT count(*), coalesce(sum(n), 0) FROM coverage_rec")
+		recorders, rows = cur.fetchone()
+		types = {}
+		if rows:
+			cur.execute("CREATE INDEX ON coverage_rec (recorder)")
+			for dschema, dtable, key in docs:
+				cur.execute('SELECT count(*), coalesce(sum(r.n), 0) FROM coverage_rec r JOIN "%s"."%s" d ON d."%s"::text = r.recorder'
+				            % (dschema, dtable, key))
+				n, total = cur.fetchone()
+				if n:
+					types["%s.%s" % (dschema, dtable)] = {"docs": int(n), "rows": int(total)}
+			known = sum(t["rows"] for t in types.values())
+			if rows - known:
+				types["(no document)"] = {"docs": None, "rows": int(rows - known)}
+		out[code] = {"kind": kind, "rows": int(rows), "types": types}
+	return out
+
+
+def report(measured, bound, engine, show_all=False, previous=None):
+	for code in sorted(measured):
+		reg = measured[code]
+		if not reg["rows"]:
+			continue
+		tables = bound.get(code, set())
+		covered = sum(t["rows"] for name, t in reg["types"].items() if name in tables)
+		print("\n%s (%s) - %s rows, %s by bound types (%.1f%%)" % (
+			code, reg["kind"], _n(reg["rows"]), _n(covered), 100.0 * covered / reg["rows"]))
+		before = (previous or {}).get(code, {}).get("types", {})
+		listed = sorted(reg["types"].items(), key=lambda kv: -kv[1]["rows"])
+		for name, t in listed:
+			if not show_all and name in tables and len(listed) > 12:
+				continue
+			mark = "bound" if name in tables else "-"
+			checks = engine.get(name, {})
+			check = "  ".join("%s %d/%d %s" % (k, v["equal"], v["checked"], v["date"]) for k, v in sorted(checks.items()))
+			moved = ""
+			if name in before and before[name]["rows"] != t["rows"]:
+				moved = "  (was %s)" % _n(before[name]["rows"])
+			print("  %-58s %-6s docs %8s  rows %11s%s  %s" % (
+				name, mark, _n(t["docs"]) if t["docs"] is not None else "", _n(t["rows"]), moved, check))
+		if not show_all and len(listed) > 12:
+			hidden = [n for n, _ in listed if n in tables]
+			if hidden:
+				print("  ... and %d bound types (--all lists them)" % len(hidden))
+
+
+def _n(value):
+	return "{:,}".format(value)
+
+
+def main():
+	parser = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0])
+	parser.add_argument("registers", nargs="*", help="register codes (module.register); every one when omitted")
+	parser.add_argument("--since", help="the window's first date (default: 365 days back)")
+	parser.add_argument("--all", action="store_true", help="list bound types too where a register has many")
+	parser.add_argument("--save", action="store_true", help="keep this run as a snapshot")
+	args = parser.parse_args()
+	since = args.since or (datetime.date.today() - datetime.timedelta(days=365)).isoformat()
+
+	migrate.load_env()
+	pg = migrate.pg_connect()
+	pg.autocommit = True
+	print("window from %s" % since)
+	measured = measure(pg, since, set(args.registers) or None)
+	bound = bindings(pg.cursor())
+	engine = _read(ENGINE) or {}
+
+	previous = None
+	if os.path.isdir(FOLDER):
+		snaps = sorted(n for n in os.listdir(FOLDER) if n[:4].isdigit() and n.endswith(".json"))
+		for name in reversed(snaps):
+			snap = _read(os.path.join(FOLDER, name))
+			if snap and snap.get("since") == since:
+				previous = snap["registers"]
+				print("compared with the snapshot %s" % name)
+				break
+	report(measured, bound, engine, args.all, previous)
+
+	if args.save:
+		if not os.path.isdir(FOLDER):
+			os.makedirs(FOLDER)
+		path = os.path.join(FOLDER, datetime.date.today().isoformat() + ".json")
+		_write(path, {"since": since, "registers": measured,
+		              "bound": dict((k, sorted(v)) for k, v in bound.items())})
+		print("\nsaved %s" % path)
+	return 0
+
+
+if __name__ == "__main__":
+	sys.exit(main())

package/skills/dforge-1c-import/scripts/lib/custom.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Separate the client's own development from the stock UTP configuration.
+
+Stock 1C objects are CamelCase starting with a capital; this integrator prefixes
+everything it added with a lowercase tag. Whatever carries a tag *and* holds data
+is the real porting scope - the rest is vendor code we are not copying anyway.
+"""
+from __future__ import print_function
+
+import codecs
+import collections
+import json
+import os
+import re
+import sys
+
+import paths
+
+OUT = paths.OUT
+LOWER_PREFIX = re.compile(r"^([a-zа-яёіїє]{2,6})(?=[A-ZА-Я])")
+FAMILIES = ("Reference", "Document", "InfoRg", "AccumRg", "Enum", "Chrc", "Acc", "AccRg")
+
+
+def load(name):
+	with codecs.open(os.path.join(OUT, name), "r", "utf-8") as f:
+		return json.load(f)
+
+
+def has_data(row):
+	return bool((row["rows"] or 0) or row["childRows"])
+
+
+def main():
+	rows = load("report.json")
+
+	prefixes = collections.Counter()
+	for row in rows:
+		m = LOWER_PREFIX.match(row["name"] or "")
+		if m:
+			prefixes[m.group(1)] += 1
+	print("integrator prefixes found:")
+	for tag, n in prefixes.most_common(10):
+		print("   %-8s %d objects" % (tag, n))
+
+	tags = [t for t, n in prefixes.most_common() if n >= 3]
+	is_custom = lambda row: bool(LOWER_PREFIX.match(row["name"] or "")
+	                             and LOWER_PREFIX.match(row["name"]).group(1) in tags)
+
+	print("\n%-14s %7s %7s %11s %11s" % ("family", "total", "custom", "custom+data", "stock+data"))
+	print("-" * 56)
+	scope = []
+	for family in FAMILIES:
+		members = [r for r in rows if r["family"] == family]
+		if not members:
+			continue
+		custom = [r for r in members if is_custom(r)]
+		custom_used = [r for r in custom if has_data(r)]
+		stock_used = [r for r in members if has_data(r) and not is_custom(r)]
+		scope.extend(custom_used)
+		print("%-14s %7d %7d %11d %11d"
+		      % (family, len(members), len(custom), len(custom_used), len(stock_used)))
+
+	scope.sort(key=lambda r: -((r["rows"] or 0) + r["childRows"]))
+	with codecs.open(os.path.join(OUT, "scope.md"), "w", "utf-8") as f:
+		f.write("# Client-specific objects that hold data\n\n")
+		f.write("| family | name | synonym | table | rows | child rows | cols |\n")
+		f.write("|---|---|---|---|---:|---:|---:|\n")
+		for r in scope:
+			f.write("| %s | %s | %s | %s | %s | %s | %s |\n"
+			        % (r["family"], r["name"], r["synonym"] or "", r["physical"],
+			           "{:,}".format(r["rows"] or 0), "{:,}".format(r["childRows"]), r["columns"]))
+	print("\ncustom objects with data: %d -> out/scope.md" % len(scope))
+
+	top = int(sys.argv[1]) if len(sys.argv) > 1 else 20
+	print()
+	for r in scope[:top]:
+		print("   %-10s %-42s %s" % (r["family"], r["name"], r["synonym"] or ""))
+
+
+if __name__ == "__main__":
+	main()