@dforge-core/dforge-mcp 0.2.45 → 0.2.47

package/skills/dforge-1c-import/scripts/lib/handlers.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""What each document's posting handler actually writes.
+
+The first measurement of this counted the body of `ОбработкаПроведения` and stopped there,
+and that is not what a handler writes. A four-line handler that calls a procedure writing
+two registers per line is not a four-line handler, and a document that writes movements
+from `ПриЗаписи` does not write them only when posted. Both shapes are in this base, and
+between them they moved the "ports one for one" list by more than a name.
+
+So this follows the calls. For every document module that has a posting handler:
+
+  * the registers the handler names directly;
+  * the registers named by any procedure in the same module that the handler calls;
+  * the registers written outside posting altogether, from `ПриЗаписи` / `ПередЗаписью`.
+
+A name only counts when the metadata has a register by it. `Движения.` also reaches
+accounting-movement collections and a few properties, and counting those inflated an
+earlier figure by half.
+
+    python tools/1c.py handlers            # the summary, and out/handlers.json
+    python tools/1c.py handlers --client   # the client's own documents, classified
+"""
+from __future__ import print_function
+
+import codecs
+import io
+import json
+import os
+import re
+import sys
+
+import binding
+import paths
+
+CODE = os.path.join(paths.OUT, "code", "Document")
+
+POST = u"ОбработкаПроведения"
+ON_WRITE = u"ПриЗаписи"
+BEFORE_WRITE = u"ПередЗаписью"
+
+# The client's own development carries the project's prefixes; everything else is the
+# vendor's configuration, whose registers the port deliberately does not carry.
+CLIENT = binding.client_prefixes()
+
+WORD = u"[А-Яа-яёЁA-Za-z_][А-Яа-яёЁA-Za-z_0-9]*"
+PROC = re.compile(u"^[ \t]*(?:Процедура|Функция)[ \t]+(%s)" % WORD, re.M | re.U)
+END = re.compile(u"^[ \t]*(?:КонецПроцедуры|КонецФункции)", re.M | re.U)
+REGISTER = re.compile(u"Движения\\.(%s)" % WORD, re.U)
+
+
+def procedures(text):
+	"""Procedure name -> its source. 1C does not nest them, so this is a flat scan."""
+	found = {}
+	lines = text.splitlines()
+	index = 0
+	while index < len(lines):
+		match = PROC.match(lines[index])
+		if not match:
+			index += 1
+			continue
+		end = index + 1
+		while end < len(lines) and not END.match(lines[end]):
+			end += 1
+		found[match.group(1)] = "\n".join(lines[index:end + 1])
+		index = end + 1
+	return found
+
+
+def calls(body, name):
+	"""Does `body` call `name`? A bare mention in a comment or a longer name is not one."""
+	return bool(re.search(u"(?<![А-Яа-яёЁA-Za-z_0-9])%s[ \t]*\\(" % re.escape(name),
+	                      body, re.U))
+
+
+def scan():
+	"""One row per document that has a posting handler."""
+	rows = []
+	for document in sorted(os.listdir(CODE)):
+		directory = os.path.join(CODE, document)
+		if not os.path.isdir(directory):
+			continue
+		for name in sorted(os.listdir(directory)):
+			if not name.endswith(".bsl"):
+				continue
+			with io.open(os.path.join(directory, name), encoding="utf-8",
+			             errors="replace") as handle:
+				text = handle.read()
+			if POST not in text:
+				continue
+			bodies = procedures(text)
+			if POST not in bodies:
+				continue
+
+			direct = set(REGISTER.findall(bodies[POST]))
+			via = {}
+			for procedure, body in bodies.items():
+				if procedure == POST or not REGISTER.search(body):
+					continue
+				if calls(bodies[POST], procedure):
+					via[procedure] = sorted(set(REGISTER.findall(body)))
+
+			outside = {}
+			for hook in (ON_WRITE, BEFORE_WRITE):
+				if hook in bodies and REGISTER.search(bodies[hook]):
+					outside[hook] = sorted(set(REGISTER.findall(bodies[hook])))
+
+			rows.append({
+				"doc": document,
+				"module": name,
+				"postLines": len(bodies[POST].splitlines()),
+				"direct": sorted(direct),
+				"viaCalls": via,
+				"outsidePosting": outside,
+			})
+			break
+	return rows
+
+
+def declared_registers():
+	with codecs.open(os.path.join(paths.OUT, "objects_all.json"), "r", "utf-8") as handle:
+		objects = json.load(handle)
+	return set(o["name"] for o in objects
+	           if o.get("family") in ("InfoRg", "AccumRg", "AccRg") and o.get("name"))
+
+
+def classify(rows, registers):
+	"""The client's documents, by whose registers they write."""
+	out = []
+	for row in rows:
+		if not row["doc"].startswith(CLIENT):
+			continue
+		direct = set(row["direct"])
+		via = set()
+		for names in row["viaCalls"].values():
+			via |= set(names)
+		outside = set()
+		for names in row["outsidePosting"].values():
+			outside |= set(names)
+
+		every = (direct | via | outside) & registers
+		if not every:
+			continue
+		out.append({
+			"doc": row["doc"],
+			"postLines": row["postLines"],
+			"client": sorted(n for n in every if n.startswith(CLIENT)),
+			"vendor": sorted(n for n in every if not n.startswith(CLIENT)),
+			"viaCalls": sorted((via & registers) - direct),
+			"outsidePosting": sorted(outside & registers),
+		})
+	return out
+
+
+def write_json(name, payload):
+	path = os.path.join(paths.OUT, name)
+	with codecs.open(path, "w", "utf-8") as handle:
+		handle.write(json.dumps(payload, ensure_ascii=False, indent="\t", sort_keys=True))
+	return path
+
+
+def main():
+	rows = scan()
+	registers = declared_registers()
+	client = classify(rows, registers)
+
+	write_json("handlers.json", rows)
+	write_json("handlers-client.json", client)
+
+	print("registers the metadata declares:                   %d" % len(registers))
+	print("documents with a posting handler:                  %d" % len(rows))
+	print("  whose handler calls a procedure writing a register: %d"
+	      % len([r for r in rows if r["viaCalls"]]))
+	print("  writing movements outside posting as well:          %d"
+	      % len([r for r in rows if r["outsidePosting"]]))
+
+	pure = [r for r in client if not r["vendor"]]
+	mixed = [r for r in client if r["vendor"] and r["client"]]
+	vendor = [r for r in client if r["vendor"] and not r["client"]]
+	print()
+	print("the client's documents that write a register:      %d" % len(client))
+	print("  only the client's own registers:                 %d" % len(pure))
+	print("  mixed:                                           %d" % len(mixed))
+	print("  only the vendor's:                               %d" % len(vendor))
+
+	if "--client" not in sys.argv:
+		return
+
+	print()
+	print("Ports one for one, shortest first:")
+	for row in sorted(pure, key=lambda r: r["postLines"]):
+		print("  %-54s %3d  %s"
+		      % (row["doc"], row["postLines"], ", ".join(row["client"])))
+		if row["viaCalls"]:
+			print("  %-54s      via calls: %s" % ("", ", ".join(row["viaCalls"])))
+		if row["outsidePosting"]:
+			print("  %-54s      OUTSIDE POSTING: %s"
+			      % ("", ", ".join(row["outsidePosting"])))
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/handmap.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Propose the 1C source of every column in a hand-written spec.
+
+`fleet`, `shipping` and `masterdata` were authored, so their column names were chosen
+rather than generated, and nothing in the package records which 1C attribute each one
+came from. The migration needs exactly that.
+
+Most of it can be proposed rather than typed: the names were chosen *from* the 1C
+attributes, so running the attribute through `naming.code` lands on the same word most of
+the time - `БазоваяЕдиницаИзмерения` gives `base_uom_measure` against a column called
+`base_uom`. This scores those candidates and prints what it is sure of, what it guessed
+and what it could not place, so the confirmed result can go into `sources.py` and the
+remainder can be decided by eye.
+
+    python tools/handmap.py                  # every hand-written module
+    python tools/handmap.py masterdata       # one of them
+"""
+from __future__ import print_function
+
+import collections
+import sys
+
+import autospec
+import naming
+import binding
+
+sources = binding.sources()
+
+# The authored modules, whichever the project happens to have.
+SPECS = binding.specs()
+
+# Columns the 1C object carries in its own right rather than as an attribute; these are
+# resolved from the physical table, exactly as `autospec` resolves them.
+STANDARD = set(autospec.STANDARD_PHYSICAL)
+
+
+def tokens(code):
+	return set(code.split("_"))
+
+
+def score(column, candidate):
+	"""How well a 1C attribute's generated code matches a hand-chosen column name."""
+	if column == candidate:
+		return 100
+	ours, theirs = tokens(column), tokens(candidate)
+	if not ours or not theirs:
+		return 0
+	shared = ours & theirs
+	if not shared:
+		return 0
+	# Every word of the shorter name present in the longer one is a strong signal:
+	# `base_uom` against `base_uom_measure`.
+	if ours <= theirs or theirs <= ours:
+		return 90 - abs(len(ours) - len(theirs))
+	return int(60 * len(shared) / float(len(ours | theirs)))
+
+
+def propose(module, entity, spec, model):
+	"""(confirmed, guessed, unplaced) for one entity."""
+	declared = sources.SOURCES.get(module, {}).get(entity, {})
+	object_name = declared.get("object")
+	if not object_name:
+		return None
+
+	family = declared.get("family")
+	section = declared.get("section")
+	fields = [f for f in model.fields(object_name, family)
+	          if (f.get("section") or None) == section and not naming.is_dead(f["name"])]
+	candidates = [(naming.code(f["name"], 48), f) for f in fields]
+
+	manual = declared.get("columns", {})
+	confirmed, guessed, unplaced = [], [], []
+	taken = set(manual.values())
+
+	for item in spec["fields"]:
+		column = item[0]
+		if column in manual:
+			confirmed.append((column, manual[column], "declared"))
+			continue
+		if column in STANDARD:
+			confirmed.append((column, "(standard)", "standard"))
+			continue
+		# Sorted on the score and the code only. Two candidates scoring the same
+		# under the same code made the tuple comparison reach the field dicts, which
+		# do not compare - a latent defect until an object turned up whose attributes
+		# generate the same code twice.
+		ranked = sorted(
+			((score(column, code), code, field) for code, field in candidates
+			 if field["name"] not in taken),
+			key=lambda ranked_row: (ranked_row[0], ranked_row[1]), reverse=True)
+		if ranked and ranked[0][0] >= 90:
+			confirmed.append((column, ranked[0][2]["name"], "matched"))
+			taken.add(ranked[0][2]["name"])
+		elif ranked and ranked[0][0] >= 40:
+			guessed.append((column, ranked[0][2]["name"], ranked[0][0]))
+		else:
+			unplaced.append(column)
+
+	spare = [f["name"] for _, f in candidates if f["name"] not in taken]
+	return confirmed, guessed, unplaced, spare, object_name
+
+
+def main():
+	model = autospec.Model()
+	wanted = sys.argv[1] if len(sys.argv) > 1 else None
+	totals = collections.Counter()
+
+	for module, spec in SPECS.items():
+		if wanted and module != wanted:
+			continue
+		print("\n=== %s ===" % module)
+		for entity, entity_spec in spec["entities"].items():
+			found = propose(module, entity, entity_spec, model)
+			if found is None:
+				print("\n%-28s no 1C object declared in sources.py" % entity)
+				totals["undeclared"] += len(entity_spec["fields"])
+				continue
+			confirmed, guessed, unplaced, spare, object_name = found
+			totals["confirmed"] += len(confirmed)
+			totals["guessed"] += len(guessed)
+			totals["unplaced"] += len(unplaced)
+			print("\n%-28s <- %s   (%d confirmed, %d to check, %d unplaced)"
+			      % (entity, object_name, len(confirmed), len(guessed), len(unplaced)))
+			for column, name, how in confirmed:
+				if how == "matched":
+					print("   ok    %-34s %s" % (column, name))
+			for column, name, points in guessed:
+				print("   ?%-3d  %-34s %s" % (points, column, name))
+			for column in unplaced:
+				print("   --    %-34s" % column)
+			if spare:
+				print("   1C attributes nothing claimed: %s" % ", ".join(spare[:8]))
+
+	print("\n%d confirmed, %d to check by eye, %d unplaced, %d in undeclared entities"
+	      % (totals["confirmed"], totals["guessed"], totals["unplaced"],
+	         totals["undeclared"]))
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/i18n_coverage.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+How much of the configuration is actually translated.
+
+1C synonyms are per-language: {2,"ru","Водители","uk","Водії"}. Whether a given object
+or field carries a `uk` synonym at all decides whether the port can lift its Ukrainian
+labels from the infobase or has to translate them itself.
+
+    python tools/i18n_coverage.py                 # whole configuration, by family
+    python tools/i18n_coverage.py <names...>      # named objects, per field
+"""
+from __future__ import print_function
+
+import collections
+import sys
+
+import attrs
+import meta
+
+
+def synonyms(node):
+	"""Identity block -> {lang: text}. The block is [count, lang, text, lang, text, ...]."""
+	block = node[1][3] if len(node[1]) > 3 else None
+	if not isinstance(block, list) or len(block) < 3:
+		return {}
+	body = block[1:]
+	return {body[i]: body[i + 1] for i in range(0, len(body) - 1, 2)
+	        if isinstance(body[i], str) and len(body[i]) == 2}
+
+
+def object_languages(guid, name):
+	"""(object synonyms, [(field, {lang: text}), ...])"""
+	text = meta.load(guid)
+	if text is None:
+		return {}, []
+	tree = meta.parse(text)
+	own = {}
+	for node in meta.walk(tree):
+		if attrs.is_identity(node) and node[2] == name:
+			own = synonyms(["", node])
+			break
+	fields = []
+	for node in meta.walk(tree):
+		if attrs.is_field(node):
+			fields.append((attrs.name_of(node), synonyms(node)))
+	return own, fields
+
+
+def main():
+	objects = attrs.load_json("objects.json")
+	report = {r["name"]: r for r in attrs.load_json("report.json")}
+	names = sys.argv[1:]
+
+	if names:
+		for name in names:
+			guid = next((o["guid"] for o in objects if o.get("name") == name), None)
+			own, fields = object_languages(guid, name)
+			print("\n== %s  %s" % (name, " ".join("%s=%s" % kv for kv in sorted(own.items()))))
+			for field, langs in fields:
+				print("   %-34s %s" % (field, " | ".join(
+					"%s: %s" % (lang, langs[lang]) for lang in sorted(langs))))
+		return
+
+	stats = collections.defaultdict(lambda: {"objects": 0, "obj_uk": 0,
+	                                         "fields": 0, "fld_uk": 0})
+	for obj in objects:
+		name = obj.get("name")
+		if not name:
+			continue
+		family = report.get(name, {}).get("family") or obj["family"]
+		own, fields = object_languages(obj["guid"], name)
+		row = stats[family]
+		row["objects"] += 1
+		row["obj_uk"] += 1 if "uk" in own else 0
+		row["fields"] += len(fields)
+		row["fld_uk"] += sum(1 for _, langs in fields if "uk" in langs)
+
+	print("%-18s %8s %8s %10s %8s" % ("family", "objects", "obj uk", "fields", "fld uk"))
+	print("-" * 56)
+	for family, row in sorted(stats.items(), key=lambda kv: -kv[1]["fields"]):
+		print("%-18s %8d %8d %10d %8d"
+		      % (family, row["objects"], row["obj_uk"], row["fields"], row["fld_uk"]))
+
+
+if __name__ == "__main__":
+	main()