@dforge-core/dforge-mcp 0.2.44 → 0.2.46-next

package/skills/dforge-1c-import/scripts/lib/recorders.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Which document types write a register, and how much.
+
+A register with _RecorderTRef is written by several document kinds; the TRef is the
+target's type id, which appears in the reference columns of any table pointing at the
+same type. Matching it back to a document is done through DBNames: the type id is the
+low bytes of the table number, so comparing against every document's table number
+identifies it.
+
+    python tools/recorders.py _AccumRg17050
+"""
+from __future__ import print_function
+
+import codecs
+import json
+import os
+import sys
+
+import paths
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import dump1c
+
+OUT = paths.OUT
+
+
+def load_report():
+	with codecs.open(os.path.join(OUT, "report.json"), "r", "utf-8") as f:
+		return json.load(f)
+
+
+def probe_single(cur, table):
+	"""Without a TRef the recorder's type is implicit, so find the one document table
+	whose PK matches the recorders we sampled."""
+	cur.execute("SELECT DISTINCT TOP (20) _RecorderRRef FROM [%s]" % table)
+	refs = [bytes(r[0]) for r in cur.fetchall()]
+	if not refs:
+		print("   (register is empty)")
+		return
+	placeholders = ",".join("?" * len(refs))
+	for row in load_report():
+		physical = row.get("physical") or ""
+		suffix = physical[len("_Document"):] if physical.startswith("_Document") else ""
+		if not suffix.isdigit() or not row.get("rows"):
+			continue
+		cur.execute("SELECT COUNT(*) FROM [%s] WHERE _IDRRef IN (%s)"
+		            % (physical, placeholders), *refs)
+		hits = cur.fetchone()[0]
+		if hits:
+			print("   %s - %s  (%d/%d sampled recorders matched)"
+			      % (row["name"], row["synonym"] or "", hits, len(refs)))
+			return
+	print("   no document table matched the sampled recorders")
+
+
+def main():
+	table = sys.argv[1]
+	cur = dump1c.connect(paths.required("DF1C_SERVER", "which MS SQL server holds it"),
+	                     paths.required("DF1C_DB",
+	                                    "which MS SQL infobase to read")).cursor()
+	cur.execute("""SELECT COUNT(*) FROM sys.columns
+	               WHERE object_id = OBJECT_ID(?) AND name = '_RecorderTRef'""", table)
+	if not cur.fetchone()[0]:
+		print("%s has no _RecorderTRef - one document type writes it; probing for it"
+		      % table)
+		probe_single(cur, table)
+		return
+
+	cur.execute("SELECT _RecorderTRef, COUNT(*) FROM [%s] GROUP BY _RecorderTRef "
+	            "ORDER BY 2 DESC" % table)
+	rows = cur.fetchall()
+
+	# a document's type id is its table number; _RecorderTRef stores it big-endian
+	by_number = {}
+	for row in load_report():
+		physical = row.get("physical") or ""
+		suffix = physical[len("_Document"):] if physical.startswith("_Document") else ""
+		if suffix.isdigit():                      # skips _DocumentJournal<N>
+			by_number[int(suffix)] = row
+
+	print("%-12s %-12s %s" % ("TRef", "movements", "document"))
+	for tref, n in rows:
+		number = int.from_bytes(bytes(tref), "big") & 0xFFFF
+		doc = by_number.get(number)
+		print("%-12s %-12s %s" % (bytes(tref).hex(), "{:,}".format(n),
+		                          "%s - %s" % (doc["name"], doc["synonym"] or "") if doc
+		                          else "number=%d (unmatched)" % number))
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/reftargets.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Does every authored reference point where the 1C attribute actually points?
+
+The generated modules resolve a reference from the attribute's declared type, so they
+cannot get this wrong. The authored ones name their target by hand, and a name is a poor
+guide: `БазоваяЕдиницаИзмерения` reads like the product's own units and points at the unit
+classifier, and `МестоОтгрузки` reads like a counterparty and is not one.
+
+A wrong target costs nothing until the data arrives, and then it is every row: 7,981 of
+8,400 products dangled on the first load of `masterdata`.
+
+    python tools/reftargets.py
+"""
+from __future__ import print_function
+
+import codecs
+import json
+import os
+import sys
+
+import attrs
+import autospec
+import handmap
+import binding
+
+sources = binding.sources()
+
+
+def ported(mapping):
+	"""1C object name -> the entity the port gave it."""
+	out = {}
+	for module, entities in mapping.items():
+		for entity, found in entities.items():
+			name = (found.get("object") or "").split(" / ")[0]
+			if name and "columns" in found:
+				out.setdefault(name, "%s.%s" % (module, entity))
+	return out
+
+
+def main():
+	model = autospec.Model()
+	path = os.path.join(attrs.OUT, "columnmap.json")
+	with codecs.open(path, "r", "utf-8") as f:
+		mapping = json.load(f)
+	owner = ported(mapping)
+
+	wrong = 0
+	for module in sorted(handmap.SPECS):
+		for entity, spec in handmap.SPECS[module]["entities"].items():
+			declared = sources.SOURCES.get(module, {}).get(entity, {})
+			name = declared.get("object")
+			if not name:
+				continue
+			family, section = declared.get("family"), declared.get("section")
+			_, columns = model.physical(name, family)
+			by_physical = {}
+			for field in model.fields(name, family):
+				if (field.get("section") or None) != section:
+					continue
+				physical = autospec.physical_of(field, model, columns)
+				if physical:
+					by_physical[physical] = field
+
+			for item in spec["fields"]:
+				if not item[1].startswith("ref:"):
+					continue
+				target = item[1][len("ref:"):]
+				if "." not in target:
+					target = module + "." + target
+				physical = mapping[module][entity]["columns"].get(item[0] + "_id")
+				field = by_physical.get(physical)
+				if not field:
+					continue
+				declared_targets = [p.strip()[3:].strip()
+				                    for p in (field["type"] or "").split(",")
+				                    if p.strip().startswith("->")]
+				real = [t.split(".", 1)[1] for t in declared_targets if "." in t]
+				if not real:
+					continue
+				expected = [owner.get(r) for r in real]
+				if target in expected:
+					continue
+				wrong += 1
+				print("%-10s %-22s %-20s declared %-38s 1C: %s -> %s"
+				      % (module, entity, item[0], target, ", ".join(real),
+				         ", ".join(str(e) for e in expected)))
+
+	print("\n%d reference(s) pointing at the wrong entity" % wrong)
+	return 1 if wrong else 0
+
+
+if __name__ == "__main__":
+	sys.exit(main())

package/skills/dforge-1c-import/scripts/lib/reportgen.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+"""Write a module's ported 1C reports from one compact spec.
+
+A report over a stored procedure is declared in four places that have to agree - the
+report (`ui/reports.json`), the procedure's call signature and columns
+(`logic/stored_procedures.json`), the Ukrainian labels of both (`translations/uk-UA.json`)
+and the rights to run them (`security/roles.json`) - and the platform binds them by code,
+so a parameter spelled one way in the report and another in the procedure binds NULL
+without a word. This writes all four from one spec, into the project's `authored/<module>/`
+tree, which `genmodule` lays over the generated module; the menu gets a Reports section
+from the reports file (genmodule.auto_menu).
+
+A spec module (tools/project/reports_<module>.py) defines MODULE and REPORTS, an ordered
+list of `report(...)`. The SQL of each procedure is the project's own file under
+`authored/<module>/logic/reports/`.
+
+    python tools/lib/reportgen.py tools/project/reports_production.py
+"""
+from __future__ import print_function
+
+import codecs
+import collections
+import imp
+import json
+import os
+import sys
+
+O = collections.OrderedDict
+
+KINDS = {
+	"money": ("currency", "number", 140),
+	"qty": ("number", "number", 120),
+	"int": ("number", "number", 70),
+	"date": ("date", "date", 110),
+	"datetime": ("datetime", "timestamp", 150),
+	"text": ("text", "string", 160),
+	"name": ("text", "string", 380),
+	"bool": ("checkbox", "bool", 80),
+}
+
+PG_FIELD = {"date": "date", "integer": "number", "numeric": "number", "boolean": "checkbox",
+            "text": "text", "uuid": "lookup"}
+
+
+def col(code, en, uk, kind="text", width=None):
+	"""A result column: its code, English and Ukrainian labels, and kind (KINDS)."""
+	return dict(code=code, en=en, uk=uk, kind=kind, width=width)
+
+
+def param(code, pg, en, uk, required=False, default=None, link=None):
+	"""A parameter: the procedure's argument and the report's, the same code in both. `link`
+	names the entity a uuid parameter picks from (`masterdata.warehouse`)."""
+	return dict(code=code, pg=pg, en=en, uk=uk, required=required, default=default, link=link)
+
+
+def report(code, en, uk, sp, description, params, columns, source=None):
+	"""A report over one procedure. `source` names the 1C report it ports."""
+	return dict(code=code, en=en, uk=uk, sp=sp, description=description, params=params,
+	            columns=columns, source=source)
+
+
+def date_from(required=True):
+	return param("p_date_from", "date", "From", u"Початок періоду", required, "=STARTMONTH(NOW())")
+
+
+def date_to(required=True):
+	return param("p_date_to", "date", "To", u"Кінець періоду", required, "=ENDMONTH(NOW())")
+
+
+def _field(p):
+	field = O([("fieldTypeCd", "lookup" if p["link"] else PG_FIELD.get(p["pg"], "text")),
+	           ("label", p["en"]), ("required", p["required"])])
+	if p["default"] is not None:
+		field["default"] = p["default"]
+	if p["link"]:
+		field["params"] = {"link": {"entity": p["link"]}}
+	return field
+
+
+def _read(path):
+	if not os.path.isfile(path):
+		return O()
+	with codecs.open(path, "r", "utf-8") as f:
+		return json.load(f, object_pairs_hook=O)
+
+
+def _write(path, doc):
+	directory = os.path.dirname(path)
+	if not os.path.isdir(directory):
+		os.makedirs(directory)
+	with codecs.open(path, "w", "utf-8") as f:
+		f.write(json.dumps(doc, ensure_ascii=False, indent="\t") + "\n")
+
+
+def build(module, reports, roles, root):
+	"""Write the four files for `module` under `root` (authored/<module>/)."""
+	ui, sps, uk_reports, uk_sps, rights = O(), O(), O(), O(), O()
+	for r in reports:
+		columns = []
+		for c in r["columns"]:
+			field, base, width = KINDS[c["kind"]]
+			columns.append(O([("columnCd", c["code"]), ("label", c["en"]), ("fieldTypeCd", field),
+			                  ("baseDatatypeCd", base), ("width", c["width"] or width)]))
+		sps[r["sp"]] = O([
+			("description", r["description"]), ("functionName", r["sp"]),
+			("params", [O([("paramCd", p["code"]), ("pgType", p["pg"]),
+			               ("fieldTypeCd", _field(p)["fieldTypeCd"]), ("label", p["en"]),
+			               ("required", p["required"])]) for p in r["params"]]),
+			("columns", columns)])
+		uk_sps[r["sp"]] = {"columns": O((c["code"], {"label": c["uk"]}) for c in r["columns"])}
+		ui[r["code"]] = O([
+			("label", r["en"]),
+			("description", r["description"] + (" (1C %s)" % r["source"] if r["source"] else "")),
+			("parameters", O((p["code"], _field(p)) for p in r["params"])),
+			("layout", {"panels": [O([("vizType", "table"), ("datasetCd", "rows"), ("title", r["en"])])]}),
+			("datasets", {"rows": O([("caption", r["en"]), ("datasetType", "S"), ("spCd", r["sp"])])})])
+		uk_reports[r["code"]] = O([("label", r["uk"]),
+		                           ("params", O((p["code"], {"label": p["uk"]}) for p in r["params"])),
+		                           ("datasets", {"rows": {"caption": r["uk"]}})])
+		rights["report:" + r["code"]] = "E"
+		rights["sp:" + r["sp"]] = "E"
+
+	_write(os.path.join(root, "ui", "reports.json"), ui)
+	# the print forms' procedures (printgen, prn_*) share the file and are kept
+	kept = _read(os.path.join(root, "logic", "stored_procedures.json"))
+	sps.update((k, v) for k, v in kept.items() if k.startswith("prn_"))
+	_write(os.path.join(root, "logic", "stored_procedures.json"), sps)
+
+	path = os.path.join(root, "translations", "uk-UA.json")
+	translations = _read(path)
+	translations["reports"] = uk_reports
+	translations["storedProcedures"] = uk_sps
+	_write(path, translations)
+
+	path = os.path.join(root, "security", "roles.json")
+	doc = _read(path)
+	for role in roles:
+		entry = doc.setdefault(role, O([("rights", O())]))
+		entry.setdefault("rights", O())
+		for key in [k for k in entry["rights"] if k.startswith(("report:", "sp:")) and not k.startswith("sp:prn_")]:
+			del entry["rights"][key]
+		entry["rights"].update(rights)
+	_write(path, doc)
+	return len(reports)
+
+
+def main(argv):
+	here = os.path.dirname(os.path.abspath(__file__))
+	for spec_path in argv[1:]:
+		spec = imp.load_source(os.path.splitext(os.path.basename(spec_path))[0], spec_path)
+		root = os.path.join(here, "..", "project", "authored", spec.MODULE)
+		count = build(spec.MODULE, spec.REPORTS, spec.ROLES, os.path.normpath(root))
+		print("%s: %d report(s)" % (spec.MODULE, count))
+
+
+if __name__ == "__main__":
+	main(sys.argv)

package/skills/dforge-1c-import/scripts/lib/roles.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""List the distinct collection class GUIDs across a set of objects, with samples,
+so each one can be labelled (dimension / resource / attribute / tabular column)."""
+from __future__ import print_function
+
+import collections
+import sys
+
+import attrs
+
+if __name__ == "__main__":
+	objects = attrs.load_json("objects.json")
+	types = attrs.load_json("typemap.json")
+	report = attrs.load_json("report.json")
+
+	names = sys.argv[1:] or [r["name"] for r in report if r.get("name")]
+	seen = collections.OrderedDict()
+	for name in names:
+		guid, fields = attrs.fields_of(name, objects, types)
+		if not fields:
+			continue
+		family = next((r["family"] for r in report if r["name"] == name), "?")
+		for f in fields:
+			key = (family, f["collection"])
+			entry = seen.setdefault(key, {"count": 0, "samples": []})
+			entry["count"] += 1
+			if len(entry["samples"]) < 3:
+				entry["samples"].append("%s/%s" % (name, f["name"]))
+
+	for (family, guid), entry in sorted(seen.items()):
+		print("%-16s %-40s %5d  %s" % (family, guid, entry["count"],
+		                               ", ".join(entry["samples"])[:90]))