@dforge-core/dforge-mcp 0.2.45 → 0.2.47

package/skills/dforge-1c-import/scripts/lib/idioms.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+u"""What shapes the movements are written in, weighted by the rows 1C holds for each.
+
+The generator that turns a document's posting into declarative bindings will be written
+one idiom at a time, and it is worth knowing before writing any of it how far each idiom
+goes. So every movement 1C recorded - a (document, register) pair, with its row count from
+`recorded.json` - is traced to the write sites the call graph finds behind it
+(`callgraph.py`), and each site is classified by its shape:
+
+  row       `Запись = Набор.Добавить()` and the columns assigned one by one
+  load      `Набор.Загрузить(Таблица)` - a whole table at once
+  module    `Набор.мТаблицаДвижений = Таблица; Набор.ВыполнитьПриход()` - the register's
+            own record-set module does the writing from a table the document prepared
+  method    another method of the register's own record-set module
+  write     only `Набор.Записать()` here; the set was filled some other way
+
+and, for `load` and `module`, by where the table comes from:
+
+  query     the result of a query (`Запрос.Выполнить().Выгрузить()`)
+  section   a tabular section's `Выгрузить()` - the document's own lines
+  copy      `Скопировать()` of another table
+  built     `Новый ТаблицаЗначений`, filled by the code
+  param     handed in by a caller
+  function  returned by a call this does not follow
+
+A pair written through sites of more than one shape is `mixed`. Its rows are counted once,
+under `mixed`, so the percentages add up.
+
+    python tools/1c.py idioms           # the table; writes out/idioms.json
+"""
+from __future__ import print_function
+
+import codecs
+import collections
+import io
+import json
+import os
+import sys
+
+import bsl
+import callgraph
+import paths
+
+TARGET = os.path.join(paths.OUT, "idioms.json")
+
+ROW_OPS = frozenset(bsl.norm(x) for x in (u"Добавить", u"ДобавитьПриход", u"ДобавитьРасход",
+                                          u"ДобавитьДвижение", u"Add"))
+LOAD_OPS = frozenset(bsl.norm(x) for x in (u"Загрузить", u"Load"))
+MODULE_OPS = frozenset(bsl.norm(x) for x in (u"ВыполнитьПриход", u"ВыполнитьРасход",
+                                             u"ВыполнитьДвижения", u"ВыполнитьДвижениеПриход",
+                                             u"ВыполнитьДвижениеРасход"))
+WRITE_ONLY = frozenset(bsl.norm(x) for x in (u"Записать", u"Write"))
+
+UNLOAD = bsl.norm(u"Выгрузить")
+COPY = bsl.norm(u"Скопировать")
+EXECUTE = bsl.norm(u"Выполнить")
+FILL = bsl.norm(u"ЗаполнитьЗначения")
+COLLAPSE = bsl.norm(u"Свернуть")
+MOVEMENT_TABLE = bsl.norm(u"мТаблицаДвижений")
+COLUMNS = bsl.norm(u"ВыгрузитьКолонки")
+LOAD_BY_NAME = bsl.norm(u"ЗагрузитьВТаблицуЗначений")
+
+
+def shape(op):
+	if op in ROW_OPS:
+		return "row"
+	if op in LOAD_OPS:
+		return "load"
+	if op in MODULE_OPS:
+		return "module"
+	if op in WRITE_ONLY:
+		return "write"
+	return "method"
+
+
+class Locator(object):
+	"""Finds the method around a (module path, line) and reads what feeds a table."""
+
+	def __init__(self, modules):
+		self.by_path = dict((m.path, m) for m in modules)
+
+	def method_at(self, path, line):
+		module = self.by_path.get(path)
+		if module is None:
+			return None, None
+		for method in module.methods.values():
+			if method[6] <= line <= method[7]:
+				return module, method
+		return module, None
+
+	@staticmethod
+	def call_at(method, line, op):
+		for st in bsl.statements(method[5]):
+			if st[1] != line:
+				continue
+			for expr in bsl.expressions(st):
+				for sub in bsl.subexpressions(expr):
+					if sub[0] == "callx" and sub[1][0] == "member" and bsl.norm(sub[1][2]) == op:
+						return sub
+		return None
+
+	@staticmethod
+	def origin(method, expr, depth=0):
+		"""Where a table expression comes from, read off the method's assignments."""
+		if expr is None or depth > 3:
+			return "function"
+		if expr[0] == "callx":
+			callee = expr[1]
+			if callee[0] == "member":
+				op = bsl.norm(callee[2])
+				if op == UNLOAD:
+					receiver = callee[1]
+					inner = [s for s in bsl.subexpressions(receiver) if s[0] == "callx"
+					         and s[1][0] == "member" and bsl.norm(s[1][2]) == EXECUTE]
+					if inner:
+						return "query"
+					parts = bsl.dotted(receiver)
+					if parts and any(u"запрос" in bsl.norm(p) or u"результат" in bsl.norm(p)
+					                 or u"выборка" in bsl.norm(p) for p in parts):
+						return "query"
+					return "section"
+				if op == COPY:
+					return "copy"
+				if op == COLUMNS:
+					return "columns"
+			return "function"
+		if expr[0] == "new":
+			return "built"
+		if expr[0] == "name":
+			name = bsl.norm(expr[1])
+			if name in set(bsl.norm(p[0]) for p in method[3]):
+				return "param"
+			sources = [st[3] for st in bsl.statements(method[5])
+			           if st[0] == "assign" and st[2][0] == "name" and bsl.norm(st[2][1]) == name]
+			if not sources:
+				return "function"
+			kinds = set(Locator.origin(method, s, depth + 1) for s in sources)
+			if "columns" in kinds:
+				# УТП's own step: an empty table of the register's columns, filled by
+				# column name from another table - which is the real source
+				kinds.discard("columns")
+				for st in bsl.statements(method[5]):
+					for e in bsl.expressions(st):
+						for sub in bsl.subexpressions(e):
+							if sub[0] == "callx" and len(sub[2]) >= 2 and sub[2][1] is not None \
+									and sub[2][1][0] == "name" and bsl.norm(sub[2][1][1]) == name:
+								callee = sub[1]
+								called = bsl.norm(callee[2] if callee[0] == "member"
+								                  else callee[1] if callee[0] == "name" else u"")
+								if called == LOAD_BY_NAME:
+									kinds.add("columns<-" + Locator.origin(method, sub[2][0],
+									                                       depth + 1))
+				if not kinds:
+					kinds.add("columns")
+			return kinds.pop() if len(kinds) == 1 else "+".join(sorted(kinds))
+		return "function"
+
+	@staticmethod
+	def treated(method, expr):
+		"""Does the method fill a column from the header, or collapse, on this table?"""
+		if expr is None or expr[0] != "name":
+			return set()
+		name = bsl.norm(expr[1])
+		found = set()
+		for st in bsl.statements(method[5]):
+			for e in bsl.expressions(st):
+				for sub in bsl.subexpressions(e):
+					if sub[0] == "callx" and sub[1][0] == "member" and sub[1][1] == ("name", expr[1]) \
+							or (sub[0] == "callx" and sub[1][0] == "member"
+							    and sub[1][1][0] == "name" and bsl.norm(sub[1][1][1]) == name):
+						op = bsl.norm(sub[1][2])
+						if op == FILL:
+							found.add("header-fill")
+						elif op == COLLAPSE:
+							found.add("collapse")
+		return found
+
+	def table_of(self, method, line, op):
+		"""The table a `load` or `module` site writes from."""
+		if op in LOAD_OPS:
+			call = self.call_at(method, line, op)
+			return call[2][0] if call is not None and call[2] else None
+		# module: the table was handed over as `<set>.мТаблицаДвижений = <table>` - the
+		# last such hand-over before the site; a method can write several sets in turn
+		found = None
+		for st in bsl.statements(method[5]):
+			if st[0] == "assign" and st[2][0] == "member" \
+					and bsl.norm(st[2][2]) == MOVEMENT_TABLE and st[1] <= line:
+				if found is None or st[1] >= found[1]:
+					found = st
+		return found[3] if found is not None else None
+
+
+def classify(graph, locator, sites):
+	"""(shape label, detail counter) for one movement's write sites."""
+	shapes = set()
+	details = collections.Counter()
+	for path, line, op in sites:
+		kind = shape(op)
+		shapes.add(kind)
+		if kind in ("load", "module"):
+			_, method = locator.method_at(path, line)
+			if method is None:
+				details[kind + ":unknown"] += 1
+				continue
+			table = locator.table_of(method, line, op)
+			details[kind + ":" + locator.origin(method, table)] += 1
+			for treatment in locator.treated(method, table):
+				details[kind + "+" + treatment] += 1
+	real = shapes - {"write"} or shapes
+	label = real.pop() if len(real) == 1 else "mixed"
+	return label, details
+
+
+def main():
+	out = codecs.getwriter("utf-8")(sys.stdout.buffer)
+	modules, _ = callgraph.load()
+	graph = callgraph.Graph(modules, callgraph.register_names())
+	graph.run()
+	documents = graph.document_writes(callgraph.subscriptions(modules),
+	                                  callgraph.declared_movements())
+	with io.open(os.path.join(paths.OUT, "recorded.json"), encoding="utf-8") as f:
+		recorded = json.load(f)["documents"]
+	locator = Locator(modules)
+
+	by_shape = collections.defaultdict(lambda: [0, 0])       # label -> [pairs, rows]
+	origins = collections.Counter()                          # detail -> rows
+	pairs = []
+	unreached = [0, 0]
+	for document, registers in sorted(recorded.items()):
+		for register, rows in sorted(registers.items()):
+			key = bsl.norm(register)
+			sites = set()
+			for event in ("posting", "on-write", "before-write"):
+				sites |= graph.document_sites.get((document, event, key), set())
+			if not sites:
+				unreached[0] += 1
+				unreached[1] += rows
+				pairs.append({"document": document, "register": register, "rows": rows,
+				              "shape": "not reached"})
+				continue
+			label, details = classify(graph, locator, sites)
+			by_shape[label][0] += 1
+			by_shape[label][1] += rows
+			for detail, n in details.items():
+				origins[detail] += rows
+			pairs.append({"document": document, "register": register, "rows": rows,
+			              "shape": label, "sites": len(sites),
+			              "details": dict(details)})
+	total = sum(v[1] for v in by_shape.values()) + unreached[1]
+
+	with codecs.open(TARGET, "w", "utf-8") as f:
+		json.dump({"by_shape": by_shape, "origins": origins, "pairs": pairs}, f,
+		          ensure_ascii=False, indent="\t")
+
+	out.write(u"%d movements (document, register) holding %s rows\n\n"
+	          % (len(pairs), "{:,}".format(total)))
+	out.write(u"%-12s %6s %14s %6s\n" % (u"shape", u"pairs", u"rows", u"share"))
+	for label, (n, rows) in sorted(by_shape.items(), key=lambda kv: -kv[1][1]):
+		out.write(u"%-12s %6d %14s %5.1f%%\n" % (label, n, "{:,}".format(rows),
+		                                          100.0 * rows / total))
+	out.write(u"%-12s %6d %14s %5.1f%%\n\n" % (u"not reached", unreached[0],
+	                                            "{:,}".format(unreached[1]),
+	                                            100.0 * unreached[1] / total))
+	out.write(u"table origins and treatments, by rows of the movements they feed:\n")
+	for detail, rows in origins.most_common():
+		out.write(u"  %-28s %14s\n" % (detail, "{:,}".format(rows)))
+	out.write(u"\nwritten %s\n" % TARGET)
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/integrity.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Ask the database what the metadata cannot answer.
+
+`migrate.py verify` reads the column map and `"dForge".entity_column`, so it can only
+check a reference the metadata knows about - and it will agree with the metadata even
+when the table disagrees with both. Four things it cannot see:
+
+  * a foreign key whose rows point at nothing. The loader runs with
+    `session_replication_role = 'replica'`, so Postgres checks not one of them on the
+    way in; this asks afterwards.
+  * a foreign key that points somewhere the metadata no longer does. A module upgrade
+    moves the reference column's `ref_entity_id` and leaves the constraint alone, so the
+    card offers one catalog and the table insists on another. That is how
+    `FK_OrganizationId` went on naming `masterdata.party` for 88,126 `shipping_order`
+    rows after the spec had re-pointed the column at `masterdata.organization`.
+  * a reference with no foreign key behind it. An extension column is exactly
+    that: the platform builds the 1:1 side table with a key back to the host and
+    leaves the reference itself unconstrained, so the first check cannot see it.
+  * a physical column the metadata no longer accounts for.
+
+    python tools/integrity.py                 # all four, every module schema
+    python tools/integrity.py --schema shipping
+    python tools/integrity.py --retarget      # recreate the keys that disagree
+
+`--retarget` only touches a key no row would break: if anything still dangles against
+the target the metadata names, the constraint stays and the row is reported instead.
+
+Connection details come from the same place as `migrate.py` - DF_PG_* and DF_TENANT.
+"""
+from __future__ import print_function
+
+import argparse
+import sys
+
+import migrate
+
+# Their own tables pair columns by names that coincide, so the reference check reads
+# them as retargeted when they are nothing of the kind.
+PLATFORM = ("dForge", "audit", "public", "information_schema", "pg_catalog")
+
+# R, S and F are virtual: the column they name is not a column of the table.
+VIRTUAL = ("R", "S", "F")
+
+
+def module_schemas(cur, only=None):
+	cur.execute("""SELECT DISTINCT e.schema_name FROM "dForge".entity e
+	               WHERE e.schema_name <> ALL(%s) ORDER BY 1""", (list(PLATFORM),))
+	found = [row[0] for row in cur.fetchall()]
+	return [s for s in found if not only or s == only]
+
+
+def foreign_keys(cur, schemas):
+	"""Every single-column foreign key out of these schemas, with what it points at."""
+	cur.execute("""
+		SELECT con.conname, src_ns.nspname, src.relname, src_col.attname,
+		       tgt_ns.nspname, tgt.relname, tgt_col.attname
+		FROM pg_constraint con
+		JOIN pg_class src ON src.oid = con.conrelid
+		JOIN pg_namespace src_ns ON src_ns.oid = src.relnamespace
+		JOIN pg_class tgt ON tgt.oid = con.confrelid
+		JOIN pg_namespace tgt_ns ON tgt_ns.oid = tgt.relnamespace
+		JOIN pg_attribute src_col ON src_col.attrelid = con.conrelid
+		                         AND src_col.attnum = con.conkey[1]
+		JOIN pg_attribute tgt_col ON tgt_col.attrelid = con.confrelid
+		                         AND tgt_col.attnum = con.confkey[1]
+		WHERE con.contype = 'f' AND array_length(con.conkey, 1) = 1
+		  AND src_ns.nspname = ANY(%s)
+		ORDER BY src_ns.nspname, src.relname, src_col.attname
+	""", (list(schemas),))
+	return cur.fetchall()
+
+
+def dangling(cur, schema, table, column, target_schema, target_table, target_column):
+	cur.execute('SELECT count(*) FROM "%s"."%s" s WHERE s."%s" IS NOT NULL'
+	            ' AND NOT EXISTS (SELECT 1 FROM "%s"."%s" t WHERE t."%s" = s."%s")'
+	            % (schema, table, column,
+	               target_schema, target_table, target_column, column))
+	return cur.fetchone()[0]
+
+
+def check_rows(cur, schemas):
+	"""Rows whose foreign key points at a row that is not there."""
+	print("\nforeign keys whose rows point at nothing")
+	print("-" * 78)
+	broken = 0
+	for name, sn, st, sc, tn, tt, tc in foreign_keys(cur, schemas):
+		count = dangling(cur, sn, st, sc, tn, tt, tc)
+		if count:
+			broken += 1
+			print("%-40s %-26s %8d" % ("%s.%s.%s" % (sn, st, sc), name, count))
+	print("-" * 78)
+	print("%d foreign key(s) with rows pointing at nothing" % broken)
+	return broken
+
+
+def check_targets(cur, schemas, retarget=False):
+	"""Foreign keys that point somewhere the metadata no longer does."""
+	cur.execute("""
+		SELECT con.conname, src_ns.nspname, src.relname, src_col.attname,
+		       tgt_ns.nspname, tgt.relname,
+		       ref.schema_name, lower(ref.db_object), lower(ec.other_key)
+		FROM pg_constraint con
+		JOIN pg_class src ON src.oid = con.conrelid
+		JOIN pg_namespace src_ns ON src_ns.oid = src.relnamespace
+		JOIN pg_class tgt ON tgt.oid = con.confrelid
+		JOIN pg_namespace tgt_ns ON tgt_ns.oid = tgt.relnamespace
+		JOIN pg_attribute src_col ON src_col.attrelid = con.conrelid
+		                         AND src_col.attnum = con.conkey[1]
+		JOIN "dForge".entity e ON e.schema_name = src_ns.nspname
+		                      AND lower(e.db_object) = src.relname
+		JOIN "dForge".entity_column ec ON ec.entity_id = e.entity_id
+		                              AND ec.column_type = 'R'
+		                              AND lower(ec.this_key) = src_col.attname
+		JOIN "dForge".entity ref ON ref.entity_id = ec.ref_entity_id
+		WHERE con.contype = 'f' AND array_length(con.conkey, 1) = 1
+		  AND src_ns.nspname = ANY(%s)
+		  AND (lower(ref.db_object) IS DISTINCT FROM tgt.relname
+		       OR ref.schema_name IS DISTINCT FROM tgt_ns.nspname)
+		ORDER BY src_ns.nspname, src.relname, src_col.attname
+	""", (list(schemas),))
+	rows = cur.fetchall()
+
+	print("\nforeign keys that disagree with the metadata")
+	print("-" * 78)
+	for name, sn, st, sc, fkn, fkt, rn, rt, rpk in rows:
+		count = dangling(cur, sn, st, sc, rn, rt, rpk)
+		print("%-34s %-22s %s.%s -> %s.%s" % ("%s.%s" % (st, sc), name, fkn, fkt, rn, rt))
+		if count:
+			print("   %d row(s) dangle against the target the metadata names - left alone"
+			      % count)
+			continue
+		if retarget:
+			cur.execute('ALTER TABLE "%s"."%s" DROP CONSTRAINT "%s"' % (sn, st, name))
+			cur.execute('ALTER TABLE "%s"."%s" ADD CONSTRAINT "%s" FOREIGN KEY ("%s")'
+			            ' REFERENCES "%s"."%s" ("%s")'
+			            % (sn, st, name, sc, rn, rt, rpk))
+			print("   recreated")
+	print("-" * 78)
+	print("%d foreign key(s) pointing where the metadata does not" % len(rows))
+	return len(rows)
+
+
+def check_unbacked(cur, schemas):
+	"""Reference columns the database is not enforcing.
+
+	`check_rows` walks `pg_constraint`, so a reference with no foreign key behind it is
+	invisible to it - and an extension column is exactly that: the platform builds the
+	1:1 side table with a key back to the host and leaves the reference itself
+	unconstrained. This asks the metadata instead of the catalogue, and counts.
+	"""
+	cur.execute("""
+		SELECT e.schema_name, coalesce(held.storage_table, e.db_object), ec.this_key,
+		       ref.schema_name, lower(ref.db_object), lower(ec.other_key)
+		FROM "dForge".entity_column ec
+		JOIN "dForge".entity e ON e.entity_id = ec.entity_id
+		JOIN "dForge".entity ref ON ref.entity_id = ec.ref_entity_id
+		-- The physical column is the one named by `this_key`, and it is that row -
+		-- not the reference beside it - that says which table actually holds it.
+		LEFT JOIN "dForge".entity_column held ON held.entity_id = ec.entity_id
+		                                     AND held.column_cd = ec.this_key
+		WHERE ec.column_type = 'R' AND ec.this_key IS NOT NULL
+		  AND ec.other_key IS NOT NULL
+		  AND e.schema_name = ANY(%s)
+		ORDER BY 1, 2, 3
+	""", (list(schemas),))
+
+	print("\nreferences with no foreign key behind them")
+	print("-" * 78)
+	loose = 0
+	for schema, storage, column, ref_schema, ref_table, ref_key in cur.fetchall():
+		if "." in (storage or ""):
+			schema, storage = [p.strip().strip('"') for p in storage.split(".")]
+		storage = storage.lower()
+		cur.execute("""SELECT 1 FROM pg_constraint con
+		               JOIN pg_class src ON src.oid = con.conrelid
+		               JOIN pg_namespace ns ON ns.oid = src.relnamespace
+		               JOIN pg_attribute a ON a.attrelid = con.conrelid
+		                                  AND a.attnum = con.conkey[1]
+		               WHERE con.contype = 'f' AND ns.nspname = %s
+		                 AND src.relname = %s AND a.attname = %s""",
+		            (schema, storage, column.lower()))
+		if cur.fetchone():
+			continue
+		cur.execute("""SELECT 1 FROM information_schema.columns
+		               WHERE table_schema = %s AND table_name = %s
+		                 AND column_name = %s""", (schema, storage, column.lower()))
+		if not cur.fetchone():
+			continue
+		loose += 1
+		cur.execute('SELECT count(*) FROM "%s"."%s" s WHERE s."%s" IS NOT NULL'
+		            ' AND NOT EXISTS (SELECT 1 FROM "%s"."%s" t WHERE t."%s" = s."%s")'
+		            % (schema, storage, column, ref_schema, ref_table, ref_key, column))
+		count = cur.fetchone()[0]
+		print("%-46s -> %-22s %s"
+		      % ("%s.%s.%s" % (schema, storage, column),
+		         "%s.%s" % (ref_schema, ref_table),
+		         "%d dangling" % count if count else "ok"))
+	print("-" * 78)
+	print("%d reference(s) the database does not enforce" % loose)
+	return 0
+
+
+def check_columns(cur, schemas):
+	"""Physical columns the metadata no longer accounts for."""
+	cur.execute("""
+		SELECT c.table_schema, c.table_name, c.column_name, c.data_type
+		FROM information_schema.columns c
+		JOIN "dForge".entity e ON e.schema_name = c.table_schema
+		                      AND lower(e.db_object) = c.table_name
+		WHERE c.table_schema = ANY(%s)
+		  AND NOT EXISTS (
+			SELECT 1 FROM "dForge".entity_column ec
+			WHERE ec.entity_id = e.entity_id AND lower(ec.column_cd) = c.column_name
+			  AND coalesce(ec.column_type, ' ') <> ALL(%s))
+		  AND NOT EXISTS (
+			SELECT 1 FROM "dForge".entity_column ec
+			WHERE lower(coalesce(ec.storage_table, '')) LIKE '%%"' || c.table_name || '"'
+			  AND lower(ec.column_cd) = c.column_name)
+		ORDER BY 1, 2, 3
+	""", (list(schemas), list(VIRTUAL)))
+	rows = cur.fetchall()
+
+	print("\ncolumns the metadata no longer accounts for")
+	print("-" * 78)
+	for schema, table, column, kind in rows:
+		cur.execute('SELECT count(*) FROM "%s"."%s"' % (schema, table))
+		held = cur.fetchone()[0]
+		print("%-40s %-22s %-10s %s"
+		      % ("%s.%s" % (schema, table), column, kind,
+		         "empty" if not held else "holds %d row(s)" % held))
+	print("-" * 78)
+	print("%d column(s) left without metadata" % len(rows))
+	return len(rows)
+
+
+def main():
+	parser = argparse.ArgumentParser(description=__doc__)
+	parser.add_argument("--schema", help="one module schema instead of all of them")
+	parser.add_argument("--retarget", action="store_true",
+	                    help="recreate the foreign keys that disagree with the metadata")
+	args = parser.parse_args()
+
+	migrate.load_env()
+	pg = migrate.pg_connect()
+	cur = pg.cursor()
+	schemas = module_schemas(cur, args.schema)
+	if not schemas:
+		print("no module schema to look at")
+		return 1
+	print("%d schema(s): %s" % (len(schemas), ", ".join(schemas)))
+
+	problems = check_rows(cur, schemas)
+	problems += check_targets(cur, schemas, args.retarget)
+	problems += check_unbacked(cur, schemas)
+	problems += check_columns(cur, schemas)
+	if args.retarget:
+		pg.commit()
+	return 1 if problems else 0
+
+
+if __name__ == "__main__":
+	sys.exit(main())