@dforge-core/dforge-mcp 0.2.44 → 0.2.46-next

package/skills/dforge-1c-import/scripts/lib/deps.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Module dependency graph, and whether it can be installed.
+
+dForge installs modules in dependency order, so a cycle between two modules is not a
+style problem - it is a package that cannot be installed at all. A cycle is broken with
+an extension or a bridge module (see MODULE_CONVENTIONS.md), but first it has to be seen.
+
+    python tools/deps.py
+"""
+from __future__ import print_function
+
+import codecs
+import collections
+import json
+import os
+
+import paths
+
+ROOT = paths.ROOT
+MODULES = paths.MODULES
+
+
+def load(path):
+	with codecs.open(path, "r", "utf-8") as f:
+		return json.load(f)
+
+
+def graph():
+	out = collections.OrderedDict()
+	for code in sorted(os.listdir(MODULES)):
+		manifest = os.path.join(MODULES, code, "manifest.json")
+		if os.path.isfile(manifest):
+			out[code] = set(load(manifest).get("dependencies", {}))
+	return out
+
+
+def referenced(code):
+	"""Modules this one actually points at: `extends`, and every `entity` in a link,
+	reference or set. A dependency the manifest forgets to declare still binds the
+	install order - masterdata carried a lookup into shipping while declaring nothing,
+	and the pair was uninstallable in either order."""
+	out = set()
+
+	def walk(node):
+		if isinstance(node, dict):
+			for key, value in node.items():
+				if key in ("entity", "extends") and isinstance(value, str) and "." in value:
+					out.add(value.split(".", 1)[0])
+				else:
+					walk(value)
+		elif isinstance(node, list):
+			for item in node:
+				walk(item)
+
+	directory = os.path.join(MODULES, code, "entities")
+	if os.path.isdir(directory):
+		for name in sorted(os.listdir(directory)):
+			walk(load(os.path.join(directory, name)))
+	return out - set([code])
+
+
+def cycles(edges):
+	"""Every cycle reachable in the graph, as a list of module names."""
+	found, path, state = [], [], {}
+
+	def walk(node):
+		state[node] = "open"
+		path.append(node)
+		for target in sorted(edges.get(node, ())):
+			if target not in edges:
+				continue
+			if state.get(target) == "open":
+				found.append(path[path.index(target):] + [target])
+			elif state.get(target) != "done":
+				walk(target)
+		path.pop()
+		state[node] = "done"
+
+	for node in edges:
+		if state.get(node) != "done":
+			walk(node)
+	return found
+
+
+def order(edges):
+	"""Install order: a module after everything it depends on."""
+	done, out = set(), []
+	remaining = dict((k, set(v) & set(edges)) for k, v in edges.items())
+	while remaining:
+		ready = sorted(k for k, v in remaining.items() if v <= done)
+		if not ready:
+			return out, sorted(remaining)
+		for code in ready:
+			out.append(code)
+			done.add(code)
+			del remaining[code]
+	return out, []
+
+
+def main():
+	edges = graph()
+	print("%-12s %s" % ("module", "depends on"))
+	print("-" * 60)
+	undeclared = collections.OrderedDict()
+	for code, deps in edges.items():
+		missing = sorted(referenced(code) - deps - set([code]))
+		if missing:
+			undeclared[code] = missing
+			deps.update(missing)
+		print("%-12s %s%s" % (code, ", ".join(sorted(deps)) or "-",
+		                      "   (undeclared: %s)" % ", ".join(missing) if missing else ""))
+
+	if undeclared:
+		print("\n%d module(s) reference a module they do not declare - add the"
+		      " dependency, or move the reference to the other side:" % len(undeclared))
+		for code, missing in undeclared.items():
+			print("   %s -> %s" % (code, ", ".join(missing)))
+
+	found = cycles(edges)
+	if found:
+		print("\n%d cycle(s) - these cannot be installed as they stand:" % len(found))
+		for cycle in found:
+			print("   %s" % " -> ".join(cycle))
+	else:
+		print("\nno cycles")
+
+	sequence, stuck = order(edges)
+	print("\ninstall order:\n   %s" % " -> ".join(sequence))
+	if stuck:
+		print("stuck: %s" % ", ".join(stuck))
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/docusage.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""How much each document type is used: last date, last twelve months, posted.
+
+Usage does not narrow what gets ported - a document nobody has posted in years is still
+done - but it orders the work, and it says which types have a history to prove a port on.
+A type with no posted documents has nothing for the oracle to compare against.
+
+The twelve months are counted back from the newest document in the infobase, not from
+today: an extraction is a snapshot, and "the last year" should mean the last year the
+client actually worked in it.
+
+    python tools/1c.py docusage              # the table, and out/docusage.json
+
+Read-only. The physical table comes from `DBNames` by the object's GUID, never guessed.
+"""
+from __future__ import print_function
+
+import codecs
+import io
+import json
+import os
+import sys
+
+import handlers
+import meta
+import migrate
+import paths
+
+TARGET = os.path.join(paths.OUT, "docusage.json")
+
+
+def documents():
+	"""(name, physical table) for every document in the configuration."""
+	with io.open(os.path.join(paths.OUT, "objects.json"), encoding="utf-8") as f:
+		objects = json.load(f)
+	index = meta.dbnames_index()
+	found = []
+	for obj in objects:
+		if obj.get("family") != "Document":
+			continue
+		entry = index.get(obj["guid"])
+		if entry and entry["type"] == "Document":
+			found.append((obj["name"], entry["physical"]))
+	return sorted(found)
+
+
+def measure(cur, table):
+	cur.execute("SELECT COUNT(*), MAX(_Date_Time) FROM dbo.[%s]" % table)
+	rows, last = cur.fetchone()
+	return rows, last
+
+
+def main():
+	migrate.load_env()
+	cur = migrate.mssql_connect().cursor()
+	docs = documents()
+	newest = None
+	raw = []
+	for name, table in docs:
+		rows, last = measure(cur, table)
+		raw.append((name, table, rows, last))
+		if last is not None and (newest is None or last > newest):
+			newest = last
+	cutoff = newest.replace(year=newest.year - 1) if newest else None
+	result = []
+	for name, table, rows, last in raw:
+		recent = posted = 0
+		if rows:
+			cur.execute("SELECT SUM(CASE WHEN _Date_Time >= ? THEN 1 ELSE 0 END),"
+			            " SUM(CASE WHEN _Posted = 0x01 THEN 1 ELSE 0 END) FROM dbo.[%s]"
+			            % table, cutoff)
+			recent, posted = [int(v or 0) for v in cur.fetchone()]
+		result.append({
+			"name": name,
+			"table": table,
+			"client": name.startswith(handlers.CLIENT),
+			"rows": rows,
+			"posted": posted,
+			"last12m": recent,
+			# 1C stores dates shifted by the year offset; the snapshot keeps them as stored.
+			"last": last.strftime("%Y-%m-%d") if last else None,
+		})
+	result.sort(key=lambda r: (-r["last12m"], -r["posted"], r["name"]))
+	with codecs.open(TARGET, "w", "utf-8") as f:
+		json.dump({"newest": newest.strftime("%Y-%m-%d") if newest else None,
+		           "cutoff": cutoff.strftime("%Y-%m-%d") if cutoff else None,
+		           "documents": result}, f, ensure_ascii=False, indent="\t")
+	out = codecs.getwriter("utf-8")(sys.stdout.buffer)
+	vendor = [r for r in result if not r["client"]]
+	out.write("%d document types, %d vendor; newest document %s, last 12 months from %s\n"
+	          % (len(result), len(vendor), newest, cutoff))
+	out.write("vendor with rows %d, posted %d, used in last 12m %d\n\n" % (
+		sum(1 for r in vendor if r["rows"]), sum(1 for r in vendor if r["posted"]),
+		sum(1 for r in vendor if r["last12m"])))
+	for r in result:
+		if r["rows"]:
+			out.write("%-6s %-52s %9d %9d %8d  %s\n" % (
+				"client" if r["client"] else "", r["name"], r["rows"], r["posted"],
+				r["last12m"], r["last"]))
+	out.write("\nwritten %s\n" % TARGET)
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/dump1c.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Read a 1C:Enterprise 8 infobase hosted on MS SQL Server *without* any 1C software.
+
+Everything needed to rebuild the data model lives in two service tables:
+  Params  -> row FileName='DBNames': GUID -> physical table/column name map
+  Config  -> the configuration itself, one row (8.2) or several PartNo rows (8.3) per file
+
+File bodies are raw-deflate compressed (no zlib header).
+
+Credentials come from env DF1C_USER / DF1C_PASS; without them, Windows auth is used.
+Server and database default to DF1C_SERVER / DF1C_DB.
+"""
+import argparse
+import codecs
+import json
+import os
+import re
+import struct
+import sys
+import zlib
+
+import paths
+
+OUT = paths.OUT
+
+PREFERRED_DRIVERS = (
+	"ODBC Driver 18 for SQL Server",
+	"ODBC Driver 17 for SQL Server",
+	"ODBC Driver 13 for SQL Server",
+	"SQL Server Native Client 11.0",
+	"SQL Server",
+)
+
+
+def connect(server, db):
+	import pyodbc
+	available = set(pyodbc.drivers())
+	driver = next((d for d in PREFERRED_DRIVERS if d in available), None)
+	if driver is None:
+		raise RuntimeError("no SQL Server ODBC driver found: %r" % sorted(available))
+	parts = ["DRIVER={%s}" % driver, "SERVER=%s" % server, "DATABASE=%s" % db,
+	         "TrustServerCertificate=yes"]
+	user, pwd = os.environ.get("DF1C_USER"), os.environ.get("DF1C_PASS")
+	if user:
+		parts += ["UID=%s" % user, "PWD=%s" % pwd]
+	else:
+		parts += ["Trusted_Connection=yes"]
+	sys.stderr.write("[i] %s / %s via %s\n" % (server, db, driver))
+	return pyodbc.connect(";".join(parts), autocommit=True)
+
+
+def inflate(data):
+	"""1C stores file bodies as raw deflate; a few small ones are stored plain."""
+	if not data:
+		return b""
+	data = bytes(data)
+	for attempt in (lambda: zlib.decompressobj(-15).decompress(data), lambda: zlib.decompress(data)):
+		try:
+			return attempt()
+		except zlib.error:
+			continue
+	return data
+
+
+_HAS_PARTNO = {}
+
+
+def has_partno(cur, table):
+	"""8.3 splits large files across rows by PartNo; 8.2 stores one row per file."""
+	if table not in _HAS_PARTNO:
+		cur.execute("SELECT COUNT(*) FROM sys.columns WHERE object_id = OBJECT_ID(?) AND name = 'PartNo'", table)
+		_HAS_PARTNO[table] = cur.fetchone()[0] > 0
+	return _HAS_PARTNO[table]
+
+
+def read_file(cur, table, name):
+	"""Concatenate a file's parts in order, then inflate."""
+	order = " ORDER BY PartNo" if has_partno(cur, table) else ""
+	cur.execute("SELECT BinaryData FROM %s WHERE FileName = ?%s" % (table, order), name)
+	blob = b"".join(bytes(r[0]) for r in cur.fetchall() if r[0] is not None)
+	return inflate(blob)
+
+
+# --------------------------------------------------------------------------- DBNames
+
+DBNAME_ENTRY = re.compile(r'\{([0-9a-fA-F-]{36}),"([^"]*)",(\d+)\}')
+
+
+def parse_dbnames(buf):
+	"""8.2 writes DBNames as UTF-8 text: {ver, {count, {guid,"Kind",N}, ...}}."""
+	if buf.startswith(codecs.BOM_UTF8) or buf[:1] == b"{":
+		txt = buf.decode("utf-8-sig", "replace")
+		entries = [{"guid": g, "type": k, "number": int(n)} for g, k, n in DBNAME_ENTRY.findall(txt)]
+		if entries:
+			return entries
+		raise ValueError("text DBNames matched no entries")
+	return parse_dbnames_binary(buf)
+
+
+def parse_dbnames_binary(buf):
+	"""8.3 layout: <u32 count> then count * { <u32 chars> utf16le, <u32 chars> utf16le, <u32 n> }"""
+	out, pos = [], 0
+	(count,) = struct.unpack_from("<I", buf, pos)
+	pos += 4
+	if count == 0 or count > 500000:
+		raise ValueError("implausible entry count %d" % count)
+	for _ in range(count):
+		vals = []
+		for _ in range(2):
+			(ln,) = struct.unpack_from("<I", buf, pos)
+			pos += 4
+			vals.append(buf[pos:pos + ln * 2].decode("utf-16-le"))
+			pos += ln * 2
+		(num,) = struct.unpack_from("<I", buf, pos)
+		pos += 4
+		out.append({"guid": vals[0], "type": vals[1], "number": num})
+	return out
+
+
+def physical_name(entry):
+	"""DBNames kind + number -> the SQL object name 1C generated for it."""
+	kind, number = entry.get("type"), entry.get("number")
+	if not kind or number is None:
+		return None
+	return "_%s%d" % (kind, number)
+
+
+# --------------------------------------------------------------------------- commands
+
+def cmd_databases(cur, args):
+	"""Which databases does this login actually see? Separates a bad password from missing rights."""
+	cur.execute("SELECT name, state_desc, HAS_DBACCESS(name) FROM sys.databases ORDER BY name")
+	for name, state, access in cur.fetchall():
+		print("%-44s %-10s access=%s" % (name, state, access))
+
+
+def cmd_probe(cur, args):
+	cur.execute("SELECT @@VERSION")
+	print(cur.fetchone()[0].splitlines()[0])
+	for table in ("Config", "ConfigSave", "Params", "DBSchema", "_YearOffset"):
+		try:
+			cur.execute("SELECT COUNT(*) FROM [%s]" % table)
+			print("  %-12s rows=%s" % (table, cur.fetchone()[0]))
+		except Exception as exc:
+			print("  %-12s -- %s" % (table, str(exc).split("]")[-1].strip()[:70]))
+	cur.execute("""SELECT LEFT(name, PATINDEX('%[0-9]%', name + '0') - 1) AS kind, COUNT(*)
+	               FROM sys.tables WHERE name LIKE '[_]%'
+	               GROUP BY LEFT(name, PATINDEX('%[0-9]%', name + '0') - 1)
+	               ORDER BY 2 DESC""")
+	print("\n  physical table families:")
+	for kind, n in cur.fetchall():
+		print("    %-24s %d" % (kind, n))
+
+
+def cmd_schema(cur, args):
+	cur.execute("""SELECT t.name, c.name, ty.name, c.max_length, c.precision, c.scale, c.is_nullable
+	               FROM sys.tables t
+	               JOIN sys.columns c ON c.object_id = t.object_id
+	               JOIN sys.types ty ON ty.user_type_id = c.user_type_id
+	               ORDER BY t.name, c.column_id""")
+	tables = {}
+	for tn, cn, ty, ml, pr, sc, nullable in cur.fetchall():
+		tables.setdefault(tn, []).append({
+			"column": cn, "type": ty, "len": ml,
+			"precision": pr, "scale": sc, "nullable": bool(nullable),
+		})
+	cur.execute("""SELECT t.name, SUM(p.rows) FROM sys.tables t
+	               JOIN sys.partitions p ON p.object_id = t.object_id AND p.index_id IN (0, 1)
+	               GROUP BY t.name""")
+	rows = dict(cur.fetchall())
+	write_json("schema.json", {"tables": [
+		{"name": name, "rows": rows.get(name), "columns": cols}
+		for name, cols in sorted(tables.items())
+	]})
+	print("tables=%d" % len(tables))
+
+
+def cmd_dbnames(cur, args):
+	buf = read_file(cur, "Params", "DBNames")
+	save("dbnames.bin", buf)
+	entries = parse_dbnames(buf)
+	for entry in entries:
+		entry["physical"] = physical_name(entry)
+	write_json("dbnames.json", {"count": len(entries), "entries": entries})
+	kinds = {}
+	for entry in entries:
+		kinds[entry["type"]] = kinds.get(entry["type"], 0) + 1
+	print("entries=%d  distinct kinds=%d" % (len(entries), len(kinds)))
+	for kind, n in sorted(kinds.items(), key=lambda kv: -kv[1]):
+		print("   %-28s %d" % (kind, n))
+
+
+def cmd_extract(cur, args):
+	target = os.path.join(OUT, "config")
+	if not os.path.isdir(target):
+		os.makedirs(target)
+	cur.execute("SELECT DISTINCT FileName FROM Config")
+	names = [r[0] for r in cur.fetchall()]
+	text = binary = 0
+	for i, name in enumerate(names):
+		buf = read_file(cur, "Config", name)
+		safe = re.sub(r"[^0-9A-Za-z._-]", "_", name)[:120]
+		if buf.startswith(codecs.BOM_UTF8) or b"{" in buf[:64]:
+			suffix, text = ".txt", text + 1
+		else:
+			suffix, binary = ".bin", binary + 1
+		with open(os.path.join(target, safe + suffix), "wb") as f:
+			f.write(buf)
+		if i and i % 2000 == 0:
+			sys.stderr.write("[i] %d/%d\n" % (i, len(names)))
+	print("files=%d text=%d binary=%d -> %s" % (len(names), text, binary, target))
+
+
+def save(name, data):
+	if not os.path.isdir(OUT):
+		os.makedirs(OUT)
+	with open(os.path.join(OUT, name), "wb") as f:
+		f.write(data)
+
+
+def write_json(name, doc):
+	if not os.path.isdir(OUT):
+		os.makedirs(OUT)
+	with codecs.open(os.path.join(OUT, name), "w", "utf-8") as f:
+		json.dump(doc, f, ensure_ascii=False, indent="\t")
+
+
+COMMANDS = {
+	"databases": cmd_databases,
+	"probe": cmd_probe,
+	"schema": cmd_schema,
+	"dbnames": cmd_dbnames,
+	"extract": cmd_extract,
+}
+
+
+def main():
+	ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+	ap.add_argument("command", choices=sorted(COMMANDS))
+	ap.add_argument("--server", default=os.environ.get("DF1C_SERVER")
+	                or paths.required("DF1C_SERVER", "which MS SQL server holds it"))
+	ap.add_argument("--db", default=os.environ.get("DF1C_DB")
+	                or paths.required("DF1C_DB", "which MS SQL infobase to read"))
+	args = ap.parse_args()
+	COMMANDS[args.command](connect(args.server, args.db).cursor(), args)
+
+
+if __name__ == "__main__":
+	main()

package/skills/dforge-1c-import/scripts/lib/explore.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Dump one metadata object's parse tree so its attribute layout can be read off."""
+from __future__ import print_function
+
+import codecs
+import json
+import os
+import sys
+
+import meta
+
+OUT = meta.OUT
+
+
+def find(name):
+	with codecs.open(os.path.join(OUT, "objects.json"), "r", "utf-8") as f:
+		for row in json.load(f):
+			if row.get("name") == name:
+				return row["guid"]
+	return None
+
+
+def shape(node, depth, max_depth, path):
+	pad = "  " * depth
+	if isinstance(node, list):
+		strings = [x for x in node if isinstance(x, str) and x and not meta.GUID.match(x)
+		           and not x.isdigit()]
+		hint = (" <- " + ", ".join(repr(s)[:34] for s in strings[:4])) if strings else ""
+		print("%s%s list[%d]%s" % (pad, path, len(node), hint))
+		if depth >= max_depth:
+			return
+		for i, item in enumerate(node):
+			if isinstance(item, list):
+				shape(item, depth + 1, max_depth, "[%d]" % i)
+	else:
+		if node:
+			print("%s%s = %s" % (pad, path, repr(node)[:70]))
+
+
+if __name__ == "__main__":
+	name = sys.argv[1]
+	depth = int(sys.argv[2]) if len(sys.argv) > 2 else 4
+	guid = find(name)
+	print("%s -> %s\n" % (name, guid))
+	node = meta.parse(meta.load(guid))
+	label = "root"
+	if len(sys.argv) > 3:                       # drill into e.g. 6.2
+		for part in sys.argv[3].split("."):
+			node = node[int(part)]
+		label = "root." + sys.argv[3]
+	if "--json" in sys.argv:
+		print(json.dumps(node, ensure_ascii=False, indent=1))
+	else:
+		shape(node, 0, depth, label)