@dforge-core/dforge-mcp 0.2.46-next → 0.2.48

package/skills/dforge-1c-import/scripts/lib/dump1c.py

@@ -1,54 +1,62 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """
-Read a 1C:Enterprise 8 infobase hosted on MS SQL Server *without* any 1C software.
+Export a 1C:Enterprise 8 infobase to `out/` *without* any 1C software - the one layer that
+reads 1C itself. Everything past it reads the export, so it does not matter which kind of
+infobase the export came from.
+
+Two kinds of infobase, one export:
+
+    a server infobase on MS SQL   DF1C_SERVER + DF1C_DB      (source_mssql.py)
+    a file infobase, 1Cv8.1CD     DF1C_FILE=<path to .1CD>   (source_file.py, onecd.py)
+
+What it writes:
+
+    config/<file>.txt|.bin  the configuration, one file per Config row, inflated  (extract)
+    dbnames.json            DBNames: GUID -> physical table/column name           (dbnames)
+    schema.json             every table, its columns in MS SQL's terms, row count  (schema)
+    data/<table>.parquet    the rows of every `_` table, as 1C stores them         (data)
+    export.json             the manifest: which infobase, when, its year offset,
+                            and per table the rows and the horizon taken before them
 
 Everything needed to rebuild the data model lives in two service tables:
   Params  -> row FileName='DBNames': GUID -> physical table/column name map
   Config  -> the configuration itself, one row (8.2) or several PartNo rows (8.3) per file
+File bodies are raw-deflate compressed (no zlib header), in both kinds of infobase.
 
-File bodies are raw-deflate compressed (no zlib header).
+The rows are written as 1C stores them - dates still carrying a server infobase's year
+offset, booleans as binary(1) - and the manifest records the offset, so a project query
+written against the infobase reads the export unchanged.
 
-Credentials come from env DF1C_USER / DF1C_PASS; without them, Windows auth is used.
-Server and database default to DF1C_SERVER / DF1C_DB.
+    python 1c.py dump1c all                      # everything, in order
+    python 1c.py dump1c data _Reference12 ...    # re-export these tables only
+    python 1c.py dump1c data --skip "ChngR|Settings"
+    python 1c.py dump1c data --changes           # only what 1C changed since (MS SQL; changes.py)
+    python 1c.py dump1c probe                    # what the source is, before reading it
 """
 import argparse
 import codecs
+import datetime
 import json
 import os
 import re
 import struct
 import sys
+import time
 import zlib
 
 import paths
 
 OUT = paths.OUT
+DATA = os.path.join(OUT, "data")
+MANIFEST = os.path.join(OUT, "export.json")
+CHUNK = 50000
 
-PREFERRED_DRIVERS = (
-	"ODBC Driver 18 for SQL Server",
-	"ODBC Driver 17 for SQL Server",
-	"ODBC Driver 13 for SQL Server",
-	"SQL Server Native Client 11.0",
-	"SQL Server",
-)
-
-
-def connect(server, db):
-	import pyodbc
-	available = set(pyodbc.drivers())
-	driver = next((d for d in PREFERRED_DRIVERS if d in available), None)
-	if driver is None:
-		raise RuntimeError("no SQL Server ODBC driver found: %r" % sorted(available))
-	parts = ["DRIVER={%s}" % driver, "SERVER=%s" % server, "DATABASE=%s" % db,
-	         "TrustServerCertificate=yes"]
-	user, pwd = os.environ.get("DF1C_USER"), os.environ.get("DF1C_PASS")
-	if user:
-		parts += ["UID=%s" % user, "PWD=%s" % pwd]
-	else:
-		parts += ["Trusted_Connection=yes"]
-	sys.stderr.write("[i] %s / %s via %s\n" % (server, db, driver))
-	return pyodbc.connect(";".join(parts), autocommit=True)
+# What 1C keeps only as a consequence of other tables: register totals and the document
+# journals. 53.7 M rows of a 313 M-row infobase, and no `_Version` to read their change by,
+# so each `--changes` would read them all again - 14 minutes of a cutover window that has
+# 10. Nothing loads them; `seedstate verify` reads totals, and says which to export.
+DERIVED = re.compile(r"^_(AccumRgTn?|AccRgAT|AccRgCT|DocumentJournal)\d+$")
 
 
 def inflate(data):
@@ -64,23 +72,20 @@ def inflate(data):
 	return data
 
 
-_HAS_PARTNO = {}
-
-
-def has_partno(cur, table):
-	"""8.3 splits large files across rows by PartNo; 8.2 stores one row per file."""
-	if table not in _HAS_PARTNO:
-		cur.execute("SELECT COUNT(*) FROM sys.columns WHERE object_id = OBJECT_ID(?) AND name = 'PartNo'", table)
-		_HAS_PARTNO[table] = cur.fetchone()[0] > 0
-	return _HAS_PARTNO[table]
-
-
-def read_file(cur, table, name):
-	"""Concatenate a file's parts in order, then inflate."""
-	order = " ORDER BY PartNo" if has_partno(cur, table) else ""
-	cur.execute("SELECT BinaryData FROM %s WHERE FileName = ?%s" % (table, order), name)
-	blob = b"".join(bytes(r[0]) for r in cur.fetchall() if r[0] is not None)
-	return inflate(blob)
+def open_source():
+	"""The infobase the environment names: a .1CD file, or a server infobase."""
+	path = os.environ.get("DF1C_FILE")
+	if path:
+		if os.environ.get("DF1C_SERVER") or os.environ.get("DF1C_DB"):
+			raise SystemExit("DF1C_FILE and DF1C_SERVER/DF1C_DB are both set - say which infobase to read")
+		if os.path.isdir(path):
+			path = os.path.join(path, "1Cv8.1CD")
+		import source_file
+		return source_file.FileSource(path)
+	import source_mssql
+	return source_mssql.MssqlSource(
+		paths.required("DF1C_SERVER", "which MS SQL server holds it (or DF1C_FILE for a file infobase)"),
+		paths.required("DF1C_DB", "which MS SQL infobase to read"))
 
 
 # --------------------------------------------------------------------------- DBNames
@@ -127,58 +132,75 @@ def physical_name(entry):
 	return "_%s%d" % (kind, number)
 
 
+# --------------------------------------------------------------------------- the manifest
+
+def manifest():
+	if not os.path.exists(MANIFEST):
+		return None
+	with codecs.open(MANIFEST, "r", "utf-8") as f:
+		return json.load(f)
+
+
+def begin(source):
+	"""The manifest this run adds to. One export is one infobase: adding a second one's
+	tables to it would make a mix no check could tell from the real thing."""
+	described = source.describe()
+	held = manifest()
+	if held and held.get("source") != described:
+		raise SystemExit("out/ holds an export of %s, not %s.\n"
+		                 "    Point DF1C_OUT somewhere else, or remove %s to start over."
+		                 % (json.dumps(held.get("source")), json.dumps(described), MANIFEST))
+	held = held or {"source": described, "tables": {}}
+	held["year_offset"] = source.year_offset
+	held["consistency"] = source.consistent
+	return held
+
+
+def commit(held):
+	held["updated"] = datetime.datetime.now().replace(microsecond=0).isoformat()
+	write_json("export.json", held)
+
+
 # --------------------------------------------------------------------------- commands
 
-def cmd_databases(cur, args):
-	"""Which databases does this login actually see? Separates a bad password from missing rights."""
-	cur.execute("SELECT name, state_desc, HAS_DBACCESS(name) FROM sys.databases ORDER BY name")
-	for name, state, access in cur.fetchall():
+def cmd_databases(source, args):
+	if not hasattr(source, "databases"):
+		raise SystemExit("databases: a file infobase is the one file")
+	for name, state, access in source.databases():
 		print("%-44s %-10s access=%s" % (name, state, access))
 
 
-def cmd_probe(cur, args):
-	cur.execute("SELECT @@VERSION")
-	print(cur.fetchone()[0].splitlines()[0])
-	for table in ("Config", "ConfigSave", "Params", "DBSchema", "_YearOffset"):
-		try:
-			cur.execute("SELECT COUNT(*) FROM [%s]" % table)
-			print("  %-12s rows=%s" % (table, cur.fetchone()[0]))
-		except Exception as exc:
-			print("  %-12s -- %s" % (table, str(exc).split("]")[-1].strip()[:70]))
-	cur.execute("""SELECT LEFT(name, PATINDEX('%[0-9]%', name + '0') - 1) AS kind, COUNT(*)
-	               FROM sys.tables WHERE name LIKE '[_]%'
-	               GROUP BY LEFT(name, PATINDEX('%[0-9]%', name + '0') - 1)
-	               ORDER BY 2 DESC""")
+def cmd_probe(source, args):
+	print(json.dumps(source.describe(), ensure_ascii=False))
+	print("year offset %d" % source.year_offset)
+	if hasattr(source, "probe"):
+		for line in source.probe():
+			print(line)
+	families = {}
+	for name, _, rows in source.tables():
+		if name.startswith("_"):
+			kind = re.match(r"^_([A-Za-z]+)", name).group(1)
+			families[kind] = families.get(kind, 0) + 1
 	print("\n  physical table families:")
-	for kind, n in cur.fetchall():
+	for kind, n in sorted(families.items(), key=lambda kv: -kv[1]):
 		print("    %-24s %d" % (kind, n))
 
 
-def cmd_schema(cur, args):
-	cur.execute("""SELECT t.name, c.name, ty.name, c.max_length, c.precision, c.scale, c.is_nullable
-	               FROM sys.tables t
-	               JOIN sys.columns c ON c.object_id = t.object_id
-	               JOIN sys.types ty ON ty.user_type_id = c.user_type_id
-	               ORDER BY t.name, c.column_id""")
-	tables = {}
-	for tn, cn, ty, ml, pr, sc, nullable in cur.fetchall():
-		tables.setdefault(tn, []).append({
-			"column": cn, "type": ty, "len": ml,
-			"precision": pr, "scale": sc, "nullable": bool(nullable),
-		})
-	cur.execute("""SELECT t.name, SUM(p.rows) FROM sys.tables t
-	               JOIN sys.partitions p ON p.object_id = t.object_id AND p.index_id IN (0, 1)
-	               GROUP BY t.name""")
-	rows = dict(cur.fetchall())
+def cmd_schema(source, args):
+	tables = source.tables()
 	write_json("schema.json", {"tables": [
-		{"name": name, "rows": rows.get(name), "columns": cols}
-		for name, cols in sorted(tables.items())
+		{"name": name, "rows": rows, "columns": cols} for name, cols, rows in sorted(tables)
 	]})
+	guessed = getattr(getattr(source, "names", None), "guessed", None)
+	if guessed:
+		held = begin(source)
+		held["names_guessed"] = sorted(guessed)
+		commit(held)
 	print("tables=%d" % len(tables))
 
 
-def cmd_dbnames(cur, args):
-	buf = read_file(cur, "Params", "DBNames")
+def cmd_dbnames(source, args):
+	buf = source.read_file("Params", "DBNames")
 	save("dbnames.bin", buf)
 	entries = parse_dbnames(buf)
 	for entry in entries:
@@ -192,15 +214,14 @@ def cmd_dbnames(cur, args):
 		print("   %-28s %d" % (kind, n))
 
 
-def cmd_extract(cur, args):
+def cmd_extract(source, args):
 	target = os.path.join(OUT, "config")
 	if not os.path.isdir(target):
 		os.makedirs(target)
-	cur.execute("SELECT DISTINCT FileName FROM Config")
-	names = [r[0] for r in cur.fetchall()]
+	names = source.files("Config")
 	text = binary = 0
 	for i, name in enumerate(names):
-		buf = read_file(cur, "Config", name)
+		buf = source.read_file("Config", name)
 		safe = re.sub(r"[^0-9A-Za-z._-]", "_", name)[:120]
 		if buf.startswith(codecs.BOM_UTF8) or b"{" in buf[:64]:
 			suffix, text = ".txt", text + 1
@@ -213,6 +234,135 @@ def cmd_extract(cur, args):
 	print("files=%d text=%d binary=%d -> %s" % (len(names), text, binary, target))
 
 
+def export_table(source, name, columns, expected=None):
+	"""One table's rows, in full, into `data/<name>.parquet`. The number written."""
+	import pyarrow.parquet as pq
+	if not os.path.isdir(DATA):
+		os.makedirs(DATA)
+	arrow = arrow_schema(columns)
+	tmp = os.path.join(DATA, name + ".parquet.tmp")
+	writer = pq.ParquetWriter(tmp, arrow, compression="zstd")
+	written = 0
+	try:
+		for batch in source.rows(name, [c["column"] for c in columns], CHUNK):
+			writer.write_table(arrow_batch(arrow, columns, batch))
+			written += len(batch)
+			if (expected or 0) > CHUNK:
+				sys.stderr.write("\r    %-30s %9d / %d" % (name, written, expected))
+	finally:
+		writer.close()
+	if (expected or 0) > CHUNK:
+		sys.stderr.write("\n")
+	os.replace(tmp, os.path.join(DATA, name + ".parquet"))
+	return written
+
+
+def cmd_data(source, args):
+	held = begin(source)
+	schema = dict((name, (cols, rows)) for name, cols, rows in source.tables())
+	if args.changes:
+		import changes
+		try:
+			changes.run(source, held, schema, args,
+			            lambda name, columns: export_table(source, name, columns, schema[name][1]))
+		finally:
+			commit(held)
+		return
+	if args.since:
+		raise SystemExit("data: --since goes with --changes")
+	named = list(args.tables)
+	if args.objects:
+		# Every catalog, document and chart - what `delta` reads the keys of. Small: 407
+		# tables and 1.6 M rows of a 313 M-row infobase.
+		import changes
+		objects = [n for n in sorted(schema) if n not in named
+		           and changes.classify(n, schema[n][0], schema)[0] == "object"]
+		if not objects:
+			# A file infobase: no rowversion, so nothing is an object in that sense - and
+			# an empty list would otherwise mean "every table".
+			raise SystemExit("data --objects: no table here has a rowversion - this is for an"
+			                 " MS SQL export, which `delta` needs")
+		named += objects
+	args.tables = named
+	wanted = args.tables or sorted(n for n in schema if n.startswith("_")
+	                               and (args.derived or not DERIVED.match(n)))
+	if not args.tables and not args.derived:
+		left = [n for n in schema if DERIVED.match(n)]
+		print("left out %d table(s) 1C derives from others - totals, journals (%d rows);"
+		      " name one, or --derived, to export it" % (len(left), sum(schema[n][1] or 0 for n in left)))
+	unknown = [t for t in wanted if t not in schema]
+	if unknown:
+		raise SystemExit("data: no such table: %s" % ", ".join(unknown))
+	if args.tables:
+		# A partial export still takes every enum: they are a few rows each, and the loader
+		# decodes an enum column by reading its table - left out, a module's load stops at
+		# its first enum with "_Enum727 does not exist".
+		enums = [n for n in sorted(schema) if re.match(r"^_Enum\d+$", n) and n not in wanted]
+		wanted += enums
+	if args.skip:
+		wanted = [t for t in wanted if not re.search(args.skip, t)]
+	started, total = time.time(), 0
+	for at, name in enumerate(wanted):
+		columns, expected = schema[name]
+		horizon = source.horizon()
+		written = export_table(source, name, columns, expected)
+		held["tables"][name] = {"rows": written,
+		                        "horizon": horizon.hex() if horizon is not None else None,
+		                        "exported": datetime.datetime.now().replace(microsecond=0).isoformat()}
+		total += written
+		# The manifest is written as the tables are, so an export stopped half-way says
+		# truthfully what it holds.
+		if at % 50 == 0 or at == len(wanted) - 1:
+			commit(held)
+	print("tables=%d rows=%d in %.0fs -> %s" % (len(wanted), total, time.time() - started, DATA))
+
+
+def cmd_all(source, args):
+	for step in (cmd_dbnames, cmd_schema, cmd_extract, cmd_data):
+		step(source, args)
+
+
+# --------------------------------------------------------------------------- parquet
+
+def arrow_type(column):
+	"""The schema dump's MS SQL type -> how the export stores it."""
+	import pyarrow as pa
+	kind = column["type"]
+	if kind in ("binary", "varbinary", "image", "timestamp", "rowversion"):
+		return pa.binary()
+	if kind in ("nchar", "nvarchar", "ntext", "char", "varchar", "text", "sysname", "uniqueidentifier"):
+		return pa.string()
+	if kind in ("numeric", "decimal"):
+		return pa.decimal128(column.get("precision") or 38, column.get("scale") or 0)
+	if kind in ("datetime", "datetime2", "smalldatetime", "date"):
+		return pa.timestamp("us")
+	if kind in ("int", "smallint", "tinyint", "bigint"):
+		return pa.int64()
+	if kind == "bit":
+		return pa.bool_()
+	if kind in ("float", "real"):
+		return pa.float64()
+	return pa.string()
+
+
+def arrow_schema(columns):
+	import pyarrow as pa
+	return pa.schema([pa.field(c["column"], arrow_type(c)) for c in columns])
+
+
+def arrow_batch(arrow, columns, batch):
+	import pyarrow as pa
+	arrays = []
+	for i, field in enumerate(arrow):
+		values = [row[i] for row in batch]
+		if pa.types.is_string(field.type):
+			values = [v if v is None or isinstance(v, str) else str(v) for v in values]
+		elif pa.types.is_binary(field.type):
+			values = [v if v is None else bytes(v) for v in values]
+		arrays.append(pa.array(values, type=field.type))
+	return pa.Table.from_arrays(arrays, schema=arrow)
+
+
 def save(name, data):
 	if not os.path.isdir(OUT):
 		os.makedirs(OUT)
@@ -233,18 +383,25 @@ COMMANDS = {
 	"schema": cmd_schema,
 	"dbnames": cmd_dbnames,
 	"extract": cmd_extract,
+	"data": cmd_data,
+	"all": cmd_all,
 }
 
 
 def main():
 	ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
 	ap.add_argument("command", choices=sorted(COMMANDS))
-	ap.add_argument("--server", default=os.environ.get("DF1C_SERVER")
-	                or paths.required("DF1C_SERVER", "which MS SQL server holds it"))
-	ap.add_argument("--db", default=os.environ.get("DF1C_DB")
-	                or paths.required("DF1C_DB", "which MS SQL infobase to read"))
+	ap.add_argument("tables", nargs="*", help="data: only these tables")
+	ap.add_argument("--skip", help="data: leave out tables matching this regex")
+	ap.add_argument("--objects", action="store_true",
+	                help="data: every object table (catalogs, documents, charts) besides the named ones")
+	ap.add_argument("--derived", action="store_true",
+	                help="data: export totals and journals too (left out by default)")
+	ap.add_argument("--changes", action="store_true",
+	                help="data: read only what 1C changed since the export (MS SQL; see changes.py)")
+	ap.add_argument("--since", help="data --changes: from this day (YYYY-MM-DD) or rowversion (0x...)")
 	args = ap.parse_args()
-	COMMANDS[args.command](connect(args.server, args.db).cursor(), args)
+	COMMANDS[args.command](open_source(), args)
 
 
 if __name__ == "__main__":

package/skills/dforge-1c-import/scripts/lib/external.py

@@ -40,6 +40,7 @@ import attrs
 import meta
 import migrate
 import paths
+import store
 import unpack
 
 CONTAINER = b"\xff\xff\xff\x7f"
@@ -171,10 +172,10 @@ def main():
 	enum_table = index[enum_guid]["physical"]
 	kinds = attrs.enum_values(meta.parse(meta.load(enum_guid)), u"ВидыДополнительныхВнешнихОбработок")
 
-	cur = migrate.mssql_connect().cursor()
+	cur = migrate.source_connect().cursor()
 	cur.execute("SELECT t._Code, t._Description, t._Marked, e._EnumOrder, t.%s FROM %s t "
-	            "LEFT JOIN %s e ON e._IDRRef = t.%sRRef WHERE t._Folder = 0x01"
-	            % (storage, table, enum_table, kind))
+	            "LEFT JOIN %s e ON e._IDRRef = t.%sRRef WHERE t._Folder = %s"
+	            % (storage, table, enum_table, kind, store.literal(store.TRUE)))
 	root = os.path.join(paths.OUT, "external")
 	index_rows, skipped = [], []
 	for code, name, marked, order, blob in cur.fetchall():

package/skills/dforge-1c-import/scripts/lib/meta.py

@@ -95,6 +95,16 @@ def parse(text):
 	return read_value() if pos < n else []
 
 
+def root():
+	"""The configuration's own object: `root` names it, `{2,<guid>,}`. The GUID is the
+	configuration's, so it differs from one infobase to the next."""
+	match = re.search(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", load("root") or "")
+	if not match:
+		raise SystemExit("meta: %s has no root - run `1c.py dump1c extract` first"
+		                 % os.path.join(CONFIG, "root.txt"))
+	return match.group(0)
+
+
 def load(guid):
 	path = os.path.join(CONFIG, guid + ".txt")
 	if not os.path.isfile(path):
@@ -150,7 +160,7 @@ def dbnames_index():
 
 def cmd_kinds(args):
 	"""Each group's family is whatever DBNames calls its members - no guessing from counts."""
-	tree = parse(load("1e4190e9-76c2-456e-a607-4d817110ffd9") or "")
+	tree = parse(load(root()) or "")
 	groups = collect_guid_groups(tree)
 	dbn = dbnames_index()
 
@@ -279,7 +289,7 @@ def cmd_allobjects(args):
 	common modules, roles and subsystems are missing from it - and they hold most of the
 	configuration's code.
 	"""
-	tree = parse(load("1e4190e9-76c2-456e-a607-4d817110ffd9") or "")
+	tree = parse(load(root()) or "")
 	dbn = dbnames_index()
 	result = []
 	for group in collect_guid_groups(tree):