@dforge-core/dforge-mcp 0.2.45 → 0.2.47

package/skills/dforge-1c-import/scripts/lib/source_file.py

@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+"""
+A file infobase (`1Cv8.1CD`) as an export source: `onecd` reads it, this makes what it
+reads look like what a server infobase gives, so nothing downstream asks which one it was.
+
+Three differences are folded here and nowhere else:
+
+- NAMES. A file infobase keeps every name in capitals - `_REFERENCE12`, `_FLD345RREF` - where
+  MS SQL has `_Reference12`, `_Fld345RRef`, and the column map, the project's sources and
+  the library all spell them the MS SQL way. The spelling is put back from DBNames, which
+  carries every numbered kind in its own case (`AccumRgT`, `ReferenceChngR`, `VT`, `Fld`),
+  plus the fixed names 1C gives the standard columns. A name neither covers is a service
+  table's own business that nothing reads; it is capitalised and counted in the manifest.
+- TYPES. The schema is written in MS SQL's vocabulary - `binary(16)`, `nvarchar`, `numeric`,
+  `datetime`, `len` in bytes as `sys.columns` gives it - because that is what the generator
+  and the checks read.
+- VALUES. A boolean is `binary(1)` 0x01 / 0x00 as on MS SQL; the empty date is 0001-01-01.
+  A file infobase has no year offset: its dates are the real ones, and the manifest says so.
+
+What does NOT carry over is the row version. In a file `_Version` is a per-table counter,
+not MS SQL's database-wide rowversion, so a file export has no horizon and `delta` cannot
+read changes off it. It is exported as `binary(16)`, for completeness only.
+"""
+import os
+import re
+import struct
+
+import dump1c
+import onecd
+
+SERVICE = dict((n.upper(), n) for n in (
+	"Config", "ConfigSave", "Params", "Files", "DBSchema", "IBVersion", "v8users", "_YearOffset",
+	"FileName", "Creation", "Modified", "Attributes", "DataSize", "BinaryData", "PartNo",
+	"SerializedData",
+	# v8users and IBVersion
+	"ID", "Name", "Descr", "OSName", "Changed", "RolesID", "Show", "Data", "EAuth", "AdmRole",
+	"UsSprH", "PlatformVersionReq"))
+
+# The un-numbered segments of 1C's names as MS SQL spells them, between underscores:
+# `_Fld12_TYPE` is `Fld12` and `TYPE`. Harvested from a server infobase's whole schema
+# (2,668 tables, 28,715 columns) rather than remembered, so a list that misses one is
+# fixed by harvesting again. Where two spellings fold to one - `UserId` and `UserID` both
+# occur - the commoner one is here and `migrate` matches the other regardless of case.
+WORDS = dict((w.upper(), w) for w in (
+	"APDateFrom", "APDateTill", "AccountCtRRef", "AccountDtRRef", "AccountRRef", "ActionPeriod",
+	"ActivationCondition", "Active", "ActualPeriod", "BPDateFrom", "BPDateTill", "BaseAsAct",
+	"BaseCK", "BaseCKBaseCK", "BaseCKBaseCKRRef", "BaseCKLineNo", "BusinessProcess", "CKRefRRef",
+	"CalcKindRRef", "Code", "Completed", "ConfigChngR", "ConnectPrms", "ConstID", "Correspond",
+	"Date", "Description", "DimHash", "DimIsMetadata", "DimKindRRef", "DisplNumber", "DisplacedCK",
+	"DisplacedCKDisplCKRRef", "DisplacedCKLineNo", "DocumentRRef", "DocumentTRef", "EDHashCt",
+	"EDHashDt", "EnumOrder", "Executed", "ExtDataSrcId", "ExtProps", "FileName", "FinishTime",
+	"Folder", "HeadTaskRRef", "ID", "IDRRef", "IsMetadata", "JobKey", "KeyField", "Kind",
+	"KindRRef", "LeadingCK", "LeadingCKLeadingCK", "LeadingCKLineNo", "LineNo", "MDObjID",
+	"Marked", "MessageNo", "MetadataID", "MinPeriod", "Name", "NodeRRef", "NodeTRef", "Number",
+	"NumberPrefix", "ObjectKey", "OffBalance", "OrderField", "OwnerID", "OwnerIDRRef",
+	"Parameters", "ParentIDRRef", "Period", "Periodicity", "Point", "Posted", "Predefined",
+	"PredefinedBaseCKTableLine", "PredefinedDisplCKTableLine", "PredefinedID",
+	"PredefinedLeadingCKTableLine", "PredefinedObj", "ReceivedNo", "RecordKey", "RecordKind",
+	"Recorder", "RecorderRRef", "RecorderTRef", "RegID", "RepetitionFactor",
+	"RestartAttemptNumber", "RestartCount", "RestartPeriod", "RoutePointOrder", "SentNo",
+	"SettingsData", "SettingsKey", "SettingsPresentation", "SimpleKey", "SliceUsing", "Splitter",
+	"StartTime", "Started", "State", "Storno", "TurnoverOnly", "URL", "URLHash", "Use",
+	"UseSplitter", "UseTotals", "UserId", "UserName", "Value", "Value1", "Value2", "Value3",
+	"Value4", "Version", "YearOffset",
+	# a composite reference's parts, and a column's own type tag
+	"RRRef", "RTRef", "RRef", "TRef", "L", "N", "T", "S", "B"))
+# What follows a numbered kind inside one segment: `_Fld12RRef`, `_Fld12DtRRef`.
+SUFFIXES = ("CtRRef", "DtRRef", "RRef", "TRef", "Ct", "Dt", "")
+
+
+class Names(object):
+	"""Upper-case 1CD names -> the MS SQL spelling, from DBNames' kinds."""
+
+	def __init__(self, kinds):
+		self.kinds = sorted(set(kinds), key=len, reverse=True)
+		self.numbered = [(re.compile("^%s(\\d+)(%s)$" % (k.upper(), "|".join(s.upper() for s in SUFFIXES))), k)
+		                 for k in self.kinds]
+		self.guessed = set()
+
+	def __call__(self, name):
+		if name in SERVICE:
+			return SERVICE[name]
+		parts = name.replace("DATE_TIME", "DATE\x00TIME").split("_")
+		out = []
+		for at, part in enumerate(parts):
+			if part == "DATE\x00TIME":
+				out.append("Date_Time")
+			elif not part:
+				out.append(part)
+			elif part == "TYPE":
+				# A column of its own is `_Type` (a characteristic's value type); after
+				# another segment it is a composite reference's tag, `_Fld12_TYPE`.
+				out.append("Type" if at == 1 else "TYPE")
+			else:
+				out.append(self.segment(part, name))
+		return "_".join(out)
+
+	def segment(self, part, name):
+		for pattern, kind in self.numbered:
+			match = pattern.match(part)
+			if match:
+				suffix = next(s for s in SUFFIXES if s.upper() == match.group(2))
+				return kind + match.group(1) + suffix
+		if part in WORDS:
+			return WORDS[part]
+		# A kind with no number of its own: a service table such as `_AccOpt`.
+		for kind in self.kinds:
+			if part == kind.upper():
+				return kind
+		self.guessed.add(name)
+		match = re.match(r"^([A-Z]+)(\d*)(.*)$", part)
+		return (match.group(1).capitalize() + match.group(2) + match.group(3)) if match else part
+
+
+def numeric_bytes(precision):
+	"""What `sys.columns.max_length` says for numeric(p): the schema dump carries it as-is."""
+	return 5 if precision <= 9 else 9 if precision <= 19 else 13 if precision <= 28 else 17
+
+
+def column_type(field):
+	"""A 1CD field -> the schema dump's (type, len, precision, scale) in MS SQL terms."""
+	kind, n = field.type, field.length
+	if kind == "B":
+		return "binary", n, 0, 0
+	if kind == "L":
+		return "binary", 1, 0, 0
+	if kind == "N":
+		return "numeric", numeric_bytes(n), n, field.precision
+	if kind == "NC":
+		return "nchar", n * 2, 0, 0
+	if kind == "NVC":
+		return "nvarchar", n * 2, 0, 0
+	if kind == "NT":
+		return "nvarchar", -1, 0, 0
+	if kind == "I":
+		return "varbinary", -1, 0, 0
+	if kind == "DT":
+		return "datetime", 8, 23, 3
+	if kind == "RV":
+		return "binary", 16, 0, 0
+	raise ValueError("1CD field type %s" % kind)
+
+
+def canonical(field):
+	"""A function turning one of `onecd`'s values for this field into MS SQL's."""
+	if field.type == "L":
+		return lambda v: None if v is None else (b"\x01" if v else b"\x00")
+	if field.type == "RV":
+		return lambda v: None if v is None else struct.pack(">4I", *v)
+	return None
+
+
+class FileSource(object):
+	kind = "file"
+	year_offset = 0
+	consistent = "the file as it was read; copy it, or close the infobase, before exporting"
+
+	def __init__(self, path):
+		self.path = os.path.abspath(path)
+		self.db = onecd.Database(self.path)
+		self._files = {}
+		self._tables = None
+		self.names = Names(e["type"] for e in dump1c.parse_dbnames(self.read_file("Params", "DBNames")))
+		self.by_name = dict((self.names(t), t) for t in self.db.tables)
+
+	def describe(self):
+		return {"kind": self.kind, "path": self.path, "format": self.db.version,
+		        "page_size": self.db.page_size, "locale": self.db.locale}
+
+	def _table(self, name):
+		return self.db.tables[self.by_name[name] if name in self.by_name else name.upper()]
+
+	def _service(self, table):
+		"""A service table's files, {name: [(part, compressed body)]}, read in one pass: a
+		blob is reached only through its row, so a read per file would read them all again."""
+		table = table.upper()
+		if table not in self._files:
+			found = {}
+			for name, part, body in self.db.tables[table].rows(["FILENAME", "PARTNO", "BINARYDATA"]):
+				found.setdefault(name, []).append((part or 0, body or b""))
+			self._files[table] = found
+		return self._files[table]
+
+	def files(self, table):
+		return sorted(self._service(table))
+
+	def read_file(self, table, name):
+		parts = sorted(self._service(table).get(name, ()))
+		return dump1c.inflate(b"".join(body for _, body in parts))
+
+	def tables(self):
+		"""[(name, columns, rows)] - the schema dump's shape, every table in the file.
+
+		Counting a table's rows reads all of it - a file keeps no row count - so this is
+		worked out once per run, however many commands of `dump1c all` ask for it."""
+		if self._tables is None:
+			self._tables = self._read_tables()
+		return self._tables
+
+	def _read_tables(self):
+		out = []
+		for raw, table in self.db.tables.items():
+			columns = []
+			for field in table.fields.values():
+				kind, length, precision, scale = column_type(field)
+				columns.append({"column": self.names(field.name), "type": kind, "len": length,
+				                "precision": precision, "scale": scale, "nullable": field.nullable})
+			out.append((self.names(raw), columns, table.count()))
+		return out
+
+	def rows(self, table, columns, chunk):
+		"""Batches of tuples, in `columns` order (the MS SQL spelling)."""
+		t = self._table(table)
+		by_name = dict((self.names(f), f) for f in t.fields)
+		raw = [by_name[c] for c in columns]
+		fix = [canonical(t.fields[r]) for r in raw]
+		batch = []
+		for row in t.rows(raw):
+			batch.append(tuple(f(v) if f else v for f, v in zip(fix, row)))
+			if len(batch) >= chunk:
+				yield batch
+				batch = []
+		if batch:
+			yield batch
+
+	def horizon(self):
+		return None

package/skills/dforge-1c-import/scripts/lib/source_mssql.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+"""
+A server infobase on MS SQL as an export source. The only module in the library that
+speaks T-SQL to 1C; everything past the export reads `out/` (see `store.py`).
+
+Credentials come from env DF1C_USER / DF1C_PASS; without them, Windows auth is used.
+
+The export takes 1C's horizon - `MIN_ACTIVE_ROWVERSION()` - before it reads a table, so a
+delta read off the next export picks up everything this one may have missed.
+"""
+import os
+import sys
+
+import dump1c
+
+PREFERRED_DRIVERS = (
+	"ODBC Driver 18 for SQL Server",
+	"ODBC Driver 17 for SQL Server",
+	"ODBC Driver 13 for SQL Server",
+	"SQL Server Native Client 11.0",
+	"SQL Server",
+)
+
+# A server infobase created without the offset option still has the table; one that has
+# lost it is old enough to have been created with the offset.
+DEFAULT_YEAR_OFFSET = 2000
+
+
+def connect(server, db):
+	import pyodbc
+	available = set(pyodbc.drivers())
+	driver = next((d for d in PREFERRED_DRIVERS if d in available), None)
+	if driver is None:
+		raise RuntimeError("no SQL Server ODBC driver found: %r" % sorted(available))
+	parts = ["DRIVER={%s}" % driver, "SERVER=%s" % server, "DATABASE=%s" % db,
+	         "TrustServerCertificate=yes"]
+	user, pwd = os.environ.get("DF1C_USER"), os.environ.get("DF1C_PASS")
+	if user:
+		parts += ["UID=%s" % user, "PWD=%s" % pwd]
+	else:
+		parts += ["Trusted_Connection=yes"]
+	sys.stderr.write("[i] %s / %s via %s\n" % (server, db, driver))
+	return pyodbc.connect(";".join(parts), autocommit=True)
+
+
+class MssqlSource(object):
+	kind = "mssql"
+	consistent = "per table: each is read after its horizon is taken"
+
+	def __init__(self, server, db):
+		self.server, self.db = server, db
+		self.conn = connect(server, db)
+		self.cur = self.conn.cursor()
+		self._partno = {}
+		self._offset = None
+
+	def describe(self):
+		return {"kind": self.kind, "server": self.server, "database": self.db}
+
+	@property
+	def year_offset(self):
+		if self._offset is None:
+			try:
+				self.cur.execute("SELECT Offset FROM _YearOffset")
+				row = self.cur.fetchone()
+				self._offset = int(row[0]) if row else DEFAULT_YEAR_OFFSET
+			except Exception:
+				sys.stderr.write("[!] no _YearOffset; assuming %d\n" % DEFAULT_YEAR_OFFSET)
+				self._offset = DEFAULT_YEAR_OFFSET
+		return self._offset
+
+	def has_partno(self, table):
+		"""8.3 splits large files across rows by PartNo; 8.2 stores one row per file."""
+		if table not in self._partno:
+			self.cur.execute("SELECT COUNT(*) FROM sys.columns WHERE object_id = OBJECT_ID(?)"
+			                 " AND name = 'PartNo'", table)
+			self._partno[table] = self.cur.fetchone()[0] > 0
+		return self._partno[table]
+
+	def files(self, table):
+		self.cur.execute("SELECT DISTINCT FileName FROM %s" % table)
+		return sorted(r[0] for r in self.cur.fetchall())
+
+	def read_file(self, table, name):
+		"""Concatenate a file's parts in order, then inflate."""
+		order = " ORDER BY PartNo" if self.has_partno(table) else ""
+		self.cur.execute("SELECT BinaryData FROM %s WHERE FileName = ?%s" % (table, order), name)
+		blob = b"".join(bytes(r[0]) for r in self.cur.fetchall() if r[0] is not None)
+		return dump1c.inflate(blob)
+
+	def tables(self):
+		self.cur.execute("""SELECT t.name, c.name, ty.name, c.max_length, c.precision, c.scale, c.is_nullable
+		                    FROM sys.tables t
+		                    JOIN sys.columns c ON c.object_id = t.object_id
+		                    JOIN sys.types ty ON ty.user_type_id = c.user_type_id
+		                    ORDER BY t.name, c.column_id""")
+		tables = {}
+		for tn, cn, ty, ml, pr, sc, nullable in self.cur.fetchall():
+			tables.setdefault(tn, []).append({
+				"column": cn, "type": ty, "len": ml,
+				"precision": pr, "scale": sc, "nullable": bool(nullable),
+			})
+		self.cur.execute("""SELECT t.name, SUM(p.rows) FROM sys.tables t
+		                    JOIN sys.partitions p ON p.object_id = t.object_id AND p.index_id IN (0, 1)
+		                    GROUP BY t.name""")
+		rows = dict(self.cur.fetchall())
+		return [(name, cols, rows.get(name)) for name, cols in sorted(tables.items())]
+
+	def rows(self, table, columns, chunk, where=None):
+		"""Batches of rows; `where` is a T-SQL condition on the table's own columns."""
+		cur = self.conn.cursor()
+		cur.execute("SELECT %s FROM [%s]%s" % (", ".join("[%s]" % c for c in columns), table,
+		                                       " WHERE " + where if where else ""))
+		while True:
+			batch = cur.fetchmany(chunk)
+			if not batch:
+				break
+			yield [tuple(r) for r in batch]
+
+	def column(self, sql):
+		"""The first column of a query, every row."""
+		cur = self.conn.cursor()
+		cur.execute(sql)
+		out = []
+		while True:
+			batch = cur.fetchmany(100000)
+			if not batch:
+				return out
+			out.extend(r[0] for r in batch)
+
+	def triples(self, sql):
+		"""The first three columns of a query, every row."""
+		cur = self.conn.cursor()
+		cur.execute(sql)
+		return [(r[0], r[1], r[2]) for r in cur.fetchall()]
+
+	def horizon(self):
+		"""Below it every transaction had committed: a read after it misses nothing older."""
+		self.cur.execute("SELECT MIN_ACTIVE_ROWVERSION()")
+		return bytes(self.cur.fetchone()[0])
+
+	# ------------------------------------------------------------------- diagnostics
+
+	def databases(self):
+		"""Which databases does this login actually see? Separates a bad password from missing rights."""
+		self.cur.execute("SELECT name, state_desc, HAS_DBACCESS(name) FROM sys.databases ORDER BY name")
+		return self.cur.fetchall()
+
+	def probe(self):
+		self.cur.execute("SELECT @@VERSION")
+		lines = [self.cur.fetchone()[0].splitlines()[0]]
+		for table in ("Config", "ConfigSave", "Params", "DBSchema", "_YearOffset"):
+			try:
+				self.cur.execute("SELECT COUNT(*) FROM [%s]" % table)
+				lines.append("  %-12s rows=%s" % (table, self.cur.fetchone()[0]))
+			except Exception as exc:
+				lines.append("  %-12s -- %s" % (table, str(exc).split("]")[-1].strip()[:70]))
+		return lines