@dforge-core/dforge-mcp 0.2.46-next → 0.2.48

package/skills/dforge-1c-import/scripts/lib/peek.py

@@ -22,7 +22,7 @@ import paths
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-import dump1c
+import store
 
 OUT = paths.OUT
 
@@ -52,10 +52,8 @@ def main():
 
 	table = sys.argv[1]
 	limit = int(sys.argv[2]) if len(sys.argv) > 2 else 5
-	cur = dump1c.connect(paths.required("DF1C_SERVER", "which MS SQL server holds it"),
-	                     paths.required("DF1C_DB",
-	                                    "which MS SQL infobase to read")).cursor()
-	cur.execute("SELECT TOP (%d) * FROM [%s]" % (limit, table))
+	cur = store.connect().cursor()
+	cur.execute("SELECT * FROM %s LIMIT %d" % (store.quote(table), limit))
 	columns = [d[0] for d in cur.description]
 	rows = cur.fetchall()
 	# drop columns that are empty across every sampled row - they only add noise

package/skills/dforge-1c-import/scripts/lib/recorded.py

@@ -17,7 +17,7 @@ the one document table whose keys its recorders match.
     python tools/1c.py recorded            # writes out/recorded.json
     python tools/1c.py recorded --compare  # and scores out/callgraph.json against it
 
-Read-only, and `NOLOCK`: a count does not need to wait for anyone.
+Read off the export (`store.py`), so it asks nothing of the running infobase.
 """
 from __future__ import print_function
 
@@ -31,6 +31,7 @@ import sys
 import meta
 import migrate
 import paths
+import store
 
 TARGET = os.path.join(paths.OUT, "recorded.json")
 
@@ -65,20 +66,19 @@ def documents():
 	return found
 
 
-def columns(cur, table):
-	cur.execute("SELECT name FROM sys.columns WHERE object_id = OBJECT_ID(?)", "dbo." + table)
-	return set(row[0] for row in cur.fetchall())
+def columns(table):
+	return migrate.source_schema()[table]["columns"]
 
 
 def single_recorder(cur, table, docs):
 	"""The one document type a TRef-less register's recorders belong to."""
-	cur.execute("SELECT DISTINCT TOP (20) _RecorderRRef FROM dbo.[%s] WITH (NOLOCK)" % table)
+	cur.execute("SELECT DISTINCT _RecorderRRef FROM %s LIMIT 20" % store.quote(table))
 	refs = [bytes(r[0]) for r in cur.fetchall()]
 	if not refs:
 		return None
 	marks = ",".join("?" * len(refs))
 	for number, (name, physical) in sorted(docs.items()):
-		cur.execute("SELECT COUNT(*) FROM dbo.[%s] WHERE _IDRRef IN (%s)" % (physical, marks),
+		cur.execute("SELECT COUNT(*) FROM %s WHERE _IDRRef IN (%s)" % (store.quote(physical), marks),
 		            *refs)
 		if cur.fetchone()[0]:
 			return number
@@ -87,17 +87,17 @@ def single_recorder(cur, table, docs):
 
 def measure():
 	migrate.load_env()
-	cur = migrate.mssql_connect().cursor()
+	cur = migrate.source_connect().cursor()
 	docs = documents()
 	result = collections.defaultdict(dict)       # document -> register -> rows
 	unmatched = []
 	for family, name, table in registers():
-		cols = columns(cur, table)
+		cols = columns(table)
 		if "_RecorderRRef" not in cols:
 			continue                             # not subordinate to a recorder
 		if "_RecorderTRef" in cols:
-			cur.execute("SELECT _RecorderTRef, COUNT_BIG(*) FROM dbo.[%s] WITH (NOLOCK)"
-			            " GROUP BY _RecorderTRef" % table)
+			cur.execute("SELECT _RecorderTRef, COUNT(*) FROM %s"
+			            " GROUP BY _RecorderTRef" % store.quote(table))
 			for tref, rows in cur.fetchall():
 				number = int.from_bytes(bytes(tref), "big") & 0xFFFF
 				if number in docs:
@@ -105,7 +105,7 @@ def measure():
 				else:
 					unmatched.append((name, number, int(rows)))
 		else:
-			cur.execute("SELECT COUNT_BIG(*) FROM dbo.[%s] WITH (NOLOCK)" % table)
+			cur.execute("SELECT COUNT(*) FROM %s" % store.quote(table))
 			rows = int(cur.fetchone()[0])
 			if not rows:
 				continue

package/skills/dforge-1c-import/scripts/lib/recorders.py

@@ -22,7 +22,7 @@ import paths
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-import dump1c
+import store
 
 OUT = paths.OUT
 
@@ -35,7 +35,7 @@ def load_report():
 def probe_single(cur, table):
 	"""Without a TRef the recorder's type is implicit, so find the one document table
 	whose PK matches the recorders we sampled."""
-	cur.execute("SELECT DISTINCT TOP (20) _RecorderRRef FROM [%s]" % table)
+	cur.execute("SELECT DISTINCT _RecorderRRef FROM %s LIMIT 20" % store.quote(table))
 	refs = [bytes(r[0]) for r in cur.fetchall()]
 	if not refs:
 		print("   (register is empty)")
@@ -46,8 +46,8 @@ def probe_single(cur, table):
 		suffix = physical[len("_Document"):] if physical.startswith("_Document") else ""
 		if not suffix.isdigit() or not row.get("rows"):
 			continue
-		cur.execute("SELECT COUNT(*) FROM [%s] WHERE _IDRRef IN (%s)"
-		            % (physical, placeholders), *refs)
+		cur.execute("SELECT COUNT(*) FROM %s WHERE _IDRRef IN (%s)"
+		            % (store.quote(physical), placeholders), *refs)
 		hits = cur.fetchone()[0]
 		if hits:
 			print("   %s - %s  (%d/%d sampled recorders matched)"
@@ -58,19 +58,16 @@ def probe_single(cur, table):
 
 def main():
 	table = sys.argv[1]
-	cur = dump1c.connect(paths.required("DF1C_SERVER", "which MS SQL server holds it"),
-	                     paths.required("DF1C_DB",
-	                                    "which MS SQL infobase to read")).cursor()
-	cur.execute("""SELECT COUNT(*) FROM sys.columns
-	               WHERE object_id = OBJECT_ID(?) AND name = '_RecorderTRef'""", table)
-	if not cur.fetchone()[0]:
+	cur = store.connect().cursor()
+	cur.execute("SELECT * FROM %s LIMIT 0" % store.quote(table))
+	if "_RecorderTRef" not in [d[0] for d in cur.description]:
 		print("%s has no _RecorderTRef - one document type writes it; probing for it"
 		      % table)
 		probe_single(cur, table)
 		return
 
-	cur.execute("SELECT _RecorderTRef, COUNT(*) FROM [%s] GROUP BY _RecorderTRef "
-	            "ORDER BY 2 DESC" % table)
+	cur.execute("SELECT _RecorderTRef, COUNT(*) FROM %s GROUP BY _RecorderTRef "
+	            "ORDER BY 2 DESC" % store.quote(table))
 	rows = cur.fetchall()
 
 	# a document's type id is its table number; _RecorderTRef stores it big-endian

package/skills/dforge-1c-import/scripts/lib/seedstate.py

@@ -47,6 +47,7 @@ import sys
 
 import attrs
 import migrate
+import store
 
 Register = collections.namedtuple("Register", [
 	"module", "code", "state", "journal", "journal_entity", "direction", "date",
@@ -170,7 +171,7 @@ def totals_table(table):
 	return found[0] if found else None
 
 
-def one_c_balance(mssql, pg, reg, mapping, source, period=None):
+def one_c_balance(export, pg, reg, mapping, source, period=None):
 	"""(totals table, period, {tuple: resources}) - 1C's totals at a period, non-zero only.
 
 	No period means the table's last one: 5999-11-01, where 1C keeps its CURRENT totals. A
@@ -184,6 +185,10 @@ def one_c_balance(mssql, pg, reg, mapping, source, period=None):
 	table = totals_table(mapped["table"])
 	if not table:
 		raise SystemExit("seedstate: %s has no totals table in DBNames" % mapped["table"])
+	if table not in store.manifest()["tables"]:
+		# Totals are derived, so a default export leaves them out (dump1c.DERIVED).
+		raise SystemExit("seedstate: %s's totals %s are not in the export - `1c.py dump1c data %s`"
+		                 % (mapped["table"], table, table))
 	schema, name = [p.strip('"') for p in reg.journal.split(".")]
 	types = dict((c[0], c[1]) for c in migrate.table_columns(pg, schema, name))
 	held = source[mapped["table"]]["columns"]
@@ -199,13 +204,13 @@ def one_c_balance(mssql, pg, reg, mapping, source, period=None):
 			converters.append(migrate.converter(types[ours], held[physical]["type"]))
 
 	if period is None:
-		mssql.execute("SELECT MAX(_Period) FROM [%s]" % table)
-		period = mssql.fetchone()[0]
-	mssql.execute("SELECT %s FROM [%s] WHERE _Period = ?"
-	              % (", ".join("[%s]" % c for c in columns), table), period)
+		export.execute("SELECT MAX(_Period) FROM %s" % store.quote(table))
+		period = export.fetchone()[0]
+	export.execute("SELECT %s FROM %s WHERE _Period = ?"
+	              % (", ".join(store.quote(c) for c in columns), store.quote(table)), period)
 	width = len(reg.dimensions)
 	balance = collections.defaultdict(lambda: [decimal.Decimal(0)] * len(reg.resources))
-	for row in mssql.fetchall():
+	for row in export.fetchall():
 		values = [normal(f(v), types[c])
 		          for f, v, c in zip(converters, row, reg.dimensions + reg.resources)]
 		held_now = balance[tuple(values[:width])]
@@ -259,17 +264,17 @@ def as_of_balance(pg, reg, until):
 	return out
 
 
-def verify(pg, reg, mapping, source, mssql, month=None):
+def verify(pg, reg, mapping, source, export, month=None):
 	"""The state against 1C's current totals, or - given a month - the as-of read at its
 	end against 1C's totals row for the month after."""
 	if month:
 		year, number = [int(p) for p in month.split("-")]
 		until = datetime.date(year + number // 12, number % 12 + 1, 1)
-		period = datetime.datetime(until.year + migrate.YEAR_OFFSET, until.month, 1)
-		table, period, theirs = one_c_balance(mssql, pg, reg, mapping, source, period)
+		period = datetime.datetime(until.year + store.year_offset(), until.month, 1)
+		table, period, theirs = one_c_balance(export, pg, reg, mapping, source, period)
 		ours = as_of_balance(pg, reg, until)
 	else:
-		table, period, theirs = one_c_balance(mssql, pg, reg, mapping, source)
+		table, period, theirs = one_c_balance(export, pg, reg, mapping, source)
 		ours = state_balance(pg, reg)
 	both = set(theirs) & set(ours)
 	differ = [k for k in both if theirs[k] != ours[k]]
@@ -303,7 +308,7 @@ def month_end(month):
 	year, number = [int(p) for p in month.split("-")]
 	start = datetime.date(year + number // 12, number % 12 + 1, 1)
 	return (start - datetime.timedelta(days=1),
-	        datetime.datetime(start.year + migrate.YEAR_OFFSET, start.month, 1))
+	        datetime.datetime(start.year + store.year_offset(), start.month, 1))
 
 
 def sample(reg, tuples, size):
@@ -370,7 +375,7 @@ def expected(reg, theirs, dims):
 	return total
 
 
-def probe(pg, reg, mapping, source, mssql, size, months):
+def probe(pg, reg, mapping, source, export, size, months):
 	import posting
 	cur = pg.cursor()
 	cur.execute("""SELECT a.action_id FROM "dForge".action a
@@ -392,7 +397,7 @@ def probe(pg, reg, mapping, source, mssql, size, months):
 	dates = [(None, None)] + [month_end(m) for m in months]
 	theirs = {}
 	for as_of, period in dates:
-		theirs[as_of] = one_c_balance(mssql, pg, reg, mapping, source, period)[2]
+		theirs[as_of] = one_c_balance(export, pg, reg, mapping, source, period)[2]
 	pool = {}
 	for got in theirs.values():
 		for t, v in got.items():
@@ -474,13 +479,13 @@ def main():
 			      % ("%s.%s" % (reg.module, reg.code), rows, bad, undated, off))
 	else:
 		mapping, source = migrate.column_map(), migrate.source_schema()
-		mssql = migrate.mssql_connect().cursor()
+		export = migrate.source_connect().cursor()
 		for reg in found:
 			if args.command == "probe":
-				clean = probe(pg, reg, mapping, source, mssql, args.sample,
+				clean = probe(pg, reg, mapping, source, export, args.sample,
 				              [args.asof] if args.asof else PROBE_MONTHS) and clean
 			else:
-				clean = verify(pg, reg, mapping, source, mssql, args.asof) and clean
+				clean = verify(pg, reg, mapping, source, export, args.asof) and clean
 	return 0 if clean else 1
 
 

package/skills/dforge-1c-import/scripts/lib/source_file.py

@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+"""
+A file infobase (`1Cv8.1CD`) as an export source: `onecd` reads it, this makes what it
+reads look like what a server infobase gives, so nothing downstream asks which one it was.
+
+Three differences are folded here and nowhere else:
+
+- NAMES. A file infobase keeps every name in capitals - `_REFERENCE12`, `_FLD345RREF` - where
+  MS SQL has `_Reference12`, `_Fld345RRef`, and the column map, the project's sources and
+  the library all spell them the MS SQL way. The spelling is put back from DBNames, which
+  carries every numbered kind in its own case (`AccumRgT`, `ReferenceChngR`, `VT`, `Fld`),
+  plus the fixed names 1C gives the standard columns. A name neither covers is a service
+  table's own business that nothing reads; it is capitalised and counted in the manifest.
+- TYPES. The schema is written in MS SQL's vocabulary - `binary(16)`, `nvarchar`, `numeric`,
+  `datetime`, `len` in bytes as `sys.columns` gives it - because that is what the generator
+  and the checks read.
+- VALUES. A boolean is `binary(1)` 0x01 / 0x00 as on MS SQL; the empty date is 0001-01-01.
+  A file infobase has no year offset: its dates are the real ones, and the manifest says so.
+
+What does NOT carry over is the row version. In a file `_Version` is a per-table counter,
+not MS SQL's database-wide rowversion, so a file export has no horizon and `delta` cannot
+read changes off it. It is exported as `binary(16)`, for completeness only.
+"""
+import os
+import re
+import struct
+
+import dump1c
+import onecd
+
+SERVICE = dict((n.upper(), n) for n in (
+	"Config", "ConfigSave", "Params", "Files", "DBSchema", "IBVersion", "v8users", "_YearOffset",
+	"FileName", "Creation", "Modified", "Attributes", "DataSize", "BinaryData", "PartNo",
+	"SerializedData",
+	# v8users and IBVersion
+	"ID", "Name", "Descr", "OSName", "Changed", "RolesID", "Show", "Data", "EAuth", "AdmRole",
+	"UsSprH", "PlatformVersionReq"))
+
+# The un-numbered segments of 1C's names as MS SQL spells them, between underscores:
+# `_Fld12_TYPE` is `Fld12` and `TYPE`. Harvested from a server infobase's whole schema
+# (2,668 tables, 28,715 columns) rather than remembered, so a list that misses one is
+# fixed by harvesting again. Where two spellings fold to one - `UserId` and `UserID` both
+# occur - the commoner one is here and `migrate` matches the other regardless of case.
+WORDS = dict((w.upper(), w) for w in (
+	"APDateFrom", "APDateTill", "AccountCtRRef", "AccountDtRRef", "AccountRRef", "ActionPeriod",
+	"ActivationCondition", "Active", "ActualPeriod", "BPDateFrom", "BPDateTill", "BaseAsAct",
+	"BaseCK", "BaseCKBaseCK", "BaseCKBaseCKRRef", "BaseCKLineNo", "BusinessProcess", "CKRefRRef",
+	"CalcKindRRef", "Code", "Completed", "ConfigChngR", "ConnectPrms", "ConstID", "Correspond",
+	"Date", "Description", "DimHash", "DimIsMetadata", "DimKindRRef", "DisplNumber", "DisplacedCK",
+	"DisplacedCKDisplCKRRef", "DisplacedCKLineNo", "DocumentRRef", "DocumentTRef", "EDHashCt",
+	"EDHashDt", "EnumOrder", "Executed", "ExtDataSrcId", "ExtProps", "FileName", "FinishTime",
+	"Folder", "HeadTaskRRef", "ID", "IDRRef", "IsMetadata", "JobKey", "KeyField", "Kind",
+	"KindRRef", "LeadingCK", "LeadingCKLeadingCK", "LeadingCKLineNo", "LineNo", "MDObjID",
+	"Marked", "MessageNo", "MetadataID", "MinPeriod", "Name", "NodeRRef", "NodeTRef", "Number",
+	"NumberPrefix", "ObjectKey", "OffBalance", "OrderField", "OwnerID", "OwnerIDRRef",
+	"Parameters", "ParentIDRRef", "Period", "Periodicity", "Point", "Posted", "Predefined",
+	"PredefinedBaseCKTableLine", "PredefinedDisplCKTableLine", "PredefinedID",
+	"PredefinedLeadingCKTableLine", "PredefinedObj", "ReceivedNo", "RecordKey", "RecordKind",
+	"Recorder", "RecorderRRef", "RecorderTRef", "RegID", "RepetitionFactor",
+	"RestartAttemptNumber", "RestartCount", "RestartPeriod", "RoutePointOrder", "SentNo",
+	"SettingsData", "SettingsKey", "SettingsPresentation", "SimpleKey", "SliceUsing", "Splitter",
+	"StartTime", "Started", "State", "Storno", "TurnoverOnly", "URL", "URLHash", "Use",
+	"UseSplitter", "UseTotals", "UserId", "UserName", "Value", "Value1", "Value2", "Value3",
+	"Value4", "Version", "YearOffset",
+	# a composite reference's parts, and a column's own type tag
+	"RRRef", "RTRef", "RRef", "TRef", "L", "N", "T", "S", "B"))
+# What follows a numbered kind inside one segment: `_Fld12RRef`, `_Fld12DtRRef`.
+SUFFIXES = ("CtRRef", "DtRRef", "RRef", "TRef", "Ct", "Dt", "")
+
+
+class Names(object):
+	"""Upper-case 1CD names -> the MS SQL spelling, from DBNames' kinds."""
+
+	def __init__(self, kinds):
+		self.kinds = sorted(set(kinds), key=len, reverse=True)
+		self.numbered = [(re.compile("^%s(\\d+)(%s)$" % (k.upper(), "|".join(s.upper() for s in SUFFIXES))), k)
+		                 for k in self.kinds]
+		self.guessed = set()
+
+	def __call__(self, name):
+		if name in SERVICE:
+			return SERVICE[name]
+		parts = name.replace("DATE_TIME", "DATE\x00TIME").split("_")
+		out = []
+		for at, part in enumerate(parts):
+			if part == "DATE\x00TIME":
+				out.append("Date_Time")
+			elif not part:
+				out.append(part)
+			elif part == "TYPE":
+				# A column of its own is `_Type` (a characteristic's value type); after
+				# another segment it is a composite reference's tag, `_Fld12_TYPE`.
+				out.append("Type" if at == 1 else "TYPE")
+			else:
+				out.append(self.segment(part, name))
+		return "_".join(out)
+
+	def segment(self, part, name):
+		for pattern, kind in self.numbered:
+			match = pattern.match(part)
+			if match:
+				suffix = next(s for s in SUFFIXES if s.upper() == match.group(2))
+				return kind + match.group(1) + suffix
+		if part in WORDS:
+			return WORDS[part]
+		# A kind with no number of its own: a service table such as `_AccOpt`.
+		for kind in self.kinds:
+			if part == kind.upper():
+				return kind
+		self.guessed.add(name)
+		match = re.match(r"^([A-Z]+)(\d*)(.*)$", part)
+		return (match.group(1).capitalize() + match.group(2) + match.group(3)) if match else part
+
+
+def numeric_bytes(precision):
+	"""What `sys.columns.max_length` says for numeric(p): the schema dump carries it as-is."""
+	return 5 if precision <= 9 else 9 if precision <= 19 else 13 if precision <= 28 else 17
+
+
+def column_type(field):
+	"""A 1CD field -> the schema dump's (type, len, precision, scale) in MS SQL terms."""
+	kind, n = field.type, field.length
+	if kind == "B":
+		return "binary", n, 0, 0
+	if kind == "L":
+		return "binary", 1, 0, 0
+	if kind == "N":
+		return "numeric", numeric_bytes(n), n, field.precision
+	if kind == "NC":
+		return "nchar", n * 2, 0, 0
+	if kind == "NVC":
+		return "nvarchar", n * 2, 0, 0
+	if kind == "NT":
+		return "nvarchar", -1, 0, 0
+	if kind == "I":
+		return "varbinary", -1, 0, 0
+	if kind == "DT":
+		return "datetime", 8, 23, 3
+	if kind == "RV":
+		return "binary", 16, 0, 0
+	raise ValueError("1CD field type %s" % kind)
+
+
+def canonical(field):
+	"""A function turning one of `onecd`'s values for this field into MS SQL's."""
+	if field.type == "L":
+		return lambda v: None if v is None else (b"\x01" if v else b"\x00")
+	if field.type == "RV":
+		return lambda v: None if v is None else struct.pack(">4I", *v)
+	return None
+
+
+class FileSource(object):
+	kind = "file"
+	year_offset = 0
+	consistent = "the file as it was read; copy it, or close the infobase, before exporting"
+
+	def __init__(self, path):
+		self.path = os.path.abspath(path)
+		self.db = onecd.Database(self.path)
+		self._files = {}
+		self._tables = None
+		self.names = Names(e["type"] for e in dump1c.parse_dbnames(self.read_file("Params", "DBNames")))
+		self.by_name = dict((self.names(t), t) for t in self.db.tables)
+
+	def describe(self):
+		return {"kind": self.kind, "path": self.path, "format": self.db.version,
+		        "page_size": self.db.page_size, "locale": self.db.locale}
+
+	def _table(self, name):
+		return self.db.tables[self.by_name[name] if name in self.by_name else name.upper()]
+
+	def _service(self, table):
+		"""A service table's files, {name: [(part, compressed body)]}, read in one pass: a
+		blob is reached only through its row, so a read per file would read them all again."""
+		table = table.upper()
+		if table not in self._files:
+			found = {}
+			for name, part, body in self.db.tables[table].rows(["FILENAME", "PARTNO", "BINARYDATA"]):
+				found.setdefault(name, []).append((part or 0, body or b""))
+			self._files[table] = found
+		return self._files[table]
+
+	def files(self, table):
+		return sorted(self._service(table))
+
+	def read_file(self, table, name):
+		parts = sorted(self._service(table).get(name, ()))
+		return dump1c.inflate(b"".join(body for _, body in parts))
+
+	def tables(self):
+		"""[(name, columns, rows)] - the schema dump's shape, every table in the file.
+
+		Counting a table's rows reads all of it - a file keeps no row count - so this is
+		worked out once per run, however many commands of `dump1c all` ask for it."""
+		if self._tables is None:
+			self._tables = self._read_tables()
+		return self._tables
+
+	def _read_tables(self):
+		out = []
+		for raw, table in self.db.tables.items():
+			columns = []
+			for field in table.fields.values():
+				kind, length, precision, scale = column_type(field)
+				columns.append({"column": self.names(field.name), "type": kind, "len": length,
+				                "precision": precision, "scale": scale, "nullable": field.nullable})
+			out.append((self.names(raw), columns, table.count()))
+		return out
+
+	def rows(self, table, columns, chunk):
+		"""Batches of tuples, in `columns` order (the MS SQL spelling)."""
+		t = self._table(table)
+		by_name = dict((self.names(f), f) for f in t.fields)
+		raw = [by_name[c] for c in columns]
+		fix = [canonical(t.fields[r]) for r in raw]
+		batch = []
+		for row in t.rows(raw):
+			batch.append(tuple(f(v) if f else v for f, v in zip(fix, row)))
+			if len(batch) >= chunk:
+				yield batch
+				batch = []
+		if batch:
+			yield batch
+
+	def horizon(self):
+		return None

package/skills/dforge-1c-import/scripts/lib/source_mssql.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+"""
+A server infobase on MS SQL as an export source. The only module in the library that
+speaks T-SQL to 1C; everything past the export reads `out/` (see `store.py`).
+
+Credentials come from env DF1C_USER / DF1C_PASS; without them, Windows auth is used.
+
+The export takes 1C's horizon - `MIN_ACTIVE_ROWVERSION()` - before it reads a table, so a
+delta read off the next export picks up everything this one may have missed.
+"""
+import os
+import sys
+
+import dump1c
+
+PREFERRED_DRIVERS = (
+	"ODBC Driver 18 for SQL Server",
+	"ODBC Driver 17 for SQL Server",
+	"ODBC Driver 13 for SQL Server",
+	"SQL Server Native Client 11.0",
+	"SQL Server",
+)
+
+# A server infobase created without the offset option still has the table; one that has
+# lost it is old enough to have been created with the offset.
+DEFAULT_YEAR_OFFSET = 2000
+
+
+def connect(server, db):
+	import pyodbc
+	available = set(pyodbc.drivers())
+	driver = next((d for d in PREFERRED_DRIVERS if d in available), None)
+	if driver is None:
+		raise RuntimeError("no SQL Server ODBC driver found: %r" % sorted(available))
+	parts = ["DRIVER={%s}" % driver, "SERVER=%s" % server, "DATABASE=%s" % db,
+	         "TrustServerCertificate=yes"]
+	user, pwd = os.environ.get("DF1C_USER"), os.environ.get("DF1C_PASS")
+	if user:
+		parts += ["UID=%s" % user, "PWD=%s" % pwd]
+	else:
+		parts += ["Trusted_Connection=yes"]
+	sys.stderr.write("[i] %s / %s via %s\n" % (server, db, driver))
+	return pyodbc.connect(";".join(parts), autocommit=True)
+
+
+class MssqlSource(object):
+	kind = "mssql"
+	consistent = "per table: each is read after its horizon is taken"
+
+	def __init__(self, server, db):
+		self.server, self.db = server, db
+		self.conn = connect(server, db)
+		self.cur = self.conn.cursor()
+		self._partno = {}
+		self._offset = None
+
+	def describe(self):
+		return {"kind": self.kind, "server": self.server, "database": self.db}
+
+	@property
+	def year_offset(self):
+		if self._offset is None:
+			try:
+				self.cur.execute("SELECT Offset FROM _YearOffset")
+				row = self.cur.fetchone()
+				self._offset = int(row[0]) if row else DEFAULT_YEAR_OFFSET
+			except Exception:
+				sys.stderr.write("[!] no _YearOffset; assuming %d\n" % DEFAULT_YEAR_OFFSET)
+				self._offset = DEFAULT_YEAR_OFFSET
+		return self._offset
+
+	def has_partno(self, table):
+		"""8.3 splits large files across rows by PartNo; 8.2 stores one row per file."""
+		if table not in self._partno:
+			self.cur.execute("SELECT COUNT(*) FROM sys.columns WHERE object_id = OBJECT_ID(?)"
+			                 " AND name = 'PartNo'", table)
+			self._partno[table] = self.cur.fetchone()[0] > 0
+		return self._partno[table]
+
+	def files(self, table):
+		self.cur.execute("SELECT DISTINCT FileName FROM %s" % table)
+		return sorted(r[0] for r in self.cur.fetchall())
+
+	def read_file(self, table, name):
+		"""Concatenate a file's parts in order, then inflate."""
+		order = " ORDER BY PartNo" if self.has_partno(table) else ""
+		self.cur.execute("SELECT BinaryData FROM %s WHERE FileName = ?%s" % (table, order), name)
+		blob = b"".join(bytes(r[0]) for r in self.cur.fetchall() if r[0] is not None)
+		return dump1c.inflate(blob)
+
+	def tables(self):
+		self.cur.execute("""SELECT t.name, c.name, ty.name, c.max_length, c.precision, c.scale, c.is_nullable
+		                    FROM sys.tables t
+		                    JOIN sys.columns c ON c.object_id = t.object_id
+		                    JOIN sys.types ty ON ty.user_type_id = c.user_type_id
+		                    ORDER BY t.name, c.column_id""")
+		tables = {}
+		for tn, cn, ty, ml, pr, sc, nullable in self.cur.fetchall():
+			tables.setdefault(tn, []).append({
+				"column": cn, "type": ty, "len": ml,
+				"precision": pr, "scale": sc, "nullable": bool(nullable),
+			})
+		self.cur.execute("""SELECT t.name, SUM(p.rows) FROM sys.tables t
+		                    JOIN sys.partitions p ON p.object_id = t.object_id AND p.index_id IN (0, 1)
+		                    GROUP BY t.name""")
+		rows = dict(self.cur.fetchall())
+		return [(name, cols, rows.get(name)) for name, cols in sorted(tables.items())]
+
+	def rows(self, table, columns, chunk, where=None):
+		"""Batches of rows; `where` is a T-SQL condition on the table's own columns."""
+		cur = self.conn.cursor()
+		cur.execute("SELECT %s FROM [%s]%s" % (", ".join("[%s]" % c for c in columns), table,
+		                                       " WHERE " + where if where else ""))
+		while True:
+			batch = cur.fetchmany(chunk)
+			if not batch:
+				break
+			yield [tuple(r) for r in batch]
+
+	def column(self, sql):
+		"""The first column of a query, every row."""
+		cur = self.conn.cursor()
+		cur.execute(sql)
+		out = []
+		while True:
+			batch = cur.fetchmany(100000)
+			if not batch:
+				return out
+			out.extend(r[0] for r in batch)
+
+	def triples(self, sql):
+		"""The first three columns of a query, every row."""
+		cur = self.conn.cursor()
+		cur.execute(sql)
+		return [(r[0], r[1], r[2]) for r in cur.fetchall()]
+
+	def horizon(self):
+		"""Below it every transaction had committed: a read after it misses nothing older."""
+		self.cur.execute("SELECT MIN_ACTIVE_ROWVERSION()")
+		return bytes(self.cur.fetchone()[0])
+
+	# ------------------------------------------------------------------- diagnostics
+
+	def databases(self):
+		"""Which databases does this login actually see? Separates a bad password from missing rights."""
+		self.cur.execute("SELECT name, state_desc, HAS_DBACCESS(name) FROM sys.databases ORDER BY name")
+		return self.cur.fetchall()
+
+	def probe(self):
+		self.cur.execute("SELECT @@VERSION")
+		lines = [self.cur.fetchone()[0].splitlines()[0]]
+		for table in ("Config", "ConfigSave", "Params", "DBSchema", "_YearOffset"):
+			try:
+				self.cur.execute("SELECT COUNT(*) FROM [%s]" % table)
+				lines.append("  %-12s rows=%s" % (table, self.cur.fetchone()[0]))
+			except Exception as exc:
+				lines.append("  %-12s -- %s" % (table, str(exc).split("]")[-1].strip()[:70]))
+		return lines