@dforge-core/dforge-mcp 0.2.45 → 0.2.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/README.md +2 -1
  3. package/dist/server.js +71 -3
  4. package/package.json +3 -3
  5. package/resources/docs/dsl-reference.md +1 -1
  6. package/resources/schemas/data-views.schema.json +50 -0
  7. package/scripts/install-skills.mjs +1 -0
  8. package/skills/dforge-1c-import/SKILL.md +375 -0
  9. package/skills/dforge-1c-import/references/environment.md +98 -0
  10. package/skills/dforge-1c-import/references/loading.md +164 -0
  11. package/skills/dforge-1c-import/references/measurements.md +210 -0
  12. package/skills/dforge-1c-import/references/print-forms.md +148 -0
  13. package/skills/dforge-1c-import/scripts/1c.py +178 -0
  14. package/skills/dforge-1c-import/scripts/README.md +164 -0
  15. package/skills/dforge-1c-import/scripts/lib/analyze.py +83 -0
  16. package/skills/dforge-1c-import/scripts/lib/attrs.py +278 -0
  17. package/skills/dforge-1c-import/scripts/lib/autospec.py +964 -0
  18. package/skills/dforge-1c-import/scripts/lib/binding.py +257 -0
  19. package/skills/dforge-1c-import/scripts/lib/bsl.py +811 -0
  20. package/skills/dforge-1c-import/scripts/lib/callgraph.py +717 -0
  21. package/skills/dforge-1c-import/scripts/lib/changes.py +326 -0
  22. package/skills/dforge-1c-import/scripts/lib/cli.py +53 -0
  23. package/skills/dforge-1c-import/scripts/lib/columnmap.py +346 -0
  24. package/skills/dforge-1c-import/scripts/lib/coverage.py +242 -0
  25. package/skills/dforge-1c-import/scripts/lib/custom.py +83 -0
  26. package/skills/dforge-1c-import/scripts/lib/delta.py +770 -0
  27. package/skills/dforge-1c-import/scripts/lib/deps.py +136 -0
  28. package/skills/dforge-1c-import/scripts/lib/docusage.py +106 -0
  29. package/skills/dforge-1c-import/scripts/lib/dump1c.py +408 -0
  30. package/skills/dforge-1c-import/scripts/lib/explore.py +55 -0
  31. package/skills/dforge-1c-import/scripts/lib/external.py +202 -0
  32. package/skills/dforge-1c-import/scripts/lib/extract_code.py +251 -0
  33. package/skills/dforge-1c-import/scripts/lib/forms.py +239 -0
  34. package/skills/dforge-1c-import/scripts/lib/genmodule.py +1432 -0
  35. package/skills/dforge-1c-import/scripts/lib/handlers.py +202 -0
  36. package/skills/dforge-1c-import/scripts/lib/handmap.py +142 -0
  37. package/skills/dforge-1c-import/scripts/lib/i18n_coverage.py +87 -0
  38. package/skills/dforge-1c-import/scripts/lib/idioms.py +273 -0
  39. package/skills/dforge-1c-import/scripts/lib/integrity.py +264 -0
  40. package/skills/dforge-1c-import/scripts/lib/inventory.py +535 -0
  41. package/skills/dforge-1c-import/scripts/lib/layers.py +106 -0
  42. package/skills/dforge-1c-import/scripts/lib/meta.py +325 -0
  43. package/skills/dforge-1c-import/scripts/lib/migrate.py +1318 -0
  44. package/skills/dforge-1c-import/scripts/lib/mxl.py +368 -0
  45. package/skills/dforge-1c-import/scripts/lib/naming.py +251 -0
  46. package/skills/dforge-1c-import/scripts/lib/onecd.py +269 -0
  47. package/skills/dforge-1c-import/scripts/lib/paths.py +111 -0
  48. package/skills/dforge-1c-import/scripts/lib/peek.py +68 -0
  49. package/skills/dforge-1c-import/scripts/lib/posted.py +218 -0
  50. package/skills/dforge-1c-import/scripts/lib/posting.py +655 -0
  51. package/skills/dforge-1c-import/scripts/lib/printgen.py +239 -0
  52. package/skills/dforge-1c-import/scripts/lib/rebuild.py +194 -0
  53. package/skills/dforge-1c-import/scripts/lib/recorded.py +170 -0
  54. package/skills/dforge-1c-import/scripts/lib/recorders.py +91 -0
  55. package/skills/dforge-1c-import/scripts/lib/reftargets.py +95 -0
  56. package/skills/dforge-1c-import/scripts/lib/reportgen.py +157 -0
  57. package/skills/dforge-1c-import/scripts/lib/roles.py +33 -0
  58. package/skills/dforge-1c-import/scripts/lib/seedstate.py +493 -0
  59. package/skills/dforge-1c-import/scripts/lib/sizing.py +184 -0
  60. package/skills/dforge-1c-import/scripts/lib/source_file.py +227 -0
  61. package/skills/dforge-1c-import/scripts/lib/source_mssql.py +158 -0
  62. package/skills/dforge-1c-import/scripts/lib/spec.py +400 -0
  63. package/skills/dforge-1c-import/scripts/lib/statements.py +774 -0
  64. package/skills/dforge-1c-import/scripts/lib/store.py +126 -0
  65. package/skills/dforge-1c-import/scripts/lib/subsystems.py +480 -0
  66. package/skills/dforge-1c-import/scripts/lib/terms.py +451 -0
  67. package/skills/dforge-1c-import/scripts/lib/terms_uk.py +555 -0
  68. package/skills/dforge-1c-import/scripts/lib/totals.py +387 -0
  69. package/skills/dforge-1c-import/scripts/lib/typemap.py +73 -0
  70. package/skills/dforge-1c-import/scripts/lib/unpack.py +136 -0
  71. package/skills/dforge-1c-import/scripts/lib/validate.py +508 -0
  72. package/skills/dforge-1c-import/scripts/project-template/.env.example +24 -0
  73. package/skills/dforge-1c-import/scripts/project-template/contours.py +47 -0
  74. package/skills/dforge-1c-import/scripts/project-template/sources.py +24 -0
  75. package/skills/dforge-mcp-author/SKILL.md +5 -0
  76. package/skills/dforge-mcp-author/references/action-dsl.md +1 -1
  77. package/skills/dforge-mcp-author/references/data-views.md +38 -2
  78. package/skills/dforge-mcp-author/references/formulas.md +32 -1
  79. package/skills/dforge-mcp-author/references/print-templates.md +30 -5
  80. package/skills/dforge-mcp-author/references/queries.md +17 -2
  81. package/skills/dforge-mcp-author/references/registers.md +1 -0
  82. package/skills/dforge-mcp-author/references/reports.md +1 -1
  83. package/skills/dforge-mcp-author/references/validation-checklist.md +1 -0
@@ -0,0 +1,387 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Check a register's `totals` cache against its journal, and rebuild it by hand when it is not.
5
+
6
+ The platform keeps a per-period turnover cache beside a register's journal (registers.md 8.4)
7
+ and maintains it at the posting seam: a posted movement is folded in, an unposted one taken
8
+ off. A bulk load is not a posting, so the cache does not see it. The platform REBUILDS a cache
9
+ only at install, and only when the register gains one or its shape changes
10
+ (`RegisterRegistrar.TotalsShapeChanged`); there is no standalone rebuild. So after a load:
11
+
12
+ python tools/1c.py totals check [module ...] # cache against a fresh fold, no writes
13
+ python tools/1c.py totals rebuild [module ...] # refill it from the journal
14
+
15
+ **The SQL here is the platform's, transcribed.** `check` is `BuildVerify` and `rebuild` is
16
+ `BuildRebuild`, from `dForge.Core/Services/RegisterTotals.cs` (balance / turnover) and
17
+ `RegisterAccountTotals.cs` (accounting), with the plan built from `"dForge".register` the way
18
+ `RegisterRegistrar.Totals.cs` builds it from the manifest:
19
+
20
+ - the key is (dimensions, period) - (account, cached dimensions, period) on an accounting
21
+ register, where `totals.dimensions` absent means all of them;
22
+ - the period is `date_trunc('<period>', <journalDate>)::date`, in the session's time zone,
23
+ which here is the server's (UTC) exactly as it is for the API's connections;
24
+ - a balance resource is split by the stamped direction (`> 0` in, `< 0` out); an accounting
25
+ one by side, and a compound posting is read twice - its debit side under its debit account
26
+ and side dimensions, its credit side under its credit ones - so it folds into two rows;
27
+ - a group carrying no resource value (`COUNT(field)` summed to 0) is not written;
28
+ - a row missing on either side reads as zeros, so a cached row of zeros is no disagreement.
29
+
30
+ A drift between this file and the C# is a bug in this file; re-read the two classes when the
31
+ platform changes either.
32
+
33
+ `rebuild` refuses an undated, unkeyed or undirected movement as install does. It differs from
34
+ the platform's in HOW it writes, not in what: the platform runs one `DELETE` + `INSERT ...
35
+ SELECT ... GROUP BY` inside the install transaction; this reads the same `SELECT` once (one
36
+ scan of the journal, autocommitted, nothing locked) and writes the result one year of periods
37
+ per transaction, so no transaction holds the cache for the length of a 15 GB scan. The writes
38
+ run with `session_replication_role = 'replica'`, as the loader's do: the cache carries the
39
+ audit trigger, and thousands of "inserted" rows of history describe nothing.
40
+
41
+ Which registers: every register in the tenant that declares `totals` - read from
42
+ `"dForge".register`, so nothing here names one.
43
+ """
44
+ from __future__ import print_function
45
+
46
+ import argparse
47
+ import collections
48
+ import json
49
+ import sys
50
+ import time
51
+
52
+ import migrate
53
+
54
+ GRANULARITIES = ("day", "month", "quarter", "year")
55
+
56
+ Plan = collections.namedtuple("Plan", [
57
+ "module", "code", "kind", "journal", "totals", "date", "period", "period_field",
58
+ # balance / turnover
59
+ "direction", "dimensions", "resources",
60
+ # accounting: (column, debit field, credit field) per key column, and per resource
61
+ # (debit journal field, credit journal field, debit cache field, credit cache field)
62
+ "compound", "account", "side_dimensions", "account_resources"])
63
+
64
+
65
+ def q(name):
66
+ if not name or '"' in name:
67
+ raise SystemExit("totals: invalid identifier %r" % (name,))
68
+ return '"%s"' % name
69
+
70
+
71
+ def _json(value):
72
+ if value is None or isinstance(value, (dict, list)):
73
+ return value
74
+ return json.loads(value)
75
+
76
+
77
+ def plans(cur, modules):
78
+ cur.execute("""
79
+ SELECT m.module_cd, r.register_cd, r.kind, r.journal_direction, r.journal_date,
80
+ r.dimensions, r.resources, r.accounting, r.totals,
81
+ je.schema_name, je.db_object, te.schema_name, te.db_object
82
+ FROM "dForge".register r
83
+ JOIN "dForge".module m ON m.module_id = r.module_id
84
+ JOIN "dForge".entity je ON je.entity_id = r.journal_entity_id
85
+ JOIN "dForge".entity te ON te.entity_id = r.totals_entity_id
86
+ WHERE r.totals IS NOT NULL
87
+ ORDER BY 1, 2""")
88
+ found = []
89
+ for (module, code, kind, direction, date, dims, resources, accounting, totals,
90
+ j_schema, j_table, t_schema, t_table) in cur.fetchall():
91
+ if modules and module not in modules:
92
+ continue
93
+ dims, resources = _json(dims) or [], _json(resources) or {}
94
+ accounting, totals = _json(accounting) or {}, _json(totals)
95
+ if totals["period"] not in GRANULARITIES:
96
+ raise SystemExit("totals: %s.%s - granularity %r" % (module, code, totals["period"]))
97
+ journal = "%s.%s" % (q(j_schema), q(j_table))
98
+ cache = "%s.%s" % (q(t_schema), q(t_table))
99
+ cached = totals.get("resources") or {}
100
+ if kind == "accounting":
101
+ compound = accounting.get("lineMode") == "compound"
102
+ sides = accounting.get("sideDimensions") or {}
103
+ account = ((totals["accountField"], accounting["debitAccount"], accounting["creditAccount"])
104
+ if compound else
105
+ (totals["accountField"], accounting["account"], accounting["account"]))
106
+ chosen = totals.get("dimensions")
107
+ chosen = dims if chosen is None else chosen
108
+ side_dims = [(d, sides[d]["debitField"], sides[d]["creditField"]) if d in sides
109
+ else (d, d, d) for d in chosen]
110
+ # Iterated in the declaration's order, as the C# does over def.Resources.
111
+ res = [((spec.get("debitField") or spec.get("field")),
112
+ (spec.get("creditField") or spec.get("field")),
113
+ cached[name]["debitField"], cached[name]["creditField"])
114
+ for name, spec in resources.items()]
115
+ found.append(Plan(module, code, kind, journal, cache, date, totals["period"],
116
+ totals["periodField"], None, None, None,
117
+ compound, account, side_dims, res))
118
+ else:
119
+ res = [(spec["field"], cached[name]["inField"], cached[name]["outField"])
120
+ for name, spec in resources.items()]
121
+ found.append(Plan(module, code, kind, journal, cache, date, totals["period"],
122
+ totals["periodField"], direction, dims, res,
123
+ None, None, None, None))
124
+ return found
125
+
126
+
127
+ def period_start(plan, expression):
128
+ return "date_trunc('%s', %s)::date" % (plan.period, expression)
129
+
130
+
131
+ # --------------------------------------------------------------------------- balance / turnover
132
+ # RegisterTotals.cs
133
+
134
+ def _directed(field, direction, increase):
135
+ return "SUM(%s) FILTER (WHERE %s %s 0)" % (q(field), q(direction), ">" if increase else "<")
136
+
137
+
138
+ def _carried_balance(plan):
139
+ return " + ".join("COUNT(%s)" % q(f) for f, _, _ in plan.resources)
140
+
141
+
142
+ def _balance_unplaceable(plan):
143
+ undated = "%s IS NULL" % q(plan.date)
144
+ unkeyed = " OR ".join("%s IS NULL" % q(d) for d in plan.dimensions) or "false"
145
+ undirected = "%s IS NULL" % q(plan.direction)
146
+ return ("SELECT COUNT(*) FILTER (WHERE %s), COUNT(*) FILTER (WHERE %s), "
147
+ "COUNT(*) FILTER (WHERE %s) FROM %s WHERE %s OR %s OR %s"
148
+ % (undated, unkeyed, undirected, plan.journal, undated, unkeyed, undirected))
149
+
150
+
151
+ def _balance_select(plan):
152
+ """BuildRebuild's SELECT, without its DELETE and INSERT."""
153
+ dims = [q(d) for d in plan.dimensions]
154
+ period = period_start(plan, q(plan.date))
155
+ sums = []
156
+ for field, _, _ in plan.resources:
157
+ sums.append("COALESCE(%s, 0)" % _directed(field, plan.direction, True))
158
+ sums.append("COALESCE(%s, 0)" % _directed(field, plan.direction, False))
159
+ return ("SELECT %s FROM %s GROUP BY %s HAVING %s > 0"
160
+ % (", ".join(dims + [period] + sums), plan.journal,
161
+ ", ".join(dims + [period]), _carried_balance(plan)))
162
+
163
+
164
+ def _balance_columns(plan):
165
+ key = [q(d) for d in plan.dimensions] + [q(plan.period_field)]
166
+ cached = [q(c) for _, i, o in plan.resources for c in (i, o)]
167
+ return key, cached
168
+
169
+
170
+ def _balance_verify(plan):
171
+ dims = [q(d) for d in plan.dimensions]
172
+ key, _ = _balance_columns(plan)
173
+ period = period_start(plan, q(plan.date))
174
+ computed = []
175
+ for field, i, o in plan.resources:
176
+ computed.append("COALESCE(%s, 0) AS %s" % (_directed(field, plan.direction, True), q(i)))
177
+ computed.append("COALESCE(%s, 0) AS %s" % (_directed(field, plan.direction, False), q(o)))
178
+ cols = [c for _, i, o in plan.resources for c in (i, o)]
179
+ differences = " OR ".join("COALESCE(c.%s, 0) <> COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols)
180
+ reported = key + ["COALESCE(c.%s, 0), COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols]
181
+ return ("WITH c AS (SELECT %s, %s AS %s, %s FROM %s GROUP BY %s, %s HAVING %s > 0) "
182
+ "SELECT %s FROM c FULL OUTER JOIN %s t USING (%s) WHERE %s"
183
+ % (", ".join(dims), period, q(plan.period_field), ", ".join(computed), plan.journal,
184
+ ", ".join(dims), period, _carried_balance(plan),
185
+ ", ".join(reported), plan.totals, ", ".join(key), differences))
186
+
187
+
188
+ # --------------------------------------------------------------------------- accounting
189
+ # RegisterAccountTotals.cs
190
+
191
+ def _side(plan, source, debit):
192
+ column, dt, ct = plan.account
193
+ columns = ["%s AS %s" % (q(dt if debit else ct), q(column)),
194
+ "%s AS %s" % (q(plan.date), q(plan.period_field))]
195
+ columns += ["%s AS %s" % (q(d_dt if debit else d_ct), q(d)) for d, d_dt, d_ct in plan.side_dimensions]
196
+ for j_dt, j_ct, c_dt, c_ct in plan.account_resources:
197
+ columns.append("%s AS %s" % (q(j_dt), q(c_dt)) if debit or not plan.compound
198
+ else "NULL AS %s" % q(c_dt))
199
+ columns.append("%s AS %s" % (q(j_ct), q(c_ct)) if not debit or not plan.compound
200
+ else "NULL AS %s" % q(c_ct))
201
+ return "SELECT %s FROM %s" % (", ".join(columns), source)
202
+
203
+
204
+ def _sides(plan, source):
205
+ debit = _side(plan, source, True)
206
+ return "%s UNION ALL %s" % (debit, _side(plan, source, False)) if plan.compound else debit
207
+
208
+
209
+ def _account_key(plan):
210
+ return [q(plan.account[0])] + [q(d) for d, _, _ in plan.side_dimensions] + [q(plan.period_field)]
211
+
212
+
213
+ def _account_key_select(plan):
214
+ return ([q(plan.account[0])] + [q(d) for d, _, _ in plan.side_dimensions]
215
+ + [period_start(plan, q(plan.period_field))])
216
+
217
+
218
+ def _account_cached(plan):
219
+ return [c for _, _, c_dt, c_ct in plan.account_resources for c in (c_dt, c_ct)]
220
+
221
+
222
+ def _carried_account(plan):
223
+ return " + ".join("COUNT(%s)" % q(c) for c in _account_cached(plan))
224
+
225
+
226
+ def _account_unplaceable(plan):
227
+ undated = "%s IS NULL" % q(plan.date)
228
+ columns = [plan.account[1]] + ([plan.account[2]] if plan.compound else [])
229
+ for _, d_dt, d_ct in plan.side_dimensions:
230
+ columns += [d_dt, d_ct] if plan.compound else [d_dt]
231
+ seen, unique = set(), []
232
+ for c in columns:
233
+ if c not in seen:
234
+ seen.add(c)
235
+ unique.append(c)
236
+ unkeyed = " OR ".join("%s IS NULL" % q(c) for c in unique)
237
+ return ("SELECT COUNT(*) FILTER (WHERE %s), COUNT(*) FILTER (WHERE %s), 0 FROM %s WHERE %s OR %s"
238
+ % (undated, unkeyed, plan.journal, undated, unkeyed))
239
+
240
+
241
+ def _account_select(plan):
242
+ """BuildRebuild's SELECT, without its DELETE and INSERT."""
243
+ key = _account_key(plan)
244
+ sums = ["COALESCE(SUM(%s), 0)" % q(c) for c in _account_cached(plan)]
245
+ return ("SELECT %s FROM (%s) s GROUP BY %s HAVING %s > 0"
246
+ % (", ".join(_account_key_select(plan) + sums), _sides(plan, plan.journal),
247
+ ", ".join(str(i) for i in range(1, len(key) + 1)), _carried_account(plan)))
248
+
249
+
250
+ def _account_verify(plan):
251
+ key = _account_key(plan)
252
+ cols = _account_cached(plan)
253
+ computed = ["COALESCE(SUM(%s), 0) AS %s" % (q(c), q(c)) for c in cols]
254
+ differences = " OR ".join("COALESCE(c.%s, 0) <> COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols)
255
+ reported = key + ["COALESCE(c.%s, 0), COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols]
256
+ selected = ["%s AS %s" % (e, k) for e, k in zip(_account_key_select(plan), key)] + computed
257
+ return ("WITH c AS (SELECT %s FROM (%s) s GROUP BY %s HAVING %s > 0) "
258
+ "SELECT %s FROM c FULL OUTER JOIN %s t USING (%s) WHERE %s"
259
+ % (", ".join(selected), _sides(plan, plan.journal),
260
+ ", ".join(str(i) for i in range(1, len(key) + 1)), _carried_account(plan),
261
+ ", ".join(reported), plan.totals, ", ".join(key), differences))
262
+
263
+
264
+ # --------------------------------------------------------------------------- the commands
265
+
266
+ def unplaceable(plan):
267
+ return _account_unplaceable(plan) if plan.kind == "accounting" else _balance_unplaceable(plan)
268
+
269
+
270
+ def select(plan):
271
+ return _account_select(plan) if plan.kind == "accounting" else _balance_select(plan)
272
+
273
+
274
+ def verify(plan):
275
+ return _account_verify(plan) if plan.kind == "accounting" else _balance_verify(plan)
276
+
277
+
278
+ def columns(plan):
279
+ if plan.kind == "accounting":
280
+ return _account_key(plan), [q(c) for c in _account_cached(plan)]
281
+ return _balance_columns(plan)
282
+
283
+
284
+ def name(plan):
285
+ return "%s.%s" % (plan.module, plan.code)
286
+
287
+
288
+ def check(pg, plan, show):
289
+ """(cached rows, disagreeing keys) - the cache against a fresh fold of the journal."""
290
+ cur = pg.cursor()
291
+ cur.execute("SELECT count(*) FROM %s" % plan.totals)
292
+ cached = cur.fetchone()[0]
293
+ started = time.time()
294
+ cur.execute(verify(plan))
295
+ rows = cur.fetchall()
296
+ key, cols = columns(plan)
297
+ print("%-28s cache %7d rows disagreeing keys %d (%.0fs)"
298
+ % (name(plan), cached, len(rows), time.time() - started))
299
+ if rows and show:
300
+ width = len(key)
301
+ print(" %s" % " ".join([k.strip('"') for k in key]
302
+ + ["%s fold/cache" % c.strip('"') for c in cols]))
303
+ for row in rows[:show]:
304
+ pairs = ["%s/%s" % (row[width + 2 * i], row[width + 2 * i + 1])
305
+ for i in range(len(cols))]
306
+ print(" %s" % " ".join([str(v) for v in row[:width]] + pairs))
307
+ if len(rows) > show:
308
+ print(" ... %d more" % (len(rows) - show))
309
+ return cached, len(rows)
310
+
311
+
312
+ def rebuild(pg, plan):
313
+ cur = pg.cursor()
314
+ started = time.time()
315
+ cur.execute(unplaceable(plan))
316
+ found = cur.fetchone()
317
+ if found and any(found):
318
+ raise SystemExit("totals: %s - the journal holds %d undated, %d unkeyed and %d undirected"
319
+ " movement(s); install would refuse the cache, and so does this"
320
+ % ((name(plan),) + tuple(found)))
321
+ scanned = time.time()
322
+ cur.execute(select(plan))
323
+ rows = cur.fetchall()
324
+ read = time.time()
325
+
326
+ key, cols = columns(plan)
327
+ at = len(key) - 1 # the period is the last key column
328
+ by_year = collections.defaultdict(list)
329
+ for row in rows:
330
+ by_year[row[at].year].append(row)
331
+ cur.execute("SELECT DISTINCT extract(year FROM %s)::int FROM %s"
332
+ % (q(plan.period_field), plan.totals))
333
+ years = sorted(set(by_year) | set(r[0] for r in cur.fetchall()))
334
+
335
+ insert = ("INSERT INTO %s (%s) VALUES (%s)"
336
+ % (plan.totals, ", ".join(key + cols), ", ".join(["%s"] * (len(key) + len(cols)))))
337
+ pg.autocommit = False
338
+ try:
339
+ for year in years:
340
+ cur.execute("SET LOCAL session_replication_role = 'replica'")
341
+ cur.execute("DELETE FROM %s WHERE %s >= %%s AND %s < %%s"
342
+ % (plan.totals, q(plan.period_field), q(plan.period_field)),
343
+ ("%d-01-01" % year, "%d-01-01" % (year + 1)))
344
+ if by_year.get(year):
345
+ cur.executemany(insert, by_year[year])
346
+ pg.commit()
347
+ finally:
348
+ pg.autocommit = True
349
+ print("%-28s %7d rows written over %d year(s) checks %.0fs, fold %.0fs, write %.0fs"
350
+ % (name(plan), len(rows), len(years), scanned - started, read - scanned,
351
+ time.time() - read))
352
+
353
+
354
+ def main():
355
+ migrate.load_env()
356
+ ap = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0])
357
+ ap.add_argument("command", choices=("check", "rebuild", "sql"))
358
+ ap.add_argument("module", nargs="*", help="limit to these modules")
359
+ ap.add_argument("--show", type=int, default=10, metavar="N",
360
+ help="check: disagreeing keys to print")
361
+ args = ap.parse_args()
362
+
363
+ pg = migrate.pg_connect()
364
+ # Reads are long scans: autocommit, so no transaction stays open across them.
365
+ pg.autocommit = True
366
+ cur = pg.cursor()
367
+ cur.execute("SHOW TimeZone")
368
+ sys.stderr.write("[i] session time zone %s (the period of a timestamptz depends on it)\n"
369
+ % cur.fetchone()[0])
370
+ found = plans(cur, set(args.module))
371
+ if not found:
372
+ print("no register declares totals%s" % (" in " + ", ".join(args.module)
373
+ if args.module else ""))
374
+ return 0
375
+ clean = True
376
+ for plan in found:
377
+ if args.command == "sql":
378
+ print("-- %s\n-- verify\n%s;\n-- rebuild select\n%s;\n" % (name(plan), verify(plan), select(plan)))
379
+ elif args.command == "check":
380
+ clean = check(pg, plan, args.show)[1] == 0 and clean
381
+ else:
382
+ rebuild(pg, plan)
383
+ return 0 if clean else 1
384
+
385
+
386
+ if __name__ == "__main__":
387
+ sys.exit(main())
@@ -0,0 +1,73 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Build typeGuid -> object map.
5
+
6
+ An attribute typed "СправочникСсылка.рзАвтотранспортныеПредприятия" does not carry
7
+ the catalog's object GUID; it carries one of the *type* GUIDs the catalog declares
8
+ for itself, in a block near the top of its own file:
9
+
10
+ {1,
11
+ {53,dc8f143c-...,899caee9-...,489d7014-...,...}
12
+
13
+ Every GUID in that block belongs to that one object, so mapping all of them is
14
+ enough to resolve any reference back to its target.
15
+ """
16
+ from __future__ import print_function
17
+
18
+ import codecs
19
+ import collections
20
+ import json
21
+ import os
22
+ import re
23
+ import sys
24
+
25
+ import meta
26
+
27
+ OUT = meta.OUT
28
+
29
+ # Every object file opens as {1,\r\n{53,<its own type guids>,{0,{1,...identity...
30
+ # The type guids are the unbroken GUID run that follows the two leading counts.
31
+ HEAD = re.compile(r"^\{\d+,\s*\{\d+,\s*((?:[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
32
+ r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12},\s*)+)")
33
+
34
+
35
+ def own_type_guids(text):
36
+ m = HEAD.match(text)
37
+ if not m:
38
+ return []
39
+ return re.findall(r"[0-9a-fA-F-]{36}", m.group(1))
40
+
41
+
42
+ def main():
43
+ with codecs.open(os.path.join(OUT, "objects.json"), "r", "utf-8") as f:
44
+ objects = json.load(f)
45
+
46
+ type_map = {}
47
+ clashes = 0
48
+ for obj in objects:
49
+ text = meta.load(obj["guid"])
50
+ if text is None:
51
+ continue
52
+ guids = own_type_guids(text)
53
+ for i, g in enumerate(guids):
54
+ if g in type_map and type_map[g]["guid"] != obj["guid"]:
55
+ clashes += 1
56
+ continue
57
+ type_map[g] = {"guid": obj["guid"], "name": obj["name"],
58
+ "family": obj["family"], "index": i + 1}
59
+ # the object GUID itself is sometimes used directly as a type
60
+ type_map.setdefault(obj["guid"], {"guid": obj["guid"], "name": obj["name"],
61
+ "family": obj["family"], "index": 0})
62
+
63
+ with codecs.open(os.path.join(OUT, "typemap.json"), "w", "utf-8") as f:
64
+ json.dump(type_map, f, ensure_ascii=False, indent="\t")
65
+ print("objects=%d type guids mapped=%d clashes=%d" % (len(objects), len(type_map), clashes))
66
+
67
+ # sanity check against a reference we already resolved by hand
68
+ probe = type_map.get("489d7014-f13e-4dc7-ae5e-e5c51da99ea2")
69
+ print("probe 489d7014 ->", probe)
70
+
71
+
72
+ if __name__ == "__main__":
73
+ main()
@@ -0,0 +1,136 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Unpack a 1C inner container - the `<guid>.0` entries of the Config table.
5
+
6
+ Those hold the object's modules and forms. The format is a paged container:
7
+
8
+ header FF FF FF 7F | nextPage u32 | pageSize u32 | storageVer u32
9
+ block \\r\\n <8 hex docSize> ' ' <8 hex pageSize> ' ' <8 hex nextPage> ' ' \\r\\n
10
+ followed by pageSize bytes, chained through nextPage until docSize is read
11
+ TOC the block at offset 16: 12-byte entries of (attrsAddr, dataAddr, reserved)
12
+ attrs created u64 | modified u64 | reserved u32 | name, UTF-16LE
13
+
14
+ Each entry's data may itself be raw deflate.
15
+
16
+ python tools/unpack.py <ObjectName> # list what the object's container holds
17
+ python tools/unpack.py <ObjectName> --dump DIR # write every entry to DIR
18
+ python tools/unpack.py <ObjectName> --code # print the module texts
19
+ """
20
+ from __future__ import print_function
21
+
22
+ import codecs
23
+ import os
24
+ import struct
25
+ import sys
26
+ import zlib
27
+
28
+ import attrs
29
+ import meta
30
+
31
+ FREE = 0x7FFFFFFF
32
+ CONFIG = meta.CONFIG
33
+
34
+
35
+ def read_block(buf, addr):
36
+ head = buf[addr:addr + 31]
37
+ if head[:2] != b"\r\n":
38
+ raise ValueError("no block header at %d" % addr)
39
+ doc_size = int(head[2:10], 16)
40
+ page_size = int(head[11:19], 16)
41
+ next_addr = int(head[20:28], 16)
42
+ out = bytearray(buf[addr + 31:addr + 31 + page_size])
43
+ while next_addr != FREE and len(out) < doc_size:
44
+ head = buf[next_addr:next_addr + 31]
45
+ page_size = int(head[11:19], 16)
46
+ following = int(head[20:28], 16)
47
+ out += buf[next_addr + 31:next_addr + 31 + page_size]
48
+ next_addr = following
49
+ return bytes(out[:doc_size])
50
+
51
+
52
+ def inflate(data):
53
+ try:
54
+ return zlib.decompressobj(-15).decompress(data)
55
+ except zlib.error:
56
+ return data
57
+
58
+
59
+ def entries(buf):
60
+ """[(name, content), ...] for one container."""
61
+ if buf[:4] != b"\xff\xff\xff\x7f":
62
+ return []
63
+ toc = read_block(buf, 16)
64
+ out = []
65
+ for offset in range(0, len(toc) - 11, 12):
66
+ attrs_addr, data_addr, _ = struct.unpack_from("<III", toc, offset)
67
+ if attrs_addr == FREE or data_addr == FREE:
68
+ continue
69
+ header = read_block(buf, attrs_addr)
70
+ name = header[20:].decode("utf-16-le", "replace").rstrip("\x00")
71
+ out.append((name, inflate(read_block(buf, data_addr))))
72
+ return out
73
+
74
+
75
+ def containers_of(guid):
76
+ """Every Config file belonging to this object that is a container."""
77
+ found = []
78
+ for suffix in (".0", ".1", ".2"):
79
+ path = os.path.join(CONFIG, guid + suffix + ".bin")
80
+ if os.path.isfile(path):
81
+ with open(path, "rb") as f:
82
+ found.append((guid + suffix, f.read()))
83
+ return found
84
+
85
+
86
+ def as_text(blob):
87
+ for encoding in ("utf-8-sig", "utf-16-le", "cp1251"):
88
+ try:
89
+ text = blob.decode(encoding)
90
+ except (UnicodeDecodeError, LookupError):
91
+ continue
92
+ if text.count("�") == 0:
93
+ return text
94
+ return None
95
+
96
+
97
+ def main():
98
+ name = sys.argv[1]
99
+ objects = attrs.load_json("objects.json")
100
+ guid = next((o["guid"] for o in objects if o.get("name") == name), None)
101
+ if not guid:
102
+ print("no such object: %s" % name)
103
+ return
104
+
105
+ found = containers_of(guid)
106
+ if not found:
107
+ print("%s has no container files" % name)
108
+ return
109
+
110
+ dump_dir = None
111
+ if "--dump" in sys.argv:
112
+ dump_dir = sys.argv[sys.argv.index("--dump") + 1]
113
+ if not os.path.isdir(dump_dir):
114
+ os.makedirs(dump_dir)
115
+
116
+ for container, blob in found:
117
+ items = entries(blob)
118
+ print("== %s (%d entries, %d bytes)" % (container, len(items), len(blob)))
119
+ for entry_name, content in items:
120
+ text = as_text(content)
121
+ kind = "text %d chars" % len(text) if text else "binary %d bytes" % len(content)
122
+ print(" %-28s %s" % (entry_name or "(unnamed)", kind))
123
+ if dump_dir:
124
+ safe = (entry_name or "unnamed").replace("/", "_")
125
+ path = os.path.join(dump_dir, "%s.%s" % (container, safe))
126
+ with open(path, "wb") as f:
127
+ f.write(content)
128
+ if "--code" in sys.argv and text and "Процедура" in text or \
129
+ "--code" in sys.argv and text and "Функция" in text:
130
+ print("-" * 70)
131
+ print(text[:4000])
132
+ print("-" * 70)
133
+
134
+
135
+ if __name__ == "__main__":
136
+ main()