@dforge-core/dforge-mcp 0.2.45 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +2 -1
- package/dist/server.js +71 -3
- package/package.json +3 -3
- package/resources/docs/dsl-reference.md +1 -1
- package/resources/schemas/data-views.schema.json +50 -0
- package/scripts/install-skills.mjs +1 -0
- package/skills/dforge-1c-import/SKILL.md +375 -0
- package/skills/dforge-1c-import/references/environment.md +98 -0
- package/skills/dforge-1c-import/references/loading.md +164 -0
- package/skills/dforge-1c-import/references/measurements.md +210 -0
- package/skills/dforge-1c-import/references/print-forms.md +148 -0
- package/skills/dforge-1c-import/scripts/1c.py +178 -0
- package/skills/dforge-1c-import/scripts/README.md +164 -0
- package/skills/dforge-1c-import/scripts/lib/analyze.py +83 -0
- package/skills/dforge-1c-import/scripts/lib/attrs.py +278 -0
- package/skills/dforge-1c-import/scripts/lib/autospec.py +964 -0
- package/skills/dforge-1c-import/scripts/lib/binding.py +257 -0
- package/skills/dforge-1c-import/scripts/lib/bsl.py +811 -0
- package/skills/dforge-1c-import/scripts/lib/callgraph.py +717 -0
- package/skills/dforge-1c-import/scripts/lib/changes.py +326 -0
- package/skills/dforge-1c-import/scripts/lib/cli.py +53 -0
- package/skills/dforge-1c-import/scripts/lib/columnmap.py +346 -0
- package/skills/dforge-1c-import/scripts/lib/coverage.py +242 -0
- package/skills/dforge-1c-import/scripts/lib/custom.py +83 -0
- package/skills/dforge-1c-import/scripts/lib/delta.py +770 -0
- package/skills/dforge-1c-import/scripts/lib/deps.py +136 -0
- package/skills/dforge-1c-import/scripts/lib/docusage.py +106 -0
- package/skills/dforge-1c-import/scripts/lib/dump1c.py +408 -0
- package/skills/dforge-1c-import/scripts/lib/explore.py +55 -0
- package/skills/dforge-1c-import/scripts/lib/external.py +202 -0
- package/skills/dforge-1c-import/scripts/lib/extract_code.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/forms.py +239 -0
- package/skills/dforge-1c-import/scripts/lib/genmodule.py +1432 -0
- package/skills/dforge-1c-import/scripts/lib/handlers.py +202 -0
- package/skills/dforge-1c-import/scripts/lib/handmap.py +142 -0
- package/skills/dforge-1c-import/scripts/lib/i18n_coverage.py +87 -0
- package/skills/dforge-1c-import/scripts/lib/idioms.py +273 -0
- package/skills/dforge-1c-import/scripts/lib/integrity.py +264 -0
- package/skills/dforge-1c-import/scripts/lib/inventory.py +535 -0
- package/skills/dforge-1c-import/scripts/lib/layers.py +106 -0
- package/skills/dforge-1c-import/scripts/lib/meta.py +325 -0
- package/skills/dforge-1c-import/scripts/lib/migrate.py +1318 -0
- package/skills/dforge-1c-import/scripts/lib/mxl.py +368 -0
- package/skills/dforge-1c-import/scripts/lib/naming.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/onecd.py +269 -0
- package/skills/dforge-1c-import/scripts/lib/paths.py +111 -0
- package/skills/dforge-1c-import/scripts/lib/peek.py +68 -0
- package/skills/dforge-1c-import/scripts/lib/posted.py +218 -0
- package/skills/dforge-1c-import/scripts/lib/posting.py +655 -0
- package/skills/dforge-1c-import/scripts/lib/printgen.py +239 -0
- package/skills/dforge-1c-import/scripts/lib/rebuild.py +194 -0
- package/skills/dforge-1c-import/scripts/lib/recorded.py +170 -0
- package/skills/dforge-1c-import/scripts/lib/recorders.py +91 -0
- package/skills/dforge-1c-import/scripts/lib/reftargets.py +95 -0
- package/skills/dforge-1c-import/scripts/lib/reportgen.py +157 -0
- package/skills/dforge-1c-import/scripts/lib/roles.py +33 -0
- package/skills/dforge-1c-import/scripts/lib/seedstate.py +493 -0
- package/skills/dforge-1c-import/scripts/lib/sizing.py +184 -0
- package/skills/dforge-1c-import/scripts/lib/source_file.py +227 -0
- package/skills/dforge-1c-import/scripts/lib/source_mssql.py +158 -0
- package/skills/dforge-1c-import/scripts/lib/spec.py +400 -0
- package/skills/dforge-1c-import/scripts/lib/statements.py +774 -0
- package/skills/dforge-1c-import/scripts/lib/store.py +126 -0
- package/skills/dforge-1c-import/scripts/lib/subsystems.py +480 -0
- package/skills/dforge-1c-import/scripts/lib/terms.py +451 -0
- package/skills/dforge-1c-import/scripts/lib/terms_uk.py +555 -0
- package/skills/dforge-1c-import/scripts/lib/totals.py +387 -0
- package/skills/dforge-1c-import/scripts/lib/typemap.py +73 -0
- package/skills/dforge-1c-import/scripts/lib/unpack.py +136 -0
- package/skills/dforge-1c-import/scripts/lib/validate.py +508 -0
- package/skills/dforge-1c-import/scripts/project-template/.env.example +24 -0
- package/skills/dforge-1c-import/scripts/project-template/contours.py +47 -0
- package/skills/dforge-1c-import/scripts/project-template/sources.py +24 -0
- package/skills/dforge-mcp-author/SKILL.md +5 -0
- package/skills/dforge-mcp-author/references/action-dsl.md +1 -1
- package/skills/dforge-mcp-author/references/data-views.md +38 -2
- package/skills/dforge-mcp-author/references/formulas.md +32 -1
- package/skills/dforge-mcp-author/references/print-templates.md +30 -5
- package/skills/dforge-mcp-author/references/queries.md +17 -2
- package/skills/dforge-mcp-author/references/registers.md +1 -0
- package/skills/dforge-mcp-author/references/reports.md +1 -1
- package/skills/dforge-mcp-author/references/validation-checklist.md +1 -0
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Check a register's `totals` cache against its journal, and rebuild it by hand when it is not.
|
|
5
|
+
|
|
6
|
+
The platform keeps a per-period turnover cache beside a register's journal (registers.md 8.4)
|
|
7
|
+
and maintains it at the posting seam: a posted movement is folded in, an unposted one taken
|
|
8
|
+
off. A bulk load is not a posting, so the cache does not see it. The platform REBUILDS a cache
|
|
9
|
+
only at install, and only when the register gains one or its shape changes
|
|
10
|
+
(`RegisterRegistrar.TotalsShapeChanged`); there is no standalone rebuild. So after a load:
|
|
11
|
+
|
|
12
|
+
python tools/1c.py totals check [module ...] # cache against a fresh fold, no writes
|
|
13
|
+
python tools/1c.py totals rebuild [module ...] # refill it from the journal
|
|
14
|
+
|
|
15
|
+
**The SQL here is the platform's, transcribed.** `check` is `BuildVerify` and `rebuild` is
|
|
16
|
+
`BuildRebuild`, from `dForge.Core/Services/RegisterTotals.cs` (balance / turnover) and
|
|
17
|
+
`RegisterAccountTotals.cs` (accounting), with the plan built from `"dForge".register` the way
|
|
18
|
+
`RegisterRegistrar.Totals.cs` builds it from the manifest:
|
|
19
|
+
|
|
20
|
+
- the key is (dimensions, period) - (account, cached dimensions, period) on an accounting
|
|
21
|
+
register, where `totals.dimensions` absent means all of them;
|
|
22
|
+
- the period is `date_trunc('<period>', <journalDate>)::date`, in the session's time zone,
|
|
23
|
+
which here is the server's (UTC) exactly as it is for the API's connections;
|
|
24
|
+
- a balance resource is split by the stamped direction (`> 0` in, `< 0` out); an accounting
|
|
25
|
+
one by side, and a compound posting is read twice - its debit side under its debit account
|
|
26
|
+
and side dimensions, its credit side under its credit ones - so it folds into two rows;
|
|
27
|
+
- a group carrying no resource value (`COUNT(field)` summed to 0) is not written;
|
|
28
|
+
- a row missing on either side reads as zeros, so a cached row of zeros is no disagreement.
|
|
29
|
+
|
|
30
|
+
A drift between this file and the C# is a bug in this file; re-read the two classes when the
|
|
31
|
+
platform changes either.
|
|
32
|
+
|
|
33
|
+
`rebuild` refuses an undated, unkeyed or undirected movement as install does. It differs from
|
|
34
|
+
the platform's in HOW it writes, not in what: the platform runs one `DELETE` + `INSERT ...
|
|
35
|
+
SELECT ... GROUP BY` inside the install transaction; this reads the same `SELECT` once (one
|
|
36
|
+
scan of the journal, autocommitted, nothing locked) and writes the result one year of periods
|
|
37
|
+
per transaction, so no transaction holds the cache for the length of a 15 GB scan. The writes
|
|
38
|
+
run with `session_replication_role = 'replica'`, as the loader's do: the cache carries the
|
|
39
|
+
audit trigger, and thousands of "inserted" rows of history describe nothing.
|
|
40
|
+
|
|
41
|
+
Which registers: every register in the tenant that declares `totals` - read from
|
|
42
|
+
`"dForge".register`, so nothing here names one.
|
|
43
|
+
"""
|
|
44
|
+
from __future__ import print_function
|
|
45
|
+
|
|
46
|
+
import argparse
|
|
47
|
+
import collections
|
|
48
|
+
import json
|
|
49
|
+
import sys
|
|
50
|
+
import time
|
|
51
|
+
|
|
52
|
+
import migrate
|
|
53
|
+
|
|
54
|
+
GRANULARITIES = ("day", "month", "quarter", "year")
|
|
55
|
+
|
|
56
|
+
Plan = collections.namedtuple("Plan", [
|
|
57
|
+
"module", "code", "kind", "journal", "totals", "date", "period", "period_field",
|
|
58
|
+
# balance / turnover
|
|
59
|
+
"direction", "dimensions", "resources",
|
|
60
|
+
# accounting: (column, debit field, credit field) per key column, and per resource
|
|
61
|
+
# (debit journal field, credit journal field, debit cache field, credit cache field)
|
|
62
|
+
"compound", "account", "side_dimensions", "account_resources"])
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def q(name):
|
|
66
|
+
if not name or '"' in name:
|
|
67
|
+
raise SystemExit("totals: invalid identifier %r" % (name,))
|
|
68
|
+
return '"%s"' % name
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _json(value):
|
|
72
|
+
if value is None or isinstance(value, (dict, list)):
|
|
73
|
+
return value
|
|
74
|
+
return json.loads(value)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def plans(cur, modules):
|
|
78
|
+
cur.execute("""
|
|
79
|
+
SELECT m.module_cd, r.register_cd, r.kind, r.journal_direction, r.journal_date,
|
|
80
|
+
r.dimensions, r.resources, r.accounting, r.totals,
|
|
81
|
+
je.schema_name, je.db_object, te.schema_name, te.db_object
|
|
82
|
+
FROM "dForge".register r
|
|
83
|
+
JOIN "dForge".module m ON m.module_id = r.module_id
|
|
84
|
+
JOIN "dForge".entity je ON je.entity_id = r.journal_entity_id
|
|
85
|
+
JOIN "dForge".entity te ON te.entity_id = r.totals_entity_id
|
|
86
|
+
WHERE r.totals IS NOT NULL
|
|
87
|
+
ORDER BY 1, 2""")
|
|
88
|
+
found = []
|
|
89
|
+
for (module, code, kind, direction, date, dims, resources, accounting, totals,
|
|
90
|
+
j_schema, j_table, t_schema, t_table) in cur.fetchall():
|
|
91
|
+
if modules and module not in modules:
|
|
92
|
+
continue
|
|
93
|
+
dims, resources = _json(dims) or [], _json(resources) or {}
|
|
94
|
+
accounting, totals = _json(accounting) or {}, _json(totals)
|
|
95
|
+
if totals["period"] not in GRANULARITIES:
|
|
96
|
+
raise SystemExit("totals: %s.%s - granularity %r" % (module, code, totals["period"]))
|
|
97
|
+
journal = "%s.%s" % (q(j_schema), q(j_table))
|
|
98
|
+
cache = "%s.%s" % (q(t_schema), q(t_table))
|
|
99
|
+
cached = totals.get("resources") or {}
|
|
100
|
+
if kind == "accounting":
|
|
101
|
+
compound = accounting.get("lineMode") == "compound"
|
|
102
|
+
sides = accounting.get("sideDimensions") or {}
|
|
103
|
+
account = ((totals["accountField"], accounting["debitAccount"], accounting["creditAccount"])
|
|
104
|
+
if compound else
|
|
105
|
+
(totals["accountField"], accounting["account"], accounting["account"]))
|
|
106
|
+
chosen = totals.get("dimensions")
|
|
107
|
+
chosen = dims if chosen is None else chosen
|
|
108
|
+
side_dims = [(d, sides[d]["debitField"], sides[d]["creditField"]) if d in sides
|
|
109
|
+
else (d, d, d) for d in chosen]
|
|
110
|
+
# Iterated in the declaration's order, as the C# does over def.Resources.
|
|
111
|
+
res = [((spec.get("debitField") or spec.get("field")),
|
|
112
|
+
(spec.get("creditField") or spec.get("field")),
|
|
113
|
+
cached[name]["debitField"], cached[name]["creditField"])
|
|
114
|
+
for name, spec in resources.items()]
|
|
115
|
+
found.append(Plan(module, code, kind, journal, cache, date, totals["period"],
|
|
116
|
+
totals["periodField"], None, None, None,
|
|
117
|
+
compound, account, side_dims, res))
|
|
118
|
+
else:
|
|
119
|
+
res = [(spec["field"], cached[name]["inField"], cached[name]["outField"])
|
|
120
|
+
for name, spec in resources.items()]
|
|
121
|
+
found.append(Plan(module, code, kind, journal, cache, date, totals["period"],
|
|
122
|
+
totals["periodField"], direction, dims, res,
|
|
123
|
+
None, None, None, None))
|
|
124
|
+
return found
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def period_start(plan, expression):
|
|
128
|
+
return "date_trunc('%s', %s)::date" % (plan.period, expression)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# --------------------------------------------------------------------------- balance / turnover
|
|
132
|
+
# RegisterTotals.cs
|
|
133
|
+
|
|
134
|
+
def _directed(field, direction, increase):
|
|
135
|
+
return "SUM(%s) FILTER (WHERE %s %s 0)" % (q(field), q(direction), ">" if increase else "<")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _carried_balance(plan):
|
|
139
|
+
return " + ".join("COUNT(%s)" % q(f) for f, _, _ in plan.resources)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _balance_unplaceable(plan):
|
|
143
|
+
undated = "%s IS NULL" % q(plan.date)
|
|
144
|
+
unkeyed = " OR ".join("%s IS NULL" % q(d) for d in plan.dimensions) or "false"
|
|
145
|
+
undirected = "%s IS NULL" % q(plan.direction)
|
|
146
|
+
return ("SELECT COUNT(*) FILTER (WHERE %s), COUNT(*) FILTER (WHERE %s), "
|
|
147
|
+
"COUNT(*) FILTER (WHERE %s) FROM %s WHERE %s OR %s OR %s"
|
|
148
|
+
% (undated, unkeyed, undirected, plan.journal, undated, unkeyed, undirected))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _balance_select(plan):
|
|
152
|
+
"""BuildRebuild's SELECT, without its DELETE and INSERT."""
|
|
153
|
+
dims = [q(d) for d in plan.dimensions]
|
|
154
|
+
period = period_start(plan, q(plan.date))
|
|
155
|
+
sums = []
|
|
156
|
+
for field, _, _ in plan.resources:
|
|
157
|
+
sums.append("COALESCE(%s, 0)" % _directed(field, plan.direction, True))
|
|
158
|
+
sums.append("COALESCE(%s, 0)" % _directed(field, plan.direction, False))
|
|
159
|
+
return ("SELECT %s FROM %s GROUP BY %s HAVING %s > 0"
|
|
160
|
+
% (", ".join(dims + [period] + sums), plan.journal,
|
|
161
|
+
", ".join(dims + [period]), _carried_balance(plan)))
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _balance_columns(plan):
|
|
165
|
+
key = [q(d) for d in plan.dimensions] + [q(plan.period_field)]
|
|
166
|
+
cached = [q(c) for _, i, o in plan.resources for c in (i, o)]
|
|
167
|
+
return key, cached
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _balance_verify(plan):
|
|
171
|
+
dims = [q(d) for d in plan.dimensions]
|
|
172
|
+
key, _ = _balance_columns(plan)
|
|
173
|
+
period = period_start(plan, q(plan.date))
|
|
174
|
+
computed = []
|
|
175
|
+
for field, i, o in plan.resources:
|
|
176
|
+
computed.append("COALESCE(%s, 0) AS %s" % (_directed(field, plan.direction, True), q(i)))
|
|
177
|
+
computed.append("COALESCE(%s, 0) AS %s" % (_directed(field, plan.direction, False), q(o)))
|
|
178
|
+
cols = [c for _, i, o in plan.resources for c in (i, o)]
|
|
179
|
+
differences = " OR ".join("COALESCE(c.%s, 0) <> COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols)
|
|
180
|
+
reported = key + ["COALESCE(c.%s, 0), COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols]
|
|
181
|
+
return ("WITH c AS (SELECT %s, %s AS %s, %s FROM %s GROUP BY %s, %s HAVING %s > 0) "
|
|
182
|
+
"SELECT %s FROM c FULL OUTER JOIN %s t USING (%s) WHERE %s"
|
|
183
|
+
% (", ".join(dims), period, q(plan.period_field), ", ".join(computed), plan.journal,
|
|
184
|
+
", ".join(dims), period, _carried_balance(plan),
|
|
185
|
+
", ".join(reported), plan.totals, ", ".join(key), differences))
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
# --------------------------------------------------------------------------- accounting
|
|
189
|
+
# RegisterAccountTotals.cs
|
|
190
|
+
|
|
191
|
+
def _side(plan, source, debit):
|
|
192
|
+
column, dt, ct = plan.account
|
|
193
|
+
columns = ["%s AS %s" % (q(dt if debit else ct), q(column)),
|
|
194
|
+
"%s AS %s" % (q(plan.date), q(plan.period_field))]
|
|
195
|
+
columns += ["%s AS %s" % (q(d_dt if debit else d_ct), q(d)) for d, d_dt, d_ct in plan.side_dimensions]
|
|
196
|
+
for j_dt, j_ct, c_dt, c_ct in plan.account_resources:
|
|
197
|
+
columns.append("%s AS %s" % (q(j_dt), q(c_dt)) if debit or not plan.compound
|
|
198
|
+
else "NULL AS %s" % q(c_dt))
|
|
199
|
+
columns.append("%s AS %s" % (q(j_ct), q(c_ct)) if not debit or not plan.compound
|
|
200
|
+
else "NULL AS %s" % q(c_ct))
|
|
201
|
+
return "SELECT %s FROM %s" % (", ".join(columns), source)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _sides(plan, source):
|
|
205
|
+
debit = _side(plan, source, True)
|
|
206
|
+
return "%s UNION ALL %s" % (debit, _side(plan, source, False)) if plan.compound else debit
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _account_key(plan):
|
|
210
|
+
return [q(plan.account[0])] + [q(d) for d, _, _ in plan.side_dimensions] + [q(plan.period_field)]
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _account_key_select(plan):
|
|
214
|
+
return ([q(plan.account[0])] + [q(d) for d, _, _ in plan.side_dimensions]
|
|
215
|
+
+ [period_start(plan, q(plan.period_field))])
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _account_cached(plan):
|
|
219
|
+
return [c for _, _, c_dt, c_ct in plan.account_resources for c in (c_dt, c_ct)]
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _carried_account(plan):
|
|
223
|
+
return " + ".join("COUNT(%s)" % q(c) for c in _account_cached(plan))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _account_unplaceable(plan):
|
|
227
|
+
undated = "%s IS NULL" % q(plan.date)
|
|
228
|
+
columns = [plan.account[1]] + ([plan.account[2]] if plan.compound else [])
|
|
229
|
+
for _, d_dt, d_ct in plan.side_dimensions:
|
|
230
|
+
columns += [d_dt, d_ct] if plan.compound else [d_dt]
|
|
231
|
+
seen, unique = set(), []
|
|
232
|
+
for c in columns:
|
|
233
|
+
if c not in seen:
|
|
234
|
+
seen.add(c)
|
|
235
|
+
unique.append(c)
|
|
236
|
+
unkeyed = " OR ".join("%s IS NULL" % q(c) for c in unique)
|
|
237
|
+
return ("SELECT COUNT(*) FILTER (WHERE %s), COUNT(*) FILTER (WHERE %s), 0 FROM %s WHERE %s OR %s"
|
|
238
|
+
% (undated, unkeyed, plan.journal, undated, unkeyed))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _account_select(plan):
|
|
242
|
+
"""BuildRebuild's SELECT, without its DELETE and INSERT."""
|
|
243
|
+
key = _account_key(plan)
|
|
244
|
+
sums = ["COALESCE(SUM(%s), 0)" % q(c) for c in _account_cached(plan)]
|
|
245
|
+
return ("SELECT %s FROM (%s) s GROUP BY %s HAVING %s > 0"
|
|
246
|
+
% (", ".join(_account_key_select(plan) + sums), _sides(plan, plan.journal),
|
|
247
|
+
", ".join(str(i) for i in range(1, len(key) + 1)), _carried_account(plan)))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _account_verify(plan):
|
|
251
|
+
key = _account_key(plan)
|
|
252
|
+
cols = _account_cached(plan)
|
|
253
|
+
computed = ["COALESCE(SUM(%s), 0) AS %s" % (q(c), q(c)) for c in cols]
|
|
254
|
+
differences = " OR ".join("COALESCE(c.%s, 0) <> COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols)
|
|
255
|
+
reported = key + ["COALESCE(c.%s, 0), COALESCE(t.%s, 0)" % (q(c), q(c)) for c in cols]
|
|
256
|
+
selected = ["%s AS %s" % (e, k) for e, k in zip(_account_key_select(plan), key)] + computed
|
|
257
|
+
return ("WITH c AS (SELECT %s FROM (%s) s GROUP BY %s HAVING %s > 0) "
|
|
258
|
+
"SELECT %s FROM c FULL OUTER JOIN %s t USING (%s) WHERE %s"
|
|
259
|
+
% (", ".join(selected), _sides(plan, plan.journal),
|
|
260
|
+
", ".join(str(i) for i in range(1, len(key) + 1)), _carried_account(plan),
|
|
261
|
+
", ".join(reported), plan.totals, ", ".join(key), differences))
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
# --------------------------------------------------------------------------- the commands
|
|
265
|
+
|
|
266
|
+
def unplaceable(plan):
|
|
267
|
+
return _account_unplaceable(plan) if plan.kind == "accounting" else _balance_unplaceable(plan)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def select(plan):
|
|
271
|
+
return _account_select(plan) if plan.kind == "accounting" else _balance_select(plan)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def verify(plan):
|
|
275
|
+
return _account_verify(plan) if plan.kind == "accounting" else _balance_verify(plan)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def columns(plan):
|
|
279
|
+
if plan.kind == "accounting":
|
|
280
|
+
return _account_key(plan), [q(c) for c in _account_cached(plan)]
|
|
281
|
+
return _balance_columns(plan)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def name(plan):
|
|
285
|
+
return "%s.%s" % (plan.module, plan.code)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def check(pg, plan, show):
|
|
289
|
+
"""(cached rows, disagreeing keys) - the cache against a fresh fold of the journal."""
|
|
290
|
+
cur = pg.cursor()
|
|
291
|
+
cur.execute("SELECT count(*) FROM %s" % plan.totals)
|
|
292
|
+
cached = cur.fetchone()[0]
|
|
293
|
+
started = time.time()
|
|
294
|
+
cur.execute(verify(plan))
|
|
295
|
+
rows = cur.fetchall()
|
|
296
|
+
key, cols = columns(plan)
|
|
297
|
+
print("%-28s cache %7d rows disagreeing keys %d (%.0fs)"
|
|
298
|
+
% (name(plan), cached, len(rows), time.time() - started))
|
|
299
|
+
if rows and show:
|
|
300
|
+
width = len(key)
|
|
301
|
+
print(" %s" % " ".join([k.strip('"') for k in key]
|
|
302
|
+
+ ["%s fold/cache" % c.strip('"') for c in cols]))
|
|
303
|
+
for row in rows[:show]:
|
|
304
|
+
pairs = ["%s/%s" % (row[width + 2 * i], row[width + 2 * i + 1])
|
|
305
|
+
for i in range(len(cols))]
|
|
306
|
+
print(" %s" % " ".join([str(v) for v in row[:width]] + pairs))
|
|
307
|
+
if len(rows) > show:
|
|
308
|
+
print(" ... %d more" % (len(rows) - show))
|
|
309
|
+
return cached, len(rows)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def rebuild(pg, plan):
|
|
313
|
+
cur = pg.cursor()
|
|
314
|
+
started = time.time()
|
|
315
|
+
cur.execute(unplaceable(plan))
|
|
316
|
+
found = cur.fetchone()
|
|
317
|
+
if found and any(found):
|
|
318
|
+
raise SystemExit("totals: %s - the journal holds %d undated, %d unkeyed and %d undirected"
|
|
319
|
+
" movement(s); install would refuse the cache, and so does this"
|
|
320
|
+
% ((name(plan),) + tuple(found)))
|
|
321
|
+
scanned = time.time()
|
|
322
|
+
cur.execute(select(plan))
|
|
323
|
+
rows = cur.fetchall()
|
|
324
|
+
read = time.time()
|
|
325
|
+
|
|
326
|
+
key, cols = columns(plan)
|
|
327
|
+
at = len(key) - 1 # the period is the last key column
|
|
328
|
+
by_year = collections.defaultdict(list)
|
|
329
|
+
for row in rows:
|
|
330
|
+
by_year[row[at].year].append(row)
|
|
331
|
+
cur.execute("SELECT DISTINCT extract(year FROM %s)::int FROM %s"
|
|
332
|
+
% (q(plan.period_field), plan.totals))
|
|
333
|
+
years = sorted(set(by_year) | set(r[0] for r in cur.fetchall()))
|
|
334
|
+
|
|
335
|
+
insert = ("INSERT INTO %s (%s) VALUES (%s)"
|
|
336
|
+
% (plan.totals, ", ".join(key + cols), ", ".join(["%s"] * (len(key) + len(cols)))))
|
|
337
|
+
pg.autocommit = False
|
|
338
|
+
try:
|
|
339
|
+
for year in years:
|
|
340
|
+
cur.execute("SET LOCAL session_replication_role = 'replica'")
|
|
341
|
+
cur.execute("DELETE FROM %s WHERE %s >= %%s AND %s < %%s"
|
|
342
|
+
% (plan.totals, q(plan.period_field), q(plan.period_field)),
|
|
343
|
+
("%d-01-01" % year, "%d-01-01" % (year + 1)))
|
|
344
|
+
if by_year.get(year):
|
|
345
|
+
cur.executemany(insert, by_year[year])
|
|
346
|
+
pg.commit()
|
|
347
|
+
finally:
|
|
348
|
+
pg.autocommit = True
|
|
349
|
+
print("%-28s %7d rows written over %d year(s) checks %.0fs, fold %.0fs, write %.0fs"
|
|
350
|
+
% (name(plan), len(rows), len(years), scanned - started, read - scanned,
|
|
351
|
+
time.time() - read))
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def main():
|
|
355
|
+
migrate.load_env()
|
|
356
|
+
ap = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0])
|
|
357
|
+
ap.add_argument("command", choices=("check", "rebuild", "sql"))
|
|
358
|
+
ap.add_argument("module", nargs="*", help="limit to these modules")
|
|
359
|
+
ap.add_argument("--show", type=int, default=10, metavar="N",
|
|
360
|
+
help="check: disagreeing keys to print")
|
|
361
|
+
args = ap.parse_args()
|
|
362
|
+
|
|
363
|
+
pg = migrate.pg_connect()
|
|
364
|
+
# Reads are long scans: autocommit, so no transaction stays open across them.
|
|
365
|
+
pg.autocommit = True
|
|
366
|
+
cur = pg.cursor()
|
|
367
|
+
cur.execute("SHOW TimeZone")
|
|
368
|
+
sys.stderr.write("[i] session time zone %s (the period of a timestamptz depends on it)\n"
|
|
369
|
+
% cur.fetchone()[0])
|
|
370
|
+
found = plans(cur, set(args.module))
|
|
371
|
+
if not found:
|
|
372
|
+
print("no register declares totals%s" % (" in " + ", ".join(args.module)
|
|
373
|
+
if args.module else ""))
|
|
374
|
+
return 0
|
|
375
|
+
clean = True
|
|
376
|
+
for plan in found:
|
|
377
|
+
if args.command == "sql":
|
|
378
|
+
print("-- %s\n-- verify\n%s;\n-- rebuild select\n%s;\n" % (name(plan), verify(plan), select(plan)))
|
|
379
|
+
elif args.command == "check":
|
|
380
|
+
clean = check(pg, plan, args.show)[1] == 0 and clean
|
|
381
|
+
else:
|
|
382
|
+
rebuild(pg, plan)
|
|
383
|
+
return 0 if clean else 1
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
if __name__ == "__main__":
|
|
387
|
+
sys.exit(main())
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Build typeGuid -> object map.
|
|
5
|
+
|
|
6
|
+
An attribute typed "СправочникСсылка.рзАвтотранспортныеПредприятия" does not carry
|
|
7
|
+
the catalog's object GUID; it carries one of the *type* GUIDs the catalog declares
|
|
8
|
+
for itself, in a block near the top of its own file:
|
|
9
|
+
|
|
10
|
+
{1,
|
|
11
|
+
{53,dc8f143c-...,899caee9-...,489d7014-...,...}
|
|
12
|
+
|
|
13
|
+
Every GUID in that block belongs to that one object, so mapping all of them is
|
|
14
|
+
enough to resolve any reference back to its target.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import print_function
|
|
17
|
+
|
|
18
|
+
import codecs
|
|
19
|
+
import collections
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import sys
|
|
24
|
+
|
|
25
|
+
import meta
|
|
26
|
+
|
|
27
|
+
OUT = meta.OUT
|
|
28
|
+
|
|
29
|
+
# Every object file opens as {1,\r\n{53,<its own type guids>,{0,{1,...identity...
|
|
30
|
+
# The type guids are the unbroken GUID run that follows the two leading counts.
|
|
31
|
+
HEAD = re.compile(r"^\{\d+,\s*\{\d+,\s*((?:[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
|
|
32
|
+
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12},\s*)+)")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def own_type_guids(text):
|
|
36
|
+
m = HEAD.match(text)
|
|
37
|
+
if not m:
|
|
38
|
+
return []
|
|
39
|
+
return re.findall(r"[0-9a-fA-F-]{36}", m.group(1))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def main():
|
|
43
|
+
with codecs.open(os.path.join(OUT, "objects.json"), "r", "utf-8") as f:
|
|
44
|
+
objects = json.load(f)
|
|
45
|
+
|
|
46
|
+
type_map = {}
|
|
47
|
+
clashes = 0
|
|
48
|
+
for obj in objects:
|
|
49
|
+
text = meta.load(obj["guid"])
|
|
50
|
+
if text is None:
|
|
51
|
+
continue
|
|
52
|
+
guids = own_type_guids(text)
|
|
53
|
+
for i, g in enumerate(guids):
|
|
54
|
+
if g in type_map and type_map[g]["guid"] != obj["guid"]:
|
|
55
|
+
clashes += 1
|
|
56
|
+
continue
|
|
57
|
+
type_map[g] = {"guid": obj["guid"], "name": obj["name"],
|
|
58
|
+
"family": obj["family"], "index": i + 1}
|
|
59
|
+
# the object GUID itself is sometimes used directly as a type
|
|
60
|
+
type_map.setdefault(obj["guid"], {"guid": obj["guid"], "name": obj["name"],
|
|
61
|
+
"family": obj["family"], "index": 0})
|
|
62
|
+
|
|
63
|
+
with codecs.open(os.path.join(OUT, "typemap.json"), "w", "utf-8") as f:
|
|
64
|
+
json.dump(type_map, f, ensure_ascii=False, indent="\t")
|
|
65
|
+
print("objects=%d type guids mapped=%d clashes=%d" % (len(objects), len(type_map), clashes))
|
|
66
|
+
|
|
67
|
+
# sanity check against a reference we already resolved by hand
|
|
68
|
+
probe = type_map.get("489d7014-f13e-4dc7-ae5e-e5c51da99ea2")
|
|
69
|
+
print("probe 489d7014 ->", probe)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
if __name__ == "__main__":
|
|
73
|
+
main()
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Unpack a 1C inner container - the `<guid>.0` entries of the Config table.
|
|
5
|
+
|
|
6
|
+
Those hold the object's modules and forms. The format is a paged container:
|
|
7
|
+
|
|
8
|
+
header FF FF FF 7F | nextPage u32 | pageSize u32 | storageVer u32
|
|
9
|
+
block \\r\\n <8 hex docSize> ' ' <8 hex pageSize> ' ' <8 hex nextPage> ' ' \\r\\n
|
|
10
|
+
followed by pageSize bytes, chained through nextPage until docSize is read
|
|
11
|
+
TOC the block at offset 16: 12-byte entries of (attrsAddr, dataAddr, reserved)
|
|
12
|
+
attrs created u64 | modified u64 | reserved u32 | name, UTF-16LE
|
|
13
|
+
|
|
14
|
+
Each entry's data may itself be raw deflate.
|
|
15
|
+
|
|
16
|
+
python tools/unpack.py <ObjectName> # list what the object's container holds
|
|
17
|
+
python tools/unpack.py <ObjectName> --dump DIR # write every entry to DIR
|
|
18
|
+
python tools/unpack.py <ObjectName> --code # print the module texts
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import print_function
|
|
21
|
+
|
|
22
|
+
import codecs
|
|
23
|
+
import os
|
|
24
|
+
import struct
|
|
25
|
+
import sys
|
|
26
|
+
import zlib
|
|
27
|
+
|
|
28
|
+
import attrs
|
|
29
|
+
import meta
|
|
30
|
+
|
|
31
|
+
FREE = 0x7FFFFFFF
|
|
32
|
+
CONFIG = meta.CONFIG
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def read_block(buf, addr):
|
|
36
|
+
head = buf[addr:addr + 31]
|
|
37
|
+
if head[:2] != b"\r\n":
|
|
38
|
+
raise ValueError("no block header at %d" % addr)
|
|
39
|
+
doc_size = int(head[2:10], 16)
|
|
40
|
+
page_size = int(head[11:19], 16)
|
|
41
|
+
next_addr = int(head[20:28], 16)
|
|
42
|
+
out = bytearray(buf[addr + 31:addr + 31 + page_size])
|
|
43
|
+
while next_addr != FREE and len(out) < doc_size:
|
|
44
|
+
head = buf[next_addr:next_addr + 31]
|
|
45
|
+
page_size = int(head[11:19], 16)
|
|
46
|
+
following = int(head[20:28], 16)
|
|
47
|
+
out += buf[next_addr + 31:next_addr + 31 + page_size]
|
|
48
|
+
next_addr = following
|
|
49
|
+
return bytes(out[:doc_size])
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def inflate(data):
|
|
53
|
+
try:
|
|
54
|
+
return zlib.decompressobj(-15).decompress(data)
|
|
55
|
+
except zlib.error:
|
|
56
|
+
return data
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def entries(buf):
|
|
60
|
+
"""[(name, content), ...] for one container."""
|
|
61
|
+
if buf[:4] != b"\xff\xff\xff\x7f":
|
|
62
|
+
return []
|
|
63
|
+
toc = read_block(buf, 16)
|
|
64
|
+
out = []
|
|
65
|
+
for offset in range(0, len(toc) - 11, 12):
|
|
66
|
+
attrs_addr, data_addr, _ = struct.unpack_from("<III", toc, offset)
|
|
67
|
+
if attrs_addr == FREE or data_addr == FREE:
|
|
68
|
+
continue
|
|
69
|
+
header = read_block(buf, attrs_addr)
|
|
70
|
+
name = header[20:].decode("utf-16-le", "replace").rstrip("\x00")
|
|
71
|
+
out.append((name, inflate(read_block(buf, data_addr))))
|
|
72
|
+
return out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def containers_of(guid):
|
|
76
|
+
"""Every Config file belonging to this object that is a container."""
|
|
77
|
+
found = []
|
|
78
|
+
for suffix in (".0", ".1", ".2"):
|
|
79
|
+
path = os.path.join(CONFIG, guid + suffix + ".bin")
|
|
80
|
+
if os.path.isfile(path):
|
|
81
|
+
with open(path, "rb") as f:
|
|
82
|
+
found.append((guid + suffix, f.read()))
|
|
83
|
+
return found
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def as_text(blob):
|
|
87
|
+
for encoding in ("utf-8-sig", "utf-16-le", "cp1251"):
|
|
88
|
+
try:
|
|
89
|
+
text = blob.decode(encoding)
|
|
90
|
+
except (UnicodeDecodeError, LookupError):
|
|
91
|
+
continue
|
|
92
|
+
if text.count("�") == 0:
|
|
93
|
+
return text
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def main():
|
|
98
|
+
name = sys.argv[1]
|
|
99
|
+
objects = attrs.load_json("objects.json")
|
|
100
|
+
guid = next((o["guid"] for o in objects if o.get("name") == name), None)
|
|
101
|
+
if not guid:
|
|
102
|
+
print("no such object: %s" % name)
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
found = containers_of(guid)
|
|
106
|
+
if not found:
|
|
107
|
+
print("%s has no container files" % name)
|
|
108
|
+
return
|
|
109
|
+
|
|
110
|
+
dump_dir = None
|
|
111
|
+
if "--dump" in sys.argv:
|
|
112
|
+
dump_dir = sys.argv[sys.argv.index("--dump") + 1]
|
|
113
|
+
if not os.path.isdir(dump_dir):
|
|
114
|
+
os.makedirs(dump_dir)
|
|
115
|
+
|
|
116
|
+
for container, blob in found:
|
|
117
|
+
items = entries(blob)
|
|
118
|
+
print("== %s (%d entries, %d bytes)" % (container, len(items), len(blob)))
|
|
119
|
+
for entry_name, content in items:
|
|
120
|
+
text = as_text(content)
|
|
121
|
+
kind = "text %d chars" % len(text) if text else "binary %d bytes" % len(content)
|
|
122
|
+
print(" %-28s %s" % (entry_name or "(unnamed)", kind))
|
|
123
|
+
if dump_dir:
|
|
124
|
+
safe = (entry_name or "unnamed").replace("/", "_")
|
|
125
|
+
path = os.path.join(dump_dir, "%s.%s" % (container, safe))
|
|
126
|
+
with open(path, "wb") as f:
|
|
127
|
+
f.write(content)
|
|
128
|
+
if "--code" in sys.argv and text and "Процедура" in text or \
|
|
129
|
+
"--code" in sys.argv and text and "Функция" in text:
|
|
130
|
+
print("-" * 70)
|
|
131
|
+
print(text[:4000])
|
|
132
|
+
print("-" * 70)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
if __name__ == "__main__":
|
|
136
|
+
main()
|