@dforge-core/dforge-mcp 0.2.44 → 0.2.46-next
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +3 -2
- package/dist/server.js +109 -84
- package/package.json +48 -46
- package/resources/docs/conventions.md +17 -14
- package/resources/docs/dsl-reference.md +15 -5
- package/resources/schemas/entity.schema.json +1 -1
- package/resources/schemas/registers.schema.json +8 -3
- package/scripts/install-skills.mjs +1 -0
- package/skills/dforge-1c-import/SKILL.md +374 -0
- package/skills/dforge-1c-import/references/environment.md +97 -0
- package/skills/dforge-1c-import/references/loading.md +161 -0
- package/skills/dforge-1c-import/references/measurements.md +210 -0
- package/skills/dforge-1c-import/references/print-forms.md +148 -0
- package/skills/dforge-1c-import/scripts/1c.py +178 -0
- package/skills/dforge-1c-import/scripts/README.md +110 -0
- package/skills/dforge-1c-import/scripts/lib/analyze.py +83 -0
- package/skills/dforge-1c-import/scripts/lib/attrs.py +278 -0
- package/skills/dforge-1c-import/scripts/lib/autospec.py +964 -0
- package/skills/dforge-1c-import/scripts/lib/binding.py +257 -0
- package/skills/dforge-1c-import/scripts/lib/bsl.py +811 -0
- package/skills/dforge-1c-import/scripts/lib/callgraph.py +717 -0
- package/skills/dforge-1c-import/scripts/lib/cli.py +53 -0
- package/skills/dforge-1c-import/scripts/lib/columnmap.py +346 -0
- package/skills/dforge-1c-import/scripts/lib/coverage.py +242 -0
- package/skills/dforge-1c-import/scripts/lib/custom.py +83 -0
- package/skills/dforge-1c-import/scripts/lib/delta.py +741 -0
- package/skills/dforge-1c-import/scripts/lib/deps.py +136 -0
- package/skills/dforge-1c-import/scripts/lib/docusage.py +105 -0
- package/skills/dforge-1c-import/scripts/lib/dump1c.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/explore.py +55 -0
- package/skills/dforge-1c-import/scripts/lib/external.py +201 -0
- package/skills/dforge-1c-import/scripts/lib/extract_code.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/forms.py +239 -0
- package/skills/dforge-1c-import/scripts/lib/genmodule.py +1432 -0
- package/skills/dforge-1c-import/scripts/lib/handlers.py +202 -0
- package/skills/dforge-1c-import/scripts/lib/handmap.py +142 -0
- package/skills/dforge-1c-import/scripts/lib/i18n_coverage.py +87 -0
- package/skills/dforge-1c-import/scripts/lib/idioms.py +273 -0
- package/skills/dforge-1c-import/scripts/lib/integrity.py +264 -0
- package/skills/dforge-1c-import/scripts/lib/inventory.py +535 -0
- package/skills/dforge-1c-import/scripts/lib/layers.py +106 -0
- package/skills/dforge-1c-import/scripts/lib/meta.py +315 -0
- package/skills/dforge-1c-import/scripts/lib/migrate.py +1299 -0
- package/skills/dforge-1c-import/scripts/lib/mxl.py +368 -0
- package/skills/dforge-1c-import/scripts/lib/naming.py +251 -0
- package/skills/dforge-1c-import/scripts/lib/paths.py +90 -0
- package/skills/dforge-1c-import/scripts/lib/peek.py +70 -0
- package/skills/dforge-1c-import/scripts/lib/posted.py +218 -0
- package/skills/dforge-1c-import/scripts/lib/posting.py +655 -0
- package/skills/dforge-1c-import/scripts/lib/printgen.py +239 -0
- package/skills/dforge-1c-import/scripts/lib/rebuild.py +194 -0
- package/skills/dforge-1c-import/scripts/lib/recorded.py +170 -0
- package/skills/dforge-1c-import/scripts/lib/recorders.py +94 -0
- package/skills/dforge-1c-import/scripts/lib/reftargets.py +95 -0
- package/skills/dforge-1c-import/scripts/lib/reportgen.py +157 -0
- package/skills/dforge-1c-import/scripts/lib/roles.py +33 -0
- package/skills/dforge-1c-import/scripts/lib/seedstate.py +488 -0
- package/skills/dforge-1c-import/scripts/lib/sizing.py +184 -0
- package/skills/dforge-1c-import/scripts/lib/spec.py +400 -0
- package/skills/dforge-1c-import/scripts/lib/statements.py +774 -0
- package/skills/dforge-1c-import/scripts/lib/subsystems.py +480 -0
- package/skills/dforge-1c-import/scripts/lib/terms.py +451 -0
- package/skills/dforge-1c-import/scripts/lib/terms_uk.py +555 -0
- package/skills/dforge-1c-import/scripts/lib/totals.py +387 -0
- package/skills/dforge-1c-import/scripts/lib/typemap.py +73 -0
- package/skills/dforge-1c-import/scripts/lib/unpack.py +136 -0
- package/skills/dforge-1c-import/scripts/lib/validate.py +508 -0
- package/skills/dforge-1c-import/scripts/project-template/contours.py +47 -0
- package/skills/dforge-1c-import/scripts/project-template/sources.py +14 -0
- package/skills/dforge-mcp-author/SKILL.md +8 -2
- package/skills/dforge-mcp-author/examples/column-security/entities/product.json +1 -1
- package/skills/dforge-mcp-author/examples/column-security/manifest.json +0 -1
- package/skills/dforge-mcp-author/examples/column-security/security/roles.json +2 -2
- package/skills/dforge-mcp-author/references/action-dsl.md +20 -7
- package/skills/dforge-mcp-author/references/conventions.md +7 -1
- package/skills/dforge-mcp-author/references/field-types.md +3 -2
- package/skills/dforge-mcp-author/references/formulas.md +5 -1
- package/skills/dforge-mcp-author/references/print-templates.md +5 -2
- package/skills/dforge-mcp-author/references/queries.md +1 -1
- package/skills/dforge-mcp-author/references/registers.md +18 -3
- package/skills/dforge-mcp-author/references/reports.md +4 -1
- package/skills/dforge-mcp-author/references/schema-import.md +1 -1
- package/skills/dforge-mcp-author/references/security.md +1 -1
- package/skills/dforge-mcp-author/references/translations.md +38 -15
- package/skills/dforge-mcp-author/references/validation-checklist.md +2 -3
- package/skills/dforge-module-build/SKILL.md +10 -1
- package/skills/dforge-module-ship/SKILL.md +31 -3
- package/skills/dforge-mcp-author/examples/column-security/translations/en-US.json +0 -35
|
@@ -0,0 +1,655 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""Post and unpost one real document per register binding, and check what happened.
|
|
4
|
+
|
|
5
|
+
Declaring a register is cheap and looks right in the JSON. Whether a document can actually
|
|
6
|
+
post to it is a question only the running platform answers, and the answers so far have
|
|
7
|
+
all been no, each for a different reason one line further in than the last. So this is not
|
|
8
|
+
a demonstration: every binding in every generated module is found automatically, one real
|
|
9
|
+
document is put through it, and each claim is asserted rather than printed.
|
|
10
|
+
|
|
11
|
+
Per binding:
|
|
12
|
+
|
|
13
|
+
the register holds no row this document owns, before
|
|
14
|
+
posting succeeds
|
|
15
|
+
the state flag is set
|
|
16
|
+
one register row per line (or one, from a header binding)
|
|
17
|
+
source names the writing document
|
|
18
|
+
source.line distinguishes the lines - the whole key, not its first column
|
|
19
|
+
the dimension TUPLE takes as many distinct values as there were lines
|
|
20
|
+
unposting leaves the register with none of this document's rows
|
|
21
|
+
|
|
22
|
+
The document is put back either way, and a refusal is reported with what the platform
|
|
23
|
+
said rather than swallowed. Nothing else in the tenant is touched: the register rows
|
|
24
|
+
loaded from 1C carry no `source`, so none of them can match a document's own.
|
|
25
|
+
|
|
26
|
+
DF_TOKEN=<jwt> python tools/1c.py posting # every binding
|
|
27
|
+
DF_TOKEN=<jwt> python tools/1c.py posting shipping # one module
|
|
28
|
+
|
|
29
|
+
The token is a tenant-scoped JWT. With DF_TOKEN unset the CLI's own session store
|
|
30
|
+
(~/.dforge/auth/*.json) is read, which is where `dforge-cli auth login` leaves one; it
|
|
31
|
+
lasts an hour. Nothing here holds a credential and nothing writes one down.
|
|
32
|
+
"""
|
|
33
|
+
from __future__ import print_function
|
|
34
|
+
|
|
35
|
+
import base64
|
|
36
|
+
import codecs
|
|
37
|
+
import glob
|
|
38
|
+
import io
|
|
39
|
+
import json
|
|
40
|
+
import os
|
|
41
|
+
import re
|
|
42
|
+
import sys
|
|
43
|
+
import time
|
|
44
|
+
|
|
45
|
+
import paths
|
|
46
|
+
|
|
47
|
+
try:
|
|
48
|
+
import urllib.request as request
|
|
49
|
+
import urllib.error as urlerror
|
|
50
|
+
except ImportError: # pragma: no cover
|
|
51
|
+
import urllib2 as request # noqa: F401
|
|
52
|
+
import urllib2 as urlerror # noqa: F401
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
API = os.environ.get("DF_API", "http://localhost:5001") + "/api/exec"
|
|
56
|
+
|
|
57
|
+
# How many documents a binding gated on its header is tried on before it is reported as
|
|
58
|
+
# wanting a new document rather than failed.
|
|
59
|
+
GATED_TRIES = 10
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# --------------------------------------------------------------------------- the session
|
|
63
|
+
|
|
64
|
+
def token():
|
|
65
|
+
"""A tenant-scoped JWT, from the environment or from the CLI's session store."""
|
|
66
|
+
found = os.environ.get("DF_TOKEN")
|
|
67
|
+
if found:
|
|
68
|
+
# an expired one is refused here, before a run starts changing anything
|
|
69
|
+
expires = claims(found.strip()).get("exp")
|
|
70
|
+
if expires and expires <= time.time():
|
|
71
|
+
sys.stderr.write(
|
|
72
|
+
"DF_TOKEN expired %d minute(s) ago. Sign in again with\n"
|
|
73
|
+
" dforge-cli auth login --url %s\n"
|
|
74
|
+
% (int((time.time() - expires) / 60), os.environ.get("DF_API", "http://localhost:5001")))
|
|
75
|
+
raise SystemExit(2)
|
|
76
|
+
return found.strip()
|
|
77
|
+
|
|
78
|
+
# The store holds a session per tenant signed into; the newest one may be another tenant's,
|
|
79
|
+
# and a call with it reads as "Folder not found" rather than as the wrong tenant.
|
|
80
|
+
tenant = tenant_id()
|
|
81
|
+
store = os.path.expanduser("~/.dforge/auth")
|
|
82
|
+
best = None
|
|
83
|
+
for path in glob.glob(os.path.join(store, "*.json")):
|
|
84
|
+
try:
|
|
85
|
+
with io.open(path, encoding="utf-8") as handle:
|
|
86
|
+
session = json.load(handle)
|
|
87
|
+
except (ValueError, IOError):
|
|
88
|
+
continue
|
|
89
|
+
if not session.get("AccessToken"):
|
|
90
|
+
continue
|
|
91
|
+
if (session.get("ExpiresAtUtc") or 0) <= time.time():
|
|
92
|
+
continue
|
|
93
|
+
if tenant and claims(session["AccessToken"]).get("tenant_id") != tenant:
|
|
94
|
+
continue
|
|
95
|
+
if best is None or session["ExpiresAtUtc"] > best["ExpiresAtUtc"]:
|
|
96
|
+
best = session
|
|
97
|
+
if best:
|
|
98
|
+
sys.stderr.write("[i] session for %s, %d minute(s) left\n"
|
|
99
|
+
% (best.get("UserEmail") or "?",
|
|
100
|
+
int((best["ExpiresAtUtc"] - time.time()) / 60)))
|
|
101
|
+
return best["AccessToken"]
|
|
102
|
+
|
|
103
|
+
sys.stderr.write(
|
|
104
|
+
"no token%s. Set DF_TOKEN=<jwt>, or sign in with\n"
|
|
105
|
+
" dforge-cli auth login --url %s\n"
|
|
106
|
+
% (" for tenant %s" % os.environ.get("DF_TENANT") if tenant else "",
|
|
107
|
+
os.environ.get("DF_API", "http://localhost:5001")))
|
|
108
|
+
raise SystemExit(2)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def claims(jwt):
|
|
112
|
+
"""A JWT's payload, unverified - only to tell which tenant a stored session is for."""
|
|
113
|
+
try:
|
|
114
|
+
part = jwt.split(".")[1]
|
|
115
|
+
return json.loads(base64.urlsafe_b64decode(part + "=" * (-len(part) % 4)).decode("utf-8"))
|
|
116
|
+
except (IndexError, ValueError, TypeError):
|
|
117
|
+
return {}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def tenant_id():
|
|
121
|
+
"""DF_TENANT's id from the platform's tenant table, or None when it cannot be read."""
|
|
122
|
+
code = os.environ.get("DF_TENANT")
|
|
123
|
+
if not code:
|
|
124
|
+
return None
|
|
125
|
+
try:
|
|
126
|
+
import psycopg2
|
|
127
|
+
conn = psycopg2.connect(
|
|
128
|
+
host=os.environ.get("DF_PG_HOST", "localhost"),
|
|
129
|
+
port=int(os.environ.get("DF_PG_PORT", "5438")),
|
|
130
|
+
dbname=os.environ.get("DF_AUTH_DB", "dforge"),
|
|
131
|
+
user=os.environ.get("DF_PG_USER", "dforge"),
|
|
132
|
+
password=os.environ.get("DF_PG_PASS", "dforge"))
|
|
133
|
+
try:
|
|
134
|
+
cur = conn.cursor()
|
|
135
|
+
cur.execute("select tenant_id::text from public.tenant where tenant_cd = %s", (code,))
|
|
136
|
+
row = cur.fetchone()
|
|
137
|
+
finally:
|
|
138
|
+
conn.close()
|
|
139
|
+
except Exception as e: # noqa: BLE001
|
|
140
|
+
sys.stderr.write("[!] tenant %s not resolved (%s); any stored session is taken\n" % (code, e))
|
|
141
|
+
return None
|
|
142
|
+
return row[0] if row else None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class SessionExpired(SystemExit):
|
|
146
|
+
"""The API refused the token (401): the session ran out mid-run. A SystemExit, so a script's
|
|
147
|
+
broad `except Exception` does not swallow it and go on refusing every document after."""
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def call(method, params, folder, jwt):
|
|
151
|
+
body = json.dumps({"method": method, "params": params,
|
|
152
|
+
"context": {"folderId": folder}}).encode("utf-8")
|
|
153
|
+
req = request.Request(API, data=body, headers={
|
|
154
|
+
"Content-Type": "application/json", "Authorization": "Bearer " + jwt})
|
|
155
|
+
try:
|
|
156
|
+
return json.loads(request.urlopen(req).read().decode("utf-8"))
|
|
157
|
+
except urlerror.HTTPError as error:
|
|
158
|
+
if error.code == 401:
|
|
159
|
+
sys.stderr.write(
|
|
160
|
+
"\n[!] the API refused the token (401) at %s - the session has expired. Stopped.\n"
|
|
161
|
+
" Sign in again: dforge-cli auth login --url %s\n"
|
|
162
|
+
" then run `posted --mark` to restore any marker this run took off.\n"
|
|
163
|
+
% (method, os.environ.get("DF_API", "http://localhost:5001")))
|
|
164
|
+
raise SessionExpired(3)
|
|
165
|
+
return {"httpError": error.code,
|
|
166
|
+
"body": error.read().decode("utf-8", "replace")}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# --------------------------------------------------------------------------- the bindings
|
|
170
|
+
|
|
171
|
+
def bindings(only=None):
|
|
172
|
+
"""Every A column in every generated module, with what it needs to be exercised."""
|
|
173
|
+
found = []
|
|
174
|
+
for path in sorted(glob.glob(os.path.join(paths.MODULES, "*", "entities", "*.json"))):
|
|
175
|
+
module = os.path.basename(os.path.dirname(os.path.dirname(path)))
|
|
176
|
+
if only and module != only:
|
|
177
|
+
continue
|
|
178
|
+
with io.open(path, encoding="utf-8") as handle:
|
|
179
|
+
entity = json.load(handle)
|
|
180
|
+
code = os.path.splitext(os.path.basename(path))[0]
|
|
181
|
+
for column, spec in (entity.get("fields") or {}).items():
|
|
182
|
+
if spec.get("columnType") != "A":
|
|
183
|
+
continue
|
|
184
|
+
params = spec.get("params") or {}
|
|
185
|
+
reference = params.get("register")
|
|
186
|
+
if not reference:
|
|
187
|
+
continue
|
|
188
|
+
found.append({
|
|
189
|
+
"module": module,
|
|
190
|
+
"entity": "%s.%s" % (module, code),
|
|
191
|
+
"code": code,
|
|
192
|
+
"column": column,
|
|
193
|
+
"register": reference,
|
|
194
|
+
"lineEntity": params.get("lineEntity"),
|
|
195
|
+
"lineField": params.get("lineField"),
|
|
196
|
+
"map": params.get("map") or {},
|
|
197
|
+
# Carried even though the checks below read dimensions off the register
|
|
198
|
+
# table rather than the document: a caller asking "where does this
|
|
199
|
+
# binding get each value" gets a wrong answer from `map` alone, and the
|
|
200
|
+
# first such caller silently measured nothing.
|
|
201
|
+
"headerMap": params.get("headerMap") or {},
|
|
202
|
+
"skip": params.get("skipPostingWhenEmpty") or [],
|
|
203
|
+
"group": params.get("groupLinesBy") or [],
|
|
204
|
+
})
|
|
205
|
+
return found
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def declarations():
|
|
209
|
+
"""`module.register` -> its declaration, across every module that declares one."""
|
|
210
|
+
found = {}
|
|
211
|
+
for path in sorted(glob.glob(os.path.join(paths.MODULES, "*", "registers.json"))):
|
|
212
|
+
module = os.path.basename(os.path.dirname(path))
|
|
213
|
+
with io.open(path, encoding="utf-8") as handle:
|
|
214
|
+
for code, definition in json.load(handle).items():
|
|
215
|
+
found["%s.%s" % (module, code)] = (module, definition)
|
|
216
|
+
return found
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
# --------------------------------------------------------------------------- the checks
|
|
220
|
+
|
|
221
|
+
class Run(object):
|
|
222
|
+
def __init__(self, cursor, jwt):
|
|
223
|
+
self.cursor, self.jwt, self.failed = cursor, jwt, []
|
|
224
|
+
self.folders = {}
|
|
225
|
+
|
|
226
|
+
def one(self, sql, *args):
|
|
227
|
+
self.cursor.execute(sql, args)
|
|
228
|
+
row = self.cursor.fetchone()
|
|
229
|
+
return row[0] if row else None
|
|
230
|
+
|
|
231
|
+
def folder(self, module):
|
|
232
|
+
# As a string: the RPC declares folderId as one, and a bigint straight out of
|
|
233
|
+
# psycopg2 is rejected by the model binder before the method is reached.
|
|
234
|
+
if module not in self.folders:
|
|
235
|
+
self.folders[module] = str(self.one(
|
|
236
|
+
'select f.folder_id from "dForge".folder f '
|
|
237
|
+
'join "dForge".module m on m.module_id = f.module_id '
|
|
238
|
+
'where m.module_cd = %s order by f.folder_id limit 1', module))
|
|
239
|
+
return self.folders[module]
|
|
240
|
+
|
|
241
|
+
def has_column(self, schema, table, column):
|
|
242
|
+
return self.one("""select 1 from information_schema.columns
|
|
243
|
+
where table_schema = %s and table_name = %s
|
|
244
|
+
and column_name = %s""", schema, table, column) is not None
|
|
245
|
+
|
|
246
|
+
def check(self, label, got, want):
|
|
247
|
+
ok = got == want
|
|
248
|
+
print(" %-52s %s" % (
|
|
249
|
+
label, "ok" if ok else "FAIL got %r, want %r" % (got, want)))
|
|
250
|
+
if not ok:
|
|
251
|
+
self.failed.append(label)
|
|
252
|
+
return ok
|
|
253
|
+
|
|
254
|
+
def set_state(self, binding, key, value):
|
|
255
|
+
return call("data.update",
|
|
256
|
+
{"entity": binding["entity"], "key": key,
|
|
257
|
+
"values": {binding["column"]: value}},
|
|
258
|
+
self.folder(binding["module"]), self.jwt)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def refused(result):
|
|
262
|
+
"""The platform's own words, or None when it did not refuse."""
|
|
263
|
+
if "httpError" in result:
|
|
264
|
+
return "HTTP %s %s" % (result["httpError"], result["body"][:300])
|
|
265
|
+
if result.get("code"):
|
|
266
|
+
return "%s: %s %s" % (result["code"], result.get("message", ""),
|
|
267
|
+
json.dumps(result.get("details") or {})[:300])
|
|
268
|
+
return None
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def plain(value):
|
|
272
|
+
"""A binding value that names a column of the row it is read off.
|
|
273
|
+
|
|
274
|
+
The others are not SQL this file can splice in: a quoted constant is stamped, a
|
|
275
|
+
`<reference>.<column>` path is read through a join, and a value starting with `=` is an
|
|
276
|
+
F-column formula the platform translates at posting time.
|
|
277
|
+
"""
|
|
278
|
+
return not (value.startswith("'") or value.startswith("=") or "." in value)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def explain_shortfall(run, binding, definition, document):
|
|
282
|
+
"""Why a posting wrote fewer records than the document had lines.
|
|
283
|
+
|
|
284
|
+
Two causes, and they are not the same problem. A line whose dimension is NULL is
|
|
285
|
+
dropped by the engine without a word - found the first time this ran, where two lines
|
|
286
|
+
of a 361-line draft carry no product - and a document that repeats a dimension tuple
|
|
287
|
+
has fewer keys than lines to begin with. The count alone says neither, and the
|
|
288
|
+
difference decides whether the binding is wrong or the data is.
|
|
289
|
+
"""
|
|
290
|
+
if not binding["lineEntity"]:
|
|
291
|
+
return
|
|
292
|
+
source = '"%s"."%s"' % (binding["module"], binding["lineEntity"])
|
|
293
|
+
where = "%s = %%s" % binding["lineField"]
|
|
294
|
+
# Only the dimensions the LINES supply; a headerMap one is the same on every row.
|
|
295
|
+
columns = [binding["map"][d] for d in (definition.get("dimensions") or [])
|
|
296
|
+
if d in binding["map"] and plain(binding["map"][d])]
|
|
297
|
+
if not columns:
|
|
298
|
+
return
|
|
299
|
+
blank = " or ".join("%s is null" % c for c in columns)
|
|
300
|
+
print(" of the document's lines: %s carry a NULL dimension, %s distinct key(s)"
|
|
301
|
+
% (run.one("select count(*) from %s where %s and (%s)" % (source, where, blank),
|
|
302
|
+
document),
|
|
303
|
+
run.one("select count(distinct (%s)) from %s where %s"
|
|
304
|
+
% (", ".join(columns), source, where), document)))
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
SLICE = re.compile(r"getSlice\('([\w.]+)',\s*\{\s*(\w+):\s*(?:old)?\[(\w+)\]\s*\},\s*now\(\)\)")
|
|
308
|
+
REFUSE = re.compile(r"if \(([^{]*)\)\s*\{\s*error\(")
|
|
309
|
+
TEST = re.compile(r"(\w+)\.(\w+)\s*(==|!=)\s*'([^']*)'")
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def unpost_lock(binding, declared):
|
|
313
|
+
"""A SQL test, on the candidate `d`, that a synchronous trigger will let it be unposted.
|
|
314
|
+
|
|
315
|
+
A module may freeze a posted document - production's rework act is read-only once its
|
|
316
|
+
work shift is not open - and the freeze refuses the unposting too. Posting a document
|
|
317
|
+
that cannot be unposted leaves it posted, so the lock is read off the module's own
|
|
318
|
+
triggers and the candidate chosen where it does not hold. Only the shape the triggers
|
|
319
|
+
use is understood: a slice of an info register at now(), keyed by one of the row's
|
|
320
|
+
columns, and an `error()` under a test of one of its resources. Returns (sql, words),
|
|
321
|
+
or None when no trigger on this entity guards the state flag.
|
|
322
|
+
"""
|
|
323
|
+
folder = os.path.join(paths.MODULES, binding["module"], "logic")
|
|
324
|
+
try:
|
|
325
|
+
with io.open(os.path.join(folder, "triggers.json"), encoding="utf-8") as handle:
|
|
326
|
+
triggers = json.load(handle).get("triggers") or []
|
|
327
|
+
except (IOError, ValueError):
|
|
328
|
+
return None
|
|
329
|
+
for trigger in triggers:
|
|
330
|
+
if trigger.get("entity") != binding["code"] or trigger.get("async", True) \
|
|
331
|
+
or trigger.get("event") not in ("update", "any"):
|
|
332
|
+
continue
|
|
333
|
+
with io.open(os.path.join(folder, "actions", trigger["action"] + ".dsl"),
|
|
334
|
+
encoding="utf-8") as handle:
|
|
335
|
+
script = handle.read()
|
|
336
|
+
if "old[%s]" % binding["column"] not in script:
|
|
337
|
+
continue
|
|
338
|
+
slice_, refuse = SLICE.search(script), REFUSE.search(script)
|
|
339
|
+
test = TEST.search(refuse.group(1)) if refuse else None
|
|
340
|
+
if not (slice_ and test) or slice_.group(1) not in declared:
|
|
341
|
+
print(" trigger %s guards %s in a shape this file does not read"
|
|
342
|
+
% (trigger["code"], binding["column"]))
|
|
343
|
+
continue
|
|
344
|
+
variable, resource, operator, value = test.groups()
|
|
345
|
+
module, definition = declared[slice_.group(1)]
|
|
346
|
+
field = (definition.get("resources") or {}).get(resource, {}).get("field", resource)
|
|
347
|
+
current = ('(select r.%s from "%s"."%s" r where r.%s = d.%s and r.%s <= now() '
|
|
348
|
+
'order by r.%s desc limit 1)'
|
|
349
|
+
% (field, module, definition["entity"], slice_.group(2), slice_.group(3),
|
|
350
|
+
definition["period"], definition["period"]))
|
|
351
|
+
refused_when = "%s %s '%s'" % (current, "=" if operator == "==" else "<>", value)
|
|
352
|
+
if "%s == null" % variable in refuse.group(1):
|
|
353
|
+
refused_when = "%s is null or %s" % (current, refused_when)
|
|
354
|
+
return (" and not coalesce(%s, false)" % refused_when,
|
|
355
|
+
"%s %s %s '%s' (trigger %s)" % (slice_.group(1), resource, operator, value,
|
|
356
|
+
trigger["code"]))
|
|
357
|
+
return None
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def exercise(run, binding, declared, tried=()):
|
|
361
|
+
tried = list(tried)
|
|
362
|
+
if not tried:
|
|
363
|
+
print()
|
|
364
|
+
print(" %s.%s -> %s" % (binding["entity"], binding["column"], binding["register"]))
|
|
365
|
+
|
|
366
|
+
if binding["register"] not in declared:
|
|
367
|
+
print(" binds a register no module declares")
|
|
368
|
+
run.failed.append(binding["register"])
|
|
369
|
+
return
|
|
370
|
+
|
|
371
|
+
module, definition = declared[binding["register"]]
|
|
372
|
+
# Which table holds what a posting wrote. A balance register keeps state in `entity`; a
|
|
373
|
+
# TURNOVER register keeps none and names a `journal` instead, which is the same rows to
|
|
374
|
+
# look at and a different key to find them under.
|
|
375
|
+
# A balance WITH a journal is traced through the journal: its rows carry `source`, and the
|
|
376
|
+
# state's do not.
|
|
377
|
+
held = (definition.get("journal") if definition.get("kind") == "balance"
|
|
378
|
+
and definition.get("journal") else None) \
|
|
379
|
+
or definition.get("entity") or definition.get("journal")
|
|
380
|
+
if not held:
|
|
381
|
+
print(" register declares neither 'entity' nor 'journal' - nothing to inspect")
|
|
382
|
+
return
|
|
383
|
+
table = '"%s"."%s"' % (module, held)
|
|
384
|
+
pk_column = binding["code"] + "_id"
|
|
385
|
+
# The register's KEY is the whole dimension tuple, not its first column. Asserting
|
|
386
|
+
# uniqueness on one of them held for the first three registers by accident - each had a
|
|
387
|
+
# dimension unique per line - and failed on the first one keyed by a pair, where 2,184
|
|
388
|
+
# lines share 182 groups and are told apart by the metric.
|
|
389
|
+
dimensions = definition.get("dimensions") or []
|
|
390
|
+
dimension_tuple = ", ".join('"%s"' % d for d in dimensions)
|
|
391
|
+
|
|
392
|
+
# A document nobody has posted, that has lines if this binding posts from lines.
|
|
393
|
+
#
|
|
394
|
+
# Preferring one the INFOBASE posted is not cosmetic. Those are the documents whose
|
|
395
|
+
# register rows 1C holds, so what the platform writes can be compared against them;
|
|
396
|
+
# a document 1C never posted is a draft, and a draft's lines can be half filled. The
|
|
397
|
+
# first run of this binding picked a 361-line draft whose last two lines carry no
|
|
398
|
+
# product at all, and spent the failure on that rather than on the binding.
|
|
399
|
+
# `posted` is 1C's own flag and most documents here carry it; the few that have no 1C
|
|
400
|
+
# source do not, so it is asked for rather than assumed.
|
|
401
|
+
best = "posted desc, " if run.has_column(binding["module"], binding["code"], "posted") \
|
|
402
|
+
else ""
|
|
403
|
+
# A document marked as posted in 1C (`tools/1c.py posted --mark`) is settled: the engine
|
|
404
|
+
# refuses to post it, which is the point of the marker, so it is not one to try.
|
|
405
|
+
marker = binding["column"] + "_imported"
|
|
406
|
+
unmarked = (" and not coalesce(d.%s, false)" % marker
|
|
407
|
+
if run.has_column(binding["module"], binding["code"], marker) else "")
|
|
408
|
+
# A document whose skipped dimension is empty posts nothing and stays unposted by design
|
|
409
|
+
# (`LeaveUnpostedAsync`), so it proves nothing about the binding. For a binding that posts
|
|
410
|
+
# from the document, ask for one that has a value there; on lines the check below counts
|
|
411
|
+
# what the skipped lines leave out.
|
|
412
|
+
if not binding["lineEntity"]:
|
|
413
|
+
for dimension in binding["skip"]:
|
|
414
|
+
source = binding["map"].get(dimension, dimension)
|
|
415
|
+
if plain(source):
|
|
416
|
+
unmarked += " and d.%s is not null" % source
|
|
417
|
+
# A draft whose line leaves a dimension empty that the binding does NOT skip on is refused
|
|
418
|
+
# by design - the engine will not key a row on NULL - and that proves nothing about the
|
|
419
|
+
# binding either: it is a half-filled draft. The act's repacking draft with no batch, one
|
|
420
|
+
# 1C never posted, is one; no posted act has a writing line without one.
|
|
421
|
+
half_filled = ""
|
|
422
|
+
if binding["lineEntity"]:
|
|
423
|
+
required = [binding["map"][d] for d in (definition.get("dimensions") or [])
|
|
424
|
+
if d in binding["map"] and d not in binding["skip"]
|
|
425
|
+
and plain(binding["map"][d])]
|
|
426
|
+
if required:
|
|
427
|
+
half_filled = (' and not exists (select 1 from "%s"."%s" e where e.%s = d.%s and (%s))'
|
|
428
|
+
% (binding["module"], binding["lineEntity"], binding["lineField"],
|
|
429
|
+
pk_column, " or ".join("e.%s is null" % c for c in required)))
|
|
430
|
+
# A document already tried and skipped whole - see `tried` below.
|
|
431
|
+
if tried:
|
|
432
|
+
unmarked += " and d.%s::text not in (%s)" % (
|
|
433
|
+
pk_column, ", ".join("'%s'" % t for t in tried))
|
|
434
|
+
# Nor one whose movements the infobase already holds. 1C keeps the movements of a document
|
|
435
|
+
# that was written unposted after posting - two recipes do this - and posting it again here
|
|
436
|
+
# would collide with its own loaded rows on the slice key, which says nothing about the binding.
|
|
437
|
+
if run.has_column(module, held, "recorder"):
|
|
438
|
+
unmarked += ' and not exists (select 1 from %s r where r.recorder = d.%s)' % (table, pk_column)
|
|
439
|
+
# Nor one a synchronous trigger will refuse to unpost once posted - it would be left posted.
|
|
440
|
+
lock = unpost_lock(binding, declared)
|
|
441
|
+
free = lock[0] if lock else ""
|
|
442
|
+
|
|
443
|
+
def unlockable_none(document, retry):
|
|
444
|
+
"""True, with the binding reported skipped, when only locked candidates remain."""
|
|
445
|
+
if document is None and lock and retry() is not None:
|
|
446
|
+
print(" skipped: no candidate the lock lets unpost - %s" % lock[1])
|
|
447
|
+
return True
|
|
448
|
+
return False
|
|
449
|
+
|
|
450
|
+
if binding["lineEntity"]:
|
|
451
|
+
def candidate(extra):
|
|
452
|
+
return run.one(
|
|
453
|
+
'select d.%s from "%s"."%s" d join "%s"."%s" l on l.%s = d.%s '
|
|
454
|
+
'where d.%s = false%s group by d.%s, d.%s order by %scount(*) desc, d.%s limit 1'
|
|
455
|
+
% (pk_column, binding["module"], binding["code"],
|
|
456
|
+
binding["module"], binding["lineEntity"], binding["lineField"],
|
|
457
|
+
pk_column, binding["column"], unmarked + extra, pk_column,
|
|
458
|
+
"posted" if best else pk_column, best, pk_column))
|
|
459
|
+
document = candidate(half_filled + free)
|
|
460
|
+
if unlockable_none(document, lambda: candidate(half_filled)):
|
|
461
|
+
return
|
|
462
|
+
if document is None and half_filled and candidate("") is not None:
|
|
463
|
+
print(" only drafts whose lines leave %s empty remain - refused by design; "
|
|
464
|
+
"exercise on a new document" % "/".join(required))
|
|
465
|
+
return
|
|
466
|
+
# A line whose skipped dimension is empty records nothing, by declaration, so the
|
|
467
|
+
# lines a posting must account for are the others - the canteen price document
|
|
468
|
+
# records 7 of a 205-line document, the lines whose price changed.
|
|
469
|
+
skipped = [binding["map"].get(d, d) for d in binding["skip"]
|
|
470
|
+
if d not in binding["headerMap"]]
|
|
471
|
+
filled = "".join(" and %s is not null" % c for c in skipped if plain(c))
|
|
472
|
+
# A grouped binding writes one movement per distinct value of its grouping columns.
|
|
473
|
+
counted = ("distinct (%s)" % ", ".join(binding["group"])) if binding["group"] else "*"
|
|
474
|
+
lines = run.one('select count(%s) from "%s"."%s" where %s = %%s%s'
|
|
475
|
+
% (counted, binding["module"], binding["lineEntity"], binding["lineField"],
|
|
476
|
+
filled),
|
|
477
|
+
document) if document else 0
|
|
478
|
+
else:
|
|
479
|
+
def candidate(extra):
|
|
480
|
+
return run.one('select %s from "%s"."%s" d where %s = false%s order by %s limit 1'
|
|
481
|
+
% (pk_column, binding["module"], binding["code"],
|
|
482
|
+
binding["column"], unmarked + extra, best + pk_column))
|
|
483
|
+
document = candidate(free)
|
|
484
|
+
if unlockable_none(document, lambda: candidate("")):
|
|
485
|
+
return
|
|
486
|
+
lines = 1
|
|
487
|
+
|
|
488
|
+
if document is None:
|
|
489
|
+
if tried:
|
|
490
|
+
print(" no draft that writes in %d tried, and none left - "
|
|
491
|
+
"exercise on a new document" % len(tried))
|
|
492
|
+
else:
|
|
493
|
+
print(" no unposted, unmarked document to try")
|
|
494
|
+
return
|
|
495
|
+
print(" document %s, %d line(s)" % (document, lines))
|
|
496
|
+
|
|
497
|
+
# What a posting leaves behind depends on what kind of register it is, and the two
|
|
498
|
+
# shapes need different questions asked of them.
|
|
499
|
+
#
|
|
500
|
+
# An information register's records ARE its journal: each one is stamped with the
|
|
501
|
+
# document that wrote it, so a posting can be found, counted and traced to its line.
|
|
502
|
+
#
|
|
503
|
+
# A balance register with no journal keeps only state, grouped by its dimensions. A
|
|
504
|
+
# document's contribution is not a row and cannot be found afterwards — several
|
|
505
|
+
# documents add into the same row. So the question is not "which rows are mine" but
|
|
506
|
+
# "did the totals move by what this document says, and did unposting put them back".
|
|
507
|
+
# That is a stronger check than counting rows, and it is the only one available.
|
|
508
|
+
traceable = definition.get("kind") == "info" or definition.get("journal")
|
|
509
|
+
|
|
510
|
+
resources = sorted((name, spec.get("field", name))
|
|
511
|
+
for name, spec in (definition.get("resources") or {}).items())
|
|
512
|
+
|
|
513
|
+
def totals():
|
|
514
|
+
return tuple(run.one("select coalesce(sum(%s), 0) from %s" % (field, table))
|
|
515
|
+
for _, field in resources)
|
|
516
|
+
|
|
517
|
+
def expected():
|
|
518
|
+
"""What the document itself says each resource should move by.
|
|
519
|
+
|
|
520
|
+
Read off the lines through the binding's own map, which is the point: if the map
|
|
521
|
+
names the wrong column the totals still move, and only this catches it.
|
|
522
|
+
"""
|
|
523
|
+
source = binding["lineEntity"] or binding["code"]
|
|
524
|
+
where = ("%s = %%s" % binding["lineField"]) if binding["lineEntity"] \
|
|
525
|
+
else ("%s = %%s" % pk_column)
|
|
526
|
+
moved = []
|
|
527
|
+
for name, _ in resources:
|
|
528
|
+
column = binding["map"].get(name)
|
|
529
|
+
if not column or not plain(column):
|
|
530
|
+
moved.append(0)
|
|
531
|
+
continue
|
|
532
|
+
moved.append(run.one('select coalesce(sum(%s), 0) from "%s"."%s" where %s'
|
|
533
|
+
% (column, binding["module"], source, where), document))
|
|
534
|
+
return tuple(moved)
|
|
535
|
+
|
|
536
|
+
# `source.pk` is an OBJECT keyed by the document's pk columns, not a scalar, and
|
|
537
|
+
# `source.entity` is the bare entity code without its module. Both were asserted the
|
|
538
|
+
# other way round here until a record was written and looked at - which is the whole
|
|
539
|
+
# reason this file exists rather than a paragraph claiming the binding works.
|
|
540
|
+
owned = ("select count(*) from %s where source->'pk'->>'%s' = %%s"
|
|
541
|
+
% (table, pk_column))
|
|
542
|
+
before = None
|
|
543
|
+
if traceable:
|
|
544
|
+
run.check("rows this document owns, before", run.one(owned, str(document)), 0)
|
|
545
|
+
else:
|
|
546
|
+
before = totals()
|
|
547
|
+
print(" resource totals before %s" % (before,))
|
|
548
|
+
|
|
549
|
+
key = {pk_column: str(document)}
|
|
550
|
+
result = run.set_state(binding, key, True)
|
|
551
|
+
why = refused(result)
|
|
552
|
+
if why:
|
|
553
|
+
print(" POSTING REFUSED %s" % why)
|
|
554
|
+
run.failed.append("%s.%s post" % (binding["entity"], binding["column"]))
|
|
555
|
+
return
|
|
556
|
+
|
|
557
|
+
# A dimension the DOCUMENT supplies and the binding skips on - a headerMap expression that
|
|
558
|
+
# also carries the handler's gates, as the shop-output act's does - can leave the whole
|
|
559
|
+
# document posting nothing, which is right for it and proves nothing here. Only the platform
|
|
560
|
+
# can evaluate the expression, so the document is tried, and on a skip the next one is.
|
|
561
|
+
# When every draft tried is gated - most of the act's drafts are ones 1C never posted - the
|
|
562
|
+
# binding is not exercised here at all, which is neither a pass nor a failure: the flag is off
|
|
563
|
+
# because the engine leaves it off when nothing was written, so asserting it would FAIL a
|
|
564
|
+
# binding that did what it says. It wants a new document instead.
|
|
565
|
+
header_skip = [d for d in binding["skip"] if d in binding["headerMap"]]
|
|
566
|
+
if header_skip and binding["lineEntity"] and not run.one(
|
|
567
|
+
'select %s from "%s"."%s" where %s = %%s'
|
|
568
|
+
% (binding["column"], binding["module"], binding["code"], pk_column), document) \
|
|
569
|
+
and (not traceable or run.one(owned, str(document)) == 0):
|
|
570
|
+
print(" posted nothing: %s came out empty on the document, as declared"
|
|
571
|
+
% ", ".join(header_skip))
|
|
572
|
+
if len(tried) + 1 >= GATED_TRIES:
|
|
573
|
+
print(" no draft that writes in %d tried - exercise on a new document"
|
|
574
|
+
% (len(tried) + 1))
|
|
575
|
+
return
|
|
576
|
+
exercise(run, binding, declared, tried + [str(document)])
|
|
577
|
+
return
|
|
578
|
+
|
|
579
|
+
try:
|
|
580
|
+
run.check("state flag set",
|
|
581
|
+
run.one('select %s from "%s"."%s" where %s = %%s'
|
|
582
|
+
% (binding["column"], binding["module"], binding["code"],
|
|
583
|
+
pk_column), document), True)
|
|
584
|
+
if traceable:
|
|
585
|
+
if not run.check("rows after posting", run.one(owned, str(document)), lines):
|
|
586
|
+
explain_shortfall(run, binding, definition, document)
|
|
587
|
+
run.check("source names the document",
|
|
588
|
+
run.one("select distinct source->>'entity' from %s "
|
|
589
|
+
"where source->'pk'->>'%s' = %%s" % (table, pk_column), str(document)),
|
|
590
|
+
binding["code"])
|
|
591
|
+
if binding["group"]:
|
|
592
|
+
# A group's movement names the document alone.
|
|
593
|
+
run.check("no source.line on a grouped movement",
|
|
594
|
+
run.one("select count(*) from %s where source->'pk'->>'%s' = %%s "
|
|
595
|
+
"and source ? 'line'" % (table, pk_column), str(document)), 0)
|
|
596
|
+
elif binding["lineEntity"]:
|
|
597
|
+
run.check("source.line distinguishes every line",
|
|
598
|
+
run.one("select count(distinct source->'line'->'pk') from %s "
|
|
599
|
+
"where source->'pk'->>'%s' = %%s" % (table, pk_column), str(document)),
|
|
600
|
+
lines)
|
|
601
|
+
# Grouped, two groups may map to one dimension (the packing shop's warehouse).
|
|
602
|
+
if dimension_tuple and not binding["group"]:
|
|
603
|
+
run.check("distinct (%s)" % ", ".join(dimensions),
|
|
604
|
+
run.one("select count(distinct (%s)) from %s "
|
|
605
|
+
"where source->'pk'->>'%s' = %%s"
|
|
606
|
+
% (dimension_tuple, table, pk_column),
|
|
607
|
+
str(document)), lines)
|
|
608
|
+
else:
|
|
609
|
+
after = totals()
|
|
610
|
+
moved = tuple(a - b for a, b in zip(after, before))
|
|
611
|
+
print(" resource totals after %s" % (after,))
|
|
612
|
+
run.check("totals moved by what the lines say", moved, expected())
|
|
613
|
+
finally:
|
|
614
|
+
result = run.set_state(binding, key, False)
|
|
615
|
+
why = refused(result)
|
|
616
|
+
if why:
|
|
617
|
+
print(" UNPOSTING REFUSED - DOCUMENT %s LEFT POSTED, fix by hand %s"
|
|
618
|
+
% (document, why))
|
|
619
|
+
run.failed.append("%s.%s unpost: %s %s LEFT POSTED"
|
|
620
|
+
% (binding["entity"], binding["column"], pk_column, document))
|
|
621
|
+
return
|
|
622
|
+
if traceable:
|
|
623
|
+
run.check("rows after unposting", run.one(owned, str(document)), 0)
|
|
624
|
+
else:
|
|
625
|
+
run.check("totals back where they were", totals(), before)
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def main():
|
|
629
|
+
only = next((a for a in sys.argv[1:] if not a.startswith("-")), None)
|
|
630
|
+
|
|
631
|
+
import migrate
|
|
632
|
+
migrate.load_env()
|
|
633
|
+
connection = migrate.pg_connect()
|
|
634
|
+
connection.autocommit = True
|
|
635
|
+
|
|
636
|
+
declared = declarations()
|
|
637
|
+
found = bindings(only)
|
|
638
|
+
print("%d register(s) declared, %d binding(s) to exercise"
|
|
639
|
+
% (len(declared), len(found)))
|
|
640
|
+
if not found:
|
|
641
|
+
return 0
|
|
642
|
+
|
|
643
|
+
run = Run(connection.cursor(), token())
|
|
644
|
+
for binding in found:
|
|
645
|
+
exercise(run, binding, declared)
|
|
646
|
+
|
|
647
|
+
print()
|
|
648
|
+
print("%d check(s) failed" % len(run.failed))
|
|
649
|
+
for name in run.failed:
|
|
650
|
+
print(" %s" % name)
|
|
651
|
+
return 1 if run.failed else 0
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
if __name__ == "__main__":
|
|
655
|
+
sys.exit(main())
|