dsh-plugin-opc 0.0.0-stage → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +257 -2
- package/config/overlay.example.yml +9 -0
- package/docs/acceptance-matrix.md +30 -0
- package/docs/images/01-plugins.png +0 -0
- package/docs/images/02-install-menu.png +0 -0
- package/docs/images/03-install-address.png +0 -0
- package/docs/images/04-installed.png +0 -0
- package/docs/images/05-open-sidebar.png +0 -0
- package/docs/images/06-opc-entries.png +0 -0
- package/docs/images/07-space-and-knowledge.png +0 -0
- package/docs/images/08-search-results.png +0 -0
- package/docs/images/09-document.png +0 -0
- package/docs/images/10-mention-menu.png +0 -0
- package/docs/images/11-mention-question.png +0 -0
- package/docs/images/12-answer-sources.png +0 -0
- package/docs/images/13-skills-install.png +0 -0
- package/docs/images/14-skill-enabled.png +0 -0
- package/docs/images/15-skill-slash.png +0 -0
- package/docs/images/16-mcp-list.png +0 -0
- package/docs/images/17-mcp-tool.png +0 -0
- package/docs/images/chat-mention.png +0 -0
- package/docs/images/git-source.png +0 -0
- package/docs/images/mcp-services.png +0 -0
- package/docs/implementation-status.md +40 -0
- package/lib/client.js +31892 -0
- package/lib/index.js +3477 -0
- package/package.json +90 -4
- package/runtime/app_delivery/LICENSE +18 -0
- package/runtime/app_delivery/PROVENANCE.md +1 -0
- package/runtime/app_delivery/SKILL.md +38 -0
- package/runtime/app_delivery/__init__.py +1 -0
- package/runtime/app_delivery/assets/invoke_api.py.txt +269 -0
- package/runtime/app_delivery/references/contract.md +90 -0
- package/runtime/app_delivery/references/framework-checks.md +59 -0
- package/runtime/app_delivery/references/integration.md +23 -0
- package/runtime/app_delivery/references/preparation-checks.md +26 -0
- package/runtime/app_delivery/scripts/__init__.py +1 -0
- package/runtime/app_delivery/scripts/delivery.py +650 -0
- package/runtime/app_delivery/scripts/invoke_api.py +269 -0
- package/runtime/app_delivery/scripts/sqlite_snapshot.py +139 -0
- package/runtime/app_delivery_auth.py +113 -0
- package/runtime/app_delivery_config.py +108 -0
- package/runtime/app_delivery_migration.py +2452 -0
- package/runtime/app_delivery_repair.py +48 -0
- package/runtime/app_delivery_sources.py +39 -0
- package/runtime/vendor/tomli/LICENSE +21 -0
- package/runtime/vendor/tomli/__init__.py +8 -0
- package/runtime/vendor/tomli/_parser.py +793 -0
- package/runtime/vendor/tomli/_re.py +119 -0
- package/runtime/vendor/tomli/_types.py +10 -0
- package/runtime/vendor/yaml/LICENSE +20 -0
- package/runtime/vendor/yaml/__init__.py +389 -0
- package/runtime/vendor/yaml/composer.py +138 -0
- package/runtime/vendor/yaml/constructor.py +748 -0
- package/runtime/vendor/yaml/cyaml.py +100 -0
- package/runtime/vendor/yaml/dumper.py +61 -0
- package/runtime/vendor/yaml/emitter.py +1137 -0
- package/runtime/vendor/yaml/error.py +74 -0
- package/runtime/vendor/yaml/events.py +85 -0
- package/runtime/vendor/yaml/loader.py +63 -0
- package/runtime/vendor/yaml/nodes.py +48 -0
- package/runtime/vendor/yaml/parser.py +588 -0
- package/runtime/vendor/yaml/reader.py +185 -0
- package/runtime/vendor/yaml/representer.py +388 -0
- package/runtime/vendor/yaml/resolver.py +226 -0
- package/runtime/vendor/yaml/scanner.py +1435 -0
- package/runtime/vendor/yaml/serializer.py +110 -0
- package/runtime/vendor/yaml/tokens.py +103 -0
- package/runtime/worker.py +574 -0
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Invoke a documented application operation without embedding credentials."""
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import re
|
|
10
|
+
import sys
|
|
11
|
+
import urllib.error
|
|
12
|
+
import urllib.parse
|
|
13
|
+
import urllib.request
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
17
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def valid_origin(value):
|
|
22
|
+
if not isinstance(value, str) or any(c.isspace() or ord(c) < 32 for c in value) or "\\" in value:
|
|
23
|
+
raise ValueError("base_url must be an HTTP(S) origin without whitespace or backslashes")
|
|
24
|
+
u = urllib.parse.urlsplit(value)
|
|
25
|
+
if u.scheme not in {"http", "https"} or not u.hostname or u.username is not None or u.password is not None:
|
|
26
|
+
raise ValueError("base_url must be a complete HTTP(S) URL without credentials")
|
|
27
|
+
if u.query or u.fragment or u.path not in {"", "/"} or (u.port is not None and not 1 <= u.port <= 65535):
|
|
28
|
+
raise ValueError("base_url must be an application origin, not a path/query or invalid port")
|
|
29
|
+
return u
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def read_json(value, filename):
|
|
33
|
+
if filename:
|
|
34
|
+
return json.load(sys.stdin) if filename == "-" else json.loads(Path(filename).read_text())
|
|
35
|
+
return json.loads(value)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def validate_basic(value, schema, label):
|
|
39
|
+
"""Validate common JSON shapes, not the whole JSON Schema standard."""
|
|
40
|
+
types = {"string": str, "integer": int, "number": (int, float), "boolean": bool, "object": dict, "array": list, "null": type(None)}
|
|
41
|
+
kind = schema.get("type")
|
|
42
|
+
if kind in types:
|
|
43
|
+
if not isinstance(value, types[kind]) or (kind in {"integer", "number"} and isinstance(value, bool)):
|
|
44
|
+
raise ValueError(label + " has the wrong JSON type")
|
|
45
|
+
if "enum" in schema and value not in schema["enum"]:
|
|
46
|
+
raise ValueError(label + " is outside the documented enum")
|
|
47
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
48
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
49
|
+
raise ValueError(label + " must be a finite JSON number")
|
|
50
|
+
if "minimum" in schema and value < schema["minimum"]:
|
|
51
|
+
raise ValueError(label + " is below the documented minimum")
|
|
52
|
+
if "maximum" in schema and value > schema["maximum"]:
|
|
53
|
+
raise ValueError(label + " is above the documented maximum")
|
|
54
|
+
if isinstance(value, dict):
|
|
55
|
+
missing = set(schema.get("required", [])) - set(value)
|
|
56
|
+
if missing:
|
|
57
|
+
raise ValueError("missing required fields in " + label + ": " + ", ".join(sorted(missing)))
|
|
58
|
+
if schema.get("additionalProperties") is False and set(value) - set(schema.get("properties", {})):
|
|
59
|
+
raise ValueError("unknown fields in " + label)
|
|
60
|
+
for key, child_schema in schema.get("properties", {}).items():
|
|
61
|
+
if key in value:
|
|
62
|
+
validate_basic(value[key], child_schema, label + "." + key)
|
|
63
|
+
if isinstance(value, list) and isinstance(schema.get("items"), dict):
|
|
64
|
+
for item in value:
|
|
65
|
+
validate_basic(item, schema["items"], label + "[]")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _resolve_contract_auth_credential(contract, auth, credential_env=None):
|
|
69
|
+
"""Use a supplied session token or exchange supplied credentials at the app login endpoint."""
|
|
70
|
+
environment = os.environ if credential_env is None else credential_env
|
|
71
|
+
profile_id = auth.get("profile")
|
|
72
|
+
profile = contract.get("auth_profiles", {}).get(profile_id) if profile_id else contract.get("auth_profile")
|
|
73
|
+
if profile_id and not isinstance(profile, dict):
|
|
74
|
+
raise ValueError("unknown authentication profile: " + str(profile_id))
|
|
75
|
+
if not isinstance(profile, dict):
|
|
76
|
+
return ""
|
|
77
|
+
token_key = profile.get("token_env")
|
|
78
|
+
if token_key and environment.get(token_key):
|
|
79
|
+
return str(environment[token_key])
|
|
80
|
+
|
|
81
|
+
u_key = profile.get("username_env")
|
|
82
|
+
p_key = profile.get("password_env")
|
|
83
|
+
username = environment.get(u_key, "") if u_key else ""
|
|
84
|
+
password = environment.get(p_key, "") if p_key else ""
|
|
85
|
+
|
|
86
|
+
login_url = str(profile.get("login_url") or "").strip()
|
|
87
|
+
if login_url and (not login_url.startswith("/") or login_url.startswith("//") or "\\" in login_url or any(c in login_url for c in "\r\n#")):
|
|
88
|
+
raise ValueError("login URL must be a same-origin application path")
|
|
89
|
+
if login_url and (u_key or p_key) and (not u_key or username) and (not p_key or password):
|
|
90
|
+
try:
|
|
91
|
+
body_tpl = profile.get("login_body_template") or profile.get("login_body")
|
|
92
|
+
if not isinstance(body_tpl, dict) or not body_tpl:
|
|
93
|
+
body_tpl = {"username": username, "password": password}
|
|
94
|
+
resolved_body = {}
|
|
95
|
+
for k, v in body_tpl.items():
|
|
96
|
+
if v in ("${username}", f"env:{u_key}", "env:USERNAME"):
|
|
97
|
+
resolved_body[k] = username
|
|
98
|
+
elif v in ("${password}", f"env:{p_key}", "env:PASSWORD"):
|
|
99
|
+
resolved_body[k] = password
|
|
100
|
+
elif isinstance(v, str) and v.startswith("env:"):
|
|
101
|
+
env_k = v[4:]
|
|
102
|
+
resolved_body[k] = environment.get(env_k) or (username if "USER" in env_k else password)
|
|
103
|
+
else:
|
|
104
|
+
resolved_body[k] = v
|
|
105
|
+
|
|
106
|
+
req_url = contract["base_url"].rstrip("/") + login_url
|
|
107
|
+
req = urllib.request.Request(
|
|
108
|
+
req_url,
|
|
109
|
+
data=json.dumps(resolved_body).encode("utf-8"),
|
|
110
|
+
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
|
111
|
+
method=str(profile.get("login_method") or "POST").upper(),
|
|
112
|
+
)
|
|
113
|
+
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
|
|
114
|
+
with opener.open(req, timeout=10) as resp:
|
|
115
|
+
if auth.get("type") == "header" and auth.get("header", "").lower() == "cookie":
|
|
116
|
+
cookies = resp.headers.get_all("Set-Cookie") or []
|
|
117
|
+
if cookies:
|
|
118
|
+
return "; ".join(c.split(";", 1)[0] for c in cookies)
|
|
119
|
+
raw = json.loads(resp.read(65536).decode("utf-8"))
|
|
120
|
+
token_path = str(profile.get("token_json_path") or "token").strip()
|
|
121
|
+
cur = raw
|
|
122
|
+
for part in token_path.split("."):
|
|
123
|
+
if isinstance(cur, dict) and part in cur:
|
|
124
|
+
cur = cur[part]
|
|
125
|
+
else:
|
|
126
|
+
cur = None
|
|
127
|
+
break
|
|
128
|
+
if isinstance(cur, str) and cur.strip():
|
|
129
|
+
return cur.strip()
|
|
130
|
+
if isinstance(raw, dict):
|
|
131
|
+
for candidate_key in ("token", "access_token", "jwt", "session_token"):
|
|
132
|
+
if isinstance(raw.get(candidate_key), str) and raw[candidate_key].strip():
|
|
133
|
+
return raw[candidate_key].strip()
|
|
134
|
+
if isinstance(raw.get("data"), dict) and isinstance(raw["data"].get(candidate_key), str):
|
|
135
|
+
return raw["data"][candidate_key].strip()
|
|
136
|
+
except (urllib.error.URLError, ValueError, OSError) as exc:
|
|
137
|
+
raise ValueError("application login failed; check the configured account connection (credentials omitted)") from exc
|
|
138
|
+
if username and password and auth.get("type") == "header" and auth.get("header", "").lower() == "authorization":
|
|
139
|
+
import base64
|
|
140
|
+
return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
|
141
|
+
return ""
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def build_request(contract, operation, params, body, approve_write=False, credential_env=None):
|
|
145
|
+
origin = valid_origin(contract["base_url"])
|
|
146
|
+
matches = [a for a in contract["apis"] if a["id"] == operation]
|
|
147
|
+
if len(matches) != 1:
|
|
148
|
+
raise ValueError("unknown operation; consult references/operations.json")
|
|
149
|
+
api = matches[0]
|
|
150
|
+
if api.get("effect") not in {"read", "write", "destructive", "external"}:
|
|
151
|
+
raise ValueError("operation must declare its business side effects")
|
|
152
|
+
if api["effect"] != "read" and not approve_write:
|
|
153
|
+
raise ValueError("operation changes business state; obtain user approval before --approve-write")
|
|
154
|
+
if not isinstance(params, dict):
|
|
155
|
+
raise ValueError("parameters must be a JSON object")
|
|
156
|
+
path = api["path"]
|
|
157
|
+
query, headers = {}, {"Accept": "application/json"}
|
|
158
|
+
known = {p["name"] for p in api.get("parameters", [])}
|
|
159
|
+
if set(params) - known:
|
|
160
|
+
raise ValueError("unknown parameters: " + ", ".join(sorted(set(params) - known)))
|
|
161
|
+
for param in api.get("parameters", []):
|
|
162
|
+
key = param["name"]
|
|
163
|
+
if param.get("required") and (key not in params or params[key] is None):
|
|
164
|
+
raise ValueError("missing required parameter: " + key)
|
|
165
|
+
if key not in params:
|
|
166
|
+
continue
|
|
167
|
+
value = params[key]
|
|
168
|
+
validate_basic(value, param, "parameter " + key)
|
|
169
|
+
if isinstance(value, (dict, list)):
|
|
170
|
+
raise ValueError("complex parameter encoding must be defined in the skill: " + key)
|
|
171
|
+
value = str(value).lower() if isinstance(value, bool) else str(value)
|
|
172
|
+
if param["in"] == "path":
|
|
173
|
+
if value in {".", "..", ""}:
|
|
174
|
+
raise ValueError("path parameter must be a nonempty non-traversing segment")
|
|
175
|
+
path = path.replace("{" + key + "}", urllib.parse.quote(value, safe=""))
|
|
176
|
+
elif param["in"] == "query":
|
|
177
|
+
query[key] = value
|
|
178
|
+
elif param["in"] == "header":
|
|
179
|
+
if key.lower() in {"host", "authorization", "proxy-authorization", "cookie", "content-length", "transfer-encoding", "connection", "upgrade"}:
|
|
180
|
+
raise ValueError("authentication headers must use the declared auth environment variable")
|
|
181
|
+
headers[key] = value
|
|
182
|
+
if re.search(r"\{[^}]+\}", path):
|
|
183
|
+
raise ValueError("unresolved path parameter")
|
|
184
|
+
if not path.startswith("/") or path.startswith("//") or "?" in path or "#" in path:
|
|
185
|
+
raise ValueError("invalid operation path")
|
|
186
|
+
auth = api.get("auth", {"type": "none"})
|
|
187
|
+
if auth.get("type") not in {"none", "bearer", "header"}:
|
|
188
|
+
raise ValueError("unsupported authentication; a dedicated client is required")
|
|
189
|
+
if auth["type"] != "none":
|
|
190
|
+
credential = (os.environ if credential_env is None else credential_env).get(auth.get("env", ""), "")
|
|
191
|
+
if not credential:
|
|
192
|
+
credential = _resolve_contract_auth_credential(contract, auth, credential_env)
|
|
193
|
+
if not credential:
|
|
194
|
+
raise ValueError("missing credential connection / 应用账号连接不可用,请检查认证方案:" + str(auth.get("profile") or auth.get("env") or "default"))
|
|
195
|
+
if auth["type"] == "bearer":
|
|
196
|
+
headers["Authorization"] = "Bearer " + credential
|
|
197
|
+
else:
|
|
198
|
+
if auth["header"].lower() in {"host", "proxy-authorization", "content-length", "transfer-encoding", "connection", "upgrade"}:
|
|
199
|
+
raise ValueError("invalid authentication header name")
|
|
200
|
+
headers[auth["header"]] = credential
|
|
201
|
+
spec = api.get("request_body")
|
|
202
|
+
if spec and spec.get("required") and body is None:
|
|
203
|
+
raise ValueError("request body is required")
|
|
204
|
+
if body is not None:
|
|
205
|
+
if not spec:
|
|
206
|
+
raise ValueError("operation does not declare a request body")
|
|
207
|
+
schema = spec.get("schema", {})
|
|
208
|
+
validate_basic(body, schema, "request body")
|
|
209
|
+
headers["Content-Type"] = "application/json"
|
|
210
|
+
url = contract["base_url"].rstrip("/") + path
|
|
211
|
+
if query:
|
|
212
|
+
url += "?" + urllib.parse.urlencode(query)
|
|
213
|
+
target = urllib.parse.urlsplit(url)
|
|
214
|
+
if (target.scheme, target.netloc) != (origin.scheme, origin.netloc):
|
|
215
|
+
raise ValueError("request escaped the declared application origin")
|
|
216
|
+
for key, value in headers.items():
|
|
217
|
+
if not re.fullmatch(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+", key) or any(c in value for c in "\r\n\x00"):
|
|
218
|
+
raise ValueError("invalid header name/value; credential values are not logged")
|
|
219
|
+
try:
|
|
220
|
+
value.encode("latin-1")
|
|
221
|
+
except UnicodeEncodeError:
|
|
222
|
+
raise ValueError("header value requires explicit encoding; credential values are not logged") from None
|
|
223
|
+
payload = None if body is None else json.dumps(body, allow_nan=False).encode()
|
|
224
|
+
return urllib.request.Request(url, data=payload, headers=headers, method=api["method"])
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def execute(contract, operation, params, body=None, approve_write=False, credential_env=None):
|
|
228
|
+
request = build_request(contract, operation, params, body, approve_write, credential_env)
|
|
229
|
+
# Do not forward credentials through redirects or system proxies.
|
|
230
|
+
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
|
|
231
|
+
try:
|
|
232
|
+
with opener.open(request, timeout=180) as response:
|
|
233
|
+
return response.status, response.headers.get("Content-Type", ""), response.read(100_000_001)
|
|
234
|
+
except urllib.error.HTTPError as exc:
|
|
235
|
+
try:
|
|
236
|
+
return exc.code, exc.headers.get("Content-Type", ""), exc.read(100_000_001)
|
|
237
|
+
finally:
|
|
238
|
+
exc.close()
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def main():
|
|
242
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
243
|
+
parser.add_argument("--operation", required=True)
|
|
244
|
+
group = parser.add_mutually_exclusive_group()
|
|
245
|
+
group.add_argument("--params", default="{}")
|
|
246
|
+
group.add_argument("--params-file")
|
|
247
|
+
parser.add_argument("--body-file")
|
|
248
|
+
parser.add_argument("--approve-write", action="store_true")
|
|
249
|
+
args = parser.parse_args()
|
|
250
|
+
try:
|
|
251
|
+
root = Path(__file__).resolve().parents[1]
|
|
252
|
+
if (root / ".work-delivery-pending").exists():
|
|
253
|
+
raise ValueError("Work application deployment has not passed verification; this skill is disabled")
|
|
254
|
+
contract = json.loads((root / "references/operations.json").read_text())
|
|
255
|
+
params = read_json(args.params, args.params_file)
|
|
256
|
+
body = read_json("null", args.body_file)
|
|
257
|
+
status, content_type, content = execute(contract, args.operation, params, body, args.approve_write)
|
|
258
|
+
if len(content) > 100_000_000:
|
|
259
|
+
raise ValueError("response exceeds 100 MB; narrow the query")
|
|
260
|
+
print(content.decode("utf-8", errors="replace"))
|
|
261
|
+
return 0 if 200 <= status < 300 else 1
|
|
262
|
+
except (ValueError, KeyError, OSError, urllib.error.URLError) as exc:
|
|
263
|
+
# Avoid logging request objects, credential values, or credential-bearing URLs.
|
|
264
|
+
print("Invocation failed: " + (str(exc) if isinstance(exc, ValueError) else type(exc).__name__), file=sys.stderr)
|
|
265
|
+
return 2
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
if __name__ == "__main__":
|
|
269
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Consistent SQLite snapshots; never copy a live main file without its WAL."""
|
|
2
|
+
from contextlib import closing
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import sqlite3
|
|
6
|
+
import struct
|
|
7
|
+
import tempfile
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def is_sqlite(path):
|
|
12
|
+
with Path(path).open('rb') as stream:
|
|
13
|
+
return stream.read(16) == b'SQLite format 3\x00'
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def validate_wal(path):
|
|
17
|
+
"""Fail closed on malformed archived WALs which SQLite may silently ignore."""
|
|
18
|
+
data = Path(path).read_bytes()
|
|
19
|
+
if not data:
|
|
20
|
+
return
|
|
21
|
+
if len(data) < 32:
|
|
22
|
+
raise ValueError('WAL 文件不完整,请关闭原应用后重新导出')
|
|
23
|
+
magic, version, page_size, checkpoint_seq, salt1, salt2 = struct.unpack('>IIIIII', data[:24])
|
|
24
|
+
if magic not in (0x377f0682, 0x377f0683) or version != 3007000 or page_size < 512 or page_size > 65536 or page_size & (page_size - 1):
|
|
25
|
+
raise ValueError('WAL 文件头无效,请重新导出完整数据库')
|
|
26
|
+
endian = '<' if magic == 0x377f0682 else '>'
|
|
27
|
+
|
|
28
|
+
def checksum(block, state=(0, 0)):
|
|
29
|
+
a, b = state
|
|
30
|
+
words = struct.unpack(endian + str(len(block) // 4) + 'I', block)
|
|
31
|
+
for index in range(0, len(words), 2):
|
|
32
|
+
a = (a + words[index] + b) & 0xffffffff
|
|
33
|
+
b = (b + words[index + 1] + a) & 0xffffffff
|
|
34
|
+
return a, b
|
|
35
|
+
|
|
36
|
+
state = checksum(data[:24])
|
|
37
|
+
frame_size = page_size + 24
|
|
38
|
+
if state != struct.unpack('>II', data[24:32]) or (len(data) - 32) % frame_size:
|
|
39
|
+
raise ValueError('WAL 校验失败或快照不完整,请重新导出')
|
|
40
|
+
if len(data) == 32:
|
|
41
|
+
return
|
|
42
|
+
header_salt = data[16:24]
|
|
43
|
+
active_frames = 0
|
|
44
|
+
last_active_commit = False
|
|
45
|
+
in_recycled_tail = False
|
|
46
|
+
for offset in range(32, len(data), frame_size):
|
|
47
|
+
frame = data[offset:offset + frame_size]
|
|
48
|
+
frame_salt = frame[8:16]
|
|
49
|
+
if frame_salt == header_salt:
|
|
50
|
+
if in_recycled_tail:
|
|
51
|
+
raise ValueError('WAL 帧顺序异常,请关闭原应用后重新导出')
|
|
52
|
+
state = checksum(frame[:8] + frame[24:], state)
|
|
53
|
+
if state != struct.unpack('>II', frame[16:24]):
|
|
54
|
+
raise ValueError('WAL 帧校验失败,请重新导出')
|
|
55
|
+
page_no, db_size = struct.unpack('>II', frame[:8])
|
|
56
|
+
if page_no == 0:
|
|
57
|
+
raise ValueError('WAL 帧页号无效,请重新导出')
|
|
58
|
+
active_frames += 1
|
|
59
|
+
last_active_commit = db_size > 0
|
|
60
|
+
else:
|
|
61
|
+
in_recycled_tail = True
|
|
62
|
+
if active_frames == 0 and offset == 32:
|
|
63
|
+
frame_salt1, frame_salt2 = struct.unpack('>II', frame_salt)
|
|
64
|
+
is_zero_tail = frame == b'\x00' * frame_size
|
|
65
|
+
is_prior_cycle = checkpoint_seq > 0 and frame_salt1 < salt1 and frame_salt2 != salt2
|
|
66
|
+
if not (is_zero_tail or is_prior_cycle):
|
|
67
|
+
raise ValueError('WAL 帧不一致,请关闭原应用后重新导出')
|
|
68
|
+
if active_frames > 0 and not last_active_commit:
|
|
69
|
+
raise ValueError('WAL 含未提交或截断事务,请关闭原应用后重新导出')
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def snapshot_database(source, target, timeout_seconds=30):
|
|
73
|
+
source_path = Path(source)
|
|
74
|
+
target_path = Path(target)
|
|
75
|
+
wal_path = source_path.with_name(source_path.name + '-wal')
|
|
76
|
+
if wal_path.is_file():
|
|
77
|
+
validate_wal(wal_path)
|
|
78
|
+
started = time.monotonic()
|
|
79
|
+
|
|
80
|
+
def progress(status, remaining, total):
|
|
81
|
+
if time.monotonic() - started > timeout_seconds:
|
|
82
|
+
raise ValueError('数据库持续忙碌,请暂停写入后重新导入')
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
connect_timeout = max(0.1, min(5.0, float(timeout_seconds)))
|
|
86
|
+
with closing(sqlite3.connect(source_path.resolve().as_uri() + '?mode=ro', uri=True, timeout=connect_timeout)) as original:
|
|
87
|
+
with closing(sqlite3.connect(str(target_path), timeout=connect_timeout)) as output:
|
|
88
|
+
original.backup(output, pages=256, progress=progress, sleep=0.05)
|
|
89
|
+
if output.execute('PRAGMA quick_check').fetchone() != ('ok',):
|
|
90
|
+
raise ValueError('数据库快照校验失败')
|
|
91
|
+
mode_row = output.execute('PRAGMA journal_mode').fetchone()
|
|
92
|
+
if not mode_row or str(mode_row[0]).lower() != 'delete':
|
|
93
|
+
output.execute('PRAGMA journal_mode=DELETE')
|
|
94
|
+
if not wal_path.exists():
|
|
95
|
+
src_bytes = source_path.read_bytes()
|
|
96
|
+
dst_bytes = bytearray(target_path.read_bytes())
|
|
97
|
+
if len(src_bytes) == len(dst_bytes) >= 100 and src_bytes[18:20] == b'\x01\x01':
|
|
98
|
+
dst_bytes[24:28] = src_bytes[24:28]
|
|
99
|
+
dst_bytes[92:96] = src_bytes[92:96]
|
|
100
|
+
if bytes(dst_bytes) == src_bytes:
|
|
101
|
+
target_path.write_bytes(dst_bytes)
|
|
102
|
+
except sqlite3.Error as error:
|
|
103
|
+
raise ValueError('无法生成一致的 SQLite 快照,请关闭应用后重新导出') from error
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def validate_sidecars(source):
|
|
107
|
+
for item in Path(source).rglob('*'):
|
|
108
|
+
if item.is_symlink():
|
|
109
|
+
raise ValueError('source contains symlink')
|
|
110
|
+
if not item.is_file():
|
|
111
|
+
continue
|
|
112
|
+
if item.name.endswith('-wal'):
|
|
113
|
+
database = item.with_name(item.name[:-4])
|
|
114
|
+
if not database.is_file() or not is_sqlite(database):
|
|
115
|
+
raise ValueError('发现缺少原数据库的 WAL 文件:' + item.name)
|
|
116
|
+
if item.name.endswith('-journal') and item.stat().st_size:
|
|
117
|
+
raise ValueError('存在数据库事务日志,请关闭原应用后重新导出:' + item.name)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def normalize_extracted_databases(source):
|
|
121
|
+
validate_sidecars(source)
|
|
122
|
+
for database in list(Path(source).rglob('*')):
|
|
123
|
+
if not database.is_file() or not is_sqlite(database):
|
|
124
|
+
continue
|
|
125
|
+
wal = database.with_name(database.name + '-wal')
|
|
126
|
+
if wal.exists():
|
|
127
|
+
validate_wal(wal)
|
|
128
|
+
# SHM is a derived index, not data; reconstruct it from the archived WAL.
|
|
129
|
+
database.with_name(database.name + '-shm').unlink(missing_ok=True)
|
|
130
|
+
with tempfile.TemporaryDirectory(prefix='opc-sqlite-') as temp:
|
|
131
|
+
snapshot = Path(temp) / 'snapshot.db'
|
|
132
|
+
snapshot_database(database, snapshot)
|
|
133
|
+
mode = database.stat().st_mode & 0o777
|
|
134
|
+
with snapshot.open('rb') as inp, database.open('wb') as out:
|
|
135
|
+
import shutil
|
|
136
|
+
shutil.copyfileobj(inp, out)
|
|
137
|
+
os.chmod(database, mode)
|
|
138
|
+
wal.unlink(missing_ok=True)
|
|
139
|
+
database.with_name(database.name + '-shm').unlink(missing_ok=True)
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Resolve scanner uncertainty using explicit, evidence-backed authentication rules."""
|
|
2
|
+
import copy
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from app_delivery.scripts import delivery
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def resolve_authentication(contract):
|
|
9
|
+
rules = contract.get('auth_rules', [])
|
|
10
|
+
if not isinstance(rules, list):
|
|
11
|
+
raise ValueError('auth_rules 必须是数组')
|
|
12
|
+
for rule in rules:
|
|
13
|
+
if not isinstance(rule, dict) or rule.get('target') not in {'api', 'page'} or not rule.get('evidence'):
|
|
14
|
+
raise ValueError('认证规则必须声明 api/page 范围与源码证据')
|
|
15
|
+
if not isinstance(rule.get('auth'), dict) or rule['auth'].get('type') not in {'none', 'header', 'bearer'}:
|
|
16
|
+
raise ValueError('认证规则必须使用已确认的 none/header/bearer 方式')
|
|
17
|
+
if not isinstance(rule.get('evidence'), list) or not all(isinstance(e, str) and e for e in rule['evidence']):
|
|
18
|
+
raise ValueError('认证规则 evidence 必须是源码位置数组')
|
|
19
|
+
location = rule.get('path') or rule.get('prefix')
|
|
20
|
+
if not isinstance(location, str) or not location.startswith('/'):
|
|
21
|
+
raise ValueError('认证规则路径必须以 / 开头')
|
|
22
|
+
if bool(rule.get('path')) == bool(rule.get('prefix')):
|
|
23
|
+
raise ValueError('认证规则必须选择精确 path 或路径 prefix')
|
|
24
|
+
for target, key in [('api', 'apis'), ('page', 'pages')]:
|
|
25
|
+
for item in contract.get(key, []):
|
|
26
|
+
matches = [r for r in rules if r['target'] == target
|
|
27
|
+
and (not r.get('method') or r['method'].upper() == item.get('method', '').upper())
|
|
28
|
+
and (r.get('path') == item['path'] or
|
|
29
|
+
(r.get('prefix') and item['path'].startswith(r['prefix'])))]
|
|
30
|
+
if matches:
|
|
31
|
+
ranked = sorted(matches, key=lambda r: (bool(r.get('path')), len(r.get('prefix', '')), bool(r.get('method'))), reverse=True)
|
|
32
|
+
best = ranked[0]
|
|
33
|
+
rank = lambda r: (bool(r.get('path')), len(r.get('prefix', '')), bool(r.get('method')))
|
|
34
|
+
if any(rank(r) == rank(best) and r.get('auth') != best.get('auth') for r in ranked[1:]):
|
|
35
|
+
raise ValueError('存在互相冲突的认证规则:' + item['path'])
|
|
36
|
+
item['auth'] = {**copy.deepcopy(best.get('auth', {})), 'evidence': best['evidence']}
|
|
37
|
+
auth = item.get('auth', {'type': 'unknown'})
|
|
38
|
+
if auth.get('type') == 'unknown':
|
|
39
|
+
raise ValueError('认证尚未分析:' + item['path'] + '。请读取登录入口、统一中间件和权限检查,填写有证据的 auth 或 auth_rules;不要向用户索要猜测的令牌。')
|
|
40
|
+
return contract
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def validate_auth_evidence(source, proposal):
|
|
44
|
+
"""Check citations in the exact source snapshot the proposal describes."""
|
|
45
|
+
contract = proposal["contract"]
|
|
46
|
+
changes = {change["path"]: change["content"] for change in proposal.get("changes", [])}
|
|
47
|
+
line_counts = {}
|
|
48
|
+
for entry in [*contract.get("auth_rules", []), *contract.get("auth_blockers", [])]:
|
|
49
|
+
if not isinstance(entry, dict) or not isinstance(entry.get("evidence"), list) or not entry["evidence"]:
|
|
50
|
+
raise ValueError("认证证据必须是非空的 文件:行号 数组")
|
|
51
|
+
for evidence in entry["evidence"]:
|
|
52
|
+
match = re.fullmatch(r"(.+):(\d+)(?:-(\d+))?", evidence) if isinstance(evidence, str) else None
|
|
53
|
+
if not match:
|
|
54
|
+
raise ValueError("认证证据必须引用源码读取工具返回的 文件:行号")
|
|
55
|
+
path, start, end = match.groups()
|
|
56
|
+
if path not in line_counts:
|
|
57
|
+
source_path = delivery.relative(source, path)
|
|
58
|
+
content = changes.get(path)
|
|
59
|
+
if content is None and source_path.is_file():
|
|
60
|
+
content = delivery.text_of(source_path)
|
|
61
|
+
line_counts[path] = len(content.splitlines()) if content is not None else 0
|
|
62
|
+
count = line_counts[path]
|
|
63
|
+
if not 1 <= int(start) <= int(end or start) <= count:
|
|
64
|
+
location = "拟提交文件" if path in changes else "当前导入源码"
|
|
65
|
+
raise ValueError(
|
|
66
|
+
f"认证证据位置不存在:{evidence}。{location} {path} 共 {count} 行。"
|
|
67
|
+
"请只重读该文件的登录处理、会话读取或权限检查,用实际行号修正证据后重新提交。"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def verification_credential_fields(contract):
|
|
72
|
+
"""Describe the credential references the verification form may accept."""
|
|
73
|
+
fields = {}
|
|
74
|
+
for api in contract.get("apis", []):
|
|
75
|
+
auth = api.get("auth", {})
|
|
76
|
+
if auth.get("type") in {"header", "bearer"} and auth.get("env"):
|
|
77
|
+
fields[auth["env"]] = "session"
|
|
78
|
+
for profile in [contract.get("auth_profile", {}), *contract.get("auth_profiles", {}).values()]:
|
|
79
|
+
for key, kind in (("token_env", "session"), ("username_env", "username"), ("password_env", "password")):
|
|
80
|
+
if profile.get(key):
|
|
81
|
+
fields[profile[key]] = kind
|
|
82
|
+
return [{"key": key, "kind": kind} for key, kind in fields.items()]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def validate_auth_questions(proposal):
|
|
86
|
+
"""Technical uncertainty is returned to the model, not turned into a user prompt."""
|
|
87
|
+
contract = proposal.get('contract', {})
|
|
88
|
+
questions = set(proposal.get('unresolved', []) + contract.get('unresolved', []))
|
|
89
|
+
blockers = contract.get('auth_blockers', [])
|
|
90
|
+
if not isinstance(blockers, list):
|
|
91
|
+
raise ValueError('auth_blockers 必须是数组')
|
|
92
|
+
for question in questions:
|
|
93
|
+
if re.search(r'令牌|token|登录|账密|凭据|授权码', question, re.I):
|
|
94
|
+
if not any(b.get('question') == question and b.get('evidence') and b.get('affected')
|
|
95
|
+
and b.get('action') and b.get('reason') in {'external_authorization', 'missing_credential', 'role_change'}
|
|
96
|
+
for b in blockers if isinstance(b, dict)):
|
|
97
|
+
raise ValueError('认证提问缺少证据:请先核对实际认证代码;确实需要外部授权时提供 auth_blockers 的 question/evidence/affected/action/reason,不要重复询问推测的管理令牌。')
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def validate_auth_probe_coverage(contract):
|
|
101
|
+
"""New auth plans must exercise every protected API identity with a read-only case."""
|
|
102
|
+
if not contract.get('auth_rules') and not contract.get('auth_profiles'):
|
|
103
|
+
return # Existing packaged skills keep their original verification contract.
|
|
104
|
+
def identity(api):
|
|
105
|
+
auth = api.get('auth', {})
|
|
106
|
+
return (auth.get('profile', 'default'), auth.get('type'), auth.get('header', ''), auth.get('env', ''), tuple(auth.get('roles', [])))
|
|
107
|
+
protected = [a for a in contract.get('apis', []) if a.get('auth', {}).get('type') in {'header', 'bearer'}]
|
|
108
|
+
operations = {c.get('operation') for c in contract.get('verification', {}).get('cases', [])}
|
|
109
|
+
covered = {identity(a) for a in protected if a.get('id') in operations and a.get('effect') == 'read'
|
|
110
|
+
and a.get('path') != contract.get('health', {}).get('path')}
|
|
111
|
+
missing = {identity(a) for a in protected} - covered
|
|
112
|
+
if missing:
|
|
113
|
+
raise ValueError('认证验证计划不完整:每种认证身份/权限范围需要只读业务用例;不能用公开健康接口代替。请自动补齐验证计划。')
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Read original configuration locally; credential values never enter model context."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import re
|
|
8
|
+
try:
|
|
9
|
+
import tomllib
|
|
10
|
+
except ImportError:
|
|
11
|
+
from vendor import tomli as tomllib
|
|
12
|
+
|
|
13
|
+
from vendor import yaml
|
|
14
|
+
|
|
15
|
+
from app_delivery.scripts import delivery
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def document(source: Path, name: str):
|
|
19
|
+
path = delivery.relative(source, name)
|
|
20
|
+
text = delivery.text_of(path)
|
|
21
|
+
suffix = path.suffix.lower()
|
|
22
|
+
try:
|
|
23
|
+
if suffix in {".yaml", ".yml"}:
|
|
24
|
+
return yaml.safe_load(text)
|
|
25
|
+
if suffix == ".json":
|
|
26
|
+
return json.loads(text)
|
|
27
|
+
if suffix == ".toml":
|
|
28
|
+
return tomllib.loads(text)
|
|
29
|
+
if path.name.startswith(".env") or suffix == ".properties":
|
|
30
|
+
result = {}
|
|
31
|
+
for line in text.splitlines():
|
|
32
|
+
match = re.match(r"\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_.-]*)\s*=\s*(.*?)\s*$", line)
|
|
33
|
+
if match:
|
|
34
|
+
value = match[2]
|
|
35
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
36
|
+
value = value[1:-1]
|
|
37
|
+
result[match[1]] = value
|
|
38
|
+
return result
|
|
39
|
+
except Exception:
|
|
40
|
+
raise ValueError("无法读取原配置结构:" + name) from None
|
|
41
|
+
raise ValueError("原配置引用仅支持 YAML、JSON、TOML、.env 和 properties:" + name)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def original_value(source: Path, reference: dict) -> str:
|
|
45
|
+
if not isinstance(reference, dict) or not isinstance(reference.get("path"), str):
|
|
46
|
+
raise ValueError("原配置引用需要 path 和 keys")
|
|
47
|
+
keys = reference.get("keys")
|
|
48
|
+
if not isinstance(keys, list) or not keys or any(type(k) not in {str, int} for k in keys):
|
|
49
|
+
raise ValueError("原配置引用的 keys 必须是非空路径数组")
|
|
50
|
+
value = document(source, reference["path"])
|
|
51
|
+
try:
|
|
52
|
+
for key in keys:
|
|
53
|
+
if isinstance(value, list):
|
|
54
|
+
if type(key) is not int or key < 0:
|
|
55
|
+
raise KeyError(key)
|
|
56
|
+
elif not isinstance(value, dict):
|
|
57
|
+
raise KeyError(key)
|
|
58
|
+
value = value[key]
|
|
59
|
+
except (KeyError, IndexError, TypeError):
|
|
60
|
+
raise ValueError("找不到原配置字段:" + reference["path"] + " / " + ".".join(map(str, keys))) from None
|
|
61
|
+
if value is None:
|
|
62
|
+
return ""
|
|
63
|
+
if type(value) not in {str, int, float, bool}:
|
|
64
|
+
raise ValueError("原配置引用必须指向单个值")
|
|
65
|
+
return str(value).lower() if type(value) is bool else str(value)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def loopback_connection(key: str, value: str) -> bool:
|
|
69
|
+
"""Conservative address hint, never returns credentials or assumes network reachability."""
|
|
70
|
+
if key.upper() in {"HOST", "PORT", "BIND_HOST", "LISTEN_HOST"} or not re.search(r"URL|URI|DSN|HOST|ENDPOINT|CONNECTION|SERVER", key, re.I):
|
|
71
|
+
return False
|
|
72
|
+
return bool(re.search(r"(?:^|://|@|tcp\(|(?:host|server)\s*=)(?:localhost|127\.\d+\.\d+\.\d+|\[::1\])(?=[:/,;)]|$)", value.strip(), re.I))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def original_fields(source: Path, name: str) -> list[dict]:
|
|
76
|
+
"""Metadata only, including exact paths for masked fields; no original values."""
|
|
77
|
+
try:
|
|
78
|
+
root = document(source, name)
|
|
79
|
+
except (ValueError, OSError):
|
|
80
|
+
return []
|
|
81
|
+
fields = []
|
|
82
|
+
|
|
83
|
+
def walk(value, keys):
|
|
84
|
+
if len(fields) >= 200 or len(keys) > 24:
|
|
85
|
+
return
|
|
86
|
+
if isinstance(value, dict):
|
|
87
|
+
for key, item in value.items():
|
|
88
|
+
if type(key) in {str, int}:
|
|
89
|
+
walk(item, [*keys, key])
|
|
90
|
+
elif isinstance(value, list):
|
|
91
|
+
for key, item in enumerate(value):
|
|
92
|
+
walk(item, [*keys, key])
|
|
93
|
+
elif keys and (value is None or type(value) in {str, int, float, bool}):
|
|
94
|
+
metadata = {"source": {"path": name, "keys": keys}, "has_value": value is not None and str(value) != ""}
|
|
95
|
+
if loopback_connection("_".join(map(str, keys)), str(value or "")):
|
|
96
|
+
metadata["connection_hint"] = "loopback: points to the container itself after deployment; inspect whether dependency is in that same container"
|
|
97
|
+
fields.append(metadata)
|
|
98
|
+
|
|
99
|
+
walk(root, [])
|
|
100
|
+
return fields
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def runtime_configs(source: Path, configs: list[dict]) -> list[dict]:
|
|
104
|
+
result = []
|
|
105
|
+
for config in configs:
|
|
106
|
+
value = original_value(source, config["source"]) if config.get("source") else config.get("default", "")
|
|
107
|
+
result.append({"key": config["key"], "value": value, "secret": config["secret"], "required": config["required"], "description": config.get("description", ""), "value_source": "original" if config.get("source") else "default", "source_label": config["source"]["path"] + " / " + ".".join(map(str, config["source"]["keys"])) if config.get("source") else ""})
|
|
108
|
+
return result
|