blun-king-cli 9.1.597 → 9.1.600
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +17 -1
- package/agent-spine-plugin/CHANGELOG.md +1 -1
- package/agent-spine-plugin/docs/host-integration.md +1 -15
- package/agent-spine-plugin/docs/preflight-recall.md +1 -1
- package/agent-spine-plugin/scripts/check-install.js +1 -6
- package/agent-spine-plugin/scripts/hermetic-process.js +8 -0
- package/agent-spine-plugin/scripts/release-check.js +2 -3
- package/agent-spine-plugin/scripts/run-tests-hermetic.js +8 -11
- package/agent-spine-plugin/src/hook.js +40 -40
- package/agent-spine-plugin/src/lib/briefing.js +6 -3
- package/agent-spine-plugin/src/lib/hook-context.js +63 -64
- package/agent-spine-plugin/src/lib/hook-output.js +115 -94
- package/agent-spine-plugin/src/lib/hook-timeline.js +20 -11
- package/agent-spine-plugin/src/lib/host-instruction-budget.js +8 -14
- package/agent-spine-plugin/src/lib/mcp-timeline-tools.js +0 -7
- package/agent-spine-plugin/src/lib/pre-answer-timeline-recall.js +104 -0
- package/agent-spine-plugin/src/lib/session-timeline-codex.js +0 -4
- package/agent-spine-plugin/src/lib/session-timeline-enrollment-source.js +0 -3
- package/agent-spine-plugin/src/lib/session-timeline-enrollment-storage.js +0 -4
- package/agent-spine-plugin/src/lib/session-timeline-enrollment-transport.js +0 -2
- package/agent-spine-plugin/src/lib/session-timeline-enrollment.js +0 -22
- package/agent-spine-plugin/src/lib/session-timeline-host-origin.js +0 -4
- package/agent-spine-plugin/src/lib/session-timeline-prior.js +42 -50
- package/agent-spine-plugin/src/lib/session-timeline-provider.js +0 -3
- package/agent-spine-plugin/src/lib/session-timeline-query.js +0 -1
- package/agent-spine-plugin/src/lib/session-timeline-sid-acl.js +0 -4
- package/agent-spine-plugin/src/lib/session-timeline-source.js +0 -3
- package/agent-spine-plugin/src/lib/session-timeline-transport.js +0 -8
- package/agent-spine-plugin/src/lib/session-timeline-windows-acl.js +0 -3
- package/agent-spine-plugin/src/lib/session-timeline.js +370 -408
- package/agent-spine-plugin/src/lib/source-roots.js +26 -16
- package/agent-spine-plugin/src/lib/timeline-user-feedback.js +0 -1
- package/bin/core-bootstrap.js +18 -10
- package/bin/launcher-mode.js +5 -0
- package/bin/launcher-runtime.js +27 -0
- package/bin/verify-bundled-agent-sources.cjs +28 -2
- package/blun.mjs +798 -143
- package/bundled-agent-sources.json +192 -155
- package/package.json +1 -1
- package/standard-skills/translate-native/README.md +1334 -1
- package/standard-skills/translate-native/SKILL.md +1 -1
- package/standard-skills/translate-native/VERSION +1 -1
- package/standard-skills/translate-native/docs/PREMORTEM.md +901 -0
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION.md +208 -41
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_API.md +1319 -77
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_EVIDENCE_HTTP.md +66 -11
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_HTTP_PROVIDER.md +19 -1
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_RECEIPT_VERIFIER_HTTP.md +24 -6
- package/standard-skills/translate-native/integrations/commercial_localization_profile.py +23 -0
- package/standard-skills/translate-native/integrations/website_localization.py +234 -10
- package/standard-skills/translate-native/integrations/website_localization_benchmark.py +39 -3
- package/standard-skills/translate-native/integrations/website_localization_cms.py +512 -10
- package/standard-skills/translate-native/integrations/website_localization_cms_client.py +116 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_receiver.py +101 -11
- package/standard-skills/translate-native/integrations/website_localization_cms_receiver_runtime.py +12 -1
- package/standard-skills/translate-native/integrations/website_localization_cms_receiver_store.py +186 -24
- package/standard-skills/translate-native/integrations/website_localization_cms_source_client.py +64 -1
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery.py +528 -5
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_auth.py +918 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_auth_runtime.py +609 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_client.py +885 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_http.py +1244 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_runtime.py +220 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_sidecar_adapter.py +331 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_capabilities_http.py +444 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_client.py +673 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_dispatch.py +1317 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_dispatch_client.py +826 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_dispatch_http.py +1152 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_dispatch_openapi.py +896 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_dispatch_runtime.py +621 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_http.py +1180 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_submission_runtime.py +1479 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_http.py +133 -17
- package/standard-skills/translate-native/integrations/website_localization_cms_source_runtime.py +319 -4
- package/standard-skills/translate-native/integrations/website_localization_cms_source_service.py +72 -2
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification_http.py +13 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification_receiver.py +56 -13
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification_receiver_openapi.py +335 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification_receiver_runtime.py +66 -1
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_processing_monitor.py +73 -7
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_receiver_client.py +93 -9
- package/standard-skills/translate-native/integrations/website_localization_evidence_http.py +221 -11
- package/standard-skills/translate-native/integrations/website_localization_health.py +6 -0
- package/standard-skills/translate-native/integrations/website_localization_http_provider.py +1 -1
- package/standard-skills/translate-native/integrations/website_localization_quality_profiles.py +350 -0
- package/standard-skills/translate-native/integrations/website_localization_queue.py +126 -56
- package/standard-skills/translate-native/integrations/website_localization_receipt_verifier_http.py +166 -13
- package/standard-skills/translate-native/integrations/website_localization_release.py +423 -17
- package/standard-skills/translate-native/integrations/website_localization_release_coordinator.py +168 -13
- package/standard-skills/translate-native/integrations/website_localization_runner.py +10 -0
- package/standard-skills/translate-native/integrations/website_localization_worker.py +213 -9
- package/standard-skills/translate-native/provenance.json +2 -2
- package/standard-skills/translate-native/references/commercial-localization.md +96 -17
- package/standard-skills/translate-native/scripts/blun_language_guard.py +3 -1
- package/standard-skills/translate-native/scripts/check_commercial_review.py +69 -9
- package/standard-skills/translate-native/scripts/commercial_localization_profile.py +829 -31
- package/standard-tools/language-guard/VERSION +1 -1
- package/standard-tools/language-guard/blun_language_guard.py +3 -1
- package/standard-tools/language-guard/check_commercial_review.py +69 -9
- package/standard-tools/language-guard/commercial_localization_profile.py +829 -31
- package/standard-tools/language-guard/provenance.json +2 -2
- package/standard-tools/manifest.json +1 -1
- package/worker-host.mjs +215 -82
- package/agent-spine-plugin/scripts/check-install-king.js +0 -37
|
@@ -0,0 +1,1317 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Durable caller-owned outbox for the public website submission client.
|
|
3
|
+
|
|
4
|
+
The CMS backend commits one immutable change, cancellation, or tombstone before
|
|
5
|
+
network access. One leased attempt performs one client call. A crash after
|
|
6
|
+
remote acceptance safely replays the same request identity after lease expiry;
|
|
7
|
+
the public website edge remains the idempotency authority.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import importlib.util
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
import secrets
|
|
18
|
+
import sqlite3
|
|
19
|
+
import sys
|
|
20
|
+
from contextlib import contextmanager
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Iterator, Mapping
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
SCHEMA_VERSION = 2
|
|
27
|
+
MAX_ATTEMPTS = 20
|
|
28
|
+
MAX_LEASE_SECONDS = 86_400.0
|
|
29
|
+
MAX_DELAY_SECONDS = 86_400.0
|
|
30
|
+
STATUSES = ("pending", "leased", "retry_wait", "accepted", "failed")
|
|
31
|
+
OPERATIONS = ("change", "cancellation", "tombstone")
|
|
32
|
+
TOKEN = re.compile(r"^[A-Za-z0-9_.:-]{1,256}$")
|
|
33
|
+
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
34
|
+
ERROR_CODE = re.compile(r"^[a-z][a-z0-9_.-]{0,127}$")
|
|
35
|
+
_META_COLUMNS = (
|
|
36
|
+
"singleton", "schema_version", "expected_capabilities_sha256",
|
|
37
|
+
)
|
|
38
|
+
_COLUMNS = (
|
|
39
|
+
"operation", "request_id", "event_id", "site_id", "payload_json",
|
|
40
|
+
"payload_sha256", "commercial_contract_binding_json",
|
|
41
|
+
"source_max_attempts", "delivery_max_attempts",
|
|
42
|
+
"client_max_attempts", "status", "attempts", "next_attempt_at",
|
|
43
|
+
"lease_owner", "lease_token", "lease_expires_at", "last_error_code",
|
|
44
|
+
"remote_status", "remote_attempts", "remote_capabilities_sha256",
|
|
45
|
+
"remote_binding_sha256", "response_json", "response_sha256", "created_at",
|
|
46
|
+
"updated_at",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _load_module(name: str, path: Path):
|
|
51
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
52
|
+
if spec is None or spec.loader is None:
|
|
53
|
+
raise RuntimeError(f"cannot load submission dispatch dependency: {path.name}")
|
|
54
|
+
module = importlib.util.module_from_spec(spec)
|
|
55
|
+
sys.modules[spec.name] = module
|
|
56
|
+
spec.loader.exec_module(module)
|
|
57
|
+
return module
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_ROOT = Path(__file__).resolve().parents[1]
|
|
61
|
+
_CLIENT = _load_module(
|
|
62
|
+
"blun_website_localization_submission_dispatch_client",
|
|
63
|
+
_ROOT
|
|
64
|
+
/ "integrations"
|
|
65
|
+
/ "website_localization_cms_source_delivery_submission_client.py",
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class CMSSourceDeliverySubmissionDispatchBlocked(RuntimeError):
|
|
70
|
+
"""Stable content-free failure of the caller-owned submission outbox."""
|
|
71
|
+
|
|
72
|
+
def __init__(self, code: str):
|
|
73
|
+
if not isinstance(code, str) or ERROR_CODE.fullmatch(code) is None:
|
|
74
|
+
raise ValueError("submission dispatch error code is invalid")
|
|
75
|
+
super().__init__(code)
|
|
76
|
+
self.code = code
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True)
|
|
80
|
+
class SubmissionDispatchClaim:
|
|
81
|
+
operation: str
|
|
82
|
+
request_id: str
|
|
83
|
+
event_id: str
|
|
84
|
+
site_id: str
|
|
85
|
+
payload_sha256: str
|
|
86
|
+
commercial_contract_binding: dict[str, Any] | None
|
|
87
|
+
source_max_attempts: int
|
|
88
|
+
delivery_max_attempts: int
|
|
89
|
+
attempt: int
|
|
90
|
+
client_max_attempts: int
|
|
91
|
+
lease_owner: str
|
|
92
|
+
lease_token: str
|
|
93
|
+
lease_expires_at: float
|
|
94
|
+
payload_json: str = field(repr=False, compare=False)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@dataclass(frozen=True)
|
|
98
|
+
class SubmissionDispatchStatus:
|
|
99
|
+
operation: str
|
|
100
|
+
request_id: str
|
|
101
|
+
event_id: str
|
|
102
|
+
site_id: str
|
|
103
|
+
payload_sha256: str
|
|
104
|
+
commercial_contract_binding: dict[str, Any] | None
|
|
105
|
+
source_max_attempts: int
|
|
106
|
+
delivery_max_attempts: int
|
|
107
|
+
status: str
|
|
108
|
+
attempts: int
|
|
109
|
+
client_max_attempts: int
|
|
110
|
+
next_attempt_at: float
|
|
111
|
+
lease_expires_at: float | None
|
|
112
|
+
lease_expired: bool
|
|
113
|
+
last_error_code: str | None
|
|
114
|
+
remote_status: str | None
|
|
115
|
+
remote_attempts: int | None
|
|
116
|
+
remote_capabilities_sha256: str | None
|
|
117
|
+
remote_binding_sha256: str | None
|
|
118
|
+
remote_website_capability_binding: dict[str, Any] | None
|
|
119
|
+
response_sha256: str | None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class SubmissionDispatchOutcome:
|
|
124
|
+
operation: str
|
|
125
|
+
request_id: str
|
|
126
|
+
status: str
|
|
127
|
+
attempt: int
|
|
128
|
+
client_max_attempts: int
|
|
129
|
+
next_attempt_at: float
|
|
130
|
+
error_code: str | None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass(frozen=True)
|
|
134
|
+
class SubmissionDispatchHealth:
|
|
135
|
+
status: str
|
|
136
|
+
counts: dict[str, int]
|
|
137
|
+
operations: dict[str, int]
|
|
138
|
+
due: int
|
|
139
|
+
expired_leases: int
|
|
140
|
+
failed: int
|
|
141
|
+
expected_capabilities_sha256: str
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(frozen=True)
|
|
145
|
+
class SubmissionDispatchLifecycle:
|
|
146
|
+
dispatch_status: SubmissionDispatchStatus
|
|
147
|
+
source_lifecycle: dict[str, Any]
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True)
|
|
151
|
+
class SubmissionDispatchCommercialProfile:
|
|
152
|
+
commercial_profile: dict[str, Any]
|
|
153
|
+
commercial_rendering_registry: dict[str, Any]
|
|
154
|
+
website_capability_binding: dict[str, Any]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _blocked(code: str) -> CMSSourceDeliverySubmissionDispatchBlocked:
|
|
158
|
+
return CMSSourceDeliverySubmissionDispatchBlocked(
|
|
159
|
+
"source_delivery_submission_dispatch." + code
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _token(value: Any, code: str, *, limit: int = 256) -> str:
|
|
164
|
+
if (
|
|
165
|
+
not isinstance(value, str)
|
|
166
|
+
or len(value) > limit
|
|
167
|
+
or TOKEN.fullmatch(value) is None
|
|
168
|
+
):
|
|
169
|
+
raise _blocked(code)
|
|
170
|
+
return value
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _sha256(value: Any, code: str) -> str:
|
|
174
|
+
if not isinstance(value, str) or SHA256.fullmatch(value) is None:
|
|
175
|
+
raise _blocked(code)
|
|
176
|
+
return value
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _count(value: Any, code: str) -> int:
|
|
180
|
+
if (
|
|
181
|
+
isinstance(value, bool)
|
|
182
|
+
or not isinstance(value, int)
|
|
183
|
+
or not 1 <= value <= MAX_ATTEMPTS
|
|
184
|
+
):
|
|
185
|
+
raise _blocked(code)
|
|
186
|
+
return value
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _timestamp(value: Any, code: str) -> float:
|
|
190
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
191
|
+
raise _blocked(code)
|
|
192
|
+
result = float(value)
|
|
193
|
+
if result < 0 or not math.isfinite(result):
|
|
194
|
+
raise _blocked(code)
|
|
195
|
+
return result
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _duration(value: Any, code: str, *, maximum: float) -> float:
|
|
199
|
+
result = _timestamp(value, code)
|
|
200
|
+
if result <= 0 or result > maximum:
|
|
201
|
+
raise _blocked(code)
|
|
202
|
+
return result
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _canonical(value: Any, code: str) -> str:
|
|
206
|
+
try:
|
|
207
|
+
raw = _CLIENT._canonical(value)
|
|
208
|
+
return raw.decode("utf-8")
|
|
209
|
+
except Exception:
|
|
210
|
+
raise _blocked(code) from None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _digest(value: str) -> str:
|
|
214
|
+
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _commercial_contract() -> dict[str, dict[str, Any]]:
|
|
218
|
+
"""Build and validate the content-free public commercial generation."""
|
|
219
|
+
|
|
220
|
+
try:
|
|
221
|
+
commercial = _CLIENT._CMS._COMMERCIAL
|
|
222
|
+
planner = _CLIENT._CMS._PLANNER
|
|
223
|
+
profile = commercial.public_profile(planner.COMMERCIAL_PROFILE)
|
|
224
|
+
registry = planner.commercial_rendering_registry()
|
|
225
|
+
profile_unsigned = dict(profile)
|
|
226
|
+
profile_sha256 = profile_unsigned.pop("sha256")
|
|
227
|
+
registry_unsigned = dict(registry)
|
|
228
|
+
registry_sha256 = registry_unsigned.pop("sha256")
|
|
229
|
+
expected_locales = [item.locale for item in planner.EU_OFFICIAL_LOCALES]
|
|
230
|
+
if (
|
|
231
|
+
profile.get("schema") != commercial.PUBLIC_PROFILE_SCHEMA
|
|
232
|
+
or profile.get("profile") != planner.COMMERCIAL_PROFILE
|
|
233
|
+
or [item.get("name") for item in profile.get("dimensions", ())]
|
|
234
|
+
!= list(commercial.DIMENSIONS)
|
|
235
|
+
or profile.get("protected_terms") != "project-configuration-only"
|
|
236
|
+
or profile_sha256
|
|
237
|
+
!= _digest(_canonical(profile_unsigned, "commercial_profile_invalid"))
|
|
238
|
+
or registry.get("schema")
|
|
239
|
+
!= planner.COMMERCIAL_RENDERING_REGISTRY_SCHEMA
|
|
240
|
+
or registry.get("commercial_profile") != planner.COMMERCIAL_PROFILE
|
|
241
|
+
or registry.get("content_policy") != {
|
|
242
|
+
"source_text": False,
|
|
243
|
+
"target_text": False,
|
|
244
|
+
"project_prices": False,
|
|
245
|
+
"project_brands": False,
|
|
246
|
+
"credentials": False,
|
|
247
|
+
}
|
|
248
|
+
or [item.get("locale") for item in registry.get("locales", ())]
|
|
249
|
+
!= expected_locales
|
|
250
|
+
or len(set(expected_locales)) != 24
|
|
251
|
+
or registry_sha256
|
|
252
|
+
!= _digest(_canonical(registry_unsigned, "commercial_profile_invalid"))
|
|
253
|
+
):
|
|
254
|
+
raise ValueError
|
|
255
|
+
copied = json.loads(_canonical({
|
|
256
|
+
"commercial_profile": profile,
|
|
257
|
+
"commercial_rendering_registry": registry,
|
|
258
|
+
}, "commercial_profile_invalid"))
|
|
259
|
+
return copied
|
|
260
|
+
except CMSSourceDeliverySubmissionDispatchBlocked:
|
|
261
|
+
raise
|
|
262
|
+
except Exception:
|
|
263
|
+
raise _blocked("commercial_profile_invalid") from None
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _commercial_contract_binding() -> dict[str, str]:
|
|
267
|
+
contract = _commercial_contract()
|
|
268
|
+
return {
|
|
269
|
+
"schema": (
|
|
270
|
+
"blun.cms-public-submission-dispatch-"
|
|
271
|
+
"commercial-contract-binding.v1"
|
|
272
|
+
),
|
|
273
|
+
"commercial_profile": contract["commercial_profile"]["profile"],
|
|
274
|
+
"commercial_profile_sha256": contract["commercial_profile"]["sha256"],
|
|
275
|
+
"commercial_rendering_registry_sha256": contract[
|
|
276
|
+
"commercial_rendering_registry"
|
|
277
|
+
]["sha256"],
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _commercial_binding_for_payload(payload: Mapping[str, Any]) -> dict[str, str] | None:
|
|
282
|
+
commercial = (
|
|
283
|
+
payload.get("schema") == _CLIENT._CMS.CHANGE_SCHEMA
|
|
284
|
+
and isinstance(payload.get("localization"), Mapping)
|
|
285
|
+
and payload["localization"].get("content_type") == "commercial"
|
|
286
|
+
)
|
|
287
|
+
return _commercial_contract_binding() if commercial else None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _stored_commercial_binding(
|
|
291
|
+
value: Any, payload: Mapping[str, Any], code: str,
|
|
292
|
+
) -> dict[str, str] | None:
|
|
293
|
+
expected = _commercial_binding_for_payload(payload)
|
|
294
|
+
if expected is None:
|
|
295
|
+
if value is not None:
|
|
296
|
+
raise _blocked(code)
|
|
297
|
+
return None
|
|
298
|
+
if not isinstance(value, str):
|
|
299
|
+
raise _blocked(code)
|
|
300
|
+
try:
|
|
301
|
+
copied = json.loads(value)
|
|
302
|
+
if (
|
|
303
|
+
copied != expected
|
|
304
|
+
or _canonical(copied, code) != value
|
|
305
|
+
):
|
|
306
|
+
raise ValueError
|
|
307
|
+
except CMSSourceDeliverySubmissionDispatchBlocked:
|
|
308
|
+
raise
|
|
309
|
+
except Exception:
|
|
310
|
+
raise _blocked(code) from None
|
|
311
|
+
return expected
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _payload(value: Any) -> tuple[dict[str, Any], dict[str, str], str]:
|
|
315
|
+
if not isinstance(value, Mapping):
|
|
316
|
+
raise _blocked("request_invalid")
|
|
317
|
+
try:
|
|
318
|
+
copied, _raw = _CLIENT._copy_payload(
|
|
319
|
+
value, change=value.get("schema") == _CLIENT._CMS.CHANGE_SCHEMA,
|
|
320
|
+
)
|
|
321
|
+
identity = _CLIENT._identity(
|
|
322
|
+
copied, change=copied.get("schema") == _CLIENT._CMS.CHANGE_SCHEMA,
|
|
323
|
+
)
|
|
324
|
+
except Exception:
|
|
325
|
+
raise _blocked("request_invalid") from None
|
|
326
|
+
return copied, identity, _canonical(copied, "request_invalid")
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
@contextmanager
|
|
330
|
+
def _transaction(connection: sqlite3.Connection) -> Iterator[None]:
|
|
331
|
+
if connection.in_transaction:
|
|
332
|
+
raise _blocked("transaction_nested")
|
|
333
|
+
try:
|
|
334
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
335
|
+
yield
|
|
336
|
+
connection.commit()
|
|
337
|
+
except Exception:
|
|
338
|
+
if connection.in_transaction:
|
|
339
|
+
connection.rollback()
|
|
340
|
+
raise
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _schema_snapshot(connection: sqlite3.Connection) -> tuple[tuple[Any, ...], ...]:
|
|
344
|
+
return tuple(tuple(row) for row in connection.execute(
|
|
345
|
+
"SELECT type, name, tbl_name, sql FROM sqlite_master "
|
|
346
|
+
"WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name"
|
|
347
|
+
).fetchall())
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _normalized_schema(
|
|
351
|
+
snapshot: tuple[tuple[Any, ...], ...],
|
|
352
|
+
) -> tuple[tuple[Any, ...], ...]:
|
|
353
|
+
return tuple(
|
|
354
|
+
(*row[:3], " ".join(row[3].split()) if isinstance(row[3], str) else row[3])
|
|
355
|
+
for row in snapshot
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _initialize_legacy_v1_schema(
|
|
360
|
+
connection: sqlite3.Connection, expected_capabilities_sha256: str,
|
|
361
|
+
) -> None:
|
|
362
|
+
connection.execute("""
|
|
363
|
+
CREATE TABLE cms_public_submission_outbox_meta (
|
|
364
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
365
|
+
schema_version INTEGER NOT NULL CHECK (schema_version = 1),
|
|
366
|
+
expected_capabilities_sha256 TEXT NOT NULL
|
|
367
|
+
)
|
|
368
|
+
""")
|
|
369
|
+
connection.execute("""
|
|
370
|
+
INSERT INTO cms_public_submission_outbox_meta
|
|
371
|
+
VALUES (1, 1, ?)
|
|
372
|
+
""", (expected_capabilities_sha256,))
|
|
373
|
+
connection.execute("""
|
|
374
|
+
CREATE TABLE cms_public_submission_outbox (
|
|
375
|
+
operation TEXT NOT NULL CHECK (
|
|
376
|
+
operation IN ('change', 'cancellation', 'tombstone')
|
|
377
|
+
),
|
|
378
|
+
request_id TEXT NOT NULL,
|
|
379
|
+
event_id TEXT NOT NULL,
|
|
380
|
+
site_id TEXT NOT NULL,
|
|
381
|
+
payload_json TEXT NOT NULL,
|
|
382
|
+
payload_sha256 TEXT NOT NULL,
|
|
383
|
+
source_max_attempts INTEGER NOT NULL CHECK (
|
|
384
|
+
source_max_attempts BETWEEN 1 AND 20
|
|
385
|
+
),
|
|
386
|
+
delivery_max_attempts INTEGER NOT NULL CHECK (
|
|
387
|
+
delivery_max_attempts BETWEEN 1 AND 20
|
|
388
|
+
),
|
|
389
|
+
client_max_attempts INTEGER NOT NULL CHECK (
|
|
390
|
+
client_max_attempts BETWEEN 1 AND 20
|
|
391
|
+
),
|
|
392
|
+
status TEXT NOT NULL CHECK (
|
|
393
|
+
status IN ('pending', 'leased', 'retry_wait',
|
|
394
|
+
'accepted', 'failed')
|
|
395
|
+
),
|
|
396
|
+
attempts INTEGER NOT NULL CHECK (attempts >= 0),
|
|
397
|
+
next_attempt_at REAL NOT NULL,
|
|
398
|
+
lease_owner TEXT,
|
|
399
|
+
lease_token TEXT,
|
|
400
|
+
lease_expires_at REAL,
|
|
401
|
+
last_error_code TEXT,
|
|
402
|
+
remote_status TEXT,
|
|
403
|
+
remote_attempts INTEGER,
|
|
404
|
+
remote_capabilities_sha256 TEXT,
|
|
405
|
+
remote_binding_sha256 TEXT,
|
|
406
|
+
response_json TEXT,
|
|
407
|
+
response_sha256 TEXT,
|
|
408
|
+
created_at REAL NOT NULL,
|
|
409
|
+
updated_at REAL NOT NULL,
|
|
410
|
+
PRIMARY KEY (operation, request_id),
|
|
411
|
+
CHECK (
|
|
412
|
+
(status = 'leased' AND lease_owner IS NOT NULL
|
|
413
|
+
AND lease_token IS NOT NULL
|
|
414
|
+
AND lease_expires_at IS NOT NULL)
|
|
415
|
+
OR
|
|
416
|
+
(status <> 'leased' AND lease_owner IS NULL
|
|
417
|
+
AND lease_token IS NULL
|
|
418
|
+
AND lease_expires_at IS NULL)
|
|
419
|
+
)
|
|
420
|
+
)
|
|
421
|
+
""")
|
|
422
|
+
connection.execute("""
|
|
423
|
+
CREATE INDEX cms_public_submission_outbox_due
|
|
424
|
+
ON cms_public_submission_outbox (
|
|
425
|
+
status, next_attempt_at, created_at, operation, request_id
|
|
426
|
+
)
|
|
427
|
+
""")
|
|
428
|
+
connection.commit()
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _legacy_v1_schema(expected_capabilities_sha256: str) -> tuple[tuple[Any, ...], ...]:
|
|
432
|
+
connection = sqlite3.connect(":memory:")
|
|
433
|
+
try:
|
|
434
|
+
_initialize_legacy_v1_schema(connection, expected_capabilities_sha256)
|
|
435
|
+
return _schema_snapshot(connection)
|
|
436
|
+
finally:
|
|
437
|
+
connection.close()
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _is_empty_legacy_v1(
|
|
441
|
+
connection: sqlite3.Connection, expected_capabilities_sha256: str,
|
|
442
|
+
) -> bool:
|
|
443
|
+
try:
|
|
444
|
+
if _normalized_schema(_schema_snapshot(connection)) != _normalized_schema(
|
|
445
|
+
_legacy_v1_schema(expected_capabilities_sha256)
|
|
446
|
+
):
|
|
447
|
+
return False
|
|
448
|
+
meta = connection.execute(
|
|
449
|
+
"SELECT * FROM cms_public_submission_outbox_meta"
|
|
450
|
+
).fetchall()
|
|
451
|
+
count = connection.execute(
|
|
452
|
+
"SELECT COUNT(*) FROM cms_public_submission_outbox"
|
|
453
|
+
).fetchone()[0]
|
|
454
|
+
return (
|
|
455
|
+
len(meta) == 1
|
|
456
|
+
and tuple(meta[0]) == (1, 1, expected_capabilities_sha256)
|
|
457
|
+
and count == 0
|
|
458
|
+
)
|
|
459
|
+
except Exception:
|
|
460
|
+
return False
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
class DurableCMSSourceDeliverySubmissionDispatcher:
|
|
464
|
+
"""Persist and submit exact public website requests at least once."""
|
|
465
|
+
|
|
466
|
+
def __init__(
|
|
467
|
+
self,
|
|
468
|
+
connection: sqlite3.Connection,
|
|
469
|
+
expected_capabilities_sha256: str,
|
|
470
|
+
*,
|
|
471
|
+
base_delay_seconds: float | int = 5,
|
|
472
|
+
max_delay_seconds: float | int = 300,
|
|
473
|
+
):
|
|
474
|
+
if not isinstance(connection, sqlite3.Connection):
|
|
475
|
+
raise TypeError("connection must be sqlite3.Connection")
|
|
476
|
+
self.connection = connection
|
|
477
|
+
self.connection.row_factory = sqlite3.Row
|
|
478
|
+
self.expected_capabilities_sha256 = _sha256(
|
|
479
|
+
expected_capabilities_sha256, "capabilities_invalid",
|
|
480
|
+
)
|
|
481
|
+
self.base_delay_seconds = _duration(
|
|
482
|
+
base_delay_seconds, "delay_invalid", maximum=MAX_DELAY_SECONDS,
|
|
483
|
+
)
|
|
484
|
+
self.max_delay_seconds = _duration(
|
|
485
|
+
max_delay_seconds, "delay_invalid", maximum=MAX_DELAY_SECONDS,
|
|
486
|
+
)
|
|
487
|
+
if self.base_delay_seconds > self.max_delay_seconds:
|
|
488
|
+
raise _blocked("delay_invalid")
|
|
489
|
+
self._initialize()
|
|
490
|
+
|
|
491
|
+
def _initialize(self) -> None:
|
|
492
|
+
existing = _schema_snapshot(self.connection)
|
|
493
|
+
if existing:
|
|
494
|
+
if not _is_empty_legacy_v1(
|
|
495
|
+
self.connection, self.expected_capabilities_sha256
|
|
496
|
+
):
|
|
497
|
+
self._validate_schema()
|
|
498
|
+
return
|
|
499
|
+
with _transaction(self.connection):
|
|
500
|
+
if existing:
|
|
501
|
+
self.connection.execute(
|
|
502
|
+
"DROP TABLE cms_public_submission_outbox"
|
|
503
|
+
)
|
|
504
|
+
self.connection.execute(
|
|
505
|
+
"DROP TABLE cms_public_submission_outbox_meta"
|
|
506
|
+
)
|
|
507
|
+
self.connection.execute("""
|
|
508
|
+
CREATE TABLE IF NOT EXISTS cms_public_submission_outbox_meta (
|
|
509
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
510
|
+
schema_version INTEGER NOT NULL CHECK (schema_version = 2),
|
|
511
|
+
expected_capabilities_sha256 TEXT NOT NULL
|
|
512
|
+
)
|
|
513
|
+
""")
|
|
514
|
+
self.connection.execute("""
|
|
515
|
+
INSERT OR IGNORE INTO cms_public_submission_outbox_meta
|
|
516
|
+
VALUES (1, 2, ?)
|
|
517
|
+
""", (self.expected_capabilities_sha256,))
|
|
518
|
+
self.connection.execute("""
|
|
519
|
+
CREATE TABLE IF NOT EXISTS cms_public_submission_outbox (
|
|
520
|
+
operation TEXT NOT NULL CHECK (
|
|
521
|
+
operation IN ('change', 'cancellation', 'tombstone')
|
|
522
|
+
),
|
|
523
|
+
request_id TEXT NOT NULL,
|
|
524
|
+
event_id TEXT NOT NULL,
|
|
525
|
+
site_id TEXT NOT NULL,
|
|
526
|
+
payload_json TEXT NOT NULL,
|
|
527
|
+
payload_sha256 TEXT NOT NULL,
|
|
528
|
+
commercial_contract_binding_json TEXT,
|
|
529
|
+
source_max_attempts INTEGER NOT NULL CHECK (
|
|
530
|
+
source_max_attempts BETWEEN 1 AND 20
|
|
531
|
+
),
|
|
532
|
+
delivery_max_attempts INTEGER NOT NULL CHECK (
|
|
533
|
+
delivery_max_attempts BETWEEN 1 AND 20
|
|
534
|
+
),
|
|
535
|
+
client_max_attempts INTEGER NOT NULL CHECK (
|
|
536
|
+
client_max_attempts BETWEEN 1 AND 20
|
|
537
|
+
),
|
|
538
|
+
status TEXT NOT NULL CHECK (
|
|
539
|
+
status IN ('pending', 'leased', 'retry_wait',
|
|
540
|
+
'accepted', 'failed')
|
|
541
|
+
),
|
|
542
|
+
attempts INTEGER NOT NULL CHECK (attempts >= 0),
|
|
543
|
+
next_attempt_at REAL NOT NULL,
|
|
544
|
+
lease_owner TEXT,
|
|
545
|
+
lease_token TEXT,
|
|
546
|
+
lease_expires_at REAL,
|
|
547
|
+
last_error_code TEXT,
|
|
548
|
+
remote_status TEXT,
|
|
549
|
+
remote_attempts INTEGER,
|
|
550
|
+
remote_capabilities_sha256 TEXT,
|
|
551
|
+
remote_binding_sha256 TEXT,
|
|
552
|
+
response_json TEXT,
|
|
553
|
+
response_sha256 TEXT,
|
|
554
|
+
created_at REAL NOT NULL,
|
|
555
|
+
updated_at REAL NOT NULL,
|
|
556
|
+
PRIMARY KEY (operation, request_id),
|
|
557
|
+
CHECK (
|
|
558
|
+
(status = 'leased' AND lease_owner IS NOT NULL
|
|
559
|
+
AND lease_token IS NOT NULL
|
|
560
|
+
AND lease_expires_at IS NOT NULL)
|
|
561
|
+
OR
|
|
562
|
+
(status <> 'leased' AND lease_owner IS NULL
|
|
563
|
+
AND lease_token IS NULL
|
|
564
|
+
AND lease_expires_at IS NULL)
|
|
565
|
+
)
|
|
566
|
+
)
|
|
567
|
+
""")
|
|
568
|
+
self.connection.execute("""
|
|
569
|
+
CREATE INDEX IF NOT EXISTS cms_public_submission_outbox_due
|
|
570
|
+
ON cms_public_submission_outbox (
|
|
571
|
+
status, next_attempt_at, created_at, operation, request_id
|
|
572
|
+
)
|
|
573
|
+
""")
|
|
574
|
+
self._validate_schema()
|
|
575
|
+
|
|
576
|
+
def _validate_schema(self) -> None:
|
|
577
|
+
meta_columns = tuple(row["name"] for row in self.connection.execute(
|
|
578
|
+
"PRAGMA table_info(cms_public_submission_outbox_meta)"
|
|
579
|
+
).fetchall())
|
|
580
|
+
columns = tuple(row["name"] for row in self.connection.execute(
|
|
581
|
+
"PRAGMA table_info(cms_public_submission_outbox)"
|
|
582
|
+
).fetchall())
|
|
583
|
+
meta = self.connection.execute(
|
|
584
|
+
"SELECT * FROM cms_public_submission_outbox_meta"
|
|
585
|
+
).fetchall()
|
|
586
|
+
if (
|
|
587
|
+
meta_columns != _META_COLUMNS
|
|
588
|
+
or columns != _COLUMNS
|
|
589
|
+
or len(meta) != 1
|
|
590
|
+
or tuple(meta[0])
|
|
591
|
+
!= (1, SCHEMA_VERSION, self.expected_capabilities_sha256)
|
|
592
|
+
):
|
|
593
|
+
raise _blocked("schema_altered")
|
|
594
|
+
|
|
595
|
+
def enqueue(
|
|
596
|
+
self,
|
|
597
|
+
payload: Mapping[str, Any],
|
|
598
|
+
*,
|
|
599
|
+
source_max_attempts: int = 5,
|
|
600
|
+
delivery_max_attempts: int = 5,
|
|
601
|
+
client_max_attempts: int = 5,
|
|
602
|
+
now: float | int,
|
|
603
|
+
) -> SubmissionDispatchStatus:
|
|
604
|
+
self._validate_schema()
|
|
605
|
+
source_max_attempts = _count(source_max_attempts, "attempts_invalid")
|
|
606
|
+
delivery_max_attempts = _count(delivery_max_attempts, "attempts_invalid")
|
|
607
|
+
client_max_attempts = _count(client_max_attempts, "attempts_invalid")
|
|
608
|
+
now = _timestamp(now, "time_invalid")
|
|
609
|
+
copied, identity, payload_json = _payload(payload)
|
|
610
|
+
commercial_binding = _commercial_binding_for_payload(copied)
|
|
611
|
+
commercial_binding_json = (
|
|
612
|
+
None if commercial_binding is None
|
|
613
|
+
else _canonical(commercial_binding, "commercial_profile_invalid")
|
|
614
|
+
)
|
|
615
|
+
operation = identity["operation"]
|
|
616
|
+
request_id = identity["request_id"]
|
|
617
|
+
with _transaction(self.connection):
|
|
618
|
+
row = self.connection.execute("""
|
|
619
|
+
SELECT payload_sha256, commercial_contract_binding_json,
|
|
620
|
+
source_max_attempts,
|
|
621
|
+
delivery_max_attempts, client_max_attempts
|
|
622
|
+
FROM cms_public_submission_outbox
|
|
623
|
+
WHERE operation = ? AND request_id = ?
|
|
624
|
+
""", (operation, request_id)).fetchone()
|
|
625
|
+
expected = (
|
|
626
|
+
identity["payload_sha256"], commercial_binding_json,
|
|
627
|
+
source_max_attempts,
|
|
628
|
+
delivery_max_attempts, client_max_attempts,
|
|
629
|
+
)
|
|
630
|
+
if row is not None and tuple(row) != expected:
|
|
631
|
+
raise _blocked("idempotency_collision")
|
|
632
|
+
if row is None:
|
|
633
|
+
self.connection.execute("""
|
|
634
|
+
INSERT INTO cms_public_submission_outbox (
|
|
635
|
+
operation, request_id, event_id, site_id, payload_json,
|
|
636
|
+
payload_sha256, commercial_contract_binding_json,
|
|
637
|
+
source_max_attempts,
|
|
638
|
+
delivery_max_attempts, client_max_attempts, status,
|
|
639
|
+
attempts, next_attempt_at, created_at, updated_at
|
|
640
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?)
|
|
641
|
+
""", (
|
|
642
|
+
operation, request_id, identity["event_id"],
|
|
643
|
+
identity["site_id"], payload_json,
|
|
644
|
+
identity["payload_sha256"], commercial_binding_json,
|
|
645
|
+
source_max_attempts,
|
|
646
|
+
delivery_max_attempts, client_max_attempts, now, now, now,
|
|
647
|
+
))
|
|
648
|
+
return self.status(operation, request_id, now=now)
|
|
649
|
+
|
|
650
|
+
def claim(
|
|
651
|
+
self,
|
|
652
|
+
worker_id: str,
|
|
653
|
+
*,
|
|
654
|
+
now: float | int,
|
|
655
|
+
lease_seconds: float | int = 600,
|
|
656
|
+
) -> SubmissionDispatchClaim | None:
|
|
657
|
+
self._validate_schema()
|
|
658
|
+
worker_id = _token(worker_id, "worker_invalid", limit=128)
|
|
659
|
+
now = _timestamp(now, "time_invalid")
|
|
660
|
+
lease_seconds = _duration(
|
|
661
|
+
lease_seconds, "lease_invalid", maximum=MAX_LEASE_SECONDS,
|
|
662
|
+
)
|
|
663
|
+
claim = None
|
|
664
|
+
integrity_error = False
|
|
665
|
+
with _transaction(self.connection):
|
|
666
|
+
self.connection.execute("""
|
|
667
|
+
UPDATE cms_public_submission_outbox
|
|
668
|
+
SET status = 'failed', lease_owner = NULL, lease_token = NULL,
|
|
669
|
+
lease_expires_at = NULL, last_error_code = 'lease_expired',
|
|
670
|
+
updated_at = ?
|
|
671
|
+
WHERE status = 'leased' AND lease_expires_at <= ?
|
|
672
|
+
AND attempts >= client_max_attempts
|
|
673
|
+
""", (now, now))
|
|
674
|
+
self.connection.execute("""
|
|
675
|
+
UPDATE cms_public_submission_outbox
|
|
676
|
+
SET status = 'retry_wait', next_attempt_at = ?,
|
|
677
|
+
lease_owner = NULL, lease_token = NULL,
|
|
678
|
+
lease_expires_at = NULL, last_error_code = 'lease_expired',
|
|
679
|
+
updated_at = ?
|
|
680
|
+
WHERE status = 'leased' AND lease_expires_at <= ?
|
|
681
|
+
AND attempts < client_max_attempts
|
|
682
|
+
""", (now, now, now))
|
|
683
|
+
row = self.connection.execute("""
|
|
684
|
+
SELECT * FROM cms_public_submission_outbox
|
|
685
|
+
WHERE status IN ('pending', 'retry_wait')
|
|
686
|
+
AND next_attempt_at <= ? AND attempts < client_max_attempts
|
|
687
|
+
ORDER BY created_at,
|
|
688
|
+
CASE WHEN operation = 'change' THEN 1 ELSE 0 END,
|
|
689
|
+
operation, request_id LIMIT 1
|
|
690
|
+
""", (now,)).fetchone()
|
|
691
|
+
if row is not None:
|
|
692
|
+
try:
|
|
693
|
+
self._validated_row(row)
|
|
694
|
+
except CMSSourceDeliverySubmissionDispatchBlocked:
|
|
695
|
+
self.connection.execute("""
|
|
696
|
+
UPDATE cms_public_submission_outbox
|
|
697
|
+
SET status = 'failed',
|
|
698
|
+
last_error_code = 'payload_integrity', updated_at = ?
|
|
699
|
+
WHERE operation = ? AND request_id = ?
|
|
700
|
+
""", (now, row["operation"], row["request_id"]))
|
|
701
|
+
integrity_error = True
|
|
702
|
+
else:
|
|
703
|
+
lease_token = secrets.token_urlsafe(32)
|
|
704
|
+
lease_expires_at = now + lease_seconds
|
|
705
|
+
updated = self.connection.execute("""
|
|
706
|
+
UPDATE cms_public_submission_outbox
|
|
707
|
+
SET status = 'leased', attempts = attempts + 1,
|
|
708
|
+
lease_owner = ?, lease_token = ?,
|
|
709
|
+
lease_expires_at = ?, last_error_code = NULL,
|
|
710
|
+
updated_at = ?
|
|
711
|
+
WHERE operation = ? AND request_id = ?
|
|
712
|
+
AND status IN ('pending', 'retry_wait')
|
|
713
|
+
""", (
|
|
714
|
+
worker_id, lease_token, lease_expires_at, now,
|
|
715
|
+
row["operation"], row["request_id"],
|
|
716
|
+
))
|
|
717
|
+
if updated.rowcount != 1:
|
|
718
|
+
raise _blocked("claim_lost")
|
|
719
|
+
claim = SubmissionDispatchClaim(
|
|
720
|
+
operation=row["operation"],
|
|
721
|
+
request_id=row["request_id"], event_id=row["event_id"],
|
|
722
|
+
site_id=row["site_id"],
|
|
723
|
+
payload_sha256=row["payload_sha256"],
|
|
724
|
+
commercial_contract_binding=_stored_commercial_binding(
|
|
725
|
+
row["commercial_contract_binding_json"],
|
|
726
|
+
json.loads(row["payload_json"]), "state_invalid",
|
|
727
|
+
),
|
|
728
|
+
source_max_attempts=row["source_max_attempts"],
|
|
729
|
+
delivery_max_attempts=row["delivery_max_attempts"],
|
|
730
|
+
attempt=row["attempts"] + 1,
|
|
731
|
+
client_max_attempts=row["client_max_attempts"],
|
|
732
|
+
lease_owner=worker_id, lease_token=lease_token,
|
|
733
|
+
lease_expires_at=lease_expires_at,
|
|
734
|
+
payload_json=row["payload_json"],
|
|
735
|
+
)
|
|
736
|
+
if integrity_error:
|
|
737
|
+
raise _blocked("payload_integrity")
|
|
738
|
+
return claim
|
|
739
|
+
|
|
740
|
+
def run_once(
|
|
741
|
+
self,
|
|
742
|
+
client: Any,
|
|
743
|
+
worker_id: str,
|
|
744
|
+
*,
|
|
745
|
+
now: float | int,
|
|
746
|
+
lease_seconds: float | int = 600,
|
|
747
|
+
) -> SubmissionDispatchOutcome | None:
|
|
748
|
+
self._validate_client(client, lease_seconds)
|
|
749
|
+
claim = self.claim(worker_id, now=now, lease_seconds=lease_seconds)
|
|
750
|
+
if claim is None:
|
|
751
|
+
return None
|
|
752
|
+
try:
|
|
753
|
+
payload = json.loads(claim.payload_json)
|
|
754
|
+
if claim.operation == "change":
|
|
755
|
+
response = client.submit_change(
|
|
756
|
+
payload,
|
|
757
|
+
source_max_attempts=claim.source_max_attempts,
|
|
758
|
+
delivery_max_attempts=claim.delivery_max_attempts,
|
|
759
|
+
)
|
|
760
|
+
else:
|
|
761
|
+
response = client.submit_removal(
|
|
762
|
+
payload,
|
|
763
|
+
source_max_attempts=claim.source_max_attempts,
|
|
764
|
+
delivery_max_attempts=claim.delivery_max_attempts,
|
|
765
|
+
)
|
|
766
|
+
status = self.complete(claim, response, now=now)
|
|
767
|
+
except Exception as error:
|
|
768
|
+
client_failure = getattr(
|
|
769
|
+
error, "cms_source_delivery_submission_client_failure", False,
|
|
770
|
+
) is True
|
|
771
|
+
code = getattr(error, "code", "client_failure")
|
|
772
|
+
if not isinstance(code, str) or ERROR_CODE.fullmatch(code) is None:
|
|
773
|
+
code = "client_failure"
|
|
774
|
+
retryable = client_failure and getattr(error, "retryable", False) is True
|
|
775
|
+
status = self._finish_error(claim, code, retryable=retryable, now=now)
|
|
776
|
+
return SubmissionDispatchOutcome(
|
|
777
|
+
operation=status.operation, request_id=status.request_id,
|
|
778
|
+
status=status.status, attempt=status.attempts,
|
|
779
|
+
client_max_attempts=status.client_max_attempts,
|
|
780
|
+
next_attempt_at=status.next_attempt_at,
|
|
781
|
+
error_code=status.last_error_code,
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
def _validate_client(self, client: Any, lease_seconds: Any) -> None:
|
|
785
|
+
if (
|
|
786
|
+
not callable(getattr(client, "submit_change", None))
|
|
787
|
+
or not callable(getattr(client, "submit_removal", None))
|
|
788
|
+
or getattr(client, "expected_capabilities_sha256", None)
|
|
789
|
+
!= self.expected_capabilities_sha256
|
|
790
|
+
):
|
|
791
|
+
raise _blocked("client_invalid")
|
|
792
|
+
lease_seconds = _duration(
|
|
793
|
+
lease_seconds, "lease_invalid", maximum=MAX_LEASE_SECONDS,
|
|
794
|
+
)
|
|
795
|
+
timeout = getattr(client, "timeout", None)
|
|
796
|
+
if (
|
|
797
|
+
isinstance(timeout, bool)
|
|
798
|
+
or not isinstance(timeout, (int, float))
|
|
799
|
+
or not math.isfinite(float(timeout))
|
|
800
|
+
or float(timeout) <= 0
|
|
801
|
+
):
|
|
802
|
+
raise _blocked("client_invalid")
|
|
803
|
+
if lease_seconds <= float(timeout):
|
|
804
|
+
raise _blocked("lease_too_short")
|
|
805
|
+
|
|
806
|
+
def complete(
|
|
807
|
+
self, claim: SubmissionDispatchClaim, response: Any, *, now: float | int,
|
|
808
|
+
) -> SubmissionDispatchStatus:
|
|
809
|
+
self._validate_schema()
|
|
810
|
+
claim = self._claim(claim)
|
|
811
|
+
now = _timestamp(now, "time_invalid")
|
|
812
|
+
if not isinstance(response, Mapping):
|
|
813
|
+
raise _blocked("response_invalid")
|
|
814
|
+
response_json = _canonical(dict(response), "response_invalid")
|
|
815
|
+
copied = json.loads(response_json)
|
|
816
|
+
fields = {
|
|
817
|
+
"schema", "api_schema", "operation", "request_id", "event_id",
|
|
818
|
+
"site_id", "payload_sha256", "status", "attempts",
|
|
819
|
+
"delivery_max_attempts", "source_max_attempts",
|
|
820
|
+
"capabilities_sha256", "website_capability_binding",
|
|
821
|
+
"accepted_implies_publication",
|
|
822
|
+
}
|
|
823
|
+
binding = copied.get("website_capability_binding")
|
|
824
|
+
expected_schema = (
|
|
825
|
+
_CLIENT._HTTP.CHANGE_RESPONSE_SCHEMA
|
|
826
|
+
if claim.operation == "change"
|
|
827
|
+
else _CLIENT._HTTP.REMOVAL_RESPONSE_SCHEMA
|
|
828
|
+
)
|
|
829
|
+
try:
|
|
830
|
+
normalized_binding = _CLIENT._HTTP._binding(binding)
|
|
831
|
+
except Exception:
|
|
832
|
+
raise _blocked("response_invalid") from None
|
|
833
|
+
valid = (
|
|
834
|
+
set(copied) == fields
|
|
835
|
+
and copied.get("schema") == expected_schema
|
|
836
|
+
and copied.get("api_schema") == _CLIENT._HTTP.API_SCHEMA
|
|
837
|
+
and copied.get("operation") == claim.operation
|
|
838
|
+
and copied.get("request_id") == claim.request_id
|
|
839
|
+
and copied.get("event_id") == claim.event_id
|
|
840
|
+
and copied.get("site_id") == claim.site_id
|
|
841
|
+
and copied.get("payload_sha256") == claim.payload_sha256
|
|
842
|
+
and copied.get("source_max_attempts") == claim.source_max_attempts
|
|
843
|
+
and copied.get("delivery_max_attempts") == claim.delivery_max_attempts
|
|
844
|
+
and copied.get("accepted_implies_publication") is False
|
|
845
|
+
and copied.get("status") in _CLIENT._HTTP.STATUSES
|
|
846
|
+
and isinstance(copied.get("attempts"), int)
|
|
847
|
+
and not isinstance(copied.get("attempts"), bool)
|
|
848
|
+
and 0 <= copied["attempts"] <= claim.delivery_max_attempts
|
|
849
|
+
and _sha256(copied.get("capabilities_sha256"), "response_invalid")
|
|
850
|
+
== copied.get("capabilities_sha256")
|
|
851
|
+
and isinstance(binding, dict)
|
|
852
|
+
and normalized_binding == binding
|
|
853
|
+
and copied.get("capabilities_sha256")
|
|
854
|
+
== binding.get("delivery_capabilities_sha256")
|
|
855
|
+
and (
|
|
856
|
+
claim.commercial_contract_binding is None
|
|
857
|
+
or binding.get("commercial_rendering_registry_sha256")
|
|
858
|
+
== claim.commercial_contract_binding[
|
|
859
|
+
"commercial_rendering_registry_sha256"
|
|
860
|
+
]
|
|
861
|
+
)
|
|
862
|
+
)
|
|
863
|
+
if not valid:
|
|
864
|
+
raise _blocked("response_invalid")
|
|
865
|
+
binding_json = _canonical(binding, "response_invalid")
|
|
866
|
+
with _transaction(self.connection):
|
|
867
|
+
self._require_live_claim(claim, now)
|
|
868
|
+
updated = self.connection.execute("""
|
|
869
|
+
UPDATE cms_public_submission_outbox
|
|
870
|
+
SET status = 'accepted', next_attempt_at = ?,
|
|
871
|
+
lease_owner = NULL, lease_token = NULL,
|
|
872
|
+
lease_expires_at = NULL, last_error_code = NULL,
|
|
873
|
+
remote_status = ?, remote_attempts = ?,
|
|
874
|
+
remote_capabilities_sha256 = ?,
|
|
875
|
+
remote_binding_sha256 = ?, response_json = ?,
|
|
876
|
+
response_sha256 = ?,
|
|
877
|
+
updated_at = ?
|
|
878
|
+
WHERE operation = ? AND request_id = ? AND status = 'leased'
|
|
879
|
+
AND lease_owner = ? AND lease_token = ?
|
|
880
|
+
""", (
|
|
881
|
+
now, copied["status"], copied["attempts"],
|
|
882
|
+
copied["capabilities_sha256"], _digest(binding_json),
|
|
883
|
+
response_json, _digest(response_json), now, claim.operation,
|
|
884
|
+
claim.request_id, claim.lease_owner, claim.lease_token,
|
|
885
|
+
))
|
|
886
|
+
if updated.rowcount != 1:
|
|
887
|
+
raise _blocked("completion_lost")
|
|
888
|
+
return self.status(claim.operation, claim.request_id, now=now)
|
|
889
|
+
|
|
890
|
+
def _finish_error(
|
|
891
|
+
self, claim: SubmissionDispatchClaim, code: str, *, retryable: bool,
|
|
892
|
+
now: float | int,
|
|
893
|
+
) -> SubmissionDispatchStatus:
|
|
894
|
+
now = _timestamp(now, "time_invalid")
|
|
895
|
+
self._claim(claim)
|
|
896
|
+
terminal = not retryable or claim.attempt >= claim.client_max_attempts
|
|
897
|
+
status = "failed" if terminal else "retry_wait"
|
|
898
|
+
delay = min(
|
|
899
|
+
self.max_delay_seconds,
|
|
900
|
+
self.base_delay_seconds * (2 ** max(0, claim.attempt - 1)),
|
|
901
|
+
)
|
|
902
|
+
next_attempt_at = now if terminal else now + delay
|
|
903
|
+
with _transaction(self.connection):
|
|
904
|
+
self._require_live_claim(claim, now)
|
|
905
|
+
updated = self.connection.execute("""
|
|
906
|
+
UPDATE cms_public_submission_outbox
|
|
907
|
+
SET status = ?, next_attempt_at = ?, lease_owner = NULL,
|
|
908
|
+
lease_token = NULL, lease_expires_at = NULL,
|
|
909
|
+
last_error_code = ?, updated_at = ?
|
|
910
|
+
WHERE operation = ? AND request_id = ? AND status = 'leased'
|
|
911
|
+
AND lease_owner = ? AND lease_token = ?
|
|
912
|
+
""", (
|
|
913
|
+
status, next_attempt_at, code, now, claim.operation,
|
|
914
|
+
claim.request_id, claim.lease_owner, claim.lease_token,
|
|
915
|
+
))
|
|
916
|
+
if updated.rowcount != 1:
|
|
917
|
+
raise _blocked("failure_lost")
|
|
918
|
+
return self.status(claim.operation, claim.request_id, now=now)
|
|
919
|
+
|
|
920
|
+
def status(
|
|
921
|
+
self, operation: str, request_id: str, *, now: float | int,
|
|
922
|
+
) -> SubmissionDispatchStatus:
|
|
923
|
+
self._validate_schema()
|
|
924
|
+
if operation not in OPERATIONS:
|
|
925
|
+
raise _blocked("identity_invalid")
|
|
926
|
+
request_id = _token(request_id, "identity_invalid")
|
|
927
|
+
now = _timestamp(now, "time_invalid")
|
|
928
|
+
row = self.connection.execute("""
|
|
929
|
+
SELECT * FROM cms_public_submission_outbox
|
|
930
|
+
WHERE operation = ? AND request_id = ?
|
|
931
|
+
""", (operation, request_id)).fetchone()
|
|
932
|
+
if row is None:
|
|
933
|
+
raise _blocked("submission_missing")
|
|
934
|
+
self._validated_row(row)
|
|
935
|
+
lease_expires_at = row["lease_expires_at"]
|
|
936
|
+
remote_binding = None
|
|
937
|
+
if row["status"] == "accepted":
|
|
938
|
+
try:
|
|
939
|
+
response = json.loads(row["response_json"])
|
|
940
|
+
remote_binding = _CLIENT._HTTP._binding(
|
|
941
|
+
response["website_capability_binding"]
|
|
942
|
+
)
|
|
943
|
+
if (
|
|
944
|
+
response["website_capability_binding"] != remote_binding
|
|
945
|
+
or _digest(_canonical(remote_binding, "state_invalid"))
|
|
946
|
+
!= row["remote_binding_sha256"]
|
|
947
|
+
):
|
|
948
|
+
raise ValueError
|
|
949
|
+
except Exception:
|
|
950
|
+
raise _blocked("state_invalid") from None
|
|
951
|
+
return SubmissionDispatchStatus(
|
|
952
|
+
operation=row["operation"], request_id=row["request_id"],
|
|
953
|
+
event_id=row["event_id"], site_id=row["site_id"],
|
|
954
|
+
payload_sha256=row["payload_sha256"],
|
|
955
|
+
commercial_contract_binding=_stored_commercial_binding(
|
|
956
|
+
row["commercial_contract_binding_json"],
|
|
957
|
+
json.loads(row["payload_json"]), "state_invalid",
|
|
958
|
+
),
|
|
959
|
+
source_max_attempts=row["source_max_attempts"],
|
|
960
|
+
delivery_max_attempts=row["delivery_max_attempts"],
|
|
961
|
+
status=row["status"], attempts=row["attempts"],
|
|
962
|
+
client_max_attempts=row["client_max_attempts"],
|
|
963
|
+
next_attempt_at=float(row["next_attempt_at"]),
|
|
964
|
+
lease_expires_at=(
|
|
965
|
+
None if lease_expires_at is None else float(lease_expires_at)
|
|
966
|
+
),
|
|
967
|
+
lease_expired=(
|
|
968
|
+
row["status"] == "leased" and lease_expires_at <= now
|
|
969
|
+
),
|
|
970
|
+
last_error_code=row["last_error_code"],
|
|
971
|
+
remote_status=row["remote_status"],
|
|
972
|
+
remote_attempts=row["remote_attempts"],
|
|
973
|
+
remote_capabilities_sha256=row["remote_capabilities_sha256"],
|
|
974
|
+
remote_binding_sha256=row["remote_binding_sha256"],
|
|
975
|
+
remote_website_capability_binding=remote_binding,
|
|
976
|
+
response_sha256=row["response_sha256"],
|
|
977
|
+
)
|
|
978
|
+
|
|
979
|
+
def lifecycle(
|
|
980
|
+
self,
|
|
981
|
+
client: Any,
|
|
982
|
+
operation: str,
|
|
983
|
+
request_id: str,
|
|
984
|
+
*,
|
|
985
|
+
now: float | int,
|
|
986
|
+
) -> SubmissionDispatchLifecycle:
|
|
987
|
+
"""Read the verified downstream lifecycle after durable website intake."""
|
|
988
|
+
|
|
989
|
+
self._validate_read_client(client)
|
|
990
|
+
status = self.status(operation, request_id, now=now)
|
|
991
|
+
if status.status != "accepted":
|
|
992
|
+
raise _blocked("lifecycle_not_accepted")
|
|
993
|
+
identity = {
|
|
994
|
+
"operation": status.operation,
|
|
995
|
+
"request_id": status.request_id,
|
|
996
|
+
"event_id": status.event_id,
|
|
997
|
+
"site_id": status.site_id,
|
|
998
|
+
"payload_sha256": status.payload_sha256,
|
|
999
|
+
}
|
|
1000
|
+
try:
|
|
1001
|
+
response = client.submission_lifecycle(**identity)
|
|
1002
|
+
if (
|
|
1003
|
+
not isinstance(response, Mapping)
|
|
1004
|
+
or set(response) != {
|
|
1005
|
+
"schema", "api_schema", "result",
|
|
1006
|
+
"accepted_implies_publication",
|
|
1007
|
+
}
|
|
1008
|
+
or response.get("schema")
|
|
1009
|
+
!= _CLIENT._HTTP._SUBMISSION.LIFECYCLE_HTTP_RESPONSE_SCHEMA
|
|
1010
|
+
or response.get("api_schema") != _CLIENT._HTTP.API_SCHEMA
|
|
1011
|
+
or response.get("accepted_implies_publication") is not False
|
|
1012
|
+
or not isinstance(response.get("result"), Mapping)
|
|
1013
|
+
):
|
|
1014
|
+
raise ValueError
|
|
1015
|
+
normalized = _CLIENT._HTTP._submission_lifecycle_payload(
|
|
1016
|
+
_CLIENT._PayloadView(response["result"]), identity,
|
|
1017
|
+
)
|
|
1018
|
+
if (
|
|
1019
|
+
normalized != response["result"]
|
|
1020
|
+
or normalized["website_capability_binding"]
|
|
1021
|
+
!= status.remote_website_capability_binding
|
|
1022
|
+
):
|
|
1023
|
+
raise ValueError
|
|
1024
|
+
copied = json.loads(_canonical(dict(response), "lifecycle_invalid"))
|
|
1025
|
+
except CMSSourceDeliverySubmissionDispatchBlocked:
|
|
1026
|
+
raise
|
|
1027
|
+
except Exception:
|
|
1028
|
+
raise _blocked("lifecycle_invalid") from None
|
|
1029
|
+
return SubmissionDispatchLifecycle(
|
|
1030
|
+
dispatch_status=status,
|
|
1031
|
+
source_lifecycle=copied,
|
|
1032
|
+
)
|
|
1033
|
+
|
|
1034
|
+
def _validate_read_client(self, client: Any) -> None:
|
|
1035
|
+
if (
|
|
1036
|
+
not callable(getattr(client, "submission_lifecycle", None))
|
|
1037
|
+
or getattr(client, "expected_capabilities_sha256", None)
|
|
1038
|
+
!= self.expected_capabilities_sha256
|
|
1039
|
+
):
|
|
1040
|
+
raise _blocked("client_invalid")
|
|
1041
|
+
timeout = getattr(client, "timeout", None)
|
|
1042
|
+
if (
|
|
1043
|
+
isinstance(timeout, bool)
|
|
1044
|
+
or not isinstance(timeout, (int, float))
|
|
1045
|
+
or not math.isfinite(float(timeout))
|
|
1046
|
+
or float(timeout) <= 0
|
|
1047
|
+
):
|
|
1048
|
+
raise _blocked("client_invalid")
|
|
1049
|
+
|
|
1050
|
+
def commercial_profile(
|
|
1051
|
+
self, client: Any,
|
|
1052
|
+
) -> SubmissionDispatchCommercialProfile:
|
|
1053
|
+
"""Return the public offer contract bound to the live website generation."""
|
|
1054
|
+
|
|
1055
|
+
if (
|
|
1056
|
+
not callable(getattr(client, "capabilities", None))
|
|
1057
|
+
or getattr(client, "expected_capabilities_sha256", None)
|
|
1058
|
+
!= self.expected_capabilities_sha256
|
|
1059
|
+
):
|
|
1060
|
+
raise _blocked("client_invalid")
|
|
1061
|
+
timeout = getattr(client, "timeout", None)
|
|
1062
|
+
if (
|
|
1063
|
+
isinstance(timeout, bool)
|
|
1064
|
+
or not isinstance(timeout, (int, float))
|
|
1065
|
+
or not math.isfinite(float(timeout))
|
|
1066
|
+
or float(timeout) <= 0
|
|
1067
|
+
):
|
|
1068
|
+
raise _blocked("client_invalid")
|
|
1069
|
+
try:
|
|
1070
|
+
response = client.capabilities()
|
|
1071
|
+
if (
|
|
1072
|
+
not isinstance(response, Mapping)
|
|
1073
|
+
or set(response) != {"schema", "api_schema", "capabilities"}
|
|
1074
|
+
or response.get("schema")
|
|
1075
|
+
!= _CLIENT._HTTP._SUBMISSION.CAPABILITIES_HTTP_RESPONSE_SCHEMA
|
|
1076
|
+
or response.get("api_schema")
|
|
1077
|
+
!= _CLIENT._HTTP._CAPABILITIES.API_SCHEMA
|
|
1078
|
+
or not isinstance(response.get("capabilities"), Mapping)
|
|
1079
|
+
):
|
|
1080
|
+
raise ValueError
|
|
1081
|
+
normalized = _CLIENT._HTTP._CAPABILITIES._capabilities(
|
|
1082
|
+
_CLIENT._PayloadView(response["capabilities"])
|
|
1083
|
+
)
|
|
1084
|
+
if (
|
|
1085
|
+
normalized != response["capabilities"]
|
|
1086
|
+
or normalized["sha256"] != self.expected_capabilities_sha256
|
|
1087
|
+
):
|
|
1088
|
+
raise ValueError
|
|
1089
|
+
contract = _commercial_contract()
|
|
1090
|
+
binding = normalized["website_capability_binding"]
|
|
1091
|
+
if (
|
|
1092
|
+
binding["commercial_rendering_registry_sha256"]
|
|
1093
|
+
!= contract["commercial_rendering_registry"]["sha256"]
|
|
1094
|
+
):
|
|
1095
|
+
raise ValueError
|
|
1096
|
+
return SubmissionDispatchCommercialProfile(
|
|
1097
|
+
commercial_profile=contract["commercial_profile"],
|
|
1098
|
+
commercial_rendering_registry=(
|
|
1099
|
+
contract["commercial_rendering_registry"]
|
|
1100
|
+
),
|
|
1101
|
+
website_capability_binding=dict(binding),
|
|
1102
|
+
)
|
|
1103
|
+
except CMSSourceDeliverySubmissionDispatchBlocked:
|
|
1104
|
+
raise
|
|
1105
|
+
except Exception:
|
|
1106
|
+
raise _blocked("commercial_profile_invalid") from None
|
|
1107
|
+
|
|
1108
|
+
def health(self, *, now: float | int) -> SubmissionDispatchHealth:
|
|
1109
|
+
now = _timestamp(now, "time_invalid")
|
|
1110
|
+
self._validate_schema()
|
|
1111
|
+
quick = self.connection.execute("PRAGMA quick_check").fetchall()
|
|
1112
|
+
if len(quick) != 1 or tuple(quick[0]) != ("ok",):
|
|
1113
|
+
raise _blocked("database_integrity")
|
|
1114
|
+
rows = self.connection.execute(
|
|
1115
|
+
"SELECT * FROM cms_public_submission_outbox "
|
|
1116
|
+
"ORDER BY operation, request_id"
|
|
1117
|
+
).fetchall()
|
|
1118
|
+
for row in rows:
|
|
1119
|
+
self._validated_row(row)
|
|
1120
|
+
counts = {status: 0 for status in STATUSES}
|
|
1121
|
+
operations = {operation: 0 for operation in OPERATIONS}
|
|
1122
|
+
for row in rows:
|
|
1123
|
+
counts[row["status"]] += 1
|
|
1124
|
+
operations[row["operation"]] += 1
|
|
1125
|
+
due = sum(
|
|
1126
|
+
row["status"] in {"pending", "retry_wait"}
|
|
1127
|
+
and row["next_attempt_at"] <= now
|
|
1128
|
+
for row in rows
|
|
1129
|
+
)
|
|
1130
|
+
expired = sum(
|
|
1131
|
+
row["status"] == "leased" and row["lease_expires_at"] <= now
|
|
1132
|
+
for row in rows
|
|
1133
|
+
)
|
|
1134
|
+
return SubmissionDispatchHealth(
|
|
1135
|
+
status="blocked" if expired or counts["failed"] else "ok",
|
|
1136
|
+
counts=counts, operations=operations, due=due,
|
|
1137
|
+
expired_leases=expired, failed=counts["failed"],
|
|
1138
|
+
expected_capabilities_sha256=self.expected_capabilities_sha256,
|
|
1139
|
+
)
|
|
1140
|
+
|
|
1141
|
+
def _validated_row(self, row: sqlite3.Row) -> None:
|
|
1142
|
+
try:
|
|
1143
|
+
copied, identity, payload_json = _payload(json.loads(row["payload_json"]))
|
|
1144
|
+
_stored_commercial_binding(
|
|
1145
|
+
row["commercial_contract_binding_json"], copied, "state_invalid",
|
|
1146
|
+
)
|
|
1147
|
+
operation = identity["operation"]
|
|
1148
|
+
request_id = identity["request_id"]
|
|
1149
|
+
created = _timestamp(row["created_at"], "state_invalid")
|
|
1150
|
+
updated = _timestamp(row["updated_at"], "state_invalid")
|
|
1151
|
+
next_at = _timestamp(row["next_attempt_at"], "state_invalid")
|
|
1152
|
+
source_attempts = _count(row["source_max_attempts"], "state_invalid")
|
|
1153
|
+
delivery_attempts = _count(row["delivery_max_attempts"], "state_invalid")
|
|
1154
|
+
client_attempts = _count(row["client_max_attempts"], "state_invalid")
|
|
1155
|
+
except Exception:
|
|
1156
|
+
raise _blocked("state_invalid") from None
|
|
1157
|
+
status = row["status"]
|
|
1158
|
+
attempts = row["attempts"]
|
|
1159
|
+
lease = (row["lease_owner"], row["lease_token"], row["lease_expires_at"])
|
|
1160
|
+
remote = (
|
|
1161
|
+
row["remote_status"], row["remote_attempts"],
|
|
1162
|
+
row["remote_capabilities_sha256"], row["remote_binding_sha256"],
|
|
1163
|
+
row["response_json"], row["response_sha256"],
|
|
1164
|
+
)
|
|
1165
|
+
error = row["last_error_code"]
|
|
1166
|
+
valid_remote = status == "accepted"
|
|
1167
|
+
if (
|
|
1168
|
+
operation != row["operation"] or request_id != row["request_id"]
|
|
1169
|
+
or identity["event_id"] != row["event_id"]
|
|
1170
|
+
or identity["site_id"] != row["site_id"]
|
|
1171
|
+
or identity["payload_sha256"] != row["payload_sha256"]
|
|
1172
|
+
or payload_json != row["payload_json"]
|
|
1173
|
+
or source_attempts != row["source_max_attempts"]
|
|
1174
|
+
or delivery_attempts != row["delivery_max_attempts"]
|
|
1175
|
+
or client_attempts != row["client_max_attempts"]
|
|
1176
|
+
or status not in STATUSES
|
|
1177
|
+
or isinstance(attempts, bool) or not isinstance(attempts, int)
|
|
1178
|
+
or not 0 <= attempts <= client_attempts
|
|
1179
|
+
or next_at < created or updated < created
|
|
1180
|
+
or (status == "leased") != all(value is not None for value in lease)
|
|
1181
|
+
or (status != "leased" and any(value is not None for value in lease))
|
|
1182
|
+
or (status == "leased" and (
|
|
1183
|
+
_token(lease[0], "state_invalid", limit=128) != lease[0]
|
|
1184
|
+
or _token(lease[1], "state_invalid") != lease[1]
|
|
1185
|
+
or _timestamp(lease[2], "state_invalid") <= updated
|
|
1186
|
+
))
|
|
1187
|
+
or (status in {"retry_wait", "failed"}) != (error is not None)
|
|
1188
|
+
or (error is not None and (
|
|
1189
|
+
not isinstance(error, str) or ERROR_CODE.fullmatch(error) is None
|
|
1190
|
+
))
|
|
1191
|
+
or valid_remote != all(value is not None for value in remote)
|
|
1192
|
+
or (not valid_remote and any(value is not None for value in remote))
|
|
1193
|
+
or (valid_remote and (
|
|
1194
|
+
remote[0] not in _CLIENT._HTTP.STATUSES
|
|
1195
|
+
or isinstance(remote[1], bool)
|
|
1196
|
+
or not isinstance(remote[1], int)
|
|
1197
|
+
or not 0 <= remote[1] <= delivery_attempts
|
|
1198
|
+
or any(
|
|
1199
|
+
not isinstance(value, str) or SHA256.fullmatch(value) is None
|
|
1200
|
+
for value in (remote[2], remote[3], remote[5])
|
|
1201
|
+
)
|
|
1202
|
+
or not isinstance(remote[4], str)
|
|
1203
|
+
or not self._valid_stored_response(row, remote[4])
|
|
1204
|
+
))
|
|
1205
|
+
):
|
|
1206
|
+
raise _blocked("state_invalid")
|
|
1207
|
+
|
|
1208
|
+
def _valid_stored_response(self, row: sqlite3.Row, response_json: str) -> bool:
|
|
1209
|
+
try:
|
|
1210
|
+
response = json.loads(response_json)
|
|
1211
|
+
payload = json.loads(row["payload_json"])
|
|
1212
|
+
commercial_binding = _stored_commercial_binding(
|
|
1213
|
+
row["commercial_contract_binding_json"], payload,
|
|
1214
|
+
"state_invalid",
|
|
1215
|
+
)
|
|
1216
|
+
binding = _CLIENT._HTTP._binding(
|
|
1217
|
+
response.get("website_capability_binding")
|
|
1218
|
+
)
|
|
1219
|
+
expected_schema = (
|
|
1220
|
+
_CLIENT._HTTP.CHANGE_RESPONSE_SCHEMA
|
|
1221
|
+
if row["operation"] == "change"
|
|
1222
|
+
else _CLIENT._HTTP.REMOVAL_RESPONSE_SCHEMA
|
|
1223
|
+
)
|
|
1224
|
+
fields = {
|
|
1225
|
+
"schema", "api_schema", "operation", "request_id", "event_id",
|
|
1226
|
+
"site_id", "payload_sha256", "status", "attempts",
|
|
1227
|
+
"delivery_max_attempts", "source_max_attempts",
|
|
1228
|
+
"capabilities_sha256", "website_capability_binding",
|
|
1229
|
+
"accepted_implies_publication",
|
|
1230
|
+
}
|
|
1231
|
+
return (
|
|
1232
|
+
isinstance(response, dict)
|
|
1233
|
+
and set(response) == fields
|
|
1234
|
+
and _canonical(response, "state_invalid") == response_json
|
|
1235
|
+
and _digest(response_json) == row["response_sha256"]
|
|
1236
|
+
and response["schema"] == expected_schema
|
|
1237
|
+
and response["api_schema"] == _CLIENT._HTTP.API_SCHEMA
|
|
1238
|
+
and response["operation"] == row["operation"]
|
|
1239
|
+
and response["request_id"] == row["request_id"]
|
|
1240
|
+
and response["event_id"] == row["event_id"]
|
|
1241
|
+
and response["site_id"] == row["site_id"]
|
|
1242
|
+
and response["payload_sha256"] == row["payload_sha256"]
|
|
1243
|
+
and response["source_max_attempts"]
|
|
1244
|
+
== row["source_max_attempts"]
|
|
1245
|
+
and response["delivery_max_attempts"]
|
|
1246
|
+
== row["delivery_max_attempts"]
|
|
1247
|
+
and response["status"] == row["remote_status"]
|
|
1248
|
+
and response["attempts"] == row["remote_attempts"]
|
|
1249
|
+
and response["capabilities_sha256"]
|
|
1250
|
+
== row["remote_capabilities_sha256"]
|
|
1251
|
+
and response["capabilities_sha256"]
|
|
1252
|
+
== binding["delivery_capabilities_sha256"]
|
|
1253
|
+
and (
|
|
1254
|
+
commercial_binding is None
|
|
1255
|
+
or binding["commercial_rendering_registry_sha256"]
|
|
1256
|
+
== commercial_binding[
|
|
1257
|
+
"commercial_rendering_registry_sha256"
|
|
1258
|
+
]
|
|
1259
|
+
)
|
|
1260
|
+
and _digest(_canonical(binding, "state_invalid"))
|
|
1261
|
+
== row["remote_binding_sha256"]
|
|
1262
|
+
and response["website_capability_binding"] == binding
|
|
1263
|
+
and response["accepted_implies_publication"] is False
|
|
1264
|
+
)
|
|
1265
|
+
except Exception:
|
|
1266
|
+
return False
|
|
1267
|
+
|
|
1268
|
+
def _claim(self, claim: Any) -> SubmissionDispatchClaim:
|
|
1269
|
+
if not isinstance(claim, SubmissionDispatchClaim):
|
|
1270
|
+
raise _blocked("claim_invalid")
|
|
1271
|
+
try:
|
|
1272
|
+
payload, identity, payload_json = _payload(json.loads(claim.payload_json))
|
|
1273
|
+
expected_commercial_binding = _commercial_binding_for_payload(payload)
|
|
1274
|
+
except Exception:
|
|
1275
|
+
raise _blocked("claim_invalid") from None
|
|
1276
|
+
if (
|
|
1277
|
+
claim.operation != identity["operation"]
|
|
1278
|
+
or claim.request_id != identity["request_id"]
|
|
1279
|
+
or claim.event_id != identity["event_id"]
|
|
1280
|
+
or claim.site_id != identity["site_id"]
|
|
1281
|
+
or claim.payload_sha256 != identity["payload_sha256"]
|
|
1282
|
+
or claim.payload_json != payload_json
|
|
1283
|
+
or claim.commercial_contract_binding != expected_commercial_binding
|
|
1284
|
+
or isinstance(claim.attempt, bool)
|
|
1285
|
+
or not isinstance(claim.attempt, int)
|
|
1286
|
+
or not 1 <= claim.attempt <= claim.client_max_attempts <= MAX_ATTEMPTS
|
|
1287
|
+
):
|
|
1288
|
+
raise _blocked("claim_invalid")
|
|
1289
|
+
_token(claim.lease_owner, "claim_invalid", limit=128)
|
|
1290
|
+
_token(claim.lease_token, "claim_invalid")
|
|
1291
|
+
_timestamp(claim.lease_expires_at, "claim_invalid")
|
|
1292
|
+
_count(claim.source_max_attempts, "claim_invalid")
|
|
1293
|
+
_count(claim.delivery_max_attempts, "claim_invalid")
|
|
1294
|
+
_count(claim.client_max_attempts, "claim_invalid")
|
|
1295
|
+
return claim
|
|
1296
|
+
|
|
1297
|
+
def _require_live_claim(
|
|
1298
|
+
self, claim: SubmissionDispatchClaim, now: float,
|
|
1299
|
+
) -> None:
|
|
1300
|
+
row = self.connection.execute("""
|
|
1301
|
+
SELECT status, lease_owner, lease_token, lease_expires_at,
|
|
1302
|
+
commercial_contract_binding_json
|
|
1303
|
+
FROM cms_public_submission_outbox
|
|
1304
|
+
WHERE operation = ? AND request_id = ?
|
|
1305
|
+
""", (claim.operation, claim.request_id)).fetchone()
|
|
1306
|
+
if (
|
|
1307
|
+
row is None or row["status"] != "leased"
|
|
1308
|
+
or row["lease_owner"] != claim.lease_owner
|
|
1309
|
+
or row["lease_token"] != claim.lease_token
|
|
1310
|
+
or row["lease_expires_at"] != claim.lease_expires_at
|
|
1311
|
+
or row["lease_expires_at"] <= now
|
|
1312
|
+
or row["commercial_contract_binding_json"] != (
|
|
1313
|
+
None if claim.commercial_contract_binding is None else
|
|
1314
|
+
_canonical(claim.commercial_contract_binding, "claim_lost")
|
|
1315
|
+
)
|
|
1316
|
+
):
|
|
1317
|
+
raise _blocked("claim_lost")
|