@withgateway/sdk 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +12 -4
- package/dist/agents/index.d.ts +12 -4
- package/dist/agents/index.js +30 -0
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/index.mjs +30 -0
- package/dist/agents/index.mjs.map +1 -1
- package/dist/cli.js +2291 -945
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2290 -944
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.js +325 -184
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +324 -183
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +3 -1
- package/dist/worlds/index.d.ts +3 -1
- package/dist/worlds/index.js +178 -67
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +177 -66
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +21 -2
- package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
- package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -0
- package/skills/connector-schema-authoring/connector-worlds.md +1 -0
- package/skills/connector-schema-authoring/world-schema.md +7 -0
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +4 -4
- package/skills/worlds-getting-started/SKILL.md +2 -2
- package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
"""Stdlib only: the runner vendors this package. Twin of factorySpine.ts and factorySpineDocument.ts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from collections import Counter
|
|
8
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
9
|
+
from types import MappingProxyType
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .bundle_manifest import _utf16_length
|
|
13
|
+
from .factory_shapes import (
|
|
14
|
+
_KIND,
|
|
15
|
+
_TEXT,
|
|
16
|
+
FACTORY_LIMITS,
|
|
17
|
+
Check,
|
|
18
|
+
_array,
|
|
19
|
+
_javascript_number,
|
|
20
|
+
_json,
|
|
21
|
+
_literal,
|
|
22
|
+
_node_id,
|
|
23
|
+
_node_shape,
|
|
24
|
+
_number,
|
|
25
|
+
_one_of,
|
|
26
|
+
_record,
|
|
27
|
+
_slug,
|
|
28
|
+
_source,
|
|
29
|
+
_strict,
|
|
30
|
+
_string,
|
|
31
|
+
_tagged,
|
|
32
|
+
canonical,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
SPINE_COLUMN_NAME = re.compile(r"^(?!__)[A-Za-z_][A-Za-z0-9_]{0,63}$")
|
|
36
|
+
SPINE_COLUMN_TYPES = ("string", "integer", "number", "boolean", "date", "instant", "json", "ref")
|
|
37
|
+
SPINE_KEY_TYPES = ("string", "integer")
|
|
38
|
+
SPINE_FORMULA_NAMES = 32
|
|
39
|
+
SPINE_ROW_ENTRY = re.compile(r"^spine/([a-z0-9][a-z0-9_-]{0,63})/rows/((?:[0-9a-f]{2})+)\.json$")
|
|
40
|
+
SPINE_DOCUMENT_ISSUE_CODES = (
|
|
41
|
+
"ref_row_missing",
|
|
42
|
+
"spine_key",
|
|
43
|
+
"spine_row_key",
|
|
44
|
+
"spine_row_duplicate",
|
|
45
|
+
"spine_row_too_large",
|
|
46
|
+
"spine_column_missing",
|
|
47
|
+
"spine_column_duplicate",
|
|
48
|
+
"spine_check_table",
|
|
49
|
+
"spine_partition_shared",
|
|
50
|
+
"spine_quota_entity",
|
|
51
|
+
)
|
|
52
|
+
SPINE_ISSUE_TEXT: Mapping[str, str] = MappingProxyType(
|
|
53
|
+
{
|
|
54
|
+
"ref_row_missing": (
|
|
55
|
+
"Node {node} refers to {subject}, which is not a table of this factory or not a row it holds."
|
|
56
|
+
),
|
|
57
|
+
"spine_key": "Table {node} is keyed by {subject}, which is not one of its string or integer columns.",
|
|
58
|
+
"spine_row_key": (
|
|
59
|
+
f"Table {{node}} has a row whose key is not a string or integer of 1 to {FACTORY_LIMITS['keyLength']} "
|
|
60
|
+
"characters that does not start with __."
|
|
61
|
+
),
|
|
62
|
+
"spine_row_duplicate": "Table {node} holds the key {subject} more than once.",
|
|
63
|
+
"spine_row_too_large": (
|
|
64
|
+
f"The row {{subject}} of table {{node}} is larger than {FACTORY_LIMITS['rowBytes']} bytes "
|
|
65
|
+
"as canonical JSON."
|
|
66
|
+
),
|
|
67
|
+
"spine_column_missing": "Table {node} uses the column {subject}, which it does not declare.",
|
|
68
|
+
"spine_column_duplicate": "Table {node} declares the column {subject} more than once.",
|
|
69
|
+
"spine_check_table": "Node {node} has a steps check; steps checks belong on a table.",
|
|
70
|
+
"spine_partition_shared": "Table {node} partitions {subject}, which another table already partitions.",
|
|
71
|
+
"spine_quota_entity": "Table {node} sets a quota on {subject}, which its partition does not own.",
|
|
72
|
+
}
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
Finding = tuple[str, "str | None"]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _column_name(value: Any) -> bool:
|
|
79
|
+
return isinstance(value, str) and SPINE_COLUMN_NAME.fullmatch(value) is not None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _pointer(value: Any) -> bool:
|
|
83
|
+
return isinstance(value, str) and (value == "" or value.startswith("/"))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _formula_text(value: Any) -> bool:
|
|
87
|
+
if not isinstance(value, str) or _utf16_length(value) < 1:
|
|
88
|
+
return False
|
|
89
|
+
return len(value.encode("utf-8", "surrogatepass")) <= FACTORY_LIMITS["formulaBytes"]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _nonnegative(value: Any) -> bool:
|
|
93
|
+
return _number(value) and value >= 0
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _weekday(value: Any) -> bool:
|
|
97
|
+
return _number(value) and float(value).is_integer() and 0 <= value <= 6
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
_NODE_REF = _strict({"kind": _literal("node"), "id": _node_id})
|
|
101
|
+
_SOURCE = _tagged(
|
|
102
|
+
"kind",
|
|
103
|
+
{
|
|
104
|
+
"node": _NODE_REF,
|
|
105
|
+
"row": _strict({"kind": _literal("row"), "node": _node_id, "key": _TEXT}),
|
|
106
|
+
},
|
|
107
|
+
)
|
|
108
|
+
_FORM_KEYS = ("copy", "formula", "literal", "from")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def spine_value(value: Any) -> bool:
|
|
112
|
+
if not _json(value):
|
|
113
|
+
return False
|
|
114
|
+
shape = _derivation_shape(value)
|
|
115
|
+
return shape is None or shape(value)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _names(value: Any) -> bool:
|
|
119
|
+
return _record(spine_value, _column_name)(value) and len(value) <= SPINE_FORMULA_NAMES
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
_FROM_PATH = _strict({"from": _SOURCE}, {"path": _pointer})
|
|
123
|
+
_FROM_COLUMN = _strict({"from": _SOURCE, "column": _column_name})
|
|
124
|
+
_COPY_PATH = _strict({"copy": _json, "from": _SOURCE}, {"path": _pointer})
|
|
125
|
+
_COPY_COLUMN = _strict({"copy": _json, "from": _SOURCE, "column": _column_name})
|
|
126
|
+
_FORMULA = _strict({"formula": _formula_text}, {"with": _names})
|
|
127
|
+
_LITERAL = _strict({"literal": _json})
|
|
128
|
+
|
|
129
|
+
_FORM_SHAPES: Mapping[str, Callable[[dict[str, Any]], Check]] = MappingProxyType(
|
|
130
|
+
{
|
|
131
|
+
"copy": lambda value: _COPY_COLUMN if "column" in value else _COPY_PATH,
|
|
132
|
+
"formula": lambda value: _FORMULA,
|
|
133
|
+
"literal": lambda value: _LITERAL,
|
|
134
|
+
"from": lambda value: _FROM_COLUMN if "column" in value else _FROM_PATH,
|
|
135
|
+
}
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _form_of(value: Any) -> str | None:
|
|
140
|
+
if not isinstance(value, dict):
|
|
141
|
+
return None
|
|
142
|
+
return next((key for key in _FORM_KEYS if key in value), None)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _derivation_shape(value: Any) -> Check | None:
|
|
146
|
+
form = _form_of(value)
|
|
147
|
+
return None if form is None else _FORM_SHAPES[form](value)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
_SOURCES_OF: Mapping[str, Callable[[dict[str, Any]], list[dict[str, Any]]]] = MappingProxyType(
|
|
151
|
+
{
|
|
152
|
+
"copy": lambda value: [value["from"]],
|
|
153
|
+
"from": lambda value: [value["from"]],
|
|
154
|
+
"literal": lambda value: [],
|
|
155
|
+
"formula": lambda value: [
|
|
156
|
+
source for item in value.get("with", {}).values() for source in derivation_sources(item)
|
|
157
|
+
],
|
|
158
|
+
}
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def derivation_sources(value: Any) -> list[dict[str, Any]]:
|
|
163
|
+
form = _form_of(value)
|
|
164
|
+
return [] if form is None else _SOURCES_OF[form](value)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
_SEVERITY = {"severity": _one_of("block", "warn")}
|
|
168
|
+
_CHECK = _tagged(
|
|
169
|
+
"type",
|
|
170
|
+
{
|
|
171
|
+
"flow": _strict(
|
|
172
|
+
{
|
|
173
|
+
"type": _literal("flow"),
|
|
174
|
+
"start": spine_value,
|
|
175
|
+
"created": spine_value,
|
|
176
|
+
"closed": spine_value,
|
|
177
|
+
"end": spine_value,
|
|
178
|
+
},
|
|
179
|
+
_SEVERITY,
|
|
180
|
+
),
|
|
181
|
+
"sum": _strict({"type": _literal("sum"), "parts": spine_value, "total": spine_value}, _SEVERITY),
|
|
182
|
+
"quota": _strict(
|
|
183
|
+
{"type": _literal("quota"), "total": spine_value, "quotas": spine_value},
|
|
184
|
+
{"weights": spine_value, **_SEVERITY},
|
|
185
|
+
),
|
|
186
|
+
"fact": _strict(
|
|
187
|
+
{"type": _literal("fact"), "claims": spine_value, "value": spine_value},
|
|
188
|
+
{"tolerance": _nonnegative, **_SEVERITY},
|
|
189
|
+
),
|
|
190
|
+
"steps": _strict(
|
|
191
|
+
{"type": _literal("steps"), "at": _column_name, "names": _column_name},
|
|
192
|
+
{
|
|
193
|
+
"business": _column_name,
|
|
194
|
+
"holidays": spine_value,
|
|
195
|
+
"weekend": _array(_weekday, max_items=7),
|
|
196
|
+
**_SEVERITY,
|
|
197
|
+
},
|
|
198
|
+
),
|
|
199
|
+
},
|
|
200
|
+
)
|
|
201
|
+
_CHECK_VALUES: Mapping[str, Callable[[dict[str, Any]], list[Any]]] = MappingProxyType(
|
|
202
|
+
{
|
|
203
|
+
"flow": lambda check: [check["start"], check["created"], check["closed"], check["end"]],
|
|
204
|
+
"sum": lambda check: [check["parts"], check["total"]],
|
|
205
|
+
"quota": lambda check: [check["total"], check["quotas"], *([check["weights"]] if "weights" in check else [])],
|
|
206
|
+
"fact": lambda check: [check["claims"], check["value"]],
|
|
207
|
+
"steps": lambda check: [check["holidays"]] if "holidays" in check else [],
|
|
208
|
+
}
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
_COLUMN = _strict(
|
|
212
|
+
{"name": _column_name, "type": _one_of(*SPINE_COLUMN_TYPES)},
|
|
213
|
+
{"ref": _NODE_REF, "sentence": _string(0, FACTORY_LIMITS["sentenceLength"])},
|
|
214
|
+
)
|
|
215
|
+
_ROW = _record(spine_value, _column_name)
|
|
216
|
+
_ENTITY_AT = _strict({"world": _slug, "entity": _TEXT})
|
|
217
|
+
_PARTITION = _strict(
|
|
218
|
+
{"owner": _source, "entities": _array(_ENTITY_AT, min_items=1)},
|
|
219
|
+
{
|
|
220
|
+
"quotas": _array(
|
|
221
|
+
_strict({"column": _column_name, "world": _slug, "entity": _TEXT}, {"tolerance": _nonnegative}),
|
|
222
|
+
)
|
|
223
|
+
},
|
|
224
|
+
)
|
|
225
|
+
_RESERVE = _strict({"column": _column_name, "world": _slug, "entity": _TEXT, "field": _TEXT})
|
|
226
|
+
_CHECKS = _array(_CHECK, max_items=FACTORY_LIMITS["checksPerNode"])
|
|
227
|
+
_SPINE_OPTIONAL = {"group": _KIND}
|
|
228
|
+
|
|
229
|
+
SPINE_NODE = _tagged(
|
|
230
|
+
"kind",
|
|
231
|
+
{
|
|
232
|
+
"table": _node_shape(
|
|
233
|
+
"spine",
|
|
234
|
+
{
|
|
235
|
+
"kind": _literal("table"),
|
|
236
|
+
"columns": _array(_COLUMN, 1, FACTORY_LIMITS["columnsPerTable"]),
|
|
237
|
+
"key": _column_name,
|
|
238
|
+
"rows": _array(_ROW, max_items=FACTORY_LIMITS["tableRows"]),
|
|
239
|
+
},
|
|
240
|
+
{
|
|
241
|
+
**_SPINE_OPTIONAL,
|
|
242
|
+
"checks": _CHECKS,
|
|
243
|
+
"partition": _PARTITION,
|
|
244
|
+
"reserves": _array(_RESERVE),
|
|
245
|
+
"generate": _source,
|
|
246
|
+
},
|
|
247
|
+
),
|
|
248
|
+
"fact": _node_shape(
|
|
249
|
+
"spine",
|
|
250
|
+
{"kind": _literal("fact"), "value": spine_value},
|
|
251
|
+
{**_SPINE_OPTIONAL, "checks": _CHECKS},
|
|
252
|
+
),
|
|
253
|
+
"check": _node_shape(
|
|
254
|
+
"spine",
|
|
255
|
+
{"kind": _literal("check"), "checks": _array(_CHECK, 1, FACTORY_LIMITS["checksPerNode"])},
|
|
256
|
+
_SPINE_OPTIONAL,
|
|
257
|
+
),
|
|
258
|
+
},
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def spine_table(value: Any) -> dict[str, Any] | None:
|
|
263
|
+
if not isinstance(value, dict) or value.get("doc") != "spine" or value.get("kind") != "table":
|
|
264
|
+
return None
|
|
265
|
+
return value
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def table_head(table: dict[str, Any]) -> dict[str, Any]:
|
|
269
|
+
return {key: item for key, item in table.items() if key != "rows"}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def key_of(row: dict[str, Any], key: Any) -> str | None:
|
|
273
|
+
cell = row.get(key) if isinstance(key, str) else None
|
|
274
|
+
if isinstance(cell, str):
|
|
275
|
+
return cell
|
|
276
|
+
if isinstance(cell, (int, float)) and _number(cell) and float(cell).is_integer():
|
|
277
|
+
return _javascript_number(cell)
|
|
278
|
+
return None
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def row_key(row: dict[str, Any], key: str) -> str | None:
|
|
282
|
+
text = key_of(row, key)
|
|
283
|
+
if text is None or not 1 <= len(text) <= FACTORY_LIMITS["keyLength"] or text.startswith("__"):
|
|
284
|
+
return None
|
|
285
|
+
return text
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def sorted_rows(rows: Iterable[dict[str, Any]], key: Any) -> list[dict[str, Any]]:
|
|
289
|
+
return sorted(rows, key=lambda row: (key_of(row, key) or "").encode("utf-8", "surrogatepass"))
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _table_values(node: dict[str, Any]) -> list[Any]:
|
|
293
|
+
return [cell for row in node["rows"] for cell in row.values()]
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
_NODE_VALUES: Mapping[str, Callable[[dict[str, Any]], list[Any]]] = MappingProxyType(
|
|
297
|
+
{
|
|
298
|
+
"table": _table_values,
|
|
299
|
+
"fact": lambda node: [node["value"]],
|
|
300
|
+
"check": lambda node: [],
|
|
301
|
+
}
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def spine_sources(node: dict[str, Any]) -> list[dict[str, Any]]:
|
|
306
|
+
checks = [value for check in node.get("checks", []) for value in _CHECK_VALUES[check["type"]](check)]
|
|
307
|
+
values = [*_NODE_VALUES[node["kind"]](node), *checks]
|
|
308
|
+
return [source for value in values for source in derivation_sources(value)]
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def spine_table_refs(node: dict[str, Any]) -> list[str]:
|
|
312
|
+
table = spine_table(node)
|
|
313
|
+
columns = table["columns"] if table else []
|
|
314
|
+
return [column["ref"]["id"] for column in columns if "ref" in column]
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _entity_name(at: Mapping[str, Any]) -> str:
|
|
318
|
+
return f"{at['world']}.{at['entity']}"
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
class SpineIndex:
|
|
322
|
+
def __init__(self, tables: Iterable[dict[str, Any]]) -> None:
|
|
323
|
+
self._tables = {table["id"]: table for table in tables}
|
|
324
|
+
self._keys: dict[str, set[str]] = {}
|
|
325
|
+
self._owners: dict[str, str] = {}
|
|
326
|
+
for table in sorted(self._tables.values(), key=lambda table: table["id"]):
|
|
327
|
+
for at in table.get("partition", {}).get("entities", []):
|
|
328
|
+
self._owners.setdefault(_entity_name(at), table["id"])
|
|
329
|
+
|
|
330
|
+
def holds(self, table_id: str, key: str | None) -> bool:
|
|
331
|
+
table = self._tables.get(table_id)
|
|
332
|
+
if table is None:
|
|
333
|
+
return False
|
|
334
|
+
return key is None or key in self._keys_of(table)
|
|
335
|
+
|
|
336
|
+
def owner_of(self, world_entity: str) -> str | None:
|
|
337
|
+
return self._owners.get(world_entity)
|
|
338
|
+
|
|
339
|
+
def _keys_of(self, table: dict[str, Any]) -> set[str]:
|
|
340
|
+
if table["id"] not in self._keys:
|
|
341
|
+
self._keys[table["id"]] = {text for row in table["rows"] if (text := key_of(row, table["key"])) is not None}
|
|
342
|
+
return self._keys[table["id"]]
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _steps(node: dict[str, Any]) -> list[dict[str, Any]]:
|
|
346
|
+
return [check for check in node.get("checks", []) if check["type"] == "steps"]
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _value_findings(node: dict[str, Any], index: SpineIndex) -> list[Finding]:
|
|
350
|
+
return [("spine_check_table", None)] if _steps(node) else []
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _column_duplicates(table: dict[str, Any]) -> list[Finding]:
|
|
354
|
+
counts = Counter(column["name"] for column in table["columns"])
|
|
355
|
+
return [("spine_column_duplicate", name) for name, count in sorted(counts.items()) if count > 1]
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _key_findings(table: dict[str, Any]) -> list[Finding]:
|
|
359
|
+
column = next((column for column in table["columns"] if column["name"] == table["key"]), None)
|
|
360
|
+
if column is not None and column["type"] in SPINE_KEY_TYPES:
|
|
361
|
+
return []
|
|
362
|
+
return [("spine_key", table["key"])]
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _row_bytes(row: dict[str, Any]) -> int:
|
|
366
|
+
return len(canonical(row).encode("utf-8"))
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _row_findings(table: dict[str, Any]) -> list[Finding]:
|
|
370
|
+
keys = [row_key(row, table["key"]) for row in table["rows"]]
|
|
371
|
+
valid = [key for key in keys if key is not None]
|
|
372
|
+
bad_keys: list[Finding] = [("spine_row_key", None)] if len(valid) < len(keys) else []
|
|
373
|
+
too_large: list[Finding] = [
|
|
374
|
+
("spine_row_too_large", key)
|
|
375
|
+
for row, key in zip(table["rows"], keys)
|
|
376
|
+
if _row_bytes(row) > FACTORY_LIMITS["rowBytes"]
|
|
377
|
+
]
|
|
378
|
+
duplicates: list[Finding] = [("spine_row_duplicate", key) for key, count in Counter(valid).items() if count > 1]
|
|
379
|
+
return [*bad_keys, *too_large, *duplicates]
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _named_columns(table: dict[str, Any]) -> list[str]:
|
|
383
|
+
partition = table.get("partition", {})
|
|
384
|
+
return [
|
|
385
|
+
*(name for row in table["rows"] for name in row),
|
|
386
|
+
*(name for check in _steps(table) for name in (check["at"], check["names"], check.get("business")) if name),
|
|
387
|
+
*(quota["column"] for quota in partition.get("quotas", [])),
|
|
388
|
+
*(reserve["column"] for reserve in table.get("reserves", [])),
|
|
389
|
+
]
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _undeclared(table: dict[str, Any]) -> list[Finding]:
|
|
393
|
+
declared = {column["name"] for column in table["columns"]}
|
|
394
|
+
return [("spine_column_missing", name) for name in sorted(set(_named_columns(table)) - declared)]
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _quota_findings(table: dict[str, Any]) -> list[Finding]:
|
|
398
|
+
partition = table.get("partition", {})
|
|
399
|
+
owned = {_entity_name(at) for at in partition.get("entities", [])}
|
|
400
|
+
quotas = {_entity_name(quota) for quota in partition.get("quotas", [])}
|
|
401
|
+
return [("spine_quota_entity", name) for name in sorted(quotas - owned)]
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _shared_findings(table: dict[str, Any], index: SpineIndex) -> list[Finding]:
|
|
405
|
+
names = {_entity_name(at) for at in table.get("partition", {}).get("entities", [])}
|
|
406
|
+
return [("spine_partition_shared", name) for name in sorted(names) if index.owner_of(name) != table["id"]]
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _table_findings(table: dict[str, Any], index: SpineIndex) -> list[Finding]:
|
|
410
|
+
return [
|
|
411
|
+
*_column_duplicates(table),
|
|
412
|
+
*_key_findings(table),
|
|
413
|
+
*_row_findings(table),
|
|
414
|
+
*_undeclared(table),
|
|
415
|
+
*_quota_findings(table),
|
|
416
|
+
*_shared_findings(table, index),
|
|
417
|
+
]
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
_FINDINGS: Mapping[str, Callable[[dict[str, Any], SpineIndex], list[Finding]]] = MappingProxyType(
|
|
421
|
+
{"table": _table_findings, "fact": _value_findings, "check": _value_findings}
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def spine_findings(node: dict[str, Any], index: SpineIndex) -> list[Finding]:
|
|
426
|
+
return _FINDINGS[node["kind"]](node, index)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _refuse_constant(name: str) -> Any:
|
|
430
|
+
raise ValueError(f"{name} is not JSON")
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _row_of(text: str) -> dict[str, Any] | None:
|
|
434
|
+
try:
|
|
435
|
+
value = json.loads(text, parse_constant=_refuse_constant)
|
|
436
|
+
except ValueError:
|
|
437
|
+
return None
|
|
438
|
+
return value if isinstance(value, dict) else None
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
class SpineRowEntries:
|
|
442
|
+
def __init__(self) -> None:
|
|
443
|
+
self._rows: dict[str, list[tuple[str, str]]] = {}
|
|
444
|
+
|
|
445
|
+
def take(self, path: str, data: bytes | str) -> bool:
|
|
446
|
+
match = SPINE_ROW_ENTRY.fullmatch(path)
|
|
447
|
+
if match is None:
|
|
448
|
+
return False
|
|
449
|
+
table_id, hex_key = match.groups()
|
|
450
|
+
text = data.decode("utf-8", "replace") if isinstance(data, bytes) else data
|
|
451
|
+
key = bytes.fromhex(hex_key).decode("utf-8", "replace")
|
|
452
|
+
self._rows.setdefault(table_id, []).append((key, text))
|
|
453
|
+
return True
|
|
454
|
+
|
|
455
|
+
def join(self, raw: list[Any]) -> tuple[list[Any], list[str]]:
|
|
456
|
+
heads = {head["id"]: head for value in raw if (head := _raw_head(value)) is not None}
|
|
457
|
+
orphans = [table_id for table_id in self._rows if table_id not in heads]
|
|
458
|
+
failures = [head["id"] for head in heads.values() if self._head_fails(head)]
|
|
459
|
+
nodes = [self._joined(value) for value in raw]
|
|
460
|
+
return nodes, [*orphans, *failures]
|
|
461
|
+
|
|
462
|
+
def _head_fails(self, head: dict[str, Any]) -> bool:
|
|
463
|
+
if "rows" in head:
|
|
464
|
+
return True
|
|
465
|
+
return any(
|
|
466
|
+
(row := _row_of(text)) is None or key_of(row, head.get("key")) != key
|
|
467
|
+
for key, text in self._rows.get(head["id"], [])
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
def _joined(self, value: Any) -> Any:
|
|
471
|
+
head = _raw_head(value)
|
|
472
|
+
if head is None:
|
|
473
|
+
return value
|
|
474
|
+
rows = [row for _, text in self._rows.get(head["id"], []) if (row := _row_of(text)) is not None]
|
|
475
|
+
return {**head, "rows": sorted_rows(rows, head.get("key"))}
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _raw_head(value: Any) -> dict[str, Any] | None:
|
|
479
|
+
table = spine_table(value)
|
|
480
|
+
return table if table is not None and isinstance(table.get("id"), str) else None
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
"""``Admission``: how many requests run on one
|
|
2
|
-
|
|
3
|
-
Every session of a
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
come
|
|
10
|
-
|
|
11
|
-
|
|
1
|
+
"""``Admission``: how many requests run on one worker key at once, and how many wait.
|
|
2
|
+
|
|
3
|
+
Every session of a :class:`WorkerKey` (a version's main, or its world states) is
|
|
4
|
+
served by one process (its version worker, or the host itself for a world with
|
|
5
|
+
no code of its own), so a burst on a key is a burst on that process. The host
|
|
6
|
+
admits a key's requests a few at a time and one per session (a session's store
|
|
7
|
+
answers one request at a time anyway); the rest wait their turn in the order
|
|
8
|
+
they came, up to a bound. A request past the bound, or one whose turn did not
|
|
9
|
+
come in time, is refused ``busy`` with when to come back, so a burst queues or
|
|
10
|
+
is told to retry and the process never holds more requests than it has room
|
|
11
|
+
for, and a version's states never take main's turns. A request held outside the
|
|
12
|
+
queue (its session moving to another host) is counted against the same bounds
|
|
13
|
+
(:meth:`Admission.waiting`).
|
|
12
14
|
"""
|
|
13
15
|
|
|
14
16
|
from __future__ import annotations
|
|
@@ -25,7 +27,7 @@ from .validation import fail
|
|
|
25
27
|
VERSION_SLOTS = 8
|
|
26
28
|
SESSION_SLOTS = 1
|
|
27
29
|
QUEUE_DEPTH = 256
|
|
28
|
-
#: A session's own share of the waiting, so one session's burst cannot fill its
|
|
30
|
+
#: A session's own share of the waiting, so one session's burst cannot fill its key's queue.
|
|
29
31
|
SESSION_QUEUE_DEPTH = 32
|
|
30
32
|
#: With a call's own budget it stays under the platform's 60 s (``conformance/world-sessions.json``).
|
|
31
33
|
QUEUE_WAIT_S = 10.0
|
|
@@ -41,14 +43,14 @@ def _less(counter: Counter, key: object) -> None:
|
|
|
41
43
|
|
|
42
44
|
@dataclass(eq=False)
|
|
43
45
|
class _Turn:
|
|
44
|
-
|
|
46
|
+
key: str
|
|
45
47
|
session: str
|
|
46
48
|
started: float | None = None
|
|
47
49
|
|
|
48
50
|
|
|
49
51
|
class Admission:
|
|
50
|
-
"""Turns on each
|
|
51
|
-
each session's slots; another
|
|
52
|
+
"""Turns on each key, first come first served within the key's and
|
|
53
|
+
each session's slots; another key never waits."""
|
|
52
54
|
|
|
53
55
|
def __init__(
|
|
54
56
|
self,
|
|
@@ -75,34 +77,34 @@ class Admission:
|
|
|
75
77
|
self._outside_by: Counter[tuple[str, str]] = Counter()
|
|
76
78
|
|
|
77
79
|
@contextmanager
|
|
78
|
-
def turn(self,
|
|
79
|
-
"""Hold one of ``
|
|
80
|
+
def turn(self, key: str, session: str) -> Iterator[None]:
|
|
81
|
+
"""Hold one of ``key``'s slots for ``session`` while the block runs; refused
|
|
80
82
|
``busy`` (a :class:`ContractError` carrying ``retryAfter``) when none comes."""
|
|
81
|
-
turn = self._enter(_Turn(
|
|
83
|
+
turn = self._enter(_Turn(key, session))
|
|
82
84
|
try:
|
|
83
85
|
yield
|
|
84
86
|
finally:
|
|
85
87
|
self._leave(turn)
|
|
86
88
|
|
|
87
89
|
@contextmanager
|
|
88
|
-
def waiting(self,
|
|
89
|
-
"""Count a request waiting outside the queue against ``
|
|
90
|
+
def waiting(self, key: str, session: str) -> Iterator[None]:
|
|
91
|
+
"""Count a request waiting outside the queue against ``key``'s and ``session``'s waiting
|
|
90
92
|
bounds while the block runs; refused ``busy`` past them, as a queued request would be."""
|
|
91
93
|
with self._lock:
|
|
92
|
-
self._outside[
|
|
93
|
-
self._outside_by[(
|
|
94
|
-
if self._overfull(
|
|
95
|
-
self._unwait(
|
|
96
|
-
self._refuse(
|
|
94
|
+
self._outside[key] += 1
|
|
95
|
+
self._outside_by[(key, session)] += 1
|
|
96
|
+
if self._overfull(key, session):
|
|
97
|
+
self._unwait(key, session)
|
|
98
|
+
self._refuse(key, "too many requests are waiting")
|
|
97
99
|
try:
|
|
98
100
|
yield
|
|
99
101
|
finally:
|
|
100
102
|
with self._lock:
|
|
101
|
-
self._unwait(
|
|
103
|
+
self._unwait(key, session)
|
|
102
104
|
|
|
103
|
-
def _unwait(self,
|
|
104
|
-
_less(self._outside,
|
|
105
|
-
_less(self._outside_by, (
|
|
105
|
+
def _unwait(self, key: str, session: str) -> None:
|
|
106
|
+
_less(self._outside, key)
|
|
107
|
+
_less(self._outside_by, (key, session))
|
|
106
108
|
|
|
107
109
|
def held_ages(self) -> dict[str, float]:
|
|
108
110
|
"""Each session holding a turn now, with how long its oldest has run, in seconds."""
|
|
@@ -116,57 +118,57 @@ class Admission:
|
|
|
116
118
|
|
|
117
119
|
def _enter(self, turn: _Turn) -> _Turn:
|
|
118
120
|
with self._lock:
|
|
119
|
-
queue = self._waiting.setdefault(turn.
|
|
121
|
+
queue = self._waiting.setdefault(turn.key, deque())
|
|
120
122
|
queue.append(turn)
|
|
121
|
-
self._admit(turn.
|
|
122
|
-
if turn.started is None and self._overfull(turn.
|
|
123
|
+
self._admit(turn.key)
|
|
124
|
+
if turn.started is None and self._overfull(turn.key, turn.session):
|
|
123
125
|
queue.remove(turn)
|
|
124
|
-
self._refuse(turn.
|
|
126
|
+
self._refuse(turn.key, f"{len(queue)} requests are waiting")
|
|
125
127
|
deadline = time.monotonic() + self.wait_s
|
|
126
128
|
while turn.started is None:
|
|
127
129
|
left = deadline - time.monotonic()
|
|
128
130
|
if left <= 0:
|
|
129
131
|
queue.remove(turn)
|
|
130
|
-
self._refuse(turn.
|
|
132
|
+
self._refuse(turn.key, f"no turn came within {self.wait_s:g} s")
|
|
131
133
|
self._changed.wait(left)
|
|
132
134
|
return turn
|
|
133
135
|
|
|
134
|
-
def _overfull(self,
|
|
135
|
-
queue = self._waiting.get(
|
|
136
|
-
mine = sum(waiting.session == session for waiting in queue) + self._outside_by[(
|
|
137
|
-
return len(queue) + self._outside[
|
|
136
|
+
def _overfull(self, key: str, session: str) -> bool:
|
|
137
|
+
queue = self._waiting.get(key, deque())
|
|
138
|
+
mine = sum(waiting.session == session for waiting in queue) + self._outside_by[(key, session)]
|
|
139
|
+
return len(queue) + self._outside[key] > self.queue or mine > self.session_queue
|
|
138
140
|
|
|
139
141
|
def _leave(self, turn: _Turn) -> None:
|
|
140
142
|
with self._lock:
|
|
141
|
-
|
|
142
|
-
self._holding[
|
|
143
|
-
if not self._holding[
|
|
144
|
-
del self._holding[
|
|
145
|
-
self._running[turn.
|
|
146
|
-
if self._running[turn.
|
|
147
|
-
del self._running[turn.
|
|
148
|
-
self._admit(turn.
|
|
149
|
-
self._forget(turn.
|
|
150
|
-
|
|
151
|
-
def _admit(self,
|
|
143
|
+
held = (turn.key, turn.session)
|
|
144
|
+
self._holding[held].remove(turn)
|
|
145
|
+
if not self._holding[held]:
|
|
146
|
+
del self._holding[held]
|
|
147
|
+
self._running[turn.key] -= 1
|
|
148
|
+
if self._running[turn.key] <= 0:
|
|
149
|
+
del self._running[turn.key]
|
|
150
|
+
self._admit(turn.key)
|
|
151
|
+
self._forget(turn.key)
|
|
152
|
+
|
|
153
|
+
def _admit(self, key: str) -> None:
|
|
152
154
|
"""Under the lock: give free slots to the earliest waiters whose session has room."""
|
|
153
|
-
queue = self._waiting.get(
|
|
155
|
+
queue = self._waiting.get(key, deque())
|
|
154
156
|
for waiting in list(queue):
|
|
155
|
-
if self._running[
|
|
157
|
+
if self._running[key] >= self.slots:
|
|
156
158
|
break
|
|
157
|
-
if len(self._holding.get((
|
|
159
|
+
if len(self._holding.get((key, waiting.session), ())) < self.per_session:
|
|
158
160
|
queue.remove(waiting)
|
|
159
161
|
waiting.started = time.monotonic()
|
|
160
|
-
self._running[
|
|
161
|
-
self._holding.setdefault((
|
|
162
|
+
self._running[key] += 1
|
|
163
|
+
self._holding.setdefault((key, waiting.session), []).append(waiting)
|
|
162
164
|
self._changed.notify_all()
|
|
163
165
|
|
|
164
|
-
def _forget(self,
|
|
165
|
-
if not self._waiting.get(
|
|
166
|
-
self._waiting.pop(
|
|
166
|
+
def _forget(self, key: str) -> None:
|
|
167
|
+
if not self._waiting.get(key) and not self._running[key]:
|
|
168
|
+
self._waiting.pop(key, None)
|
|
167
169
|
|
|
168
|
-
def _refuse(self,
|
|
169
|
-
self._forget(
|
|
170
|
+
def _refuse(self, key: str, why: str) -> None:
|
|
171
|
+
self._forget(key)
|
|
170
172
|
fail(
|
|
171
173
|
BUSY,
|
|
172
174
|
f"this world's version is busy ({why}); retry after {self.retry_after_s} s",
|