@withgateway/sdk 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/dist/agents/index.d.mts +12 -4
  2. package/dist/agents/index.d.ts +12 -4
  3. package/dist/agents/index.js +30 -0
  4. package/dist/agents/index.js.map +1 -1
  5. package/dist/agents/index.mjs +30 -0
  6. package/dist/agents/index.mjs.map +1 -1
  7. package/dist/cli.js +2291 -945
  8. package/dist/cli.js.map +1 -1
  9. package/dist/cli.mjs +2290 -944
  10. package/dist/cli.mjs.map +1 -1
  11. package/dist/files/index.js.map +1 -1
  12. package/dist/files/index.mjs.map +1 -1
  13. package/dist/index.js +325 -184
  14. package/dist/index.js.map +1 -1
  15. package/dist/index.mjs +324 -183
  16. package/dist/index.mjs.map +1 -1
  17. package/dist/traces/index.js.map +1 -1
  18. package/dist/traces/index.mjs.map +1 -1
  19. package/dist/tracing/index.js +1 -1
  20. package/dist/tracing/index.js.map +1 -1
  21. package/dist/tracing/index.mjs +1 -1
  22. package/dist/tracing/index.mjs.map +1 -1
  23. package/dist/worlds/index.d.mts +3 -1
  24. package/dist/worlds/index.d.ts +3 -1
  25. package/dist/worlds/index.js +178 -67
  26. package/dist/worlds/index.js.map +1 -1
  27. package/dist/worlds/index.mjs +177 -66
  28. package/dist/worlds/index.mjs.map +1 -1
  29. package/package.json +1 -1
  30. package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
  31. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  32. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  33. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  34. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  35. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  36. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  37. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  38. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  39. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  40. package/runtime/gatewaysdk/environment/schema/host.py +21 -2
  41. package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
  42. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  43. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  44. package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
  45. package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
  46. package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
  47. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  48. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  49. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  51. package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
  52. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  53. package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
  54. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  55. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  56. package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
  57. package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
  58. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  59. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  60. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  61. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
  62. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  63. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  64. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
  65. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
  66. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  68. package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
  69. package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
  70. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  71. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  72. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  73. package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
  74. package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
  75. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  76. package/skills/connector-schema-authoring/SKILL.md +1 -0
  77. package/skills/connector-schema-authoring/connector-worlds.md +1 -0
  78. package/skills/connector-schema-authoring/world-schema.md +7 -0
  79. package/skills/factory-orchestrator/SKILL.md +147 -0
  80. package/skills/world-data-ingestion/ROWS.md +5 -1
  81. package/skills/world-data-ingestion/SKILL.md +4 -4
  82. package/skills/worlds-getting-started/SKILL.md +2 -2
  83. package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  84. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
@@ -0,0 +1,480 @@
1
+ """Stdlib only: the runner vendors this package. Twin of factorySpine.ts and factorySpineDocument.ts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from collections import Counter
8
+ from collections.abc import Callable, Iterable, Mapping
9
+ from types import MappingProxyType
10
+ from typing import Any
11
+
12
+ from .bundle_manifest import _utf16_length
13
+ from .factory_shapes import (
14
+ _KIND,
15
+ _TEXT,
16
+ FACTORY_LIMITS,
17
+ Check,
18
+ _array,
19
+ _javascript_number,
20
+ _json,
21
+ _literal,
22
+ _node_id,
23
+ _node_shape,
24
+ _number,
25
+ _one_of,
26
+ _record,
27
+ _slug,
28
+ _source,
29
+ _strict,
30
+ _string,
31
+ _tagged,
32
+ canonical,
33
+ )
34
+
35
+ SPINE_COLUMN_NAME = re.compile(r"^(?!__)[A-Za-z_][A-Za-z0-9_]{0,63}$")
36
+ SPINE_COLUMN_TYPES = ("string", "integer", "number", "boolean", "date", "instant", "json", "ref")
37
+ SPINE_KEY_TYPES = ("string", "integer")
38
+ SPINE_FORMULA_NAMES = 32
39
+ SPINE_ROW_ENTRY = re.compile(r"^spine/([a-z0-9][a-z0-9_-]{0,63})/rows/((?:[0-9a-f]{2})+)\.json$")
40
+ SPINE_DOCUMENT_ISSUE_CODES = (
41
+ "ref_row_missing",
42
+ "spine_key",
43
+ "spine_row_key",
44
+ "spine_row_duplicate",
45
+ "spine_row_too_large",
46
+ "spine_column_missing",
47
+ "spine_column_duplicate",
48
+ "spine_check_table",
49
+ "spine_partition_shared",
50
+ "spine_quota_entity",
51
+ )
52
+ SPINE_ISSUE_TEXT: Mapping[str, str] = MappingProxyType(
53
+ {
54
+ "ref_row_missing": (
55
+ "Node {node} refers to {subject}, which is not a table of this factory or not a row it holds."
56
+ ),
57
+ "spine_key": "Table {node} is keyed by {subject}, which is not one of its string or integer columns.",
58
+ "spine_row_key": (
59
+ f"Table {{node}} has a row whose key is not a string or integer of 1 to {FACTORY_LIMITS['keyLength']} "
60
+ "characters that does not start with __."
61
+ ),
62
+ "spine_row_duplicate": "Table {node} holds the key {subject} more than once.",
63
+ "spine_row_too_large": (
64
+ f"The row {{subject}} of table {{node}} is larger than {FACTORY_LIMITS['rowBytes']} bytes "
65
+ "as canonical JSON."
66
+ ),
67
+ "spine_column_missing": "Table {node} uses the column {subject}, which it does not declare.",
68
+ "spine_column_duplicate": "Table {node} declares the column {subject} more than once.",
69
+ "spine_check_table": "Node {node} has a steps check; steps checks belong on a table.",
70
+ "spine_partition_shared": "Table {node} partitions {subject}, which another table already partitions.",
71
+ "spine_quota_entity": "Table {node} sets a quota on {subject}, which its partition does not own.",
72
+ }
73
+ )
74
+
75
+ Finding = tuple[str, "str | None"]
76
+
77
+
78
+ def _column_name(value: Any) -> bool:
79
+ return isinstance(value, str) and SPINE_COLUMN_NAME.fullmatch(value) is not None
80
+
81
+
82
+ def _pointer(value: Any) -> bool:
83
+ return isinstance(value, str) and (value == "" or value.startswith("/"))
84
+
85
+
86
+ def _formula_text(value: Any) -> bool:
87
+ if not isinstance(value, str) or _utf16_length(value) < 1:
88
+ return False
89
+ return len(value.encode("utf-8", "surrogatepass")) <= FACTORY_LIMITS["formulaBytes"]
90
+
91
+
92
+ def _nonnegative(value: Any) -> bool:
93
+ return _number(value) and value >= 0
94
+
95
+
96
+ def _weekday(value: Any) -> bool:
97
+ return _number(value) and float(value).is_integer() and 0 <= value <= 6
98
+
99
+
100
+ _NODE_REF = _strict({"kind": _literal("node"), "id": _node_id})
101
+ _SOURCE = _tagged(
102
+ "kind",
103
+ {
104
+ "node": _NODE_REF,
105
+ "row": _strict({"kind": _literal("row"), "node": _node_id, "key": _TEXT}),
106
+ },
107
+ )
108
+ _FORM_KEYS = ("copy", "formula", "literal", "from")
109
+
110
+
111
+ def spine_value(value: Any) -> bool:
112
+ if not _json(value):
113
+ return False
114
+ shape = _derivation_shape(value)
115
+ return shape is None or shape(value)
116
+
117
+
118
+ def _names(value: Any) -> bool:
119
+ return _record(spine_value, _column_name)(value) and len(value) <= SPINE_FORMULA_NAMES
120
+
121
+
122
+ _FROM_PATH = _strict({"from": _SOURCE}, {"path": _pointer})
123
+ _FROM_COLUMN = _strict({"from": _SOURCE, "column": _column_name})
124
+ _COPY_PATH = _strict({"copy": _json, "from": _SOURCE}, {"path": _pointer})
125
+ _COPY_COLUMN = _strict({"copy": _json, "from": _SOURCE, "column": _column_name})
126
+ _FORMULA = _strict({"formula": _formula_text}, {"with": _names})
127
+ _LITERAL = _strict({"literal": _json})
128
+
129
+ _FORM_SHAPES: Mapping[str, Callable[[dict[str, Any]], Check]] = MappingProxyType(
130
+ {
131
+ "copy": lambda value: _COPY_COLUMN if "column" in value else _COPY_PATH,
132
+ "formula": lambda value: _FORMULA,
133
+ "literal": lambda value: _LITERAL,
134
+ "from": lambda value: _FROM_COLUMN if "column" in value else _FROM_PATH,
135
+ }
136
+ )
137
+
138
+
139
+ def _form_of(value: Any) -> str | None:
140
+ if not isinstance(value, dict):
141
+ return None
142
+ return next((key for key in _FORM_KEYS if key in value), None)
143
+
144
+
145
+ def _derivation_shape(value: Any) -> Check | None:
146
+ form = _form_of(value)
147
+ return None if form is None else _FORM_SHAPES[form](value)
148
+
149
+
150
+ _SOURCES_OF: Mapping[str, Callable[[dict[str, Any]], list[dict[str, Any]]]] = MappingProxyType(
151
+ {
152
+ "copy": lambda value: [value["from"]],
153
+ "from": lambda value: [value["from"]],
154
+ "literal": lambda value: [],
155
+ "formula": lambda value: [
156
+ source for item in value.get("with", {}).values() for source in derivation_sources(item)
157
+ ],
158
+ }
159
+ )
160
+
161
+
162
+ def derivation_sources(value: Any) -> list[dict[str, Any]]:
163
+ form = _form_of(value)
164
+ return [] if form is None else _SOURCES_OF[form](value)
165
+
166
+
167
+ _SEVERITY = {"severity": _one_of("block", "warn")}
168
+ _CHECK = _tagged(
169
+ "type",
170
+ {
171
+ "flow": _strict(
172
+ {
173
+ "type": _literal("flow"),
174
+ "start": spine_value,
175
+ "created": spine_value,
176
+ "closed": spine_value,
177
+ "end": spine_value,
178
+ },
179
+ _SEVERITY,
180
+ ),
181
+ "sum": _strict({"type": _literal("sum"), "parts": spine_value, "total": spine_value}, _SEVERITY),
182
+ "quota": _strict(
183
+ {"type": _literal("quota"), "total": spine_value, "quotas": spine_value},
184
+ {"weights": spine_value, **_SEVERITY},
185
+ ),
186
+ "fact": _strict(
187
+ {"type": _literal("fact"), "claims": spine_value, "value": spine_value},
188
+ {"tolerance": _nonnegative, **_SEVERITY},
189
+ ),
190
+ "steps": _strict(
191
+ {"type": _literal("steps"), "at": _column_name, "names": _column_name},
192
+ {
193
+ "business": _column_name,
194
+ "holidays": spine_value,
195
+ "weekend": _array(_weekday, max_items=7),
196
+ **_SEVERITY,
197
+ },
198
+ ),
199
+ },
200
+ )
201
+ _CHECK_VALUES: Mapping[str, Callable[[dict[str, Any]], list[Any]]] = MappingProxyType(
202
+ {
203
+ "flow": lambda check: [check["start"], check["created"], check["closed"], check["end"]],
204
+ "sum": lambda check: [check["parts"], check["total"]],
205
+ "quota": lambda check: [check["total"], check["quotas"], *([check["weights"]] if "weights" in check else [])],
206
+ "fact": lambda check: [check["claims"], check["value"]],
207
+ "steps": lambda check: [check["holidays"]] if "holidays" in check else [],
208
+ }
209
+ )
210
+
211
+ _COLUMN = _strict(
212
+ {"name": _column_name, "type": _one_of(*SPINE_COLUMN_TYPES)},
213
+ {"ref": _NODE_REF, "sentence": _string(0, FACTORY_LIMITS["sentenceLength"])},
214
+ )
215
+ _ROW = _record(spine_value, _column_name)
216
+ _ENTITY_AT = _strict({"world": _slug, "entity": _TEXT})
217
+ _PARTITION = _strict(
218
+ {"owner": _source, "entities": _array(_ENTITY_AT, min_items=1)},
219
+ {
220
+ "quotas": _array(
221
+ _strict({"column": _column_name, "world": _slug, "entity": _TEXT}, {"tolerance": _nonnegative}),
222
+ )
223
+ },
224
+ )
225
+ _RESERVE = _strict({"column": _column_name, "world": _slug, "entity": _TEXT, "field": _TEXT})
226
+ _CHECKS = _array(_CHECK, max_items=FACTORY_LIMITS["checksPerNode"])
227
+ _SPINE_OPTIONAL = {"group": _KIND}
228
+
229
+ SPINE_NODE = _tagged(
230
+ "kind",
231
+ {
232
+ "table": _node_shape(
233
+ "spine",
234
+ {
235
+ "kind": _literal("table"),
236
+ "columns": _array(_COLUMN, 1, FACTORY_LIMITS["columnsPerTable"]),
237
+ "key": _column_name,
238
+ "rows": _array(_ROW, max_items=FACTORY_LIMITS["tableRows"]),
239
+ },
240
+ {
241
+ **_SPINE_OPTIONAL,
242
+ "checks": _CHECKS,
243
+ "partition": _PARTITION,
244
+ "reserves": _array(_RESERVE),
245
+ "generate": _source,
246
+ },
247
+ ),
248
+ "fact": _node_shape(
249
+ "spine",
250
+ {"kind": _literal("fact"), "value": spine_value},
251
+ {**_SPINE_OPTIONAL, "checks": _CHECKS},
252
+ ),
253
+ "check": _node_shape(
254
+ "spine",
255
+ {"kind": _literal("check"), "checks": _array(_CHECK, 1, FACTORY_LIMITS["checksPerNode"])},
256
+ _SPINE_OPTIONAL,
257
+ ),
258
+ },
259
+ )
260
+
261
+
262
+ def spine_table(value: Any) -> dict[str, Any] | None:
263
+ if not isinstance(value, dict) or value.get("doc") != "spine" or value.get("kind") != "table":
264
+ return None
265
+ return value
266
+
267
+
268
+ def table_head(table: dict[str, Any]) -> dict[str, Any]:
269
+ return {key: item for key, item in table.items() if key != "rows"}
270
+
271
+
272
+ def key_of(row: dict[str, Any], key: Any) -> str | None:
273
+ cell = row.get(key) if isinstance(key, str) else None
274
+ if isinstance(cell, str):
275
+ return cell
276
+ if isinstance(cell, (int, float)) and _number(cell) and float(cell).is_integer():
277
+ return _javascript_number(cell)
278
+ return None
279
+
280
+
281
+ def row_key(row: dict[str, Any], key: str) -> str | None:
282
+ text = key_of(row, key)
283
+ if text is None or not 1 <= len(text) <= FACTORY_LIMITS["keyLength"] or text.startswith("__"):
284
+ return None
285
+ return text
286
+
287
+
288
+ def sorted_rows(rows: Iterable[dict[str, Any]], key: Any) -> list[dict[str, Any]]:
289
+ return sorted(rows, key=lambda row: (key_of(row, key) or "").encode("utf-8", "surrogatepass"))
290
+
291
+
292
+ def _table_values(node: dict[str, Any]) -> list[Any]:
293
+ return [cell for row in node["rows"] for cell in row.values()]
294
+
295
+
296
+ _NODE_VALUES: Mapping[str, Callable[[dict[str, Any]], list[Any]]] = MappingProxyType(
297
+ {
298
+ "table": _table_values,
299
+ "fact": lambda node: [node["value"]],
300
+ "check": lambda node: [],
301
+ }
302
+ )
303
+
304
+
305
+ def spine_sources(node: dict[str, Any]) -> list[dict[str, Any]]:
306
+ checks = [value for check in node.get("checks", []) for value in _CHECK_VALUES[check["type"]](check)]
307
+ values = [*_NODE_VALUES[node["kind"]](node), *checks]
308
+ return [source for value in values for source in derivation_sources(value)]
309
+
310
+
311
+ def spine_table_refs(node: dict[str, Any]) -> list[str]:
312
+ table = spine_table(node)
313
+ columns = table["columns"] if table else []
314
+ return [column["ref"]["id"] for column in columns if "ref" in column]
315
+
316
+
317
+ def _entity_name(at: Mapping[str, Any]) -> str:
318
+ return f"{at['world']}.{at['entity']}"
319
+
320
+
321
+ class SpineIndex:
322
+ def __init__(self, tables: Iterable[dict[str, Any]]) -> None:
323
+ self._tables = {table["id"]: table for table in tables}
324
+ self._keys: dict[str, set[str]] = {}
325
+ self._owners: dict[str, str] = {}
326
+ for table in sorted(self._tables.values(), key=lambda table: table["id"]):
327
+ for at in table.get("partition", {}).get("entities", []):
328
+ self._owners.setdefault(_entity_name(at), table["id"])
329
+
330
+ def holds(self, table_id: str, key: str | None) -> bool:
331
+ table = self._tables.get(table_id)
332
+ if table is None:
333
+ return False
334
+ return key is None or key in self._keys_of(table)
335
+
336
+ def owner_of(self, world_entity: str) -> str | None:
337
+ return self._owners.get(world_entity)
338
+
339
+ def _keys_of(self, table: dict[str, Any]) -> set[str]:
340
+ if table["id"] not in self._keys:
341
+ self._keys[table["id"]] = {text for row in table["rows"] if (text := key_of(row, table["key"])) is not None}
342
+ return self._keys[table["id"]]
343
+
344
+
345
+ def _steps(node: dict[str, Any]) -> list[dict[str, Any]]:
346
+ return [check for check in node.get("checks", []) if check["type"] == "steps"]
347
+
348
+
349
+ def _value_findings(node: dict[str, Any], index: SpineIndex) -> list[Finding]:
350
+ return [("spine_check_table", None)] if _steps(node) else []
351
+
352
+
353
+ def _column_duplicates(table: dict[str, Any]) -> list[Finding]:
354
+ counts = Counter(column["name"] for column in table["columns"])
355
+ return [("spine_column_duplicate", name) for name, count in sorted(counts.items()) if count > 1]
356
+
357
+
358
+ def _key_findings(table: dict[str, Any]) -> list[Finding]:
359
+ column = next((column for column in table["columns"] if column["name"] == table["key"]), None)
360
+ if column is not None and column["type"] in SPINE_KEY_TYPES:
361
+ return []
362
+ return [("spine_key", table["key"])]
363
+
364
+
365
+ def _row_bytes(row: dict[str, Any]) -> int:
366
+ return len(canonical(row).encode("utf-8"))
367
+
368
+
369
+ def _row_findings(table: dict[str, Any]) -> list[Finding]:
370
+ keys = [row_key(row, table["key"]) for row in table["rows"]]
371
+ valid = [key for key in keys if key is not None]
372
+ bad_keys: list[Finding] = [("spine_row_key", None)] if len(valid) < len(keys) else []
373
+ too_large: list[Finding] = [
374
+ ("spine_row_too_large", key)
375
+ for row, key in zip(table["rows"], keys)
376
+ if _row_bytes(row) > FACTORY_LIMITS["rowBytes"]
377
+ ]
378
+ duplicates: list[Finding] = [("spine_row_duplicate", key) for key, count in Counter(valid).items() if count > 1]
379
+ return [*bad_keys, *too_large, *duplicates]
380
+
381
+
382
+ def _named_columns(table: dict[str, Any]) -> list[str]:
383
+ partition = table.get("partition", {})
384
+ return [
385
+ *(name for row in table["rows"] for name in row),
386
+ *(name for check in _steps(table) for name in (check["at"], check["names"], check.get("business")) if name),
387
+ *(quota["column"] for quota in partition.get("quotas", [])),
388
+ *(reserve["column"] for reserve in table.get("reserves", [])),
389
+ ]
390
+
391
+
392
+ def _undeclared(table: dict[str, Any]) -> list[Finding]:
393
+ declared = {column["name"] for column in table["columns"]}
394
+ return [("spine_column_missing", name) for name in sorted(set(_named_columns(table)) - declared)]
395
+
396
+
397
+ def _quota_findings(table: dict[str, Any]) -> list[Finding]:
398
+ partition = table.get("partition", {})
399
+ owned = {_entity_name(at) for at in partition.get("entities", [])}
400
+ quotas = {_entity_name(quota) for quota in partition.get("quotas", [])}
401
+ return [("spine_quota_entity", name) for name in sorted(quotas - owned)]
402
+
403
+
404
+ def _shared_findings(table: dict[str, Any], index: SpineIndex) -> list[Finding]:
405
+ names = {_entity_name(at) for at in table.get("partition", {}).get("entities", [])}
406
+ return [("spine_partition_shared", name) for name in sorted(names) if index.owner_of(name) != table["id"]]
407
+
408
+
409
+ def _table_findings(table: dict[str, Any], index: SpineIndex) -> list[Finding]:
410
+ return [
411
+ *_column_duplicates(table),
412
+ *_key_findings(table),
413
+ *_row_findings(table),
414
+ *_undeclared(table),
415
+ *_quota_findings(table),
416
+ *_shared_findings(table, index),
417
+ ]
418
+
419
+
420
+ _FINDINGS: Mapping[str, Callable[[dict[str, Any], SpineIndex], list[Finding]]] = MappingProxyType(
421
+ {"table": _table_findings, "fact": _value_findings, "check": _value_findings}
422
+ )
423
+
424
+
425
+ def spine_findings(node: dict[str, Any], index: SpineIndex) -> list[Finding]:
426
+ return _FINDINGS[node["kind"]](node, index)
427
+
428
+
429
+ def _refuse_constant(name: str) -> Any:
430
+ raise ValueError(f"{name} is not JSON")
431
+
432
+
433
+ def _row_of(text: str) -> dict[str, Any] | None:
434
+ try:
435
+ value = json.loads(text, parse_constant=_refuse_constant)
436
+ except ValueError:
437
+ return None
438
+ return value if isinstance(value, dict) else None
439
+
440
+
441
+ class SpineRowEntries:
442
+ def __init__(self) -> None:
443
+ self._rows: dict[str, list[tuple[str, str]]] = {}
444
+
445
+ def take(self, path: str, data: bytes | str) -> bool:
446
+ match = SPINE_ROW_ENTRY.fullmatch(path)
447
+ if match is None:
448
+ return False
449
+ table_id, hex_key = match.groups()
450
+ text = data.decode("utf-8", "replace") if isinstance(data, bytes) else data
451
+ key = bytes.fromhex(hex_key).decode("utf-8", "replace")
452
+ self._rows.setdefault(table_id, []).append((key, text))
453
+ return True
454
+
455
+ def join(self, raw: list[Any]) -> tuple[list[Any], list[str]]:
456
+ heads = {head["id"]: head for value in raw if (head := _raw_head(value)) is not None}
457
+ orphans = [table_id for table_id in self._rows if table_id not in heads]
458
+ failures = [head["id"] for head in heads.values() if self._head_fails(head)]
459
+ nodes = [self._joined(value) for value in raw]
460
+ return nodes, [*orphans, *failures]
461
+
462
+ def _head_fails(self, head: dict[str, Any]) -> bool:
463
+ if "rows" in head:
464
+ return True
465
+ return any(
466
+ (row := _row_of(text)) is None or key_of(row, head.get("key")) != key
467
+ for key, text in self._rows.get(head["id"], [])
468
+ )
469
+
470
+ def _joined(self, value: Any) -> Any:
471
+ head = _raw_head(value)
472
+ if head is None:
473
+ return value
474
+ rows = [row for _, text in self._rows.get(head["id"], []) if (row := _row_of(text)) is not None]
475
+ return {**head, "rows": sorted_rows(rows, head.get("key"))}
476
+
477
+
478
+ def _raw_head(value: Any) -> dict[str, Any] | None:
479
+ table = spine_table(value)
480
+ return table if table is not None and isinstance(table.get("id"), str) else None
@@ -1,14 +1,16 @@
1
- """``Admission``: how many requests run on one version at once, and how many wait.
2
-
3
- Every session of a version is served by one process (its version worker, or the
4
- host itself for a world with no code of its own), so a burst on a version is a
5
- burst on that process. The host admits a version's requests a few at a time and
6
- one per session (a session's store answers one request at a time anyway); the
7
- rest wait their turn in the order they came, up to a bound. A request past the
8
- bound, or one whose turn did not come in time, is refused ``busy`` with when to
9
- come back, so a burst queues or is told to retry and the process never holds
10
- more requests than it has room for. A request held outside the queue (its session moving to
11
- another host) is counted against the same bounds (:meth:`Admission.waiting`).
1
+ """``Admission``: how many requests run on one worker key at once, and how many wait.
2
+
3
+ Every session of a :class:`WorkerKey` (a version's main, or its world states) is
4
+ served by one process (its version worker, or the host itself for a world with
5
+ no code of its own), so a burst on a key is a burst on that process. The host
6
+ admits a key's requests a few at a time and one per session (a session's store
7
+ answers one request at a time anyway); the rest wait their turn in the order
8
+ they came, up to a bound. A request past the bound, or one whose turn did not
9
+ come in time, is refused ``busy`` with when to come back, so a burst queues or
10
+ is told to retry and the process never holds more requests than it has room
11
+ for, and a version's states never take main's turns. A request held outside the
12
+ queue (its session moving to another host) is counted against the same bounds
13
+ (:meth:`Admission.waiting`).
12
14
  """
13
15
 
14
16
  from __future__ import annotations
@@ -25,7 +27,7 @@ from .validation import fail
25
27
  VERSION_SLOTS = 8
26
28
  SESSION_SLOTS = 1
27
29
  QUEUE_DEPTH = 256
28
- #: A session's own share of the waiting, so one session's burst cannot fill its version's queue.
30
+ #: A session's own share of the waiting, so one session's burst cannot fill its key's queue.
29
31
  SESSION_QUEUE_DEPTH = 32
30
32
  #: With a call's own budget it stays under the platform's 60 s (``conformance/world-sessions.json``).
31
33
  QUEUE_WAIT_S = 10.0
@@ -41,14 +43,14 @@ def _less(counter: Counter, key: object) -> None:
41
43
 
42
44
  @dataclass(eq=False)
43
45
  class _Turn:
44
- version: str
46
+ key: str
45
47
  session: str
46
48
  started: float | None = None
47
49
 
48
50
 
49
51
  class Admission:
50
- """Turns on each version, first come first served within the version's and
51
- each session's slots; another version never waits."""
52
+ """Turns on each key, first come first served within the key's and
53
+ each session's slots; another key never waits."""
52
54
 
53
55
  def __init__(
54
56
  self,
@@ -75,34 +77,34 @@ class Admission:
75
77
  self._outside_by: Counter[tuple[str, str]] = Counter()
76
78
 
77
79
  @contextmanager
78
- def turn(self, version: str, session: str) -> Iterator[None]:
79
- """Hold one of ``version``'s slots for ``session`` while the block runs; refused
80
+ def turn(self, key: str, session: str) -> Iterator[None]:
81
+ """Hold one of ``key``'s slots for ``session`` while the block runs; refused
80
82
  ``busy`` (a :class:`ContractError` carrying ``retryAfter``) when none comes."""
81
- turn = self._enter(_Turn(version, session))
83
+ turn = self._enter(_Turn(key, session))
82
84
  try:
83
85
  yield
84
86
  finally:
85
87
  self._leave(turn)
86
88
 
87
89
  @contextmanager
88
- def waiting(self, version: str, session: str) -> Iterator[None]:
89
- """Count a request waiting outside the queue against ``version``'s and ``session``'s waiting
90
+ def waiting(self, key: str, session: str) -> Iterator[None]:
91
+ """Count a request waiting outside the queue against ``key``'s and ``session``'s waiting
90
92
  bounds while the block runs; refused ``busy`` past them, as a queued request would be."""
91
93
  with self._lock:
92
- self._outside[version] += 1
93
- self._outside_by[(version, session)] += 1
94
- if self._overfull(version, session):
95
- self._unwait(version, session)
96
- self._refuse(version, "too many requests are waiting")
94
+ self._outside[key] += 1
95
+ self._outside_by[(key, session)] += 1
96
+ if self._overfull(key, session):
97
+ self._unwait(key, session)
98
+ self._refuse(key, "too many requests are waiting")
97
99
  try:
98
100
  yield
99
101
  finally:
100
102
  with self._lock:
101
- self._unwait(version, session)
103
+ self._unwait(key, session)
102
104
 
103
- def _unwait(self, version: str, session: str) -> None:
104
- _less(self._outside, version)
105
- _less(self._outside_by, (version, session))
105
+ def _unwait(self, key: str, session: str) -> None:
106
+ _less(self._outside, key)
107
+ _less(self._outside_by, (key, session))
106
108
 
107
109
  def held_ages(self) -> dict[str, float]:
108
110
  """Each session holding a turn now, with how long its oldest has run, in seconds."""
@@ -116,57 +118,57 @@ class Admission:
116
118
 
117
119
  def _enter(self, turn: _Turn) -> _Turn:
118
120
  with self._lock:
119
- queue = self._waiting.setdefault(turn.version, deque())
121
+ queue = self._waiting.setdefault(turn.key, deque())
120
122
  queue.append(turn)
121
- self._admit(turn.version)
122
- if turn.started is None and self._overfull(turn.version, turn.session):
123
+ self._admit(turn.key)
124
+ if turn.started is None and self._overfull(turn.key, turn.session):
123
125
  queue.remove(turn)
124
- self._refuse(turn.version, f"{len(queue)} requests are waiting")
126
+ self._refuse(turn.key, f"{len(queue)} requests are waiting")
125
127
  deadline = time.monotonic() + self.wait_s
126
128
  while turn.started is None:
127
129
  left = deadline - time.monotonic()
128
130
  if left <= 0:
129
131
  queue.remove(turn)
130
- self._refuse(turn.version, f"no turn came within {self.wait_s:g} s")
132
+ self._refuse(turn.key, f"no turn came within {self.wait_s:g} s")
131
133
  self._changed.wait(left)
132
134
  return turn
133
135
 
134
- def _overfull(self, version: str, session: str) -> bool:
135
- queue = self._waiting.get(version, deque())
136
- mine = sum(waiting.session == session for waiting in queue) + self._outside_by[(version, session)]
137
- return len(queue) + self._outside[version] > self.queue or mine > self.session_queue
136
+ def _overfull(self, key: str, session: str) -> bool:
137
+ queue = self._waiting.get(key, deque())
138
+ mine = sum(waiting.session == session for waiting in queue) + self._outside_by[(key, session)]
139
+ return len(queue) + self._outside[key] > self.queue or mine > self.session_queue
138
140
 
139
141
  def _leave(self, turn: _Turn) -> None:
140
142
  with self._lock:
141
- key = (turn.version, turn.session)
142
- self._holding[key].remove(turn)
143
- if not self._holding[key]:
144
- del self._holding[key]
145
- self._running[turn.version] -= 1
146
- if self._running[turn.version] <= 0:
147
- del self._running[turn.version]
148
- self._admit(turn.version)
149
- self._forget(turn.version)
150
-
151
- def _admit(self, version: str) -> None:
143
+ held = (turn.key, turn.session)
144
+ self._holding[held].remove(turn)
145
+ if not self._holding[held]:
146
+ del self._holding[held]
147
+ self._running[turn.key] -= 1
148
+ if self._running[turn.key] <= 0:
149
+ del self._running[turn.key]
150
+ self._admit(turn.key)
151
+ self._forget(turn.key)
152
+
153
+ def _admit(self, key: str) -> None:
152
154
  """Under the lock: give free slots to the earliest waiters whose session has room."""
153
- queue = self._waiting.get(version, deque())
155
+ queue = self._waiting.get(key, deque())
154
156
  for waiting in list(queue):
155
- if self._running[version] >= self.slots:
157
+ if self._running[key] >= self.slots:
156
158
  break
157
- if len(self._holding.get((version, waiting.session), ())) < self.per_session:
159
+ if len(self._holding.get((key, waiting.session), ())) < self.per_session:
158
160
  queue.remove(waiting)
159
161
  waiting.started = time.monotonic()
160
- self._running[version] += 1
161
- self._holding.setdefault((version, waiting.session), []).append(waiting)
162
+ self._running[key] += 1
163
+ self._holding.setdefault((key, waiting.session), []).append(waiting)
162
164
  self._changed.notify_all()
163
165
 
164
- def _forget(self, version: str) -> None:
165
- if not self._waiting.get(version) and not self._running[version]:
166
- self._waiting.pop(version, None)
166
+ def _forget(self, key: str) -> None:
167
+ if not self._waiting.get(key) and not self._running[key]:
168
+ self._waiting.pop(key, None)
167
169
 
168
- def _refuse(self, version: str, why: str) -> None:
169
- self._forget(version)
170
+ def _refuse(self, key: str, why: str) -> None:
171
+ self._forget(key)
170
172
  fail(
171
173
  BUSY,
172
174
  f"this world's version is busy ({why}); retry after {self.retry_after_s} s",