@withgateway/sdk 0.3.0 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +6 -1
  2. package/dist/cli.js +3971 -2338
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3950 -2318
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +214 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +206 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
  13. package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1456 -564
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1433 -557
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +211 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +203 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +96 -21
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +96 -21
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +300 -123
  35. package/dist/worlds/index.d.ts +300 -123
  36. package/dist/worlds/index.js +1385 -508
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1377 -501
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -515
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  57. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  58. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  59. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  60. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  61. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  62. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  63. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  64. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  65. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  66. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  67. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  68. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  69. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  70. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  71. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  72. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  73. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  74. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  75. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  76. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  77. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  78. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  79. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  80. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  81. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  82. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  83. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  84. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  86. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  87. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  88. package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
  89. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  91. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  92. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  93. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  94. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  95. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  96. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  97. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  99. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  103. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  104. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  105. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  106. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  107. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  108. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  109. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  110. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  111. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  112. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  113. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  114. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  115. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  116. package/runtime/gatewaysdk/environment/score.py +57 -0
  117. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  118. package/runtime/gatewaysdk/environment/world.py +42 -3
  119. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  120. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  121. package/skills/connector-schema-authoring/SKILL.md +22 -14
  122. package/skills/connector-schema-authoring/connector-worlds.md +26 -10
  123. package/skills/connector-schema-authoring/world-schema.md +111 -51
  124. package/skills/world-data-ingestion/ROWS.md +29 -3
  125. package/skills/world-data-ingestion/SKILL.md +16 -16
  126. package/skills/worlds-getting-started/SKILL.md +18 -18
  127. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  128. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  129. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  130. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,33 @@
1
+ """What one caller's SQL may cost a world database."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import math
7
+ from dataclasses import dataclass
8
+ from typing import Any, Iterable
9
+
10
+ BINARY = (bytes, bytearray, memoryview)
11
+ #: ``{"base64": ""}`` around an encoded blob, and the separator after each value.
12
+ BLOB_WRAPPER = len(json.dumps({"base64": ""}))
13
+ SEPARATOR = 2
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class QueryBudget:
18
+ """Wall time for every statement in all, the largest single value the engine may make, and the
19
+ bytes the whole answer takes as JSON (what is sent, not what is counted in characters)."""
20
+
21
+ seconds: float
22
+ value_bytes: int
23
+ answer_bytes: int
24
+
25
+ @staticmethod
26
+ def size_of(row: Iterable[Any]) -> int:
27
+ return sum(QueryBudget._json_bytes(value) + SEPARATOR for value in row)
28
+
29
+ @staticmethod
30
+ def _json_bytes(value: Any) -> int:
31
+ if isinstance(value, BINARY):
32
+ return 4 * math.ceil(len(value) / 3) + BLOB_WRAPPER
33
+ return len(json.dumps(value))
@@ -0,0 +1,101 @@
1
+ """The connections world-authored code holds: a tool's read-only ``conn`` and an
2
+ ingest session's raw SQL. Neither can commit, roll back or close; the database
3
+ owns every transaction."""
4
+
5
+ from __future__ import annotations
6
+
7
+ from typing import TYPE_CHECKING, Any, Iterable
8
+
9
+ from ..validation import ContractError, fail
10
+
11
+ if TYPE_CHECKING:
12
+ from .base import WorldDatabase
13
+
14
+
15
+ class ReadCursor:
16
+ def __init__(self, rows: Iterable[Any]):
17
+ self._rows = iter(rows)
18
+ self.rowcount = -1
19
+
20
+ def fetchone(self):
21
+ return next(self._rows, None)
22
+
23
+ def fetchall(self):
24
+ return list(self._rows)
25
+
26
+ def __iter__(self):
27
+ return self._rows
28
+
29
+
30
+ class ReadConnection:
31
+ """Compatibility conn= surface: execute SELECT only; no raw writes/commit."""
32
+
33
+ def __init__(self, database: "WorldDatabase"):
34
+ self._database = database
35
+
36
+ def execute(self, sql: str, params: Any = ()) -> ReadCursor:
37
+ return ReadCursor(self._database.query(sql, tuple(params)))
38
+
39
+ def cursor(self):
40
+ fail(
41
+ "raw_write",
42
+ "Use conn.execute for reads and gateway_env.store for mutations",
43
+ )
44
+
45
+ def executescript(self, *args):
46
+ fail("raw_write", "Raw SQL scripts are unavailable in schema-backed tools")
47
+
48
+ def commit(self):
49
+ fail("raw_write", "The world owns the tool transaction")
50
+
51
+ rollback = commit
52
+ close = commit
53
+
54
+
55
+ class IngestConnection:
56
+ """Raw SQL inside one schema-gated ingest session.
57
+
58
+ Domain tables and TEMP objects are writable; the database owns the
59
+ transaction and commits only when every persisted row satisfies the contract
60
+ (known gap: a TEMP object named like a main table, see :class:`IngestSession`).
61
+ """
62
+
63
+ def __init__(self, database: "WorldDatabase"):
64
+ self._database = database
65
+ self._open = True
66
+ self.result: dict | None = None
67
+
68
+ def execute(self, sql: str, params: Any = ()) -> Any:
69
+ return self._run(lambda database: database._execute(sql, params))
70
+
71
+ def executemany(self, sql: str, rows: Any) -> Any:
72
+ return self._run(lambda database: database._executemany(sql, rows))
73
+
74
+ def script(self, sql: str) -> None:
75
+ """Run a multi-statement script one statement at a time; an engine's own
76
+ script call may commit the session's transaction first."""
77
+ for statement in self._database.split_statements(sql):
78
+ self.execute(statement)
79
+
80
+ def _run(self, operation):
81
+ if not self._open:
82
+ fail("transaction", "The ingest session has ended")
83
+ database = self._database
84
+ database._key_error = None
85
+ try:
86
+ return operation(database)
87
+ except database.native_error as error:
88
+ context = {database.engine: str(error)}
89
+ if database._key_error is not None:
90
+ context["key"] = database._key_error
91
+ raise ContractError(
92
+ "ingest_sql",
93
+ "Statement rejected; only domain tables and TEMP objects are writable",
94
+ **context,
95
+ ) from error
96
+
97
+ def commit(self):
98
+ fail("raw_write", "The store commits the ingest session when it ends")
99
+
100
+ rollback = commit
101
+ close = commit
@@ -0,0 +1,8 @@
1
+ """The engines a world database can run on; the base's factory picks among them."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .sqlite import SQLiteWorldDatabase
6
+
7
+ ENGINES = (SQLiteWorldDatabase,)
8
+ DEFAULT_ENGINE = SQLiteWorldDatabase
@@ -0,0 +1,208 @@
1
+ """Who may do what to a world database, decided without naming an engine.
2
+
3
+ An engine reports every access a statement makes as an :class:`Access`; the
4
+ :class:`Guard` answers allow or deny from the door that is open. The doors are
5
+ the store's: a read-only query, a trusted verifier read, an ingest session, the
6
+ store's own bookkeeping, and the host's connection outside any of those.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from abc import ABC, abstractmethod
12
+ from contextlib import contextmanager
13
+ from dataclasses import dataclass, replace
14
+ from enum import Enum
15
+ from typing import Iterator, Mapping
16
+
17
+ RUNTIME_TABLES = frozenset(
18
+ {
19
+ "benchmark_runtime_context",
20
+ "benchmark_runtime_parameters",
21
+ "benchmark_conversation_events",
22
+ }
23
+ )
24
+ RUNTIME_PARAMETERS = "benchmark_runtime_parameters"
25
+ PRIVATE_TABLE_ORDER = (
26
+ "_gateway_lineage",
27
+ "_gateway_batches",
28
+ "_gateway_contract",
29
+ "benchmark_runtime_context",
30
+ "benchmark_conversation_events",
31
+ )
32
+ PRIVATE_TABLES = frozenset(PRIVATE_TABLE_ORDER)
33
+ STAGING = "<staging>"
34
+
35
+
36
+ class AccessKind(Enum):
37
+ SELECT = "select"
38
+ READ = "read"
39
+ FUNCTION = "function"
40
+ RECURSIVE = "recursive"
41
+ WRITE = "write"
42
+ STAGE = "stage"
43
+ SAVEPOINT = "savepoint"
44
+ OTHER = "other"
45
+
46
+
47
+ READ_KINDS = frozenset({AccessKind.SELECT, AccessKind.READ, AccessKind.FUNCTION, AccessKind.RECURSIVE})
48
+
49
+
50
+ @dataclass(frozen=True)
51
+ class Access:
52
+ """One thing a statement does: read a table, call a function, write rows,
53
+ stage a TEMP object, open a savepoint, or anything else (DDL, PRAGMA,
54
+ ATTACH, transaction control), which every door but the trusted one refuses.
55
+ ``target`` is the table or function name, ``schema`` the database it lives in
56
+ (``None`` when the engine does not say, :data:`STAGING` for the engine's
57
+ session-private staging space). ``unsafe`` marks a function the engine knows
58
+ reaches outside the database (the file system, a native extension)."""
59
+
60
+ kind: AccessKind
61
+ target: str | None = None
62
+ schema: str | None = None
63
+ unsafe: bool = False
64
+
65
+ @property
66
+ def unsafe_function(self) -> bool:
67
+ return self.kind is AccessKind.FUNCTION and self.unsafe
68
+
69
+
70
+ class Door(ABC):
71
+ @abstractmethod
72
+ def allows(self, access: Access, entities: frozenset[str]) -> bool: ...
73
+
74
+
75
+ class ReadOnlyDoor(Door):
76
+ """``query``: reads of domain tables and the runtime parameters; with
77
+ ``private`` the runtime context and events too (a verifier's trusted read)."""
78
+
79
+ def __init__(self, private: bool) -> None:
80
+ self.private = private
81
+
82
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
83
+ if access.kind not in READ_KINDS:
84
+ return False
85
+ if access.kind is AccessKind.READ and not self._readable(access.target, entities):
86
+ return False
87
+ return not access.unsafe_function
88
+
89
+ def _readable(self, table: str | None, entities: frozenset[str]) -> bool:
90
+ if table in entities or table == RUNTIME_PARAMETERS:
91
+ return True
92
+ return self.private and table in RUNTIME_TABLES
93
+
94
+
95
+ class TrustedDoor(Door):
96
+ """The store's own statements: bookkeeping, audits, transaction control."""
97
+
98
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
99
+ return True
100
+
101
+
102
+ class IngestDoor(Door):
103
+ """Domain tables and TEMP staging are writable; transaction control, PRAGMA,
104
+ ATTACH, persistent DDL and the private bookkeeping tables stay with the store,
105
+ so a session cannot commit early, disable enforcement, or forge provenance. Known gap: a STAGE
106
+ access is allowed whatever its name, so a TEMP object named like a main table shadows it (open
107
+ bug 2026-09-28-world-store-ingest-temp-shadow-bypasses-audit, to be fixed by #474)."""
108
+
109
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
110
+ if access.kind in READ_KINDS:
111
+ return (
112
+ not (access.kind is AccessKind.READ and access.target in PRIVATE_TABLES) and not access.unsafe_function
113
+ )
114
+ if access.kind is AccessKind.WRITE:
115
+ return access.target in entities or access.schema == STAGING
116
+ return access.kind in (AccessKind.STAGE, AccessKind.SAVEPOINT)
117
+
118
+
119
+ class HostDoor(Door):
120
+ """The connection outside any store method: it reads its own file, writes only
121
+ the runtime tables, and calls no unsafe function. The store's own statements all
122
+ run behind the trusted door, so nothing of the store's passes through this one."""
123
+
124
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
125
+ if access.kind in READ_KINDS:
126
+ return not access.unsafe_function
127
+ return access.kind is AccessKind.WRITE and access.target in RUNTIME_TABLES
128
+
129
+
130
+ TRUSTED = TrustedDoor()
131
+ INGEST = IngestDoor()
132
+ HOST = HostDoor()
133
+
134
+
135
+ class Guard:
136
+ """The open door of one database connection and the attached stores it reads.
137
+
138
+ An access to an attached store is decided first, whatever door is open: only
139
+ a read of a table that store exposes passes, so nothing is ever written
140
+ through an alias. Then the read-only door wins over the trusted one, which
141
+ wins over an ingest session, and the host door decides what is left."""
142
+
143
+ def __init__(self, entities: frozenset[str], unsafe_functions: frozenset[str]) -> None:
144
+ self._entities = entities
145
+ self._unsafe = frozenset(name.lower() for name in unsafe_functions)
146
+ self._attached: dict[str, frozenset[str]] = {}
147
+ self._reading: ReadOnlyDoor | None = None
148
+ self._trusted = 0
149
+ self._ingesting = False
150
+
151
+ @property
152
+ def attached(self) -> Mapping[str, frozenset[str]]:
153
+ return dict(self._attached)
154
+
155
+ @property
156
+ def ingesting(self) -> bool:
157
+ return self._ingesting
158
+
159
+ def attach(self, name: str, tables: frozenset[str]) -> None:
160
+ self._attached[name] = tables
161
+
162
+ def is_attached(self, name: str) -> bool:
163
+ return name in self._attached
164
+
165
+ def allows(self, access: Access) -> bool:
166
+ access = self._marked(access)
167
+ if access.schema in self._attached:
168
+ return access.kind is AccessKind.READ and access.target in self._attached[access.schema]
169
+ return self.door().allows(access, self._entities)
170
+
171
+ def _marked(self, access: Access) -> Access:
172
+ """``access`` with ``unsafe`` set when it calls a function the engine declared unsafe, by name in any case."""
173
+ if access.kind is not AccessKind.FUNCTION:
174
+ return access
175
+ return replace(access, unsafe=access.unsafe or (access.target or "").lower() in self._unsafe)
176
+
177
+ def door(self) -> Door:
178
+ if self._reading is not None:
179
+ return self._reading
180
+ if self._trusted:
181
+ return TRUSTED
182
+ if self._ingesting:
183
+ return INGEST
184
+ return HOST
185
+
186
+ @contextmanager
187
+ def reading(self, private: bool) -> Iterator[None]:
188
+ self._reading = ReadOnlyDoor(private)
189
+ try:
190
+ yield
191
+ finally:
192
+ self._reading = None
193
+
194
+ @contextmanager
195
+ def trusted(self) -> Iterator[None]:
196
+ self._trusted += 1
197
+ try:
198
+ yield
199
+ finally:
200
+ self._trusted -= 1
201
+
202
+ @contextmanager
203
+ def ingest(self) -> Iterator[None]:
204
+ self._ingesting = True
205
+ try:
206
+ yield
207
+ finally:
208
+ self._ingesting = False
@@ -0,0 +1,154 @@
1
+ """One schema-gated raw SQL session: write freely, commit only if every row conforms
2
+ (known gap: a TEMP object named like a main table, see :class:`IngestSession`)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import uuid
7
+ from typing import TYPE_CHECKING, Iterator
8
+
9
+ from ..compiler import quote
10
+ from ..validation import canonical, digest, encoded, fail, members, nonempty
11
+ from .connections import IngestConnection
12
+ from .provenance import validate_source
13
+
14
+ if TYPE_CHECKING:
15
+ from .base import WorldDatabase
16
+
17
+
18
+ class IngestSession:
19
+ """The session :meth:`WorldDatabase.ingest` opens. The database's guard is on its
20
+ ingest door while the caller holds the connection; on exit every persisted row is
21
+ audited, and the changes are recorded as one batch with a lineage row per touched
22
+ key, or the whole session rolls back. Known gap: a TEMP table named like a domain or
23
+ bookkeeping table shadows it for the audit and the provenance writes (bug log
24
+ ``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, pinned xfail in the guard matrix,
25
+ to be fixed by #474)."""
26
+
27
+ def __init__(self, database: "WorldDatabase", *, source: dict, mapping: dict, batch_id: str | None, limit: int):
28
+ validate_source(source, "/ingest/source")
29
+ members(mapping, {"id", "version"}, {"id", "version"}, "/ingest/mapping")
30
+ for key, value in mapping.items():
31
+ nonempty(value, f"/ingest/mapping/{key}")
32
+ self._database = database
33
+ self._source = source
34
+ self._mapping = mapping
35
+ self._batch_id = batch_id
36
+ self._limit = limit
37
+
38
+ def run(self) -> Iterator[IngestConnection]:
39
+ database = self._database
40
+ with database._lock:
41
+ if database._depth or database._guard.ingesting:
42
+ fail(
43
+ "transaction",
44
+ "Ingest needs no active tool, batch or ingest transaction",
45
+ )
46
+ batch = self._batch_id if self._batch_id is not None else uuid.uuid4().hex
47
+ nonempty(batch, "/ingest/id")
48
+ with database._trusted():
49
+ applied = database._execute("SELECT 1 FROM _gateway_batches WHERE id = ?", (batch,)).fetchone()
50
+ if applied:
51
+ fail(
52
+ "idempotency_conflict",
53
+ "Ingest ID was already applied; raw sessions are not replayable",
54
+ "/ingest/id",
55
+ batch=batch,
56
+ )
57
+ before = self._state()
58
+ connection = IngestConnection(database)
59
+ with database._trusted():
60
+ database._begin_write()
61
+ database._depth = 1
62
+ try:
63
+ with database._guard.ingest():
64
+ changes = yield from self._session(connection, before, batch)
65
+ finally:
66
+ database._depth = 0
67
+ connection._open = False
68
+ with database._trusted():
69
+ database._drop_staging()
70
+ connection.result = self._result(batch, changes)
71
+
72
+ def _session(self, connection: IngestConnection, before: dict, batch: str):
73
+ database = self._database
74
+ try:
75
+ yield connection
76
+ violations = database._audit.violations(self._limit)
77
+ if violations:
78
+ fail(
79
+ "ingest_rejected",
80
+ "Persisted rows violate the contract; nothing was committed",
81
+ batch=batch,
82
+ violations=[error.detail for error in violations],
83
+ truncated=len(violations) >= self._limit,
84
+ )
85
+ changes = self._changes(before, self._state())
86
+ with database._trusted():
87
+ self._record(batch, changes)
88
+ database._commit()
89
+ return changes
90
+ except BaseException:
91
+ with database._trusted():
92
+ database._rollback()
93
+ raise
94
+
95
+ def _record(self, batch: str, changes: list) -> None:
96
+ if not changes:
97
+ return
98
+ database = self._database
99
+ metadata = {
100
+ "id": batch,
101
+ "kind": "ingest",
102
+ "target": database.contract.target,
103
+ "mapping": self._mapping,
104
+ "coverage": "partial",
105
+ }
106
+ database._execute(
107
+ "INSERT INTO _gateway_batches VALUES (?, ?, ?)",
108
+ (batch, digest(encoded(changes)), canonical(metadata)),
109
+ )
110
+ database._executemany(
111
+ "INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
112
+ [(batch, index, entity, key, canonical(self._source)) for index, (entity, key, _) in enumerate(changes)],
113
+ )
114
+
115
+ def _state(self) -> dict[str, dict[str, str]]:
116
+ """Persisted ``_key -> payload`` per entity, for change attribution."""
117
+ database = self._database
118
+ with database._trusted():
119
+ return {
120
+ name: dict(database._execute(f"SELECT _key, payload FROM {quote(name)}"))
121
+ for name in database.contract.entities
122
+ }
123
+
124
+ @staticmethod
125
+ def _changes(before: dict, after: dict) -> list[list[str]]:
126
+ changes = []
127
+ for name in sorted(before):
128
+ old, new = before[name], after[name]
129
+ for key in sorted(set(old) | set(new)):
130
+ if old.get(key) == new.get(key):
131
+ continue
132
+ changes.append([name, key, IngestSession._change(key, old, new)])
133
+ return changes
134
+
135
+ @staticmethod
136
+ def _change(key: str, old: dict, new: dict) -> str:
137
+ if key not in old:
138
+ return "insert"
139
+ if key not in new:
140
+ return "delete"
141
+ return "update"
142
+
143
+ @staticmethod
144
+ def _result(batch: str, changes: list) -> dict:
145
+ counts = {op: 0 for op in ("insert", "update", "delete")}
146
+ for _, _, op in changes:
147
+ counts[op] += 1
148
+ return {
149
+ "status": "applied" if changes else "unchanged",
150
+ "batch": batch,
151
+ "inserted": counts["insert"],
152
+ "updated": counts["update"],
153
+ "deleted": counts["delete"],
154
+ }
@@ -0,0 +1,58 @@
1
+ """The one rule by which a world's rows match a filter: the store's ``list_where``,
2
+ an api world's routes and ``worlds data query`` all answer with it."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from dataclasses import dataclass, field
7
+ from typing import Any, Callable, Iterable
8
+
9
+ COMPARISONS: dict[str, Callable[[str, str], bool]] = {
10
+ "equals": lambda have, wanted: have == wanted,
11
+ "contains": lambda have, wanted: wanted in have,
12
+ "prefix": lambda have, wanted: have.startswith(wanted),
13
+ }
14
+
15
+
16
+ def dig(row: Any, path: str) -> Any:
17
+ for part in path.split("."):
18
+ if not isinstance(row, dict) or part not in row:
19
+ return None
20
+ row = row[part]
21
+ return row
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class RowMatch:
26
+ """Every filter whose argument was supplied must match; a parameter left out does not narrow.
27
+
28
+ ``filters`` maps an argument to the dotted field path it reads, ``values`` the
29
+ argument to the value (or values) sent. ``modes`` picks how: ``equals``
30
+ (default), ``contains`` (case-insensitive substring, the vendor's ``icontains``)
31
+ or ``prefix``. The stored value is compared as text, case-insensitively; a list
32
+ field matches when any element does, and a parameter sent more than once
33
+ (``status[]=a&status[]=b``) matches when any of its values does. A missing field
34
+ never matches."""
35
+
36
+ filters: dict[str, str]
37
+ values: dict[str, Any]
38
+ modes: dict[str, str] = field(default_factory=dict)
39
+
40
+ @classmethod
41
+ def on_fields(cls, values: dict[str, Any]) -> "RowMatch":
42
+ return cls({name: name for name in values}, values)
43
+
44
+ def matches(self, row: dict) -> bool:
45
+ return all(self._field_matches(row, arg, path) for arg, path in self.filters.items() if arg in self.values)
46
+
47
+ def select(self, rows: Iterable[dict]) -> list[dict]:
48
+ return [row for row in rows if self.matches(row)]
49
+
50
+ def _field_matches(self, row: dict, arg: str, path: str) -> bool:
51
+ value = dig(row, path)
52
+ if value is None:
53
+ return False
54
+ compare = COMPARISONS.get(self.modes.get(arg, "equals"), COMPARISONS["equals"])
55
+ sent = self.values[arg]
56
+ wanted = [w.lower() for w in (sent if isinstance(sent, list) else [sent])]
57
+ candidates = value if isinstance(value, list) else [value]
58
+ return any(compare(str(candidate).lower(), w) for candidate in candidates for w in wanted)
@@ -0,0 +1,30 @@
1
+ """The provenance a mutation carries: the record it came from and the evidence behind it."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from ..validation import fail, members, nonempty
8
+
9
+
10
+ def validate_source(source: Any, location: str) -> None:
11
+ members(source, {"record", "evidence"}, {"record"}, location)
12
+ nonempty(source["record"], f"{location}/record")
13
+ evidence = source.get("evidence", [])
14
+ if not isinstance(evidence, list):
15
+ fail(
16
+ "provenance",
17
+ "evidence must be an array of stable reference/digest pairs",
18
+ location,
19
+ )
20
+ for index, entry in enumerate(evidence):
21
+ members(
22
+ entry,
23
+ {"reference", "digest"},
24
+ {"reference", "digest"},
25
+ f"{location}/evidence/{index}",
26
+ )
27
+ nonempty(entry["reference"], location)
28
+ value = entry["digest"]
29
+ if not isinstance(value, str) or len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
30
+ fail("provenance", "Evidence digest must be lowercase SHA-256 hex", location)
@@ -0,0 +1,53 @@
1
+ """The row a world database answers a query with, whatever engine read it."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, Iterable, Sequence
6
+
7
+
8
+ class Row(tuple):
9
+ """One result row: a tuple of its values that also answers by column name, so
10
+ ``row[0]``, ``row["name"]``, ``dict(row)`` and ``tuple(row) == (...)`` hold on
11
+ every engine. A name is matched exactly, then case-insensitively; with two
12
+ columns of one name, the first answers."""
13
+
14
+ def __new__(cls, columns: Sequence[str], values: Iterable[Any]) -> "Row":
15
+ row = super().__new__(cls, values)
16
+ row._columns = tuple(columns)
17
+ return row
18
+
19
+ @classmethod
20
+ def all(cls, description: Sequence[Sequence[Any]] | None, values: Iterable[Iterable[Any]]) -> "Rows":
21
+ """Every row of a result, named by a DB-API ``cursor.description``."""
22
+ columns = tuple(column[0] for column in description or ())
23
+ return Rows(columns, (cls(columns, row) for row in values))
24
+
25
+ def keys(self) -> list[str]:
26
+ return list(self._columns)
27
+
28
+ def __getitem__(self, key: Any) -> Any:
29
+ if not isinstance(key, str):
30
+ return super().__getitem__(key)
31
+ return super().__getitem__(self._index(key))
32
+
33
+ def _index(self, name: str) -> int:
34
+ if name in self._columns:
35
+ return self._columns.index(name)
36
+ folded = [column.lower() for column in self._columns]
37
+ if name.lower() in folded:
38
+ return folded.index(name.lower())
39
+ raise IndexError(f"No column named {name}")
40
+
41
+ def __getnewargs__(self) -> tuple:
42
+ return (self._columns, tuple(self))
43
+
44
+ def __repr__(self) -> str:
45
+ return f"Row({dict(zip(self._columns, self))!r})"
46
+
47
+
48
+ class Rows(list):
49
+ """A query's rows, which also name the columns when there are none."""
50
+
51
+ def __init__(self, columns: Sequence[str], rows: Iterable[Row]) -> None:
52
+ super().__init__(rows)
53
+ self.columns = tuple(columns)
@@ -0,0 +1,39 @@
1
+ """The statements that write the runtime tables a verifier reads: the active task,
2
+ the run's condition bindings and the turn events. A world database and a
3
+ mock-tools world's raw file run the same statements."""
4
+
5
+ from __future__ import annotations
6
+
7
+ Statement = tuple[str, tuple]
8
+
9
+ TASK_CONTEXT = "benchmark_runtime_context"
10
+ RUN_PARAMETERS = "benchmark_runtime_parameters"
11
+ CONVERSATION_EVENTS = "benchmark_conversation_events"
12
+
13
+
14
+ class RuntimeRows:
15
+ @staticmethod
16
+ def task_context(scenario_id: str) -> list[Statement]:
17
+ return [
18
+ (f"DELETE FROM {TASK_CONTEXT}", ()),
19
+ (f"INSERT INTO {TASK_CONTEXT} (context_key, scenario_id) VALUES ('active_task', ?)", (scenario_id,)),
20
+ ]
21
+
22
+ @staticmethod
23
+ def run_parameters(parameters: dict[str, str]) -> list[Statement]:
24
+ return [(f"DELETE FROM {RUN_PARAMETERS}", ())] + [
25
+ (f"INSERT INTO {RUN_PARAMETERS} (key, value) VALUES (?, ?)", (key, parameters[key]))
26
+ for key in sorted(parameters)
27
+ ]
28
+
29
+ @staticmethod
30
+ def conversation_event(
31
+ scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
32
+ ) -> list[Statement]:
33
+ return [
34
+ (
35
+ f"INSERT INTO {CONVERSATION_EVENTS} "
36
+ "(scenario_id, user_turn_index, event_type, actor, name, payload_json) VALUES (?, ?, ?, ?, ?, ?)",
37
+ (scenario_id, user_turn_index, event_type, actor, name, payload_json),
38
+ )
39
+ ]