@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,33 @@
1
+ """What one caller's SQL may cost a world database."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import math
7
+ from dataclasses import dataclass
8
+ from typing import Any, Iterable
9
+
10
+ BINARY = (bytes, bytearray, memoryview)
11
+ #: ``{"base64": ""}`` around an encoded blob, and the separator after each value.
12
+ BLOB_WRAPPER = len(json.dumps({"base64": ""}))
13
+ SEPARATOR = 2
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class QueryBudget:
18
+ """Wall time for every statement in all, the largest single value the engine may make, and the
19
+ bytes the whole answer takes as JSON (what is sent, not what is counted in characters)."""
20
+
21
+ seconds: float
22
+ value_bytes: int
23
+ answer_bytes: int
24
+
25
+ @staticmethod
26
+ def size_of(row: Iterable[Any]) -> int:
27
+ return sum(QueryBudget._json_bytes(value) + SEPARATOR for value in row)
28
+
29
+ @staticmethod
30
+ def _json_bytes(value: Any) -> int:
31
+ if isinstance(value, BINARY):
32
+ return 4 * math.ceil(len(value) / 3) + BLOB_WRAPPER
33
+ return len(json.dumps(value))
@@ -0,0 +1,101 @@
1
+ """The connections world-authored code holds: a tool's read-only ``conn`` and an
2
+ ingest session's raw SQL. Neither can commit, roll back or close; the database
3
+ owns every transaction."""
4
+
5
+ from __future__ import annotations
6
+
7
+ from typing import TYPE_CHECKING, Any, Iterable
8
+
9
+ from ..validation import ContractError, fail
10
+
11
+ if TYPE_CHECKING:
12
+ from .base import WorldDatabase
13
+
14
+
15
+ class ReadCursor:
16
+ def __init__(self, rows: Iterable[Any]):
17
+ self._rows = iter(rows)
18
+ self.rowcount = -1
19
+
20
+ def fetchone(self):
21
+ return next(self._rows, None)
22
+
23
+ def fetchall(self):
24
+ return list(self._rows)
25
+
26
+ def __iter__(self):
27
+ return self._rows
28
+
29
+
30
+ class ReadConnection:
31
+ """Compatibility conn= surface: execute SELECT only; no raw writes/commit."""
32
+
33
+ def __init__(self, database: "WorldDatabase"):
34
+ self._database = database
35
+
36
+ def execute(self, sql: str, params: Any = ()) -> ReadCursor:
37
+ return ReadCursor(self._database.query(sql, tuple(params)))
38
+
39
+ def cursor(self):
40
+ fail(
41
+ "raw_write",
42
+ "Use conn.execute for reads and gateway_env.store for mutations",
43
+ )
44
+
45
+ def executescript(self, *args):
46
+ fail("raw_write", "Raw SQL scripts are unavailable in schema-backed tools")
47
+
48
+ def commit(self):
49
+ fail("raw_write", "The world owns the tool transaction")
50
+
51
+ rollback = commit
52
+ close = commit
53
+
54
+
55
+ class IngestConnection:
56
+ """Raw SQL inside one schema-gated ingest session.
57
+
58
+ Domain tables and TEMP objects are writable; the database owns the
59
+ transaction and commits only when every persisted row satisfies the contract
60
+ (known gap: a TEMP object named like a main table, see :class:`IngestSession`).
61
+ """
62
+
63
+ def __init__(self, database: "WorldDatabase"):
64
+ self._database = database
65
+ self._open = True
66
+ self.result: dict | None = None
67
+
68
+ def execute(self, sql: str, params: Any = ()) -> Any:
69
+ return self._run(lambda database: database._execute(sql, params))
70
+
71
+ def executemany(self, sql: str, rows: Any) -> Any:
72
+ return self._run(lambda database: database._executemany(sql, rows))
73
+
74
+ def script(self, sql: str) -> None:
75
+ """Run a multi-statement script one statement at a time; an engine's own
76
+ script call may commit the session's transaction first."""
77
+ for statement in self._database.split_statements(sql):
78
+ self.execute(statement)
79
+
80
+ def _run(self, operation):
81
+ if not self._open:
82
+ fail("transaction", "The ingest session has ended")
83
+ database = self._database
84
+ database._key_error = None
85
+ try:
86
+ return operation(database)
87
+ except database.native_error as error:
88
+ context = {database.engine: str(error)}
89
+ if database._key_error is not None:
90
+ context["key"] = database._key_error
91
+ raise ContractError(
92
+ "ingest_sql",
93
+ "Statement rejected; only domain tables and TEMP objects are writable",
94
+ **context,
95
+ ) from error
96
+
97
+ def commit(self):
98
+ fail("raw_write", "The store commits the ingest session when it ends")
99
+
100
+ rollback = commit
101
+ close = commit
@@ -0,0 +1,8 @@
1
+ """The engines a world database can run on; the base's factory picks among them."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .sqlite import SQLiteWorldDatabase
6
+
7
+ ENGINES = (SQLiteWorldDatabase,)
8
+ DEFAULT_ENGINE = SQLiteWorldDatabase
@@ -0,0 +1,215 @@
1
+ from __future__ import annotations
2
+
3
+ from abc import ABC, abstractmethod
4
+ from contextlib import contextmanager
5
+ from dataclasses import dataclass, replace
6
+ from enum import Enum
7
+ from typing import Iterator, Mapping
8
+
9
+ RUNTIME_TABLES = frozenset(
10
+ {
11
+ "benchmark_runtime_context",
12
+ "benchmark_runtime_parameters",
13
+ "benchmark_conversation_events",
14
+ }
15
+ )
16
+ RUNTIME_PARAMETERS = "benchmark_runtime_parameters"
17
+ PRIVATE_TABLE_ORDER = (
18
+ "_gateway_lineage",
19
+ "_gateway_batches",
20
+ "_gateway_contract",
21
+ "benchmark_runtime_context",
22
+ "benchmark_conversation_events",
23
+ )
24
+ PRIVATE_TABLES = frozenset(PRIVATE_TABLE_ORDER)
25
+ STAGING = "<staging>"
26
+
27
+
28
+ class AccessKind(Enum):
29
+ SELECT = "select"
30
+ READ = "read"
31
+ FUNCTION = "function"
32
+ RECURSIVE = "recursive"
33
+ WRITE = "write"
34
+ STAGE = "stage"
35
+ SAVEPOINT = "savepoint"
36
+ OTHER = "other"
37
+
38
+
39
+ READ_KINDS = frozenset({AccessKind.SELECT, AccessKind.READ, AccessKind.FUNCTION, AccessKind.RECURSIVE})
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class Access:
44
+ """Persistent DDL, PRAGMA, ATTACH and transaction control are ``OTHER``, which every door but the trusted one
45
+ refuses; ``schema`` is ``None`` when the engine does not say."""
46
+
47
+ kind: AccessKind
48
+ target: str | None = None
49
+ schema: str | None = None
50
+ unsafe: bool = False
51
+ column: str | None = None
52
+
53
+ @property
54
+ def unsafe_function(self) -> bool:
55
+ return self.kind is AccessKind.FUNCTION and self.unsafe
56
+
57
+
58
+ class Door(ABC):
59
+ @abstractmethod
60
+ def allows(self, access: Access, entities: frozenset[str]) -> bool: ...
61
+
62
+
63
+ class ReadOnlyDoor(Door):
64
+ """``private`` is a verifier's trusted read: only it sees the runtime context and conversation events."""
65
+
66
+ def __init__(self, private: bool) -> None:
67
+ self.private = private
68
+
69
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
70
+ if access.kind not in READ_KINDS:
71
+ return False
72
+ if access.kind is AccessKind.READ and not self._readable(access.target, entities):
73
+ return False
74
+ return not access.unsafe_function
75
+
76
+ def _readable(self, table: str | None, entities: frozenset[str]) -> bool:
77
+ if table in entities or table == RUNTIME_PARAMETERS:
78
+ return True
79
+ return self.private and table in RUNTIME_TABLES
80
+
81
+
82
+ class TrustedDoor(Door):
83
+ """The store's own statements: bookkeeping, audits, transaction control."""
84
+
85
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
86
+ return True
87
+
88
+
89
+ class IngestDoor(Door):
90
+ """Transaction control, PRAGMA, ATTACH, persistent DDL and the bookkeeping tables stay with the store, so a session
91
+ cannot commit early, disable enforcement or forge provenance. Known gap (#474): a TEMP object named like a main
92
+ table shadows it (bug 2026-09-28-world-store-ingest-temp-shadow-bypasses-audit)."""
93
+
94
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
95
+ if access.kind in READ_KINDS:
96
+ return (
97
+ not (access.kind is AccessKind.READ and access.target in PRIVATE_TABLES) and not access.unsafe_function
98
+ )
99
+ if access.kind is AccessKind.WRITE:
100
+ return access.target in entities or access.schema == STAGING
101
+ return access.kind in (AccessKind.STAGE, AccessKind.SAVEPOINT)
102
+
103
+
104
+ class HostDoor(Door):
105
+ """The connection outside any store method; the store's own statements run behind the trusted door, so this one
106
+ stays narrow."""
107
+
108
+ def allows(self, access: Access, entities: frozenset[str]) -> bool:
109
+ if access.kind in READ_KINDS:
110
+ return not access.unsafe_function
111
+ return access.kind is AccessKind.WRITE and access.target in RUNTIME_TABLES
112
+
113
+
114
+ TRUSTED = TrustedDoor()
115
+ INGEST = IngestDoor()
116
+ HOST = HostDoor()
117
+
118
+
119
+ class Guard:
120
+ """An attached store is decided first, whatever door is open: only a read of a table it exposes passes, so nothing
121
+ is ever written through an alias. Then read-only wins over trusted, trusted over ingest; the host door decides the
122
+ rest. A ``bare`` store's tables carry names no world may declare (the ``gateway_`` prefix), so a read naming
123
+ one without its store (``count(*)``) is that store's."""
124
+
125
+ def __init__(self, entities: frozenset[str], unsafe_functions: frozenset[str]) -> None:
126
+ self._entities = entities
127
+ self._unsafe = frozenset(name.lower() for name in unsafe_functions)
128
+ self._attached: dict[str, frozenset[str]] = {}
129
+ self._bare: dict[str, str] = {}
130
+ self._reads: dict[str, int] = {}
131
+ self._columns: dict[str, set[str]] = {}
132
+ self._reading: ReadOnlyDoor | None = None
133
+ self._trusted = 0
134
+ self._ingesting = False
135
+
136
+ @property
137
+ def attached(self) -> Mapping[str, frozenset[str]]:
138
+ return dict(self._attached)
139
+
140
+ @property
141
+ def ingesting(self) -> bool:
142
+ return self._ingesting
143
+
144
+ def attach(self, name: str, tables: frozenset[str], *, bare: bool = False) -> None:
145
+ self._attached[name] = tables
146
+ self._reads[name] = 0
147
+ self._columns[name] = set()
148
+ if bare:
149
+ self._bare.update({table: name for table in tables})
150
+
151
+ def detach(self, name: str) -> None:
152
+ self._attached.pop(name, None)
153
+ self._reads.pop(name, None)
154
+ self._columns.pop(name, None)
155
+ self._bare = {table: owner for table, owner in self._bare.items() if owner != name}
156
+
157
+ def is_attached(self, name: str) -> bool:
158
+ return name in self._attached
159
+
160
+ def reads(self, name: str) -> int:
161
+ return self._reads.get(name, 0)
162
+
163
+ def columns(self, name: str) -> frozenset[str]:
164
+ return frozenset(self._columns.get(name, ()))
165
+
166
+ def allows(self, access: Access) -> bool:
167
+ access = self._marked(access)
168
+ if access.schema not in self._attached:
169
+ return self.door().allows(access, self._entities)
170
+ allowed = access.kind is AccessKind.READ and access.target in self._attached[access.schema]
171
+ self._reads[access.schema] += int(allowed)
172
+ if allowed and access.column:
173
+ self._columns[access.schema].add(access.column)
174
+ return allowed
175
+
176
+ def _marked(self, access: Access) -> Access:
177
+ """A bare name only an attached store holds reads that store; an unsafe function matches by name in any case."""
178
+ if access.kind is AccessKind.READ and access.schema is None and access.target in self._bare:
179
+ return replace(access, schema=self._bare[access.target])
180
+ if access.kind is not AccessKind.FUNCTION:
181
+ return access
182
+ return replace(access, unsafe=access.unsafe or (access.target or "").lower() in self._unsafe)
183
+
184
+ def door(self) -> Door:
185
+ if self._reading is not None:
186
+ return self._reading
187
+ if self._trusted:
188
+ return TRUSTED
189
+ if self._ingesting:
190
+ return INGEST
191
+ return HOST
192
+
193
+ @contextmanager
194
+ def reading(self, private: bool) -> Iterator[None]:
195
+ self._reading = ReadOnlyDoor(private)
196
+ try:
197
+ yield
198
+ finally:
199
+ self._reading = None
200
+
201
+ @contextmanager
202
+ def trusted(self) -> Iterator[None]:
203
+ self._trusted += 1
204
+ try:
205
+ yield
206
+ finally:
207
+ self._trusted -= 1
208
+
209
+ @contextmanager
210
+ def ingest(self) -> Iterator[None]:
211
+ self._ingesting = True
212
+ try:
213
+ yield
214
+ finally:
215
+ self._ingesting = False
@@ -0,0 +1,154 @@
1
+ """One schema-gated raw SQL session: write freely, commit only if every row conforms
2
+ (known gap: a TEMP object named like a main table, see :class:`IngestSession`)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import uuid
7
+ from typing import TYPE_CHECKING, Iterator
8
+
9
+ from ..compiler import quote
10
+ from ..validation import canonical, digest, encoded, fail, members, nonempty
11
+ from .connections import IngestConnection
12
+ from .provenance import validate_source
13
+
14
+ if TYPE_CHECKING:
15
+ from .base import WorldDatabase
16
+
17
+
18
+ class IngestSession:
19
+ """The session :meth:`WorldDatabase.ingest` opens. The database's guard is on its
20
+ ingest door while the caller holds the connection; on exit every persisted row is
21
+ audited, and the changes are recorded as one batch with a lineage row per touched
22
+ key, or the whole session rolls back. Known gap: a TEMP table named like a domain or
23
+ bookkeeping table shadows it for the audit and the provenance writes (bug log
24
+ ``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, pinned xfail in the guard matrix,
25
+ to be fixed by #474)."""
26
+
27
+ def __init__(self, database: "WorldDatabase", *, source: dict, mapping: dict, batch_id: str | None, limit: int):
28
+ validate_source(source, "/ingest/source")
29
+ members(mapping, {"id", "version"}, {"id", "version"}, "/ingest/mapping")
30
+ for key, value in mapping.items():
31
+ nonempty(value, f"/ingest/mapping/{key}")
32
+ self._database = database
33
+ self._source = source
34
+ self._mapping = mapping
35
+ self._batch_id = batch_id
36
+ self._limit = limit
37
+
38
+ def run(self) -> Iterator[IngestConnection]:
39
+ database = self._database
40
+ with database._lock:
41
+ if database._depth or database._guard.ingesting:
42
+ fail(
43
+ "transaction",
44
+ "Ingest needs no active tool, batch or ingest transaction",
45
+ )
46
+ batch = self._batch_id if self._batch_id is not None else uuid.uuid4().hex
47
+ nonempty(batch, "/ingest/id")
48
+ with database._trusted():
49
+ applied = database._execute("SELECT 1 FROM _gateway_batches WHERE id = ?", (batch,)).fetchone()
50
+ if applied:
51
+ fail(
52
+ "idempotency_conflict",
53
+ "Ingest ID was already applied; raw sessions are not replayable",
54
+ "/ingest/id",
55
+ batch=batch,
56
+ )
57
+ before = self._state()
58
+ connection = IngestConnection(database)
59
+ with database._trusted():
60
+ database._begin_write()
61
+ database._depth = 1
62
+ try:
63
+ with database._guard.ingest():
64
+ changes = yield from self._session(connection, before, batch)
65
+ finally:
66
+ database._depth = 0
67
+ connection._open = False
68
+ with database._trusted():
69
+ database._drop_staging()
70
+ connection.result = self._result(batch, changes)
71
+
72
+ def _session(self, connection: IngestConnection, before: dict, batch: str):
73
+ database = self._database
74
+ try:
75
+ yield connection
76
+ violations = database._audit.violations(self._limit)
77
+ if violations:
78
+ fail(
79
+ "ingest_rejected",
80
+ "Persisted rows violate the contract; nothing was committed",
81
+ batch=batch,
82
+ violations=[error.detail for error in violations],
83
+ truncated=len(violations) >= self._limit,
84
+ )
85
+ changes = self._changes(before, self._state())
86
+ with database._trusted():
87
+ self._record(batch, changes)
88
+ database._commit()
89
+ return changes
90
+ except BaseException:
91
+ with database._trusted():
92
+ database._rollback()
93
+ raise
94
+
95
+ def _record(self, batch: str, changes: list) -> None:
96
+ if not changes:
97
+ return
98
+ database = self._database
99
+ metadata = {
100
+ "id": batch,
101
+ "kind": "ingest",
102
+ "target": database.contract.target,
103
+ "mapping": self._mapping,
104
+ "coverage": "partial",
105
+ }
106
+ database._execute(
107
+ "INSERT INTO _gateway_batches VALUES (?, ?, ?)",
108
+ (batch, digest(encoded(changes)), canonical(metadata)),
109
+ )
110
+ database._executemany(
111
+ "INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
112
+ [(batch, index, entity, key, canonical(self._source)) for index, (entity, key, _) in enumerate(changes)],
113
+ )
114
+
115
+ def _state(self) -> dict[str, dict[str, str]]:
116
+ """Persisted ``_key -> payload`` per entity, for change attribution."""
117
+ database = self._database
118
+ with database._trusted():
119
+ return {
120
+ name: dict(database._execute(f"SELECT _key, payload FROM {quote(name)}"))
121
+ for name in database.contract.entities
122
+ }
123
+
124
+ @staticmethod
125
+ def _changes(before: dict, after: dict) -> list[list[str]]:
126
+ changes = []
127
+ for name in sorted(before):
128
+ old, new = before[name], after[name]
129
+ for key in sorted(set(old) | set(new)):
130
+ if old.get(key) == new.get(key):
131
+ continue
132
+ changes.append([name, key, IngestSession._change(key, old, new)])
133
+ return changes
134
+
135
+ @staticmethod
136
+ def _change(key: str, old: dict, new: dict) -> str:
137
+ if key not in old:
138
+ return "insert"
139
+ if key not in new:
140
+ return "delete"
141
+ return "update"
142
+
143
+ @staticmethod
144
+ def _result(batch: str, changes: list) -> dict:
145
+ counts = {op: 0 for op in ("insert", "update", "delete")}
146
+ for _, _, op in changes:
147
+ counts[op] += 1
148
+ return {
149
+ "status": "applied" if changes else "unchanged",
150
+ "batch": batch,
151
+ "inserted": counts["insert"],
152
+ "updated": counts["update"],
153
+ "deleted": counts["delete"],
154
+ }
@@ -0,0 +1,58 @@
1
+ """The one rule by which a world's rows match a filter: the store's ``list_where``,
2
+ an api world's routes and ``worlds data query`` all answer with it."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from dataclasses import dataclass, field
7
+ from typing import Any, Callable, Iterable
8
+
9
+ COMPARISONS: dict[str, Callable[[str, str], bool]] = {
10
+ "equals": lambda have, wanted: have == wanted,
11
+ "contains": lambda have, wanted: wanted in have,
12
+ "prefix": lambda have, wanted: have.startswith(wanted),
13
+ }
14
+
15
+
16
+ def dig(row: Any, path: str) -> Any:
17
+ for part in path.split("."):
18
+ if not isinstance(row, dict) or part not in row:
19
+ return None
20
+ row = row[part]
21
+ return row
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class RowMatch:
26
+ """Every filter whose argument was supplied must match; a parameter left out does not narrow.
27
+
28
+ ``filters`` maps an argument to the dotted field path it reads, ``values`` the
29
+ argument to the value (or values) sent. ``modes`` picks how: ``equals``
30
+ (default), ``contains`` (case-insensitive substring, the vendor's ``icontains``)
31
+ or ``prefix``. The stored value is compared as text, case-insensitively; a list
32
+ field matches when any element does, and a parameter sent more than once
33
+ (``status[]=a&status[]=b``) matches when any of its values does. A missing field
34
+ never matches."""
35
+
36
+ filters: dict[str, str]
37
+ values: dict[str, Any]
38
+ modes: dict[str, str] = field(default_factory=dict)
39
+
40
+ @classmethod
41
+ def on_fields(cls, values: dict[str, Any]) -> "RowMatch":
42
+ return cls({name: name for name in values}, values)
43
+
44
+ def matches(self, row: dict) -> bool:
45
+ return all(self._field_matches(row, arg, path) for arg, path in self.filters.items() if arg in self.values)
46
+
47
+ def select(self, rows: Iterable[dict]) -> list[dict]:
48
+ return [row for row in rows if self.matches(row)]
49
+
50
+ def _field_matches(self, row: dict, arg: str, path: str) -> bool:
51
+ value = dig(row, path)
52
+ if value is None:
53
+ return False
54
+ compare = COMPARISONS.get(self.modes.get(arg, "equals"), COMPARISONS["equals"])
55
+ sent = self.values[arg]
56
+ wanted = [w.lower() for w in (sent if isinstance(sent, list) else [sent])]
57
+ candidates = value if isinstance(value, list) else [value]
58
+ return any(compare(str(candidate).lower(), w) for candidate in candidates for w in wanted)
@@ -0,0 +1,30 @@
1
+ """The provenance a mutation carries: the record it came from and the evidence behind it."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from ..validation import fail, members, nonempty
8
+
9
+
10
+ def validate_source(source: Any, location: str) -> None:
11
+ members(source, {"record", "evidence"}, {"record"}, location)
12
+ nonempty(source["record"], f"{location}/record")
13
+ evidence = source.get("evidence", [])
14
+ if not isinstance(evidence, list):
15
+ fail(
16
+ "provenance",
17
+ "evidence must be an array of stable reference/digest pairs",
18
+ location,
19
+ )
20
+ for index, entry in enumerate(evidence):
21
+ members(
22
+ entry,
23
+ {"reference", "digest"},
24
+ {"reference", "digest"},
25
+ f"{location}/evidence/{index}",
26
+ )
27
+ nonempty(entry["reference"], location)
28
+ value = entry["digest"]
29
+ if not isinstance(value, str) or len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
30
+ fail("provenance", "Evidence digest must be lowercase SHA-256 hex", location)