@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""What one caller's SQL may cost a world database."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, Iterable
|
|
9
|
+
|
|
10
|
+
BINARY = (bytes, bytearray, memoryview)
|
|
11
|
+
#: ``{"base64": ""}`` around an encoded blob, and the separator after each value.
|
|
12
|
+
BLOB_WRAPPER = len(json.dumps({"base64": ""}))
|
|
13
|
+
SEPARATOR = 2
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class QueryBudget:
|
|
18
|
+
"""Wall time for every statement in all, the largest single value the engine may make, and the
|
|
19
|
+
bytes the whole answer takes as JSON (what is sent, not what is counted in characters)."""
|
|
20
|
+
|
|
21
|
+
seconds: float
|
|
22
|
+
value_bytes: int
|
|
23
|
+
answer_bytes: int
|
|
24
|
+
|
|
25
|
+
@staticmethod
|
|
26
|
+
def size_of(row: Iterable[Any]) -> int:
|
|
27
|
+
return sum(QueryBudget._json_bytes(value) + SEPARATOR for value in row)
|
|
28
|
+
|
|
29
|
+
@staticmethod
|
|
30
|
+
def _json_bytes(value: Any) -> int:
|
|
31
|
+
if isinstance(value, BINARY):
|
|
32
|
+
return 4 * math.ceil(len(value) / 3) + BLOB_WRAPPER
|
|
33
|
+
return len(json.dumps(value))
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""The connections world-authored code holds: a tool's read-only ``conn`` and an
|
|
2
|
+
ingest session's raw SQL. Neither can commit, roll back or close; the database
|
|
3
|
+
owns every transaction."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Iterable
|
|
8
|
+
|
|
9
|
+
from ..validation import ContractError, fail
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from .base import WorldDatabase
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ReadCursor:
|
|
16
|
+
def __init__(self, rows: Iterable[Any]):
|
|
17
|
+
self._rows = iter(rows)
|
|
18
|
+
self.rowcount = -1
|
|
19
|
+
|
|
20
|
+
def fetchone(self):
|
|
21
|
+
return next(self._rows, None)
|
|
22
|
+
|
|
23
|
+
def fetchall(self):
|
|
24
|
+
return list(self._rows)
|
|
25
|
+
|
|
26
|
+
def __iter__(self):
|
|
27
|
+
return self._rows
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ReadConnection:
|
|
31
|
+
"""Compatibility conn= surface: execute SELECT only; no raw writes/commit."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, database: "WorldDatabase"):
|
|
34
|
+
self._database = database
|
|
35
|
+
|
|
36
|
+
def execute(self, sql: str, params: Any = ()) -> ReadCursor:
|
|
37
|
+
return ReadCursor(self._database.query(sql, tuple(params)))
|
|
38
|
+
|
|
39
|
+
def cursor(self):
|
|
40
|
+
fail(
|
|
41
|
+
"raw_write",
|
|
42
|
+
"Use conn.execute for reads and gateway_env.store for mutations",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
def executescript(self, *args):
|
|
46
|
+
fail("raw_write", "Raw SQL scripts are unavailable in schema-backed tools")
|
|
47
|
+
|
|
48
|
+
def commit(self):
|
|
49
|
+
fail("raw_write", "The world owns the tool transaction")
|
|
50
|
+
|
|
51
|
+
rollback = commit
|
|
52
|
+
close = commit
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class IngestConnection:
|
|
56
|
+
"""Raw SQL inside one schema-gated ingest session.
|
|
57
|
+
|
|
58
|
+
Domain tables and TEMP objects are writable; the database owns the
|
|
59
|
+
transaction and commits only when every persisted row satisfies the contract
|
|
60
|
+
(known gap: a TEMP object named like a main table, see :class:`IngestSession`).
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(self, database: "WorldDatabase"):
|
|
64
|
+
self._database = database
|
|
65
|
+
self._open = True
|
|
66
|
+
self.result: dict | None = None
|
|
67
|
+
|
|
68
|
+
def execute(self, sql: str, params: Any = ()) -> Any:
|
|
69
|
+
return self._run(lambda database: database._execute(sql, params))
|
|
70
|
+
|
|
71
|
+
def executemany(self, sql: str, rows: Any) -> Any:
|
|
72
|
+
return self._run(lambda database: database._executemany(sql, rows))
|
|
73
|
+
|
|
74
|
+
def script(self, sql: str) -> None:
|
|
75
|
+
"""Run a multi-statement script one statement at a time; an engine's own
|
|
76
|
+
script call may commit the session's transaction first."""
|
|
77
|
+
for statement in self._database.split_statements(sql):
|
|
78
|
+
self.execute(statement)
|
|
79
|
+
|
|
80
|
+
def _run(self, operation):
|
|
81
|
+
if not self._open:
|
|
82
|
+
fail("transaction", "The ingest session has ended")
|
|
83
|
+
database = self._database
|
|
84
|
+
database._key_error = None
|
|
85
|
+
try:
|
|
86
|
+
return operation(database)
|
|
87
|
+
except database.native_error as error:
|
|
88
|
+
context = {database.engine: str(error)}
|
|
89
|
+
if database._key_error is not None:
|
|
90
|
+
context["key"] = database._key_error
|
|
91
|
+
raise ContractError(
|
|
92
|
+
"ingest_sql",
|
|
93
|
+
"Statement rejected; only domain tables and TEMP objects are writable",
|
|
94
|
+
**context,
|
|
95
|
+
) from error
|
|
96
|
+
|
|
97
|
+
def commit(self):
|
|
98
|
+
fail("raw_write", "The store commits the ingest session when it ends")
|
|
99
|
+
|
|
100
|
+
rollback = commit
|
|
101
|
+
close = commit
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from contextlib import contextmanager
|
|
5
|
+
from dataclasses import dataclass, replace
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Iterator, Mapping
|
|
8
|
+
|
|
9
|
+
RUNTIME_TABLES = frozenset(
|
|
10
|
+
{
|
|
11
|
+
"benchmark_runtime_context",
|
|
12
|
+
"benchmark_runtime_parameters",
|
|
13
|
+
"benchmark_conversation_events",
|
|
14
|
+
}
|
|
15
|
+
)
|
|
16
|
+
RUNTIME_PARAMETERS = "benchmark_runtime_parameters"
|
|
17
|
+
PRIVATE_TABLE_ORDER = (
|
|
18
|
+
"_gateway_lineage",
|
|
19
|
+
"_gateway_batches",
|
|
20
|
+
"_gateway_contract",
|
|
21
|
+
"benchmark_runtime_context",
|
|
22
|
+
"benchmark_conversation_events",
|
|
23
|
+
)
|
|
24
|
+
PRIVATE_TABLES = frozenset(PRIVATE_TABLE_ORDER)
|
|
25
|
+
STAGING = "<staging>"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AccessKind(Enum):
|
|
29
|
+
SELECT = "select"
|
|
30
|
+
READ = "read"
|
|
31
|
+
FUNCTION = "function"
|
|
32
|
+
RECURSIVE = "recursive"
|
|
33
|
+
WRITE = "write"
|
|
34
|
+
STAGE = "stage"
|
|
35
|
+
SAVEPOINT = "savepoint"
|
|
36
|
+
OTHER = "other"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
READ_KINDS = frozenset({AccessKind.SELECT, AccessKind.READ, AccessKind.FUNCTION, AccessKind.RECURSIVE})
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class Access:
|
|
44
|
+
"""Persistent DDL, PRAGMA, ATTACH and transaction control are ``OTHER``, which every door but the trusted one
|
|
45
|
+
refuses; ``schema`` is ``None`` when the engine does not say."""
|
|
46
|
+
|
|
47
|
+
kind: AccessKind
|
|
48
|
+
target: str | None = None
|
|
49
|
+
schema: str | None = None
|
|
50
|
+
unsafe: bool = False
|
|
51
|
+
column: str | None = None
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def unsafe_function(self) -> bool:
|
|
55
|
+
return self.kind is AccessKind.FUNCTION and self.unsafe
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Door(ABC):
|
|
59
|
+
@abstractmethod
|
|
60
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool: ...
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ReadOnlyDoor(Door):
|
|
64
|
+
"""``private`` is a verifier's trusted read: only it sees the runtime context and conversation events."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, private: bool) -> None:
|
|
67
|
+
self.private = private
|
|
68
|
+
|
|
69
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
70
|
+
if access.kind not in READ_KINDS:
|
|
71
|
+
return False
|
|
72
|
+
if access.kind is AccessKind.READ and not self._readable(access.target, entities):
|
|
73
|
+
return False
|
|
74
|
+
return not access.unsafe_function
|
|
75
|
+
|
|
76
|
+
def _readable(self, table: str | None, entities: frozenset[str]) -> bool:
|
|
77
|
+
if table in entities or table == RUNTIME_PARAMETERS:
|
|
78
|
+
return True
|
|
79
|
+
return self.private and table in RUNTIME_TABLES
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class TrustedDoor(Door):
|
|
83
|
+
"""The store's own statements: bookkeeping, audits, transaction control."""
|
|
84
|
+
|
|
85
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
86
|
+
return True
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class IngestDoor(Door):
|
|
90
|
+
"""Transaction control, PRAGMA, ATTACH, persistent DDL and the bookkeeping tables stay with the store, so a session
|
|
91
|
+
cannot commit early, disable enforcement or forge provenance. Known gap (#474): a TEMP object named like a main
|
|
92
|
+
table shadows it (bug 2026-09-28-world-store-ingest-temp-shadow-bypasses-audit)."""
|
|
93
|
+
|
|
94
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
95
|
+
if access.kind in READ_KINDS:
|
|
96
|
+
return (
|
|
97
|
+
not (access.kind is AccessKind.READ and access.target in PRIVATE_TABLES) and not access.unsafe_function
|
|
98
|
+
)
|
|
99
|
+
if access.kind is AccessKind.WRITE:
|
|
100
|
+
return access.target in entities or access.schema == STAGING
|
|
101
|
+
return access.kind in (AccessKind.STAGE, AccessKind.SAVEPOINT)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class HostDoor(Door):
|
|
105
|
+
"""The connection outside any store method; the store's own statements run behind the trusted door, so this one
|
|
106
|
+
stays narrow."""
|
|
107
|
+
|
|
108
|
+
def allows(self, access: Access, entities: frozenset[str]) -> bool:
|
|
109
|
+
if access.kind in READ_KINDS:
|
|
110
|
+
return not access.unsafe_function
|
|
111
|
+
return access.kind is AccessKind.WRITE and access.target in RUNTIME_TABLES
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
TRUSTED = TrustedDoor()
|
|
115
|
+
INGEST = IngestDoor()
|
|
116
|
+
HOST = HostDoor()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class Guard:
|
|
120
|
+
"""An attached store is decided first, whatever door is open: only a read of a table it exposes passes, so nothing
|
|
121
|
+
is ever written through an alias. Then read-only wins over trusted, trusted over ingest; the host door decides the
|
|
122
|
+
rest. A ``bare`` store's tables carry names no world may declare (the ``gateway_`` prefix), so a read naming
|
|
123
|
+
one without its store (``count(*)``) is that store's."""
|
|
124
|
+
|
|
125
|
+
def __init__(self, entities: frozenset[str], unsafe_functions: frozenset[str]) -> None:
|
|
126
|
+
self._entities = entities
|
|
127
|
+
self._unsafe = frozenset(name.lower() for name in unsafe_functions)
|
|
128
|
+
self._attached: dict[str, frozenset[str]] = {}
|
|
129
|
+
self._bare: dict[str, str] = {}
|
|
130
|
+
self._reads: dict[str, int] = {}
|
|
131
|
+
self._columns: dict[str, set[str]] = {}
|
|
132
|
+
self._reading: ReadOnlyDoor | None = None
|
|
133
|
+
self._trusted = 0
|
|
134
|
+
self._ingesting = False
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def attached(self) -> Mapping[str, frozenset[str]]:
|
|
138
|
+
return dict(self._attached)
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def ingesting(self) -> bool:
|
|
142
|
+
return self._ingesting
|
|
143
|
+
|
|
144
|
+
def attach(self, name: str, tables: frozenset[str], *, bare: bool = False) -> None:
|
|
145
|
+
self._attached[name] = tables
|
|
146
|
+
self._reads[name] = 0
|
|
147
|
+
self._columns[name] = set()
|
|
148
|
+
if bare:
|
|
149
|
+
self._bare.update({table: name for table in tables})
|
|
150
|
+
|
|
151
|
+
def detach(self, name: str) -> None:
|
|
152
|
+
self._attached.pop(name, None)
|
|
153
|
+
self._reads.pop(name, None)
|
|
154
|
+
self._columns.pop(name, None)
|
|
155
|
+
self._bare = {table: owner for table, owner in self._bare.items() if owner != name}
|
|
156
|
+
|
|
157
|
+
def is_attached(self, name: str) -> bool:
|
|
158
|
+
return name in self._attached
|
|
159
|
+
|
|
160
|
+
def reads(self, name: str) -> int:
|
|
161
|
+
return self._reads.get(name, 0)
|
|
162
|
+
|
|
163
|
+
def columns(self, name: str) -> frozenset[str]:
|
|
164
|
+
return frozenset(self._columns.get(name, ()))
|
|
165
|
+
|
|
166
|
+
def allows(self, access: Access) -> bool:
|
|
167
|
+
access = self._marked(access)
|
|
168
|
+
if access.schema not in self._attached:
|
|
169
|
+
return self.door().allows(access, self._entities)
|
|
170
|
+
allowed = access.kind is AccessKind.READ and access.target in self._attached[access.schema]
|
|
171
|
+
self._reads[access.schema] += int(allowed)
|
|
172
|
+
if allowed and access.column:
|
|
173
|
+
self._columns[access.schema].add(access.column)
|
|
174
|
+
return allowed
|
|
175
|
+
|
|
176
|
+
def _marked(self, access: Access) -> Access:
|
|
177
|
+
"""A bare name only an attached store holds reads that store; an unsafe function matches by name in any case."""
|
|
178
|
+
if access.kind is AccessKind.READ and access.schema is None and access.target in self._bare:
|
|
179
|
+
return replace(access, schema=self._bare[access.target])
|
|
180
|
+
if access.kind is not AccessKind.FUNCTION:
|
|
181
|
+
return access
|
|
182
|
+
return replace(access, unsafe=access.unsafe or (access.target or "").lower() in self._unsafe)
|
|
183
|
+
|
|
184
|
+
def door(self) -> Door:
|
|
185
|
+
if self._reading is not None:
|
|
186
|
+
return self._reading
|
|
187
|
+
if self._trusted:
|
|
188
|
+
return TRUSTED
|
|
189
|
+
if self._ingesting:
|
|
190
|
+
return INGEST
|
|
191
|
+
return HOST
|
|
192
|
+
|
|
193
|
+
@contextmanager
|
|
194
|
+
def reading(self, private: bool) -> Iterator[None]:
|
|
195
|
+
self._reading = ReadOnlyDoor(private)
|
|
196
|
+
try:
|
|
197
|
+
yield
|
|
198
|
+
finally:
|
|
199
|
+
self._reading = None
|
|
200
|
+
|
|
201
|
+
@contextmanager
|
|
202
|
+
def trusted(self) -> Iterator[None]:
|
|
203
|
+
self._trusted += 1
|
|
204
|
+
try:
|
|
205
|
+
yield
|
|
206
|
+
finally:
|
|
207
|
+
self._trusted -= 1
|
|
208
|
+
|
|
209
|
+
@contextmanager
|
|
210
|
+
def ingest(self) -> Iterator[None]:
|
|
211
|
+
self._ingesting = True
|
|
212
|
+
try:
|
|
213
|
+
yield
|
|
214
|
+
finally:
|
|
215
|
+
self._ingesting = False
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""One schema-gated raw SQL session: write freely, commit only if every row conforms
|
|
2
|
+
(known gap: a TEMP object named like a main table, see :class:`IngestSession`)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import uuid
|
|
7
|
+
from typing import TYPE_CHECKING, Iterator
|
|
8
|
+
|
|
9
|
+
from ..compiler import quote
|
|
10
|
+
from ..validation import canonical, digest, encoded, fail, members, nonempty
|
|
11
|
+
from .connections import IngestConnection
|
|
12
|
+
from .provenance import validate_source
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from .base import WorldDatabase
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class IngestSession:
|
|
19
|
+
"""The session :meth:`WorldDatabase.ingest` opens. The database's guard is on its
|
|
20
|
+
ingest door while the caller holds the connection; on exit every persisted row is
|
|
21
|
+
audited, and the changes are recorded as one batch with a lineage row per touched
|
|
22
|
+
key, or the whole session rolls back. Known gap: a TEMP table named like a domain or
|
|
23
|
+
bookkeeping table shadows it for the audit and the provenance writes (bug log
|
|
24
|
+
``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, pinned xfail in the guard matrix,
|
|
25
|
+
to be fixed by #474)."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, database: "WorldDatabase", *, source: dict, mapping: dict, batch_id: str | None, limit: int):
|
|
28
|
+
validate_source(source, "/ingest/source")
|
|
29
|
+
members(mapping, {"id", "version"}, {"id", "version"}, "/ingest/mapping")
|
|
30
|
+
for key, value in mapping.items():
|
|
31
|
+
nonempty(value, f"/ingest/mapping/{key}")
|
|
32
|
+
self._database = database
|
|
33
|
+
self._source = source
|
|
34
|
+
self._mapping = mapping
|
|
35
|
+
self._batch_id = batch_id
|
|
36
|
+
self._limit = limit
|
|
37
|
+
|
|
38
|
+
def run(self) -> Iterator[IngestConnection]:
|
|
39
|
+
database = self._database
|
|
40
|
+
with database._lock:
|
|
41
|
+
if database._depth or database._guard.ingesting:
|
|
42
|
+
fail(
|
|
43
|
+
"transaction",
|
|
44
|
+
"Ingest needs no active tool, batch or ingest transaction",
|
|
45
|
+
)
|
|
46
|
+
batch = self._batch_id if self._batch_id is not None else uuid.uuid4().hex
|
|
47
|
+
nonempty(batch, "/ingest/id")
|
|
48
|
+
with database._trusted():
|
|
49
|
+
applied = database._execute("SELECT 1 FROM _gateway_batches WHERE id = ?", (batch,)).fetchone()
|
|
50
|
+
if applied:
|
|
51
|
+
fail(
|
|
52
|
+
"idempotency_conflict",
|
|
53
|
+
"Ingest ID was already applied; raw sessions are not replayable",
|
|
54
|
+
"/ingest/id",
|
|
55
|
+
batch=batch,
|
|
56
|
+
)
|
|
57
|
+
before = self._state()
|
|
58
|
+
connection = IngestConnection(database)
|
|
59
|
+
with database._trusted():
|
|
60
|
+
database._begin_write()
|
|
61
|
+
database._depth = 1
|
|
62
|
+
try:
|
|
63
|
+
with database._guard.ingest():
|
|
64
|
+
changes = yield from self._session(connection, before, batch)
|
|
65
|
+
finally:
|
|
66
|
+
database._depth = 0
|
|
67
|
+
connection._open = False
|
|
68
|
+
with database._trusted():
|
|
69
|
+
database._drop_staging()
|
|
70
|
+
connection.result = self._result(batch, changes)
|
|
71
|
+
|
|
72
|
+
def _session(self, connection: IngestConnection, before: dict, batch: str):
|
|
73
|
+
database = self._database
|
|
74
|
+
try:
|
|
75
|
+
yield connection
|
|
76
|
+
violations = database._audit.violations(self._limit)
|
|
77
|
+
if violations:
|
|
78
|
+
fail(
|
|
79
|
+
"ingest_rejected",
|
|
80
|
+
"Persisted rows violate the contract; nothing was committed",
|
|
81
|
+
batch=batch,
|
|
82
|
+
violations=[error.detail for error in violations],
|
|
83
|
+
truncated=len(violations) >= self._limit,
|
|
84
|
+
)
|
|
85
|
+
changes = self._changes(before, self._state())
|
|
86
|
+
with database._trusted():
|
|
87
|
+
self._record(batch, changes)
|
|
88
|
+
database._commit()
|
|
89
|
+
return changes
|
|
90
|
+
except BaseException:
|
|
91
|
+
with database._trusted():
|
|
92
|
+
database._rollback()
|
|
93
|
+
raise
|
|
94
|
+
|
|
95
|
+
def _record(self, batch: str, changes: list) -> None:
|
|
96
|
+
if not changes:
|
|
97
|
+
return
|
|
98
|
+
database = self._database
|
|
99
|
+
metadata = {
|
|
100
|
+
"id": batch,
|
|
101
|
+
"kind": "ingest",
|
|
102
|
+
"target": database.contract.target,
|
|
103
|
+
"mapping": self._mapping,
|
|
104
|
+
"coverage": "partial",
|
|
105
|
+
}
|
|
106
|
+
database._execute(
|
|
107
|
+
"INSERT INTO _gateway_batches VALUES (?, ?, ?)",
|
|
108
|
+
(batch, digest(encoded(changes)), canonical(metadata)),
|
|
109
|
+
)
|
|
110
|
+
database._executemany(
|
|
111
|
+
"INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
|
|
112
|
+
[(batch, index, entity, key, canonical(self._source)) for index, (entity, key, _) in enumerate(changes)],
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def _state(self) -> dict[str, dict[str, str]]:
|
|
116
|
+
"""Persisted ``_key -> payload`` per entity, for change attribution."""
|
|
117
|
+
database = self._database
|
|
118
|
+
with database._trusted():
|
|
119
|
+
return {
|
|
120
|
+
name: dict(database._execute(f"SELECT _key, payload FROM {quote(name)}"))
|
|
121
|
+
for name in database.contract.entities
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
def _changes(before: dict, after: dict) -> list[list[str]]:
|
|
126
|
+
changes = []
|
|
127
|
+
for name in sorted(before):
|
|
128
|
+
old, new = before[name], after[name]
|
|
129
|
+
for key in sorted(set(old) | set(new)):
|
|
130
|
+
if old.get(key) == new.get(key):
|
|
131
|
+
continue
|
|
132
|
+
changes.append([name, key, IngestSession._change(key, old, new)])
|
|
133
|
+
return changes
|
|
134
|
+
|
|
135
|
+
@staticmethod
|
|
136
|
+
def _change(key: str, old: dict, new: dict) -> str:
|
|
137
|
+
if key not in old:
|
|
138
|
+
return "insert"
|
|
139
|
+
if key not in new:
|
|
140
|
+
return "delete"
|
|
141
|
+
return "update"
|
|
142
|
+
|
|
143
|
+
@staticmethod
|
|
144
|
+
def _result(batch: str, changes: list) -> dict:
|
|
145
|
+
counts = {op: 0 for op in ("insert", "update", "delete")}
|
|
146
|
+
for _, _, op in changes:
|
|
147
|
+
counts[op] += 1
|
|
148
|
+
return {
|
|
149
|
+
"status": "applied" if changes else "unchanged",
|
|
150
|
+
"batch": batch,
|
|
151
|
+
"inserted": counts["insert"],
|
|
152
|
+
"updated": counts["update"],
|
|
153
|
+
"deleted": counts["delete"],
|
|
154
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""The one rule by which a world's rows match a filter: the store's ``list_where``,
|
|
2
|
+
an api world's routes and ``worlds data query`` all answer with it."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any, Callable, Iterable
|
|
8
|
+
|
|
9
|
+
COMPARISONS: dict[str, Callable[[str, str], bool]] = {
|
|
10
|
+
"equals": lambda have, wanted: have == wanted,
|
|
11
|
+
"contains": lambda have, wanted: wanted in have,
|
|
12
|
+
"prefix": lambda have, wanted: have.startswith(wanted),
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def dig(row: Any, path: str) -> Any:
|
|
17
|
+
for part in path.split("."):
|
|
18
|
+
if not isinstance(row, dict) or part not in row:
|
|
19
|
+
return None
|
|
20
|
+
row = row[part]
|
|
21
|
+
return row
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class RowMatch:
|
|
26
|
+
"""Every filter whose argument was supplied must match; a parameter left out does not narrow.
|
|
27
|
+
|
|
28
|
+
``filters`` maps an argument to the dotted field path it reads, ``values`` the
|
|
29
|
+
argument to the value (or values) sent. ``modes`` picks how: ``equals``
|
|
30
|
+
(default), ``contains`` (case-insensitive substring, the vendor's ``icontains``)
|
|
31
|
+
or ``prefix``. The stored value is compared as text, case-insensitively; a list
|
|
32
|
+
field matches when any element does, and a parameter sent more than once
|
|
33
|
+
(``status[]=a&status[]=b``) matches when any of its values does. A missing field
|
|
34
|
+
never matches."""
|
|
35
|
+
|
|
36
|
+
filters: dict[str, str]
|
|
37
|
+
values: dict[str, Any]
|
|
38
|
+
modes: dict[str, str] = field(default_factory=dict)
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def on_fields(cls, values: dict[str, Any]) -> "RowMatch":
|
|
42
|
+
return cls({name: name for name in values}, values)
|
|
43
|
+
|
|
44
|
+
def matches(self, row: dict) -> bool:
|
|
45
|
+
return all(self._field_matches(row, arg, path) for arg, path in self.filters.items() if arg in self.values)
|
|
46
|
+
|
|
47
|
+
def select(self, rows: Iterable[dict]) -> list[dict]:
|
|
48
|
+
return [row for row in rows if self.matches(row)]
|
|
49
|
+
|
|
50
|
+
def _field_matches(self, row: dict, arg: str, path: str) -> bool:
|
|
51
|
+
value = dig(row, path)
|
|
52
|
+
if value is None:
|
|
53
|
+
return False
|
|
54
|
+
compare = COMPARISONS.get(self.modes.get(arg, "equals"), COMPARISONS["equals"])
|
|
55
|
+
sent = self.values[arg]
|
|
56
|
+
wanted = [w.lower() for w in (sent if isinstance(sent, list) else [sent])]
|
|
57
|
+
candidates = value if isinstance(value, list) else [value]
|
|
58
|
+
return any(compare(str(candidate).lower(), w) for candidate in candidates for w in wanted)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""The provenance a mutation carries: the record it came from and the evidence behind it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ..validation import fail, members, nonempty
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate_source(source: Any, location: str) -> None:
|
|
11
|
+
members(source, {"record", "evidence"}, {"record"}, location)
|
|
12
|
+
nonempty(source["record"], f"{location}/record")
|
|
13
|
+
evidence = source.get("evidence", [])
|
|
14
|
+
if not isinstance(evidence, list):
|
|
15
|
+
fail(
|
|
16
|
+
"provenance",
|
|
17
|
+
"evidence must be an array of stable reference/digest pairs",
|
|
18
|
+
location,
|
|
19
|
+
)
|
|
20
|
+
for index, entry in enumerate(evidence):
|
|
21
|
+
members(
|
|
22
|
+
entry,
|
|
23
|
+
{"reference", "digest"},
|
|
24
|
+
{"reference", "digest"},
|
|
25
|
+
f"{location}/evidence/{index}",
|
|
26
|
+
)
|
|
27
|
+
nonempty(entry["reference"], location)
|
|
28
|
+
value = entry["digest"]
|
|
29
|
+
if not isinstance(value, str) or len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
|
|
30
|
+
fail("provenance", "Evidence digest must be lowercase SHA-256 hex", location)
|