@withgateway/sdk 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +15468 -11263
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15233 -11028
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +291 -40
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +284 -37
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +203 -47
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +198 -42
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
- package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
- package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
- package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2149 -615
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2053 -566
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +268 -31
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +262 -28
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +19 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +19 -11
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +886 -117
- package/dist/worlds/index.d.ts +886 -117
- package/dist/worlds/index.js +1770 -452
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1752 -434
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +14 -10
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/_world.py +95 -28
- package/runtime/gatewaysdk/environment/runtime.py +10 -6
- package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/api.py +19 -23
- package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
- package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
- package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
- package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
- package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
- package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
- package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
- package/runtime/gatewaysdk/environment/schema/host.py +68 -115
- package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
- package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
- package/runtime/gatewaysdk/environment/schema/local.py +7 -10
- package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
- package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
- package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
- package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
- package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
- package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
- package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
- package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
- package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
- package/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/skills/connector-schema-authoring/SKILL.md +54 -4
- package/skills/connector-schema-authoring/connector-worlds.md +21 -7
- package/skills/connector-schema-authoring/world-schema.md +144 -83
- package/skills/hooks-getting-started/SKILL.md +27 -2
- package/skills/world-data-ingestion/ROWS.md +62 -0
- package/skills/world-data-ingestion/SKILL.md +91 -54
- package/skills/worlds-getting-started/SKILL.md +146 -146
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/skills/worlds-getting-started/references/openapi-import.md +90 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@withgateway/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.9",
|
|
4
4
|
"description": "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -155,16 +155,16 @@
|
|
|
155
155
|
}
|
|
156
156
|
},
|
|
157
157
|
"optionalDependencies": {
|
|
158
|
-
"@arizeai/openinference-instrumentation-anthropic": "^0.
|
|
159
|
-
"@arizeai/openinference-instrumentation-openai": "^4.
|
|
158
|
+
"@arizeai/openinference-instrumentation-anthropic": "^0.2.8",
|
|
159
|
+
"@arizeai/openinference-instrumentation-openai": "^4.2.7"
|
|
160
160
|
},
|
|
161
161
|
"devDependencies": {
|
|
162
|
-
"@anthropic-ai/sdk": "^0.
|
|
162
|
+
"@anthropic-ai/sdk": "^0.91.1",
|
|
163
163
|
"@opentelemetry/api": "^1.9.0",
|
|
164
|
-
"@opentelemetry/exporter-trace-otlp-http": "^0.
|
|
165
|
-
"@opentelemetry/resources": "^2.
|
|
166
|
-
"@opentelemetry/sdk-trace-base": "^
|
|
167
|
-
"@opentelemetry/sdk-trace-node": "^2.
|
|
164
|
+
"@opentelemetry/exporter-trace-otlp-http": "^0.222.0",
|
|
165
|
+
"@opentelemetry/resources": "^2.11.0",
|
|
166
|
+
"@opentelemetry/sdk-trace-base": "^2.11.0",
|
|
167
|
+
"@opentelemetry/sdk-trace-node": "^2.11.0",
|
|
168
168
|
"@opentelemetry/semantic-conventions": "^1.40.0",
|
|
169
169
|
"@types/node": "^20.0.0",
|
|
170
170
|
"openai": "^4.70.0",
|
|
@@ -177,7 +177,9 @@
|
|
|
177
177
|
"protobufjs": "^8.4.1",
|
|
178
178
|
"vite@6": "^6.4.3",
|
|
179
179
|
"postcss": "^8.5.18",
|
|
180
|
-
"picomatch@4": "^4.0.4"
|
|
180
|
+
"picomatch@4": "^4.0.4",
|
|
181
|
+
"nanoid": ">=3.3.18",
|
|
182
|
+
"esbuild": ">=0.28.1"
|
|
181
183
|
},
|
|
182
184
|
"pnpm": {
|
|
183
185
|
"overrides": {
|
|
@@ -185,7 +187,9 @@
|
|
|
185
187
|
"vitest": "^3.2.6",
|
|
186
188
|
"vite@6": "^6.4.3",
|
|
187
189
|
"postcss": "^8.5.18",
|
|
188
|
-
"picomatch@4": "^4.0.4"
|
|
190
|
+
"picomatch@4": "^4.0.4",
|
|
191
|
+
"nanoid": ">=3.3.18",
|
|
192
|
+
"esbuild": ">=0.28.1"
|
|
189
193
|
}
|
|
190
194
|
},
|
|
191
195
|
"bin": {
|
|
@@ -1,13 +1,6 @@
|
|
|
1
|
-
"""The one TOML reader for every Python
|
|
1
|
+
"""The one TOML reader for the SDK: the stdlib ``tomllib`` (every supported Python ships it).
|
|
2
|
+
Import it from here so a future fallback lives in one place."""
|
|
2
3
|
|
|
3
|
-
|
|
4
|
-
``tomli; python_version < '3.11'``). Import ``tomllib`` from here, never the
|
|
5
|
-
stdlib module directly, so the fallback lives in one place.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
try:
|
|
9
|
-
import tomllib
|
|
10
|
-
except ModuleNotFoundError: # pragma: no cover - python 3.10
|
|
11
|
-
import tomli as tomllib # type: ignore[no-redef]
|
|
4
|
+
import tomllib
|
|
12
5
|
|
|
13
6
|
__all__ = ["tomllib"]
|
|
@@ -51,21 +51,98 @@ class VerifierResult:
|
|
|
51
51
|
def split_sql_statements(script: str) -> List[str]:
|
|
52
52
|
"""Split a SQL script into complete statements (via
|
|
53
53
|
``sqlite3.complete_statement`` accumulation, so semicolons inside string
|
|
54
|
-
literals don't split).
|
|
54
|
+
literals don't split). Text that is only comments — a note after the last
|
|
55
|
+
semicolon, a commented-out block — is not a statement and is dropped, so
|
|
56
|
+
every reader of a verifier (check, grade, the host) sees the same list."""
|
|
55
57
|
statements: List[str] = []
|
|
56
58
|
buffer = ""
|
|
57
59
|
for line in script.splitlines(keepends=True):
|
|
58
60
|
buffer += line
|
|
59
61
|
if sqlite3.complete_statement(buffer):
|
|
60
|
-
|
|
61
|
-
if stripped:
|
|
62
|
-
statements.append(stripped)
|
|
62
|
+
_append_statement(statements, buffer)
|
|
63
63
|
buffer = ""
|
|
64
|
-
|
|
65
|
-
statements.append(buffer.strip())
|
|
64
|
+
_append_statement(statements, buffer)
|
|
66
65
|
return statements
|
|
67
66
|
|
|
68
67
|
|
|
68
|
+
def _append_statement(statements: List[str], text: str) -> None:
|
|
69
|
+
stripped = text.strip()
|
|
70
|
+
if _without_comments(stripped).strip(" \t\r\n;"):
|
|
71
|
+
statements.append(stripped)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _without_comments(sql: str) -> str:
|
|
75
|
+
"""``sql`` with ``--`` line comments and ``/* */`` blocks removed, string
|
|
76
|
+
literals left alone (a ``--`` inside quotes is text, not a comment)."""
|
|
77
|
+
out: List[str] = []
|
|
78
|
+
i, n = 0, len(sql)
|
|
79
|
+
quote: Optional[str] = None
|
|
80
|
+
while i < n:
|
|
81
|
+
ch = sql[i]
|
|
82
|
+
if quote is not None:
|
|
83
|
+
out.append(ch)
|
|
84
|
+
quote = None if ch == quote else quote
|
|
85
|
+
i += 1
|
|
86
|
+
continue
|
|
87
|
+
if ch in ("'", '"', "`"):
|
|
88
|
+
quote = ch
|
|
89
|
+
out.append(ch)
|
|
90
|
+
i += 1
|
|
91
|
+
continue
|
|
92
|
+
if sql.startswith("--", i):
|
|
93
|
+
end = sql.find("\n", i)
|
|
94
|
+
i = n if end == -1 else end
|
|
95
|
+
continue
|
|
96
|
+
if sql.startswith("/*", i):
|
|
97
|
+
end = sql.find("*/", i + 2)
|
|
98
|
+
i = n if end == -1 else end + 2
|
|
99
|
+
continue
|
|
100
|
+
out.append(ch)
|
|
101
|
+
i += 1
|
|
102
|
+
return "".join(out)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class SqlVerifier:
|
|
106
|
+
"""One ``verifiers/<name>.sql``: the statements it runs and the one score
|
|
107
|
+
rule every reader applies — ``schema check``, a local session's grade, the
|
|
108
|
+
World Host's grade. Statements run in order; the score is the first column
|
|
109
|
+
of the first row of the LAST statement that returns rows (or could). No
|
|
110
|
+
rows / NULL → ``None``, never coerced to 0."""
|
|
111
|
+
|
|
112
|
+
def __init__(self, name: str, script: str) -> None:
|
|
113
|
+
self.name = name[:-4] if name.endswith(".sql") else name
|
|
114
|
+
self.statements = split_sql_statements(script)
|
|
115
|
+
|
|
116
|
+
@classmethod
|
|
117
|
+
def read(cls, path: Path) -> "SqlVerifier":
|
|
118
|
+
return cls(path.stem, path.read_text(encoding="utf-8"))
|
|
119
|
+
|
|
120
|
+
def score(
|
|
121
|
+
self, execute: Callable[[str], Optional[List[Any]]]
|
|
122
|
+
) -> "tuple[Optional[float], Optional[str]]":
|
|
123
|
+
"""Run every statement through ``execute`` (rows, or ``None`` for a
|
|
124
|
+
statement that returns none, e.g. a temp-table DDL) and answer
|
|
125
|
+
``(score, sql)`` — the score and the statement it came from, both
|
|
126
|
+
``None`` when nothing returned rows."""
|
|
127
|
+
score: Optional[float] = None
|
|
128
|
+
last_sql: Optional[str] = None
|
|
129
|
+
for statement in self.statements:
|
|
130
|
+
rows = execute(statement)
|
|
131
|
+
if rows is None:
|
|
132
|
+
continue
|
|
133
|
+
last_sql = statement
|
|
134
|
+
value = rows[0][0] if rows else None
|
|
135
|
+
score = self._as_score(value)
|
|
136
|
+
return score, last_sql
|
|
137
|
+
|
|
138
|
+
def _as_score(self, value: Any) -> Optional[float]:
|
|
139
|
+
if value is None:
|
|
140
|
+
return None
|
|
141
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
142
|
+
raise VerifierError(f"verifier '{self.name}' returned {value!r}, not a number")
|
|
143
|
+
return float(value)
|
|
144
|
+
|
|
145
|
+
|
|
69
146
|
_NO_INPUT = object()
|
|
70
147
|
|
|
71
148
|
|
|
@@ -301,9 +378,9 @@ class World:
|
|
|
301
378
|
def verify(self, name: str) -> VerifierResult:
|
|
302
379
|
"""Run ``verifiers/<name>.sql`` against this world.
|
|
303
380
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
381
|
+
:class:`SqlVerifier` is the rule: the first column of the first row of
|
|
382
|
+
the LAST row-returning statement. NULL / no rows → score None (passed
|
|
383
|
+
None) — never coerced to 0.
|
|
307
384
|
"""
|
|
308
385
|
from .schema import ContractError
|
|
309
386
|
|
|
@@ -313,15 +390,11 @@ class World:
|
|
|
313
390
|
available = sorted(p.stem for p in (self.env.bundle_path / "verifiers").glob("*.sql"))
|
|
314
391
|
raise VerifierError(f"unknown verifier '{name}' (available: {available})")
|
|
315
392
|
|
|
316
|
-
|
|
317
|
-
score: Optional[float] = None
|
|
318
|
-
saw_rows = False
|
|
319
|
-
last_sql = ""
|
|
320
|
-
for statement in split_sql_statements(script):
|
|
393
|
+
def execute(statement: str) -> Optional[List[Any]]:
|
|
321
394
|
try:
|
|
322
395
|
if self.store is not None:
|
|
323
396
|
try:
|
|
324
|
-
|
|
397
|
+
return self.store.query(statement, private=True)
|
|
325
398
|
except ContractError as error:
|
|
326
399
|
# A verifier naming a column the contract never projected
|
|
327
400
|
# must fail with SQLite's own words and the statement, not
|
|
@@ -332,24 +405,18 @@ class World:
|
|
|
332
405
|
f"verifier '{verifier_name}' failed: {reason}",
|
|
333
406
|
sql=statement,
|
|
334
407
|
) from error
|
|
335
|
-
saw_rows = True
|
|
336
|
-
last_sql = statement
|
|
337
|
-
value = rows[0][0] if rows else None
|
|
338
|
-
score = float(value) if value is not None else None
|
|
339
|
-
continue
|
|
340
408
|
cursor = self._conn.execute(statement)
|
|
341
409
|
except sqlite3.Error as error:
|
|
342
410
|
raise VerifierError(
|
|
343
411
|
f"verifier '{verifier_name}' failed: {error}\n SQL: {statement}"
|
|
344
412
|
) from error
|
|
345
|
-
if cursor.description is
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
if not saw_rows:
|
|
413
|
+
if cursor.description is None:
|
|
414
|
+
return None
|
|
415
|
+
row = cursor.fetchone()
|
|
416
|
+
return [] if row is None else [row]
|
|
417
|
+
|
|
418
|
+
score, last_sql = SqlVerifier.read(path).score(execute)
|
|
419
|
+
if last_sql is None:
|
|
353
420
|
detail = "no row-returning statement — score is null"
|
|
354
421
|
elif score is None:
|
|
355
422
|
detail = f"null score from: {last_sql}"
|
|
@@ -28,13 +28,13 @@ import sqlite3
|
|
|
28
28
|
from contextvars import ContextVar
|
|
29
29
|
from typing import Any, Callable, Dict, Optional
|
|
30
30
|
|
|
31
|
+
from .schema.store_contract import STORE_METHOD_NAMES, unknown_store_method_detail
|
|
32
|
+
|
|
31
33
|
__all__ = ["tool", "db", "store"]
|
|
32
34
|
|
|
33
35
|
#: The active world's sqlite connection. Set by ``World.execute`` around each
|
|
34
36
|
#: handler invocation; ``db`` reads it at call time.
|
|
35
|
-
_active_connection: ContextVar[Optional[sqlite3.Connection]] = ContextVar(
|
|
36
|
-
"gateway_env_active_connection", default=None
|
|
37
|
-
)
|
|
37
|
+
_active_connection: ContextVar[Optional[sqlite3.Connection]] = ContextVar("gateway_env_active_connection", default=None)
|
|
38
38
|
|
|
39
39
|
#: The registry the loader is currently filling. ``@tool`` registers into it
|
|
40
40
|
#: at module-exec time; outside a bundle load the decorator is a no-op tagger.
|
|
@@ -47,11 +47,15 @@ _strict_registration: ContextVar[bool] = ContextVar("gateway_env_strict_registra
|
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
class _Store:
|
|
50
|
-
"""Entity operations bind to the currently executing schema-backed world.
|
|
50
|
+
"""Entity operations bind to the currently executing schema-backed world.
|
|
51
|
+
|
|
52
|
+
Exactly the methods of :data:`~gatewaysdk.environment.schema.store_contract.STORE_METHODS`
|
|
53
|
+
are reachable; any other name is a handler bug, refused with the list of
|
|
54
|
+
what the store does offer so the author never has to guess."""
|
|
51
55
|
|
|
52
56
|
def __getattr__(self, name):
|
|
53
|
-
if name not in
|
|
54
|
-
raise AttributeError(name)
|
|
57
|
+
if name not in STORE_METHOD_NAMES:
|
|
58
|
+
raise AttributeError(unknown_store_method_detail(name))
|
|
55
59
|
active = _active_store.get()
|
|
56
60
|
if active is None:
|
|
57
61
|
raise RuntimeError("gateway_env.store requires an active schema-backed world")
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Lean CLI, dependency-free: python -m gatewaysdk.environment.schema <command> PATH.
|
|
2
2
|
|
|
3
3
|
init | templates | compile | check | describe | schema-reconcile the world contract
|
|
4
|
+
init --from-openapi <spec> turn a Swagger 2.0 / OpenAPI 3.x document into a world (entities + routes)
|
|
4
5
|
ui-init declare [ui] in connector.toml and generate ui/static from the contract
|
|
5
6
|
connector-validate | connector-capture | connector-ingest | conform its vendor connection
|
|
6
7
|
data-show | data-import the rows a local world seeds with
|
|
@@ -149,6 +150,28 @@ def main(argv=None):
|
|
|
149
150
|
help="init only: connector template to start from (default: custom)",
|
|
150
151
|
)
|
|
151
152
|
parser.add_argument("--name", default=None, help="init only: environment name and package id")
|
|
153
|
+
parser.add_argument(
|
|
154
|
+
"--from-openapi",
|
|
155
|
+
default=None,
|
|
156
|
+
metavar="SOURCE",
|
|
157
|
+
help="init: build the world from a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML): a local path or an "
|
|
158
|
+
"http(s) URL fetched from here; drive:<path> needs the platform CLI. Prints the plan, writes the tree.",
|
|
159
|
+
)
|
|
160
|
+
parser.add_argument(
|
|
161
|
+
"--plan-only", action="store_true", help="init --from-openapi: print the plan and write nothing"
|
|
162
|
+
)
|
|
163
|
+
parser.add_argument(
|
|
164
|
+
"--include",
|
|
165
|
+
action="append",
|
|
166
|
+
metavar="GLOB|TAG",
|
|
167
|
+
help="init --from-openapi: only operations whose path matches the glob or that carry the tag; repeatable",
|
|
168
|
+
)
|
|
169
|
+
parser.add_argument(
|
|
170
|
+
"--exclude",
|
|
171
|
+
action="append",
|
|
172
|
+
metavar="GLOB|TAG",
|
|
173
|
+
help="init --from-openapi: drop operations whose path matches the glob or that carry the tag; repeatable",
|
|
174
|
+
)
|
|
152
175
|
parser.add_argument(
|
|
153
176
|
"--with-example-tool",
|
|
154
177
|
action="store_true",
|
|
@@ -217,6 +240,18 @@ def main(argv=None):
|
|
|
217
240
|
if args.command == "describe":
|
|
218
241
|
print(describe_bundle(args.bundle), end="")
|
|
219
242
|
return
|
|
243
|
+
if args.command == "init" and args.from_openapi:
|
|
244
|
+
from .openapi_import import OpenApiImport
|
|
245
|
+
|
|
246
|
+
if args.connector != "custom":
|
|
247
|
+
raise SystemExit("--from-openapi builds the connector from the document; drop --connector")
|
|
248
|
+
importer = OpenApiImport.read(
|
|
249
|
+
args.from_openapi, include=tuple(args.include or ()), exclude=tuple(args.exclude or ())
|
|
250
|
+
)
|
|
251
|
+
print(
|
|
252
|
+
json.dumps(importer.plan().run(args.bundle, plan_only=args.plan_only, name=args.name), sort_keys=True)
|
|
253
|
+
)
|
|
254
|
+
return
|
|
220
255
|
if args.command == "init":
|
|
221
256
|
contract = init_bundle(
|
|
222
257
|
args.bundle, connector=args.connector, name=args.name, with_example_tool=args.with_example_tool
|
|
@@ -108,6 +108,7 @@ from . import snapshot as snapshots
|
|
|
108
108
|
from .bundle_runtime import stage_runtime_package
|
|
109
109
|
from .compiler import check_bundle, compile_bundle
|
|
110
110
|
from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
|
|
111
|
+
from .credentials import basic_user
|
|
111
112
|
from .grading import keyed_entities
|
|
112
113
|
from .store import SQLiteStore
|
|
113
114
|
from .validation import ContractError, fail, read_json
|
|
@@ -295,7 +296,8 @@ def _str_map(table: dict, key: str, location: str) -> dict[str, str]:
|
|
|
295
296
|
return dict(mapping)
|
|
296
297
|
|
|
297
298
|
|
|
298
|
-
def
|
|
299
|
+
def operation_api_settings(entry: dict, operation: Operation, location: str) -> OperationSettings:
|
|
300
|
+
"""One operation's ``[operations.api]`` table, validated against the operation it belongs to."""
|
|
299
301
|
table = entry.get(FILTER_KEY, {})
|
|
300
302
|
if not isinstance(table, dict):
|
|
301
303
|
fail("spec", "[operations.api] must be a table", location)
|
|
@@ -466,11 +468,10 @@ def load_api_settings(world: Path, spec: Spec) -> ApiSettings:
|
|
|
466
468
|
handlers = world / (handlers_name or HANDLERS)
|
|
467
469
|
if handlers_name is not None and not handlers.is_file():
|
|
468
470
|
fail("spec", f"api.handlers names {handlers_name}, which is not a file in the world", "/api/handlers")
|
|
469
|
-
operations
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
operations[operation.name] = _operation_settings(entry, operation, location)
|
|
471
|
+
operations = {
|
|
472
|
+
e["name"]: operation_api_settings(e, spec.operations[e["name"]], f"/operations/{i}/api")
|
|
473
|
+
for i, e in enumerate(raw.get("operations", []))
|
|
474
|
+
}
|
|
474
475
|
before = api.get("before")
|
|
475
476
|
if before is not None and (not isinstance(before, str) or not re.match(r"^[a-z_][a-z0-9_]*$", before)):
|
|
476
477
|
fail("spec", "api.before names a function in the world's handlers module", "/api/before")
|
|
@@ -780,14 +781,14 @@ def _matches(
|
|
|
780
781
|
return True
|
|
781
782
|
|
|
782
783
|
|
|
783
|
-
def
|
|
784
|
-
"""The filter a route with no ``filter`` of its own implies: each ``{segment}``
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
return {name: name for name in operation.placeholders if name in
|
|
784
|
+
def _segment_filter(operation: Operation, ctx: ApiContext, entity: str) -> dict[str, str]:
|
|
785
|
+
"""The filter a route with no ``filter`` of its own implies: each ``{segment}`` of its
|
|
786
|
+
path that names a top-level field of its entity narrows to the rows carrying that value
|
|
787
|
+
(``GET /wallets/{id}``, ``GET /records/by-email/{email}``). A segment naming no field
|
|
788
|
+
(``/users/{userId}/labels``, a scope the caller must send) implies nothing; a declared
|
|
789
|
+
``filter`` says it all itself. Twin: ``route_bindings.implied_filters``."""
|
|
790
|
+
fields = ctx.fields_of(entity)
|
|
791
|
+
return {name: name for name in operation.placeholders if name in fields}
|
|
791
792
|
|
|
792
793
|
|
|
793
794
|
def _query_field_filter(ctx: ApiContext, entity: str, sent: dict[str, list[str]]) -> dict[str, str]:
|
|
@@ -925,11 +926,7 @@ class Router:
|
|
|
925
926
|
value = headers.get("authorization", "")
|
|
926
927
|
presented = value[7:] if value.lower().startswith("bearer ") else None
|
|
927
928
|
elif spec.scheme == "basic":
|
|
928
|
-
|
|
929
|
-
try:
|
|
930
|
-
presented = base64.b64decode(value[6:]).decode() if value.lower().startswith("basic ") else None
|
|
931
|
-
except (ValueError, UnicodeDecodeError):
|
|
932
|
-
presented = None
|
|
929
|
+
presented = basic_user(headers.get("authorization", ""))
|
|
933
930
|
elif spec.scheme == "query":
|
|
934
931
|
presented = query.get(spec.auth_name or "")
|
|
935
932
|
if not presented:
|
|
@@ -1649,7 +1646,7 @@ class WorldApp:
|
|
|
1649
1646
|
if op.action == "create":
|
|
1650
1647
|
row = ctx.insert(entity, _bind(op, ctx, body))
|
|
1651
1648
|
return op.status, row if op.single else wrap([row], ctx.settings, op.envelope)
|
|
1652
|
-
filters = op.filter or
|
|
1649
|
+
filters = op.filter or _segment_filter(operation, ctx, entity)
|
|
1653
1650
|
if not op.filter and op.action == "read":
|
|
1654
1651
|
filters = {**_query_field_filter(ctx, entity, ctx.query_all), **filters}
|
|
1655
1652
|
# A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
|
|
@@ -1839,12 +1836,11 @@ def openapi_document(spec: Spec, contract_id: str, settings: ApiSettings | None
|
|
|
1839
1836
|
paths: dict[str, dict] = {}
|
|
1840
1837
|
for operation in spec.operations.values():
|
|
1841
1838
|
op = settings.for_operation(operation.name) if settings else OperationSettings()
|
|
1839
|
+
queries = [name for name in operation.params if name not in operation.placeholders]
|
|
1842
1840
|
params = [
|
|
1843
1841
|
{"name": name, "in": "path", "required": True, "schema": {"type": "string"}}
|
|
1844
1842
|
for name in operation.placeholders
|
|
1845
|
-
] + [
|
|
1846
|
-
{"name": name, "in": "query", "required": False, "schema": {"type": "string"}} for name in operation.params
|
|
1847
|
-
]
|
|
1843
|
+
] + [{"name": name, "in": "query", "required": False, "schema": {"type": "string"}} for name in queries]
|
|
1848
1844
|
entry: dict[str, Any] = {
|
|
1849
1845
|
"operationId": operation.name,
|
|
1850
1846
|
"parameters": params,
|
|
@@ -25,9 +25,11 @@ from dataclasses import dataclass, field
|
|
|
25
25
|
from pathlib import Path
|
|
26
26
|
from typing import IO, Any, Iterable, Iterator
|
|
27
27
|
|
|
28
|
+
from .cast import TEXT_CASTER, EntityCasters
|
|
28
29
|
from .compiler import Contract, quote
|
|
29
|
-
from .
|
|
30
|
+
from .csv_rows import csv_delimiter, is_delimited_name, iter_csv_records
|
|
30
31
|
from .redaction import RowRedactor
|
|
32
|
+
from .store import SQLiteStore
|
|
31
33
|
from .validation import ContractError, fail, validate
|
|
32
34
|
|
|
33
35
|
CHUNK_BYTES = 64 * 1024 * 1024
|
|
@@ -124,13 +126,17 @@ class _Reader:
|
|
|
124
126
|
return rest
|
|
125
127
|
|
|
126
128
|
|
|
127
|
-
def _open_text(source: str | Path) -> IO[str]:
|
|
129
|
+
def _open_text(source: str | Path, *, delimited: bool = False) -> IO[str]:
|
|
130
|
+
"""The file as UTF-8 text; a delimited file keeps its line endings (``newline=""``) and
|
|
131
|
+
loses a BOM, as the csv reader wants them."""
|
|
132
|
+
encoding = "utf-8-sig" if delimited else "utf-8"
|
|
133
|
+
newline = "" if delimited else None
|
|
128
134
|
if str(source) == "-":
|
|
129
|
-
return io.TextIOWrapper(sys.stdin.buffer, encoding=
|
|
135
|
+
return io.TextIOWrapper(sys.stdin.buffer, encoding=encoding, newline=newline)
|
|
130
136
|
path = Path(source)
|
|
131
|
-
if path.suffix == ".gz":
|
|
132
|
-
return io.TextIOWrapper(gzip.open(path, "rb"), encoding=
|
|
133
|
-
return open(path, "r", encoding=
|
|
137
|
+
if path.suffix.lower() == ".gz":
|
|
138
|
+
return io.TextIOWrapper(gzip.open(path, "rb"), encoding=encoding, newline=newline)
|
|
139
|
+
return open(path, "r", encoding=encoding, newline=newline)
|
|
134
140
|
|
|
135
141
|
|
|
136
142
|
def _row(value: Any, where: str) -> dict:
|
|
@@ -255,28 +261,76 @@ def iter_file_rows(
|
|
|
255
261
|
yield from _stream_document(reader, where, entity, seen)
|
|
256
262
|
|
|
257
263
|
|
|
264
|
+
def csv_needs_entity(where: str) -> str:
|
|
265
|
+
"""Why a CSV without ``--entity`` is refused: nothing in the file says which entity its columns fill."""
|
|
266
|
+
return (
|
|
267
|
+
f"{where} is a CSV; say which entity its columns fill with --entity (the header row names the fields; "
|
|
268
|
+
"cells are cast by the entity's field types, an empty cell is null when the field allows it)"
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def iter_csv_rows(
|
|
273
|
+
stream: IO[str],
|
|
274
|
+
where: str,
|
|
275
|
+
*,
|
|
276
|
+
entity: str | None,
|
|
277
|
+
seen: dict[str, int] | None = None,
|
|
278
|
+
delimiter: str = ",",
|
|
279
|
+
casters: EntityCasters | None = None,
|
|
280
|
+
) -> Iterator[tuple[str, dict]]:
|
|
281
|
+
"""``(entity, row)`` from a delimited file, one record at a time (a quoted cell may span
|
|
282
|
+
lines): the header row names the fields, each cell is cast by the entity's field type
|
|
283
|
+
(:mod:`.cast`) or kept as text when no ``casters`` are given, an empty cell is ``None``."""
|
|
284
|
+
if entity is None:
|
|
285
|
+
fail("rows", csv_needs_entity(where), where)
|
|
286
|
+
if seen is not None:
|
|
287
|
+
seen.setdefault(entity, 0)
|
|
288
|
+
caster = (casters(entity) if casters else None) or TEXT_CASTER
|
|
289
|
+
for number, record in iter_csv_records(stream, where, delimiter):
|
|
290
|
+
if seen is not None:
|
|
291
|
+
seen[entity] += 1
|
|
292
|
+
yield entity, caster.row(record, f"{where}:{number}")
|
|
293
|
+
|
|
294
|
+
|
|
258
295
|
def iter_rows(
|
|
259
|
-
sources: Iterable[str | Path],
|
|
296
|
+
sources: Iterable[str | Path],
|
|
297
|
+
*,
|
|
298
|
+
entity: str | None = None,
|
|
299
|
+
seen: dict[str, int] | None = None,
|
|
300
|
+
delimiter: str | None = None,
|
|
301
|
+
casters: EntityCasters | None = None,
|
|
260
302
|
) -> Iterator[tuple[str, dict]]:
|
|
261
|
-
"""Stream ``(entity, row)`` from rows files (``-`` is stdin, ``.gz`` is gunzipped
|
|
262
|
-
|
|
303
|
+
"""Stream ``(entity, row)`` from rows files (``-`` is stdin, ``.gz`` is gunzipped, ``.csv``/``.tsv``
|
|
304
|
+
with ``--entity``, split by ``delimiter`` or by name, cells cast by ``casters``) in order, never
|
|
305
|
+
holding a whole file."""
|
|
263
306
|
for source in sources:
|
|
264
307
|
where = str(source)
|
|
308
|
+
delimited = is_delimited_name(where)
|
|
265
309
|
try:
|
|
266
|
-
stream = _open_text(source)
|
|
310
|
+
stream = _open_text(source, delimited=delimited)
|
|
267
311
|
except OSError:
|
|
268
312
|
fail("rows", f"Cannot read {where}", where)
|
|
269
313
|
with stream:
|
|
270
314
|
try:
|
|
271
|
-
|
|
315
|
+
if delimited:
|
|
316
|
+
split = csv_delimiter(where, delimiter)
|
|
317
|
+
yield from iter_csv_rows(stream, where, entity=entity, seen=seen, delimiter=split, casters=casters)
|
|
318
|
+
else:
|
|
319
|
+
yield from iter_file_rows(stream, where, entity=entity, seen=seen)
|
|
272
320
|
except UnicodeError:
|
|
273
321
|
fail("rows", f"Cannot read {where} as UTF-8 text", where)
|
|
274
322
|
|
|
275
323
|
|
|
276
|
-
def count_rows(
|
|
324
|
+
def count_rows(
|
|
325
|
+
sources: Iterable[str | Path],
|
|
326
|
+
*,
|
|
327
|
+
entity: str | None = None,
|
|
328
|
+
cap: int | None = None,
|
|
329
|
+
delimiter: str | None = None,
|
|
330
|
+
) -> int:
|
|
277
331
|
"""How many rows the files hold, stopping early past ``cap``."""
|
|
278
332
|
total = 0
|
|
279
|
-
for _ in iter_rows(sources, entity=entity):
|
|
333
|
+
for _ in iter_rows(sources, entity=entity, delimiter=delimiter):
|
|
280
334
|
total += 1
|
|
281
335
|
if cap is not None and total > cap:
|
|
282
336
|
break
|
|
@@ -714,7 +768,7 @@ def build_batch(
|
|
|
714
768
|
refusal found. Per chunk: one transaction per chunk, a refused chunk is left out
|
|
715
769
|
and the rest applied. ``dry_run`` finds the refusals and applies nothing.
|
|
716
770
|
``redactor`` (:func:`redaction.row_redactor`) runs over every row before staging."""
|
|
717
|
-
from .snapshot import batch_digest, file_sha256, write_sidecar
|
|
771
|
+
from .snapshot import batch_digest, contract_hash, file_sha256, write_sidecar
|
|
718
772
|
|
|
719
773
|
if mode not in ("append", "replace"):
|
|
720
774
|
fail("mode", "mode must be append or replace", "/mode")
|
|
@@ -8,7 +8,7 @@ from pathlib import Path
|
|
|
8
8
|
SCHEMA_PACKAGE = Path(__file__).parent
|
|
9
9
|
ENVIRONMENT_PACKAGE = SCHEMA_PACKAGE.parent
|
|
10
10
|
# Modules outside `schema/` that its modules import; copied beside them so `main.py` resolves them.
|
|
11
|
-
SIBLING_MODULES = ("_toml.py",)
|
|
11
|
+
SIBLING_MODULES = ("_toml.py", "world_session_contract.py")
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def stage_runtime_package(server: Path) -> Path:
|
|
@@ -18,7 +18,9 @@ def stage_runtime_package(server: Path) -> Path:
|
|
|
18
18
|
for package_dir in ("gatewaysdk", "gatewaysdk/environment"):
|
|
19
19
|
(server / package_dir / "__init__.py").write_text("")
|
|
20
20
|
for name in SIBLING_MODULES:
|
|
21
|
-
shutil.copy(
|
|
21
|
+
shutil.copy(
|
|
22
|
+
ENVIRONMENT_PACKAGE / name, server / "gatewaysdk" / "environment" / name
|
|
23
|
+
)
|
|
22
24
|
for module in sorted(SCHEMA_PACKAGE.glob("*.py")):
|
|
23
25
|
shutil.copy(module, package / module.name)
|
|
24
26
|
return package
|