@withgateway/sdk 0.3.0 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +6 -1
  2. package/dist/cli.js +3971 -2338
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3950 -2318
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +214 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +206 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
  13. package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1456 -564
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1433 -557
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +211 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +203 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +96 -21
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +96 -21
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +300 -123
  35. package/dist/worlds/index.d.ts +300 -123
  36. package/dist/worlds/index.js +1385 -508
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1377 -501
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -515
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  57. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  58. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  59. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  60. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  61. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  62. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  63. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  64. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  65. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  66. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  67. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  68. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  69. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  70. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  71. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  72. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  73. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  74. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  75. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  76. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  77. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  78. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  79. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  80. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  81. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  82. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  83. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  84. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  86. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  87. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  88. package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
  89. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  91. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  92. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  93. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  94. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  95. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  96. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  97. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  99. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  103. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  104. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  105. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  106. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  107. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  108. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  109. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  110. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  111. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  112. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  113. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  114. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  115. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  116. package/runtime/gatewaysdk/environment/score.py +57 -0
  117. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  118. package/runtime/gatewaysdk/environment/world.py +42 -3
  119. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  120. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  121. package/skills/connector-schema-authoring/SKILL.md +22 -14
  122. package/skills/connector-schema-authoring/connector-worlds.md +26 -10
  123. package/skills/connector-schema-authoring/world-schema.md +111 -51
  124. package/skills/world-data-ingestion/ROWS.md +29 -3
  125. package/skills/world-data-ingestion/SKILL.md +16 -16
  126. package/skills/worlds-getting-started/SKILL.md +18 -18
  127. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  128. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  129. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  130. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -22,15 +22,15 @@ from __future__ import annotations
22
22
 
23
23
  import re
24
24
  from collections import Counter, deque
25
+ from collections.abc import Iterator
25
26
  from dataclasses import dataclass, field
27
+ from itertools import islice
26
28
  from typing import Any
27
29
 
28
30
  from .compiler import FIELD_RESERVED, RESERVED, RESERVED_STATE_KEYS
29
- from .openapi_document import OpenApiDocument
30
- from .validation import fail, pointer
31
+ from .openapi_document import OpenApiDocument, unescape
32
+ from .validation import FIELD_NAME, fail, pointer
31
33
 
32
- IDENTIFIER = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$")
33
- FIELD_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
34
34
  RESERVED_FIELDS = frozenset({"_key", "payload"})
35
35
  #: A ``$def`` for a value the vendor leaves untyped (``{}``, ``additionalProperties: true``
36
36
  #: at the top). Not recursive on purpose: the validator wants recursive targets typed.
@@ -103,10 +103,10 @@ class Names:
103
103
  @staticmethod
104
104
  def field(source: str) -> str:
105
105
  """A property name the compiler accepts; the source name when it already is one."""
106
- name = re.sub(r"[^A-Za-z0-9_]", "_", source)
107
- if not name or not FIELD_IDENTIFIER.match(name):
106
+ name = re.sub(r"[^A-Za-z0-9_-]", "_", source)
107
+ if not name or not FIELD_NAME.match(name):
108
108
  name = "f_" + name
109
- if name in RESERVED_FIELDS or name.startswith(FIELD_RESERVED):
109
+ if name.lower() in RESERVED_FIELDS or name.lower().startswith(FIELD_RESERVED):
110
110
  name = "f_" + name
111
111
  return name
112
112
 
@@ -149,6 +149,8 @@ class EntityKey:
149
149
  """Which property is a schema's primary key, and whether it is fit to be one."""
150
150
 
151
151
  UNIQUE_EXTENSIONS = ("x-unique", "x-primary-key", "x-primaryKey", "x-identifier")
152
+ NESTED_DEPTH = 4
153
+ NESTED_VISITS = 10_000
152
154
 
153
155
  def __init__(self, document: OpenApiDocument, source: str, properties: dict[str, Any], required: set[str]):
154
156
  self.document = document
@@ -156,9 +158,8 @@ class EntityKey:
156
158
  self.properties = properties
157
159
  self.required = required
158
160
 
159
- def detect(self, *, fallback: bool = True) -> str | None:
160
- """``id``, ``<name>Id``/``<name>_id``, a ``readOnly`` property marked unique by an
161
- ``x-`` extension, else (when ``fallback``) the first required scalar."""
161
+ def declared(self) -> str | None:
162
+ """``id``, ``<name>Id``/``<name>_id``, or a ``readOnly`` property marked unique by an ``x-`` extension."""
162
163
  for name in self._id_like():
163
164
  if self._keyable(name):
164
165
  return name
@@ -170,13 +171,74 @@ class EntityKey:
170
171
  and self._keyable(name)
171
172
  ):
172
173
  return name
174
+ return None
175
+
176
+ def detect(self, *, lookups: tuple[str, ...] = (), fallback: bool = True) -> str | None:
177
+ """:meth:`declared`, else the field a route's own ``{segment}`` looks the row up by
178
+ (``/works/{doi}`` → ``DOI``; nested, ``/studies/{nctId}`` → a JSON Pointer), else
179
+ (when ``fallback``) the first required scalar."""
180
+ declared = self.declared()
181
+ if declared is not None:
182
+ return declared
183
+ for own in sorted(lookups):
184
+ found = self.top_level(own) or self.nested(own)
185
+ if found is not None:
186
+ return found
173
187
  if not fallback:
174
188
  return None
175
- for name in self.properties:
176
- if name in self.required and self._keyable(name):
177
- return name
189
+ return next((name for name in self.properties if name in self.required and self._keyable(name)), None)
190
+
191
+ def top_level(self, argument: str) -> str | None:
192
+ """The keyable property ``argument`` names, spelled however the vendor spells it."""
193
+ wanted = Names.snake(argument)
194
+ return next((n for n in self.properties if Names.snake(n) == wanted and self._keyable(n)), None)
195
+
196
+ def nested(self, argument: str) -> str | None:
197
+ """The JSON Pointer of the one keyable scalar ``argument`` names inside the
198
+ schema's objects (never inside an array or a nullable object), the shallowest
199
+ when several depths match; None when nothing does, one depth holds two, or the
200
+ search would read more than ``NESTED_VISITS`` properties (objects that share one
201
+ another many times over)."""
202
+ wanted = Names.snake(argument)
203
+ level: list[tuple[str, dict]] = [("", self.properties)]
204
+ left = self.NESTED_VISITS
205
+ for depth in range(self.NESTED_DEPTH):
206
+ members = list(islice(_members(level), left + 1))
207
+ if len(members) > left:
208
+ return None
209
+ left -= len(members)
210
+ nodes = [(here, name, raw, self._resolved(raw)) for here, name, raw in members]
211
+ found = [
212
+ here
213
+ for here, name, _, node in nodes
214
+ if depth > 0 and Names.snake(name) == wanted and self._scalar_key(node)
215
+ ]
216
+ if found:
217
+ return found[0] if len(found) == 1 else None
218
+ level = [(here, node["properties"]) for here, _, raw, node in nodes if self._step(raw, node)]
178
219
  return None
179
220
 
221
+ def leaf(self, key: str) -> dict:
222
+ """The schema of the scalar ``key`` (a property name or a JSON Pointer) names."""
223
+ properties, resolved = self.properties, {}
224
+ for part in key_parts(key):
225
+ resolved = self._resolved(properties.get(part, {}))
226
+ properties = resolved.get("properties") if isinstance(resolved.get("properties"), dict) else {}
227
+ return resolved
228
+
229
+ @staticmethod
230
+ def _scalar_key(resolved: dict) -> bool:
231
+ return type_list(resolved.get("type")) in (["string"], ["integer"]) and not _nullable(resolved)
232
+
233
+ @staticmethod
234
+ def _step(raw: Any, resolved: dict) -> bool:
235
+ """An object a nested key may pass through: never null (where it is used or where it
236
+ is defined), never a list."""
237
+ nullable = (isinstance(raw, dict) and _nullable(raw)) or _nullable(resolved)
238
+ return (
239
+ not nullable and isinstance(resolved.get("properties"), dict) and resolved.get("type") in (None, "object")
240
+ )
241
+
180
242
  def scalars(self) -> frozenset[str]:
181
243
  """Properties a filter can compare: string, integer, number or boolean."""
182
244
  return frozenset(name for name in self.properties if self._scalar_type(name) in SCALARS)
@@ -211,18 +273,21 @@ class EntityKey:
211
273
  class ConvertedDef:
212
274
  name: str
213
275
  schema: dict
214
- fields: dict[str, str] # entity roots: source property → field name
276
+ fields: dict[str, str]
215
277
 
216
278
 
217
279
  class SchemaConverter:
218
280
  """The document's schemas as world.json ``$defs``, converted once each."""
219
281
 
220
282
  def __init__(self, document: OpenApiDocument, roots: dict[str, str]):
221
- """``roots`` maps the source names that become entities to their key property."""
283
+ """``roots`` maps the source names that become entities to their key: a property
284
+ name, or a JSON Pointer (source spelling) to a scalar nested in the root."""
222
285
  self.document = document
223
286
  self.roots = roots
224
287
  self.defs: dict[str, dict] = {}
225
288
  self.fields: dict[str, dict[str, str]] = {}
289
+ self.keys: dict[str, str] = {}
290
+ self._nested: dict[str, list[str]] = {}
226
291
  self.dropped: Counter[str] = Counter()
227
292
  self.renamed: list[tuple[str, str, str]] = []
228
293
  self.warnings: list[str] = []
@@ -236,8 +301,6 @@ class SchemaConverter:
236
301
  for name in self.document.schemas:
237
302
  self._names.claim(name)
238
303
 
239
- # ------------------------------------------------------------- public
240
-
241
304
  def convert_all(self, wanted: set[str]) -> dict[str, dict]:
242
305
  """Convert ``wanted`` and everything they reference, in order of discovery."""
243
306
  for name in sorted(wanted):
@@ -247,8 +310,20 @@ class SchemaConverter:
247
310
  if name in self.defs:
248
311
  continue
249
312
  self.defs[name] = self._convert_def(name)
313
+ for root, parts in self._nested.items():
314
+ self._require_along(self.defs[root], parts)
250
315
  return dict(sorted(self.defs.items()))
251
316
 
317
+ def _require_along(self, node: dict, parts: list[str]) -> None:
318
+ """Every object on a nested key's path requires the next step, so every row carries the key."""
319
+ for part in parts:
320
+ while isinstance(node.get("$ref"), str):
321
+ node = self.defs[unescape(node["$ref"][len("#/$defs/") :])]
322
+ listed = node.setdefault("required", [])
323
+ if part not in listed:
324
+ listed.append(part)
325
+ node = node["properties"][part]
326
+
252
327
  def reference(self, name: str) -> str:
253
328
  """The ``$ref`` a world.json node uses for a top-level schema."""
254
329
  self._want(name)
@@ -264,8 +339,6 @@ class SchemaConverter:
264
339
  if name not in self.defs and name not in self._queue:
265
340
  self._queue.append(name)
266
341
 
267
- # ------------------------------------------------------------- one top-level schema
268
-
269
342
  def _convert_def(self, name: str) -> dict:
270
343
  at = f"{self.document.schemas_pointer}/{pointer(name)}"
271
344
  raw = self.document.schemas[name]
@@ -285,7 +358,7 @@ class SchemaConverter:
285
358
  def _entity_root(self, name: str, merged: dict, at: str, key: str) -> dict:
286
359
  converted = self.convert(merged, at, (name,), resolve_root=False)
287
360
  properties = converted.get("properties")
288
- if "object" not in _type_list(converted.get("type")) or not isinstance(properties, dict):
361
+ if "object" not in type_list(converted.get("type")) or not isinstance(properties, dict):
289
362
  fail("spec", f"Schema {name} is not an object schema, so it cannot be an entity", at)
290
363
  converted["type"] = "object"
291
364
  converted.setdefault("title", name)
@@ -293,10 +366,14 @@ class SchemaConverter:
293
366
  renamed = Names.fields(properties)
294
367
  self.renamed.extend((f"{at}/properties/{pointer(src)}", src, dst) for src, dst in renamed.items() if src != dst)
295
368
  converted["properties"] = {renamed[k]: v for k, v in properties.items()}
369
+ head, *rest = key_parts(key)
296
370
  required = [renamed[r] for r in converted.get("required", []) if r in renamed]
297
- if renamed[key] not in required:
298
- required.append(renamed[key])
371
+ if renamed[head] not in required:
372
+ required.append(renamed[head])
299
373
  converted["required"] = required
374
+ self.keys[name] = "/".join(["", *(pointer(p) for p in (renamed[head], *rest))]) if rest else renamed[head]
375
+ if rest:
376
+ self._nested[name] = [renamed[head], *rest]
300
377
  converted["additionalProperties"] = False
301
378
  converted.pop("anyOf", None)
302
379
  converted.pop("oneOf", None)
@@ -320,8 +397,6 @@ class SchemaConverter:
320
397
  out[key] = raw[key]
321
398
  return {"anyOf": [out, {"type": "null"}]} if _nullable(raw) else out
322
399
 
323
- # ------------------------------------------------------------- allOf
324
-
325
400
  def merged(self, resolved: dict, at: str, chain: tuple[str, ...]) -> dict:
326
401
  """:meth:`merge` of a named schema, computed once: a diamond of ``allOf``
327
402
  members (``A → B, C → D``) costs one merge of ``D``, not one per path."""
@@ -399,8 +474,6 @@ class SchemaConverter:
399
474
  at,
400
475
  )
401
476
 
402
- # ------------------------------------------------------------- one schema node
403
-
404
477
  def convert(self, raw: Any, at: str, stack: tuple[str, ...], *, resolve_root: bool = True) -> dict:
405
478
  if raw is True or raw == {}:
406
479
  return self.any_json()
@@ -463,7 +536,7 @@ class SchemaConverter:
463
536
 
464
537
  def _types(self, raw: dict, at: str) -> tuple[list[str], bool]:
465
538
  declared = raw.get("type")
466
- types = ["string" if t == "file" else t for t in _type_list(declared)]
539
+ types = ["string" if t == "file" else t for t in type_list(declared)]
467
540
  unknown = [t for t in types if t not in _TYPES]
468
541
  if unknown:
469
542
  fail("spec", f"Unknown type {unknown[0]!r}", f"{at}/type")
@@ -501,8 +574,6 @@ class SchemaConverter:
501
574
  return out
502
575
  return self.any_json()
503
576
 
504
- # ------------------------------------------------------------- per type
505
-
506
577
  def _convert_object(self, raw: dict, out: dict, at: str, stack: tuple[str, ...]) -> None:
507
578
  self._object_properties(raw, out, at, stack)
508
579
  extra = raw.get("additionalProperties")
@@ -588,10 +659,22 @@ class SchemaConverter:
588
659
  self.dropped[key] += 1
589
660
 
590
661
 
662
+ def key_parts(key: str) -> list[str]:
663
+ """A key's steps: ``[name]`` for a property, each unescaped segment of a JSON Pointer."""
664
+ return [unescape(part) for part in key[1:].split("/")] if key.startswith("/") else [key]
665
+
666
+
591
667
  def _nullable(raw: dict) -> bool:
592
668
  return raw.get("nullable") is True or raw.get("x-nullable") is True
593
669
 
594
670
 
671
+ def _members(level: list[tuple[str, dict]]) -> Iterator[tuple[str, str, Any]]:
672
+ """Every property of every object at one depth of a nested-key search, with its pointer."""
673
+ return (
674
+ (f"{prefix}/{pointer(name)}", name, raw) for prefix, properties in level for name, raw in properties.items()
675
+ )
676
+
677
+
595
678
  def _object_like(schema: dict) -> bool:
596
679
  declared = schema.get("type")
597
680
  if declared not in (None, "object"):
@@ -603,7 +686,7 @@ def _object_like(schema: dict) -> bool:
603
686
  )
604
687
 
605
688
 
606
- def _type_list(declared: Any) -> list[str]:
689
+ def type_list(declared: Any) -> list[str]:
607
690
  if isinstance(declared, str):
608
691
  return [declared]
609
692
  return [t for t in declared if isinstance(t, str)] if isinstance(declared, list) else []