@withgateway/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2880 -2032
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2982 -2135
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +180 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +172 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  13. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1309 -540
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1287 -532
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +177 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +169 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +69 -13
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +69 -13
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +249 -120
  35. package/dist/worlds/index.d.ts +249 -120
  36. package/dist/worlds/index.js +1240 -484
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1231 -476
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -514
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  57. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  58. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  59. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  60. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  61. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  62. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  63. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  64. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  65. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  66. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  67. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  68. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  69. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  70. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  71. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  72. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  73. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  74. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  75. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  76. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  77. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  78. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  79. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  80. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  81. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  82. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  83. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  84. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  86. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  87. package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
  88. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  90. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  91. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  92. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  93. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  94. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  95. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  96. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  103. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  104. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  105. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  106. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  107. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  108. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  109. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  110. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  111. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  112. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  113. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  114. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  115. package/runtime/gatewaysdk/environment/score.py +57 -0
  116. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  117. package/runtime/gatewaysdk/environment/world.py +42 -3
  118. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  119. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  120. package/skills/connector-schema-authoring/SKILL.md +19 -11
  121. package/skills/connector-schema-authoring/connector-worlds.md +13 -4
  122. package/skills/connector-schema-authoring/world-schema.md +103 -51
  123. package/skills/world-data-ingestion/ROWS.md +29 -3
  124. package/skills/world-data-ingestion/SKILL.md +14 -14
  125. package/skills/worlds-getting-started/SKILL.md +12 -12
  126. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  127. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  128. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  129. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -26,6 +26,12 @@ gateway worlds schema init ./x --from-openapi github.json --include '/repos/{own
26
26
  Inside a world sandbox (`run_gateway_cli`, `sandbox exec`) only a workspace path is accepted:
27
27
  write the document into the workspace first (`sandbox write --from spec.yaml`), then import.
28
28
 
29
+ The import itself runs in this machine's schema runtime whenever it has one; `GATEWAY_RUNTIME=local`
30
+ demands it and sends nothing to the platform. A YAML document is read by PyYAML in the Python that
31
+ runs the runtime: without it the command stops before anything is sent and names the fix
32
+ (`pip install pyyaml` there, point `GATEWAY_PYTHON` at a Python that has it, or convert the
33
+ document to JSON).
34
+
29
35
  ## What the plan says
30
36
 
31
37
  `--from-openapi` always prints the plan first (JSON, one object):
@@ -50,28 +56,51 @@ With `--plan-only` the same object carries `"written": false` and no `domain`/`n
50
56
 
51
57
  ## Conversion rules (decided; the importer documents them in `schema/provenance.json`)
52
58
 
53
- - **Entities.** Every object schema under `definitions` / `components.schemas` that some selected
54
- operation returns and that has an id-like key (`id`, `<name>Id`, `<name>_id`, a `readOnly`
55
- field the vendor marks `x-unique`/`x-primary-key`, else the first `required` string/integer field) becomes a `$defs` entry plus
56
- `x-gateway.entities.<snake_plural>` with `primary_key` (snake_case, at most 64 characters —
57
- a longer vendor name is truncated, the original kept in provenance). Keyless objects stay `$defs`,
58
- referenced inline. `allOf` is merged; `oneOf`/`anyOf` become a union the world validator
59
- accepts. Entity roots close (`additionalProperties: false`); the vendor's field names are kept
60
- as spelled (so a rows file shaped like the vendor's response imports as it is); the vendor's
61
- schema name stays the `$defs` key and `schema/provenance.json` maps every entity and operation
62
- back to its source name and JSON pointer.
59
+ - **Entities are the rows the routes carry.** The importer reads each selected operation's
60
+ response and request body through the vendor's envelope to the item schema: an array of
61
+ objects, or an object that declares no key and holds exactly one array (Crossref's
62
+ `{status, message: {items: [Work], total-results}}`, ClinicalTrials.gov's
63
+ `{studies: [Study], nextPageToken}`); a single-row response is the item when some list carries
64
+ it, it declares a key, or it names the field the route's own `{segment}` reads it by, else the
65
+ one object its wrapper holds. Wrappers never become entities. An item no planned route reads or
66
+ writes is no entity either (it stays an inline `$def`). Each entity is a `$defs` entry plus
67
+ `x-gateway.entities.<snake_plural>` with `primary_key` (snake_case, at most 64 characters, the
68
+ original kept in provenance); `allOf` is merged; `oneOf`/`anyOf` become a union the world
69
+ validator accepts. Entity roots close (`additionalProperties: false`); the vendor's field names
70
+ are kept exactly as spelled, hyphens included (`reference-count`), so a rows file shaped like
71
+ the vendor's response imports as it is.
72
+ - **Keys.** A declared key first (`id`, `<name>Id`, `<name>_id`, a `readOnly` field the vendor
73
+ marks `x-unique`/`x-primary-key`), else the field the single-row route reads the row by:
74
+ top-level and spelled the vendor's way (`/works/{doi}` → `DOI`), or nested in the row's
75
+ objects (`/studies/{nctId}` → the JSON Pointer `/protocolSection/identificationModule/nctId`,
76
+ every object on the path made required; never through an array or a nullable object). An item
77
+ only list routes carry falls back to its first required string/integer field, with a warning
78
+ that rows sharing it collapse: mark the key with `x-primary-key` or include the read route.
63
79
  - **Routes.** Every path+method in `get|post|put|patch|delete` becomes an `[[operations]]` entry:
64
80
  `name` = `operationId` snake_cased, at most 64 characters, deduped with `_2`, `_3` and a
65
- warning; `entity` = the response schema's entity when it resolves to one (an array → a list
66
- route, an object → `read|create|update|delete` by method); path parameters and declared query
67
- parameters become `params`; a `{data: [...]}` wrapper becomes `results`; on a list route a
68
- declared query parameter that names a top-level entity field becomes a `filter` (paging names
69
- such as `page`, `limit`, `offset`, `cursor` never do). A path that repeats a placeholder
70
- (`/a/{id}/{id}`) is skipped with its reason. An operation whose response is not an
71
- entity (login, upload, HTML) stays a **passthrough** route with no entity — it is served, and
72
- answers `501 {"error": "not_modelled"}` until you give it a handler.
81
+ warning; `entity` = the entity its response (or, for a write, its body or the path's `GET`)
82
+ carries, a list route or `read|create|update|delete` by method; path parameters and declared
83
+ query parameters become `params`, the vendor's names kept, dots included (`query.bibliographic`,
84
+ `filter.overallStatus`). The envelope's path becomes `results` and the route's dotted
85
+ `envelope` (`message.items`), and the rest of the envelope its `wrapper`, so the served world
86
+ answers the vendor's shape (`{status: "ok", message-type: "work-list", message: {items, …}}`).
87
+ On a list route a declared query parameter that names an entity field becomes a `filter`
88
+ (paging names never do). A single read whose path argument names no field of the row, and an
89
+ operation whose response is not an entity (login, upload, HTML), stay **passthrough** routes with
90
+ no entity, the reason in the plan — served, answering `501 {"error": "not_modelled"}` until you
91
+ give them a handler.
92
+ - **Paging.** A cursor parameter (`cursor`, `pageToken`) with a next-cursor field beside the rows
93
+ (`next-cursor`, `nextPageToken`) pages by cursor; an `offset` parameter by offset; the limit's
94
+ `default` becomes `page_size`; the count beside the rows (`total-results`, `totalCount`) is the
95
+ hits key. `[api]` pages the way most list routes do, a route paging otherwise names its own
96
+ `page`, and page-number paging (`page` + `per_page`) is named in a warning (that route answers
97
+ every row on one page).
98
+ - **Tools.** An import declares none: routes are never tools. An agent reaches the world's routes
99
+ over HTTP (the api url and token `session open --surface api` prints); declare a tool in
100
+ `x-gateway.tools`, with a handler, where an agent needs one.
73
101
  - **Skipped, with a reason:** `HEAD`/`OPTIONS`/`TRACE`; a path argument that is not an
74
- identifier; a second operation on the same method+path shape. **Refused, with a JSON pointer
102
+ identifier; a path that repeats a placeholder (`/a/{id}/{id}`); a second operation on the same
103
+ method+path shape. **Refused, with a JSON pointer
75
104
  into the document, nothing written:** no `paths`; an unknown or external `$ref`; an entity root
76
105
  that is (transitively) cyclic — model the recursion as a relationship field (`parent_id`)
77
106
  instead; a path parameter nothing declares (`--param <name>` on a hand-written route); a
@@ -85,6 +114,6 @@ With `--plan-only` the same object carries `"written": false` and no `domain`/`n
85
114
 
86
115
  The printed `next` steps are the loop, in order: `schema compile`, `schema check`, `data import
87
116
  <rows.json> --entity <first list entity>` (rows shaped like the vendor's response), `worlds serve`
88
- then `worlds call http://127.0.0.1:<port> <list_route> --args '{}'`, and `sandbox commit` when the
89
- tree lives in a sandbox. `connector conform ./x --openapi <the same document>` then checks the
117
+ then `curl -s http://127.0.0.1:<port>/<list path>` (the route over HTTP), and `sandbox commit` when
118
+ the tree lives in a sandbox. `connector conform ./x --openapi <the same document>` then checks the
90
119
  served world back against the spec.
@@ -8,14 +8,13 @@ from __future__ import annotations
8
8
 
9
9
  from copy import deepcopy
10
10
  import hashlib
11
- import json
12
11
  from pathlib import Path
13
12
  import shutil
14
13
  import tempfile
15
14
 
16
15
  from .compiler import Contract, check_bundle
17
16
  from .scaffold import init_bundle
18
- from .store import SQLiteStore
17
+ from .database import WorldDatabase
19
18
  from .validation import encoded, fail, read_json
20
19
  from ._slack_fidelity import fidelity_report, load_capabilities, slack_template_digest
21
20
  from ._slack_scenario import write_scenario
@@ -260,15 +259,12 @@ def map_slack_evidence(
260
259
  if item.get("file_access") == "check_file_info":
261
260
  discriminator = {"file_access": "check_file_info"}
262
261
  if not discriminator:
263
- # No canonical full shape or explicit unavailable discriminator: refuse invention.
264
262
  fail(
265
263
  "slack_evidence", "Incomplete file projection lacks an access discriminator", "/evidence/files"
266
264
  )
267
265
  observe("slack_unavailable_message_files", {**base, "reference": discriminator}, source)
268
266
  if "files" in value:
269
- # A message's explicitly supplied attachment collection is one value,
270
- # unlike an API page of messages. Retain file metadata as historical
271
- # observations but replace the occurrence's full/unavailable links.
267
+ # An explicit attachment collection is one value: keep file metadata as history, replace the links.
272
268
  for entity in ("slack_file_shares", "slack_message_files", "slack_unavailable_message_files"):
273
269
  retained = {
274
270
  token
@@ -671,7 +667,7 @@ def build_slack_bundle(destination: str | Path, *, evidence: dict | None = None,
671
667
  diagnostics = {}
672
668
  batch = map_slack_evidence(evidence, contract, diagnostics=diagnostics)
673
669
  with tempfile.TemporaryDirectory(prefix="gateway-slack-") as temporary:
674
- with SQLiteStore.create(Path(temporary) / "state.sqlite", contract) as store:
670
+ with WorldDatabase.create(Path(temporary) / "world.db", contract) as store:
675
671
  store.apply_batch(batch)
676
672
  if store.get("slack_users", {"workspace_id": workspace["id"], "id": principal["user_id"]}) is None:
677
673
  fail("slack_evidence", "The authenticated principal was not captured", "/evidence/principal")
@@ -724,7 +720,7 @@ def refresh_slack_bundle(destination: str | Path, evidence: dict | None = None,
724
720
  return {"status": "already_applied", "batch": batch_id, "applied": 0}
725
721
  batches = sorted((destination / "data/batches").glob("*.json"))
726
722
  with tempfile.TemporaryDirectory(prefix="gateway-slack-refresh-") as temporary:
727
- with SQLiteStore.create(Path(temporary) / "state.sqlite", contract) as store:
723
+ with WorldDatabase.create(Path(temporary) / "world.db", contract) as store:
728
724
  store.apply_batch(read_json(destination / "data/initial.json"))
729
725
  for path in batches:
730
726
  store.apply_batch(read_json(path))