@honua/mcp-server 0.0.10-alpha.0 → 0.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +259 -15
  2. package/certification/geospatial-mcp-schemas/PROVENANCE.md +41 -0
  3. package/certification/geospatial-mcp-schemas/README.md +90 -0
  4. package/certification/geospatial-mcp-schemas/common/geoprocessing-error.schema.json +45 -0
  5. package/certification/geospatial-mcp-schemas/conformance/manifest.schema.json +79 -0
  6. package/certification/geospatial-mcp-schemas/index.json +390 -0
  7. package/certification/geospatial-mcp-schemas/resources/app-package.schema.json +20 -0
  8. package/certification/geospatial-mcp-schemas/resources/app-template.schema.json +15 -0
  9. package/certification/geospatial-mcp-schemas/resources/artifact.schema.json +69 -0
  10. package/certification/geospatial-mcp-schemas/resources/deployment.schema.json +27 -0
  11. package/certification/geospatial-mcp-schemas/resources/map-package.schema.json +21 -0
  12. package/certification/geospatial-mcp-schemas/resources/map-template.schema.json +15 -0
  13. package/certification/geospatial-mcp-schemas/resources/provenance.schema.json +50 -0
  14. package/certification/geospatial-mcp-schemas/resources/published-service.schema.json +21 -0
  15. package/certification/geospatial-mcp-schemas/resources/result-package.schema.json +64 -0
  16. package/certification/geospatial-mcp-schemas/resources/style.schema.json +12 -0
  17. package/certification/geospatial-mcp-schemas/resources/theme.schema.json +12 -0
  18. package/certification/geospatial-mcp-schemas/resources/workspace.schema.json +63 -0
  19. package/certification/geospatial-mcp-schemas/tools/apply_style_preset.schema.json +27 -0
  20. package/certification/geospatial-mcp-schemas/tools/buffer_features.schema.json +60 -0
  21. package/certification/geospatial-mcp-schemas/tools/cancel_job.schema.json +15 -0
  22. package/certification/geospatial-mcp-schemas/tools/clarify_intent.schema.json +69 -0
  23. package/certification/geospatial-mcp-schemas/tools/compose_mixed_protocol_map.schema.json +21 -0
  24. package/certification/geospatial-mcp-schemas/tools/create_app_package.schema.json +33 -0
  25. package/certification/geospatial-mcp-schemas/tools/create_map_package.schema.json +41 -0
  26. package/certification/geospatial-mcp-schemas/tools/edit_features.schema.json +74 -0
  27. package/certification/geospatial-mcp-schemas/tools/execute_plan.schema.json +64 -0
  28. package/certification/geospatial-mcp-schemas/tools/export_dataset.schema.json +60 -0
  29. package/certification/geospatial-mcp-schemas/tools/geocode_address.schema.json +36 -0
  30. package/certification/geospatial-mcp-schemas/tools/geocode_addresses.schema.json +34 -0
  31. package/certification/geospatial-mcp-schemas/tools/get_style.schema.json +40 -0
  32. package/certification/geospatial-mcp-schemas/tools/ground_candidates.schema.json +66 -0
  33. package/certification/geospatial-mcp-schemas/tools/ingest_dataset.schema.json +45 -0
  34. package/certification/geospatial-mcp-schemas/tools/join_features.schema.json +73 -0
  35. package/certification/geospatial-mcp-schemas/tools/list_capabilities.schema.json +21 -0
  36. package/certification/geospatial-mcp-schemas/tools/list_layers.schema.json +14 -0
  37. package/certification/geospatial-mcp-schemas/tools/overlay_features.schema.json +53 -0
  38. package/certification/geospatial-mcp-schemas/tools/plan_analysis.schema.json +19 -0
  39. package/certification/geospatial-mcp-schemas/tools/preview_app_package.schema.json +15 -0
  40. package/certification/geospatial-mcp-schemas/tools/preview_map_package.schema.json +17 -0
  41. package/certification/geospatial-mcp-schemas/tools/preview_package.schema.json +40 -0
  42. package/certification/geospatial-mcp-schemas/tools/propose_operation.schema.json +28 -0
  43. package/certification/geospatial-mcp-schemas/tools/publish_result.schema.json +29 -0
  44. package/certification/geospatial-mcp-schemas/tools/publish_service.schema.json +59 -0
  45. package/certification/geospatial-mcp-schemas/tools/query_features.schema.json +74 -0
  46. package/certification/geospatial-mcp-schemas/tools/refine_map_package.schema.json +22 -0
  47. package/certification/geospatial-mcp-schemas/tools/render_map.schema.json +60 -0
  48. package/certification/geospatial-mcp-schemas/tools/reproject_features.schema.json +49 -0
  49. package/certification/geospatial-mcp-schemas/tools/resolve_entity.schema.json +30 -0
  50. package/certification/geospatial-mcp-schemas/tools/solve_route.schema.json +39 -0
  51. package/certification/geospatial-mcp-schemas/tools/summarize_statistics.schema.json +71 -0
  52. package/certification/geospatial-mcp-schemas/tools/validate_package.schema.json +35 -0
  53. package/certification/geospatial-mcp-schemas/tools/validate_plan.schema.json +61 -0
  54. package/dist/src/capability.d.ts +48 -0
  55. package/dist/src/capability.d.ts.map +1 -0
  56. package/dist/src/capability.js +44 -0
  57. package/dist/src/certification/census-data.d.ts +33 -0
  58. package/dist/src/certification/census-data.d.ts.map +1 -0
  59. package/dist/src/certification/census-data.js +506 -0
  60. package/dist/src/certification/census-fixture-client.d.ts +7 -0
  61. package/dist/src/certification/census-fixture-client.d.ts.map +1 -0
  62. package/dist/src/certification/census-fixture-client.js +226 -0
  63. package/dist/src/certification/certifier.d.ts +125 -0
  64. package/dist/src/certification/certifier.d.ts.map +1 -0
  65. package/dist/src/certification/certifier.js +455 -0
  66. package/dist/src/certification/cli.d.ts +3 -0
  67. package/dist/src/certification/cli.d.ts.map +1 -0
  68. package/dist/src/certification/cli.js +63 -0
  69. package/dist/src/certification/contracts.d.ts +103 -0
  70. package/dist/src/certification/contracts.d.ts.map +1 -0
  71. package/dist/src/certification/contracts.js +245 -0
  72. package/dist/src/certification/fixture-client.d.ts +4 -0
  73. package/dist/src/certification/fixture-client.d.ts.map +1 -0
  74. package/dist/src/certification/fixture-client.js +113 -0
  75. package/dist/src/certification/fixtures.d.ts +45 -0
  76. package/dist/src/certification/fixtures.d.ts.map +1 -0
  77. package/dist/src/certification/fixtures.js +92 -0
  78. package/dist/src/certification/json-schema.d.ts +71 -0
  79. package/dist/src/certification/json-schema.d.ts.map +1 -0
  80. package/dist/src/certification/json-schema.js +172 -0
  81. package/dist/src/certification/lifecycle.d.ts +14 -0
  82. package/dist/src/certification/lifecycle.d.ts.map +1 -0
  83. package/dist/src/certification/lifecycle.js +326 -0
  84. package/dist/src/certification/mock-upstream.d.ts +29 -0
  85. package/dist/src/certification/mock-upstream.d.ts.map +1 -0
  86. package/dist/src/certification/mock-upstream.js +96 -0
  87. package/dist/src/certification/operator-catalog.d.ts +90 -0
  88. package/dist/src/certification/operator-catalog.d.ts.map +1 -0
  89. package/dist/src/certification/operator-catalog.js +650 -0
  90. package/dist/src/certification/report.d.ts +4 -0
  91. package/dist/src/certification/report.d.ts.map +1 -0
  92. package/dist/src/certification/report.js +103 -0
  93. package/dist/src/certification/run.d.ts +28 -0
  94. package/dist/src/certification/run.d.ts.map +1 -0
  95. package/dist/src/certification/run.js +32 -0
  96. package/dist/src/certification/schema-index.d.ts +94 -0
  97. package/dist/src/certification/schema-index.d.ts.map +1 -0
  98. package/dist/src/certification/schema-index.js +109 -0
  99. package/dist/src/certification/target.d.ts +48 -0
  100. package/dist/src/certification/target.d.ts.map +1 -0
  101. package/dist/src/certification/target.js +219 -0
  102. package/dist/src/eval/cli.d.ts +3 -0
  103. package/dist/src/eval/cli.d.ts.map +1 -0
  104. package/dist/src/eval/cli.js +118 -0
  105. package/dist/src/eval/corpus.d.ts +34 -0
  106. package/dist/src/eval/corpus.d.ts.map +1 -0
  107. package/dist/src/eval/corpus.js +150 -0
  108. package/dist/src/eval/drivers/anthropic.d.ts +14 -0
  109. package/dist/src/eval/drivers/anthropic.d.ts.map +1 -0
  110. package/dist/src/eval/drivers/anthropic.js +120 -0
  111. package/dist/src/eval/drivers/bedrock.d.ts +54 -0
  112. package/dist/src/eval/drivers/bedrock.d.ts.map +1 -0
  113. package/dist/src/eval/drivers/bedrock.js +178 -0
  114. package/dist/src/eval/drivers/deterministic.d.ts +17 -0
  115. package/dist/src/eval/drivers/deterministic.d.ts.map +1 -0
  116. package/dist/src/eval/drivers/deterministic.js +64 -0
  117. package/dist/src/eval/drivers/index.d.ts +34 -0
  118. package/dist/src/eval/drivers/index.d.ts.map +1 -0
  119. package/dist/src/eval/drivers/index.js +74 -0
  120. package/dist/src/eval/drivers/openai.d.ts +14 -0
  121. package/dist/src/eval/drivers/openai.d.ts.map +1 -0
  122. package/dist/src/eval/drivers/openai.js +136 -0
  123. package/dist/src/eval/grade.d.ts +14 -0
  124. package/dist/src/eval/grade.d.ts.map +1 -0
  125. package/dist/src/eval/grade.js +75 -0
  126. package/dist/src/eval/northstar-corpus.d.ts +3 -0
  127. package/dist/src/eval/northstar-corpus.d.ts.map +1 -0
  128. package/dist/src/eval/northstar-corpus.js +194 -0
  129. package/dist/src/eval/operator-corpus.d.ts +3 -0
  130. package/dist/src/eval/operator-corpus.d.ts.map +1 -0
  131. package/dist/src/eval/operator-corpus.js +193 -0
  132. package/dist/src/eval/report.d.ts +109 -0
  133. package/dist/src/eval/report.d.ts.map +1 -0
  134. package/dist/src/eval/report.js +133 -0
  135. package/dist/src/eval/runner.d.ts +28 -0
  136. package/dist/src/eval/runner.d.ts.map +1 -0
  137. package/dist/src/eval/runner.js +130 -0
  138. package/dist/src/eval/standalone-corpus.d.ts +3 -0
  139. package/dist/src/eval/standalone-corpus.d.ts.map +1 -0
  140. package/dist/src/eval/standalone-corpus.js +699 -0
  141. package/dist/src/eval/types.d.ts +118 -0
  142. package/dist/src/eval/types.d.ts.map +1 -0
  143. package/dist/src/eval/types.js +10 -0
  144. package/dist/src/helpers.d.ts +11 -0
  145. package/dist/src/helpers.d.ts.map +1 -1
  146. package/dist/src/helpers.js +22 -0
  147. package/dist/src/index.d.ts +31 -0
  148. package/dist/src/index.d.ts.map +1 -1
  149. package/dist/src/index.js +50 -4
  150. package/dist/src/provenance.d.ts +44 -0
  151. package/dist/src/provenance.d.ts.map +1 -0
  152. package/dist/src/provenance.js +37 -0
  153. package/dist/src/proxy.d.ts +46 -0
  154. package/dist/src/proxy.d.ts.map +1 -0
  155. package/dist/src/proxy.js +124 -0
  156. package/dist/src/resources/styles.d.ts +29 -0
  157. package/dist/src/resources/styles.d.ts.map +1 -0
  158. package/dist/src/resources/styles.js +76 -0
  159. package/dist/src/styles.d.ts +112 -0
  160. package/dist/src/styles.d.ts.map +1 -0
  161. package/dist/src/styles.js +144 -0
  162. package/dist/src/tools/apply-style-preset.d.ts +27 -0
  163. package/dist/src/tools/apply-style-preset.d.ts.map +1 -0
  164. package/dist/src/tools/apply-style-preset.js +41 -0
  165. package/dist/src/tools/count-features.d.ts.map +1 -1
  166. package/dist/src/tools/count-features.js +5 -4
  167. package/dist/src/tools/get-extent.d.ts.map +1 -1
  168. package/dist/src/tools/get-extent.js +6 -4
  169. package/dist/src/tools/get-style.d.ts +24 -0
  170. package/dist/src/tools/get-style.d.ts.map +1 -0
  171. package/dist/src/tools/get-style.js +39 -0
  172. package/dist/src/tools/list-services.d.ts.map +1 -1
  173. package/dist/src/tools/list-services.js +16 -1
  174. package/dist/src/tools/query-features.d.ts.map +1 -1
  175. package/dist/src/tools/query-features.js +4 -1
  176. package/package.json +28 -4
package/README.md CHANGED
@@ -1,42 +1,286 @@
1
1
  # `@honua/mcp-server`
2
2
 
3
- Model Context Protocol (MCP) server for Honua geospatial feature services.
3
+ The **platform-free** Model Context Protocol (MCP) server for geospatial feature
4
+ services. Point it at **any** public Esri FeatureServer or OGC API endpoint and it
5
+ exposes discovery, query, and analysis workflows to any MCP client — with **zero**
6
+ platform lock-in and no metering. Mapbox, CARTO, and Esri each ship an MCP server
7
+ bound to their own platform; this one is bound to none.
4
8
 
5
- This package exposes a focused MCP surface for discovery and query workflows on top of Honua's FeatureServer APIs.
9
+ **Release status: beta.** Tool contracts are certified against live and fixture
10
+ targets on every release; remaining pre-1.0 work is hardening, not surface change.
6
11
 
7
- ## Requirements
12
+ Two modes:
8
13
 
9
- - Node.js `>=20`
10
- - A reachable Honua server URL
14
+ - **Standalone (platform-free) — the front door.** The `honua-mcp` bin runs the
15
+ direct-SDK surface against any public FeatureServer/OGC endpoint. No Honua
16
+ server, no admin API, no `/mcp` catalog required.
17
+ - **Honua-enhanced — the upgrade path.** The `honua-mcp-proxy` bin bridges a
18
+ Honua deployment's full `/mcp` operator catalog (planning, async jobs,
19
+ publishing) to stdio. Honua does **not** expose an AI/MCP feature-mutation tool
20
+ — AI operational data editing is not supported (honua-server ADR-0028).
11
21
 
12
- ## Environment Variables
22
+ ## Requirements
13
23
 
14
- - `HONUA_BASE_URL` (required): absolute Honua base URL, for example `https://honua.example.com`
15
- - `HONUA_TRANSPORT` (optional): `grpc-web` (default) or `rest`
16
- - `HONUA_API_KEY` (optional): admin/API key when your deployment requires it
17
- - `HONUA_TIMEOUT_MS` (optional): request timeout in milliseconds (default `30000`)
18
- - `HONUA_RETRY_MAX_RETRIES` (optional): retry attempts for transient failures (default `2`)
24
+ - Node.js `>=20`
25
+ - Any reachable FeatureServer/OGC endpoint (public or Honua)
19
26
 
20
- When `HONUA_API_KEY` is configured, use `https://` for non-localhost servers.
27
+ ## Platform-free mode (any ArcGIS / OGC endpoint)
21
28
 
22
- ## Run Locally
29
+ Point `HONUA_BASE_URL` at any origin/folder that serves the standard GeoServices
30
+ REST paths (`/rest/services`, `/FeatureServer/{id}/query`). Example — the public
31
+ US Census 2020 apportionment FeatureServer on `services.arcgis.com`:
23
32
 
24
33
  ```bash
25
34
  npm install
26
35
  npm run build
27
- HONUA_BASE_URL="https://honua.example.com" HONUA_TRANSPORT="grpc-web" node dist/src/index.js
36
+ HONUA_BASE_URL="https://services.arcgis.com/P3ePLMYs2RVChkJx/arcgis" \
37
+ HONUA_TRANSPORT="rest" node dist/src/index.js
38
+ ```
39
+
40
+ Tools that require a Honua-only surface (server-side styling via OGC API – Styles,
41
+ a `/rest/services` catalog) **degrade gracefully** on a plain endpoint: they
42
+ return a structured result
43
+
44
+ ```json
45
+ { "available": false, "surface": "OGC API - Styles", "reason": "…", "guidance": "…" }
28
46
  ```
29
47
 
48
+ instead of crashing, hanging, or returning misleading empty data. This is the
49
+ same skip-with-reason honesty the certification suite uses.
50
+
51
+ ## Environment Variables
52
+
53
+ - `HONUA_BASE_URL` (required): absolute base URL — a public ArcGIS folder
54
+ (`https://services.arcgis.com/<org>/arcgis`) or a Honua deployment
55
+ (`https://honua.example.com`).
56
+ - `HONUA_TRANSPORT` (optional): `grpc-web` (default, Honua deployments) or `rest`.
57
+ Use `rest` for plain public ArcGIS/OGC endpoints.
58
+ - `HONUA_API_KEY` (optional): API key when your deployment requires it. Public
59
+ endpoints need none.
60
+ - `HONUA_TIMEOUT_MS` (optional): request timeout in milliseconds (default `30000`)
61
+ - `HONUA_RETRY_MAX_RETRIES` (optional): retry attempts for transient failures (default `2`)
62
+
63
+ When `HONUA_API_KEY` is configured, use `https://` for non-localhost servers.
64
+
30
65
  ## MCP Tools
31
66
 
32
- - `honua_list_services`
67
+ All read-only:
68
+
69
+ - `honua_list_services` — discover services (degrades if the target has no catalog)
33
70
  - `honua_describe_layer`
34
71
  - `honua_query_features`
35
72
  - `honua_count_features`
36
73
  - `honua_get_extent`
37
74
  - `honua_statistics`
75
+ - `honua_explain_capability_gap`
76
+ - `honua_get_style`, `honua_apply_style_preset` — structured-unavailable on a plain FeatureServer
38
77
 
39
78
  ## MCP Resources
40
79
 
41
80
  - `honua://services`
42
81
  - `honua://services/{encodedServiceId}/layers/{layerId}`
82
+ - `honua://styles`, `honua://styles/{styleId}` — structured-unavailable on a plain FeatureServer
83
+
84
+ ## Certification
85
+
86
+ The package ships a **deterministic MCP certification harness** that proves the
87
+ advertised MCP surface is well-formed and conformant to the open
88
+ [`geospatial-mcp`](https://github.com/honua-io/geospatial-mcp) standard. It is
89
+ fully offline — no model/API calls, no network in the default path — and is the
90
+ evidence document for the WS-H "Provability" workstream.
91
+
92
+ For each tool the server advertises (over an in-memory MCP transport) it:
93
+
94
+ 1. Enumerates `tools/list`, `resources/list`, and `prompts/list`.
95
+ 2. Validates each `inputSchema` (and any `structuredContent` output schema) is
96
+ well-formed JSON Schema, accepting both the draft-07 dialect emitted by
97
+ zod-to-json-schema and the draft 2020-12 dialect of the standard.
98
+ 3. Where a vendored standard schema matches the advertised tool (by the
99
+ standard's `referenceToolName`), checks **conformance** — every standard
100
+ `required` property must be accepted with a compatible type. Standard tools
101
+ that are not advertised, and advertised tools outside the standard, are
102
+ recorded as **known gaps**, not failures.
103
+ 4. **Round-trips** every read-only tool (`tools/call` with a fixture input),
104
+ validating the response. Write/destructive tools are never called.
105
+
106
+ It emits a stable machine-readable JSON report plus a human-readable Markdown
107
+ summary, and exits non-zero on any conformance/round-trip failure.
108
+
109
+ ```bash
110
+ # Run the certifier against the offline fixture backend and write artifacts:
111
+ npm run certify
112
+
113
+ # CI entry points (also runnable locally):
114
+ npm run test:certification # gate: runs harness tests + certifier, exits non-zero on failure
115
+ npm run test:certification:artifact # evidence: writes artifacts, always exits 0
116
+ ```
117
+
118
+ ### Platform-free certification lane
119
+
120
+ A dedicated **standalone** target (`HONUA_MCP_CERT_TARGET=standalone`, or
121
+ `--target standalone`) certifies the platform-free surface against an in-process
122
+ fixture of a **plain public FeatureServer** — the recorded US Census 2020
123
+ apportionment layer from `services.arcgis.com`, with **no** Honua surfaces. It
124
+ proves the tools certify **green with honest skips** against a non-Honua endpoint:
125
+ the data tools round-trip against real recorded data, the Honua-only style tools
126
+ degrade to structured "not available" results, and the auth/mutation/job contracts
127
+ skip-with-reason. Fixture-backed and deterministic — no network.
128
+
129
+ ```bash
130
+ npm run certify:standalone # platform-free cert (plain FeatureServer fixture)
131
+ npm run test:certification:standalone # CI gate variant
132
+ ```
133
+
134
+ The fixture is recorded by `scripts/record-census-fixtures.mjs` into
135
+ `src/certification/census-data.ts`; the evaluator that replays it is verified for
136
+ parity against the live recordings in `test/certification/census-fixture-client.test.ts`.
137
+
138
+ Artifacts are written to the package root as
139
+ `mcp-certification-results.json` and `mcp-certification-results.md` (gitignored;
140
+ uploaded by CI). To certify against a **live** honua-server, set `HONUA_BASE_URL`
141
+ (and `HONUA_TRANSPORT`, `HONUA_MCP_SERVICE_ID`, `HONUA_MCP_LAYER_ID`); the
142
+ harness then drives a real `HonuaClient` instead of the fixture.
143
+
144
+ The standard schemas are vendored under
145
+ `certification/geospatial-mcp-schemas/` (see that directory's `PROVENANCE.md`
146
+ for the pinned source revision).
147
+
148
+ ## Transport-symmetric stdio proxy
149
+
150
+ The honua server exposes one MCP catalog over streamable-HTTP/SSE at `/mcp`.
151
+ Claude-Desktop-style clients speak **stdio**. Rather than reimplement that
152
+ catalog (which is how the HTTP and stdio surfaces historically drifted apart),
153
+ this package ships a **stdio proxy** (`honua-mcp-proxy`) that bridges a local
154
+ stdio MCP client to the remote HTTP-SSE MCP server. It connects upstream as an
155
+ MCP client and re-exposes the *same* catalog downstream over stdio — identical
156
+ tools, identical input/output schemas, identical resources and prompts, and live
157
+ `tools/list_changed` notifications. There is one source-of-truth catalog (the
158
+ server's `/mcp`); the SDK proxies it, so the two transports are symmetric by
159
+ construction. (Cross-repo: honua-io/honua-server#1950.)
160
+
161
+ ```bash
162
+ # Bridge a stdio MCP client to a remote honua /mcp surface:
163
+ HONUA_MCP_REMOTE_URL="https://demo.honua.io/mcp" honua-mcp-proxy
164
+ ```
165
+
166
+ Environment variables:
167
+
168
+ - `HONUA_MCP_REMOTE_URL` (required; alias `HONUA_MCP_URL`): the remote honua
169
+ `/mcp` endpoint to proxy.
170
+ - `HONUA_MCP_AUTH_TOKEN` (optional): sent as `Authorization: Bearer <token>`.
171
+ - `HONUA_API_KEY` (optional): sent as `x-api-key`.
172
+
173
+ A parity test (`test/proxy.test.ts`) asserts the tool/resource/template catalog
174
+ the downstream client sees is byte-identical to the upstream surface, that
175
+ `tools/call` and resource reads round-trip identically, and that `list_changed`
176
+ notifications are forwarded.
177
+
178
+ ## Cross-model workflow eval (provability)
179
+
180
+ The package ships a **cross-model workflow eval** that proves the "any client →
181
+ any workflow" claim: a held-out corpus of GIS workflows (`src/eval/corpus.ts`)
182
+ is driven through the MCP surface by different client LLMs and graded for
183
+ end-to-end success / clarification / edit rates per model. (Cross-repo:
184
+ honua-io/honua-server#1956.)
185
+
186
+ - **Deterministic control (offline, CI):** a scripted "ideal client" runs every
187
+ workflow's real `tools/call` round-trips against the offline fixture surface —
188
+ no model/API calls — and is graded identically to the live models. This is the
189
+ reproducible CI gate.
190
+ - **Live cross-model (Claude + GPT):** when `ANTHROPIC_API_KEY` /
191
+ `OPENAI_API_KEY` are set, the Claude driver (latest Opus, `claude-opus-4-8`)
192
+ and the GPT driver (GA flagship, `gpt-5.5`, override with `OPENAI_MODEL`) run
193
+ the corpus through a real agentic tool-use loop over the identical catalog. The
194
+ `@anthropic-ai/sdk` / `openai` packages are imported dynamically (not
195
+ dependencies); keys come from the environment and are never hardcoded. Set
196
+ `HONUA_MCP_REMOTE_URL` to drive a live remote `/mcp` instead of the fixture.
197
+
198
+ ```bash
199
+ npm run eval # run the eval (live models join if their keys are set)
200
+ npm run eval:offline # force the deterministic control + fixture surface
201
+ npm run test:eval # gate: harness tests + offline eval, exits non-zero on failure
202
+ npm run test:eval:artifact # evidence: writes artifacts, always exits 0
203
+ ```
204
+
205
+ Artifacts (`mcp-eval-results.json` / `mcp-eval-results.md`, gitignored, uploaded
206
+ by CI) record the per-model scorecard. The CI gate asserts the deterministic
207
+ control passes every scenario; live cross-model runs are recorded but
208
+ informational.
209
+
210
+ ### Platform-free semantic corpus (`src/eval/standalone-corpus.ts`)
211
+
212
+ The **standalone** corpus is 50+ scenarios run against the plain public
213
+ FeatureServer fixture (the census layer) with **semantic** grading — not just tool
214
+ trajectory. Each scenario asserts the *meaning* of the answer: correct feature
215
+ counts (52 rows, 4 states with ≥20 seats), correct geographic facts (California is
216
+ the most populous; Wyoming the least; the House totals 435 seats), correct tool
217
+ selection for ambiguous asks (count vs. query vs. statistics), refusal /
218
+ clarification behavior on ambiguous or unsupported requests, and anti-hallucination
219
+ guards. Because the fixture replays real recorded data, a wrong number or a
220
+ hallucinated place name fails. The grading taxonomy is documented in
221
+ [`evals/README.md`](evals/README.md).
222
+
223
+ ```bash
224
+ npm run eval:standalone # deterministic control over the census fixture (offline)
225
+ ```
226
+
227
+ ### Live lane (paid, manual only)
228
+
229
+ The **operator corpus** (`src/eval/operator-corpus.ts`, 8 harder scenarios) is
230
+ meant to run against a live honua-server **operator** `/mcp` surface with real
231
+ models. This is a **billable** lane: it makes real Anthropic / OpenAI (or AWS
232
+ Bedrock) calls. It never runs on push/PR — locally via `npm run eval:live`, or in
233
+ CI via the manual **`MCP Live Cross-Model Eval`** workflow
234
+ (`.github/workflows/mcp-eval-live.yml`, `workflow_dispatch`-only). The live model
235
+ SDKs are not package dependencies; install them first for a live run:
236
+
237
+ ```bash
238
+ npm install --no-save @anthropic-ai/sdk openai # only needed for live runs
239
+ ```
240
+
241
+ `eval:live` runs the operator corpus (`--corpus operator`) against the remote
242
+ `/mcp` and always writes artifacts (`--artifact-only`). Authentication uses the
243
+ same headers as the stdio proxy — **no dev-auth bypass**; the resolved auth mode
244
+ (`bearer` / `api-key` / `anonymous`) is recorded in the artifact so the run
245
+ proves it was authenticated. Set `HONUA_EVAL_REQUIRE_AUTH=1` to refuse an
246
+ anonymous run outright.
247
+
248
+ Shared env for every live run:
249
+
250
+ - `HONUA_MCP_REMOTE_URL` (required): the operator `/mcp` endpoint, e.g. `https://demo.honua.io/mcp`.
251
+ - `HONUA_MCP_AUTH_TOKEN`: sent as `Authorization: Bearer <token>` (preferred; ⇒ auth mode `bearer`).
252
+ - `HONUA_API_KEY`: sent as `x-api-key` (⇒ auth mode `api-key`) when the deployment uses key auth instead.
253
+ - `HONUA_EVAL_REQUIRE_AUTH=1` (recommended): fail fast if neither credential is present.
254
+
255
+ ```bash
256
+ # Anthropic Claude (default claude-opus-4-8; override with HONUA_EVAL_ANTHROPIC_MODEL):
257
+ HONUA_MCP_REMOTE_URL="https://demo.honua.io/mcp" \
258
+ HONUA_MCP_AUTH_TOKEN="$HONUA_TOKEN" \
259
+ HONUA_EVAL_REQUIRE_AUTH=1 \
260
+ ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \
261
+ npm run eval:live -- --driver anthropic
262
+
263
+ # OpenAI GPT (default gpt-5.5; override with OPENAI_MODEL, e.g. gpt-5.6-sol once you have access):
264
+ HONUA_MCP_REMOTE_URL="https://demo.honua.io/mcp" \
265
+ HONUA_MCP_AUTH_TOKEN="$HONUA_TOKEN" \
266
+ HONUA_EVAL_REQUIRE_AUTH=1 \
267
+ OPENAI_API_KEY="$OPENAI_API_KEY" \
268
+ npm run eval:live -- --driver openai
269
+
270
+ # Claude via Amazon Bedrock (AWS credential chain; default Sonnet 4.5,
271
+ # override with HONUA_EVAL_BEDROCK_MODEL — most ids need a us.* inference profile):
272
+ HONUA_MCP_REMOTE_URL="https://demo.honua.io/mcp" \
273
+ HONUA_MCP_AUTH_TOKEN="$HONUA_TOKEN" \
274
+ HONUA_EVAL_REQUIRE_AUTH=1 \
275
+ HONUA_EVAL_BEDROCK=1 AWS_REGION=us-west-2 \
276
+ HONUA_EVAL_BEDROCK_MODEL="us.anthropic.claude-sonnet-4-5-20250929-v1:0" \
277
+ npm run eval:live -- --driver bedrock
278
+ ```
279
+
280
+ Pass multiple drivers to compare them in one artifact:
281
+ `npm run eval:live -- --driver anthropic,openai`. Every driver is graded against
282
+ the **identical** operator corpus and catalog, and the artifact's
283
+ `catalog.unresolvedRequiredTools` names any required tool the live surface did
284
+ not advertise — so a scenario that fails does so for a real capability gap, not a
285
+ silent tool name-resolution bug (the runner resolves required tools against the
286
+ live `tools/list`, never against the vendored certification index).
@@ -0,0 +1,41 @@
1
+ # Vendored geospatial-mcp JSON Schemas
2
+
3
+ These JSON Schema files are a verbatim, vendored copy of the published
4
+ machine-readable schemas from the open **geospatial-mcp** standard.
5
+
6
+ - **Source repo:** https://github.com/honua-io/geospatial-mcp
7
+ - **Source path:** `spec/schemas/`
8
+ - **Source commit:** `eb53989cc61c856261cf017b4b5a8e721317dc41`
9
+ (`feat: direct geoprocessing verbs (analysis profile) + geometryPrecision/maxInlineBytes (#55)`)
10
+ - **Schema index date:** `2026-07-06`
11
+ - **Dialect:** JSON Schema draft 2020-12
12
+
13
+ > The pin is deliberately held at `eb53989` (pre geospatial-mcp#58): the #58
14
+ > platform-ops schemas are marked implemented in the manifest but honua-server
15
+ > does not serve those tools yet, so re-vendoring past #58 would introduce
16
+ > conformance failures. The post-#58 bump is owned by honua-server#2555/#2566,
17
+ > which implement the new tools and vendor their schemas together.
18
+
19
+ ## Why vendored
20
+
21
+ The MCP certification harness (`src/certification/`) must run deterministically
22
+ in CI with **zero network access and zero model/API token spend**. Vendoring the
23
+ published schemas pins the standard the Honua MCP surface is certified against to
24
+ a known, reproducible revision and removes any cross-repo fetch at certify time.
25
+
26
+ ## Refreshing
27
+
28
+ To re-pin to a newer published revision, edit the **Source commit** SHA (and the
29
+ index date) above, then run `scripts/sync-schemas.sh --write` from the repo root
30
+ to re-copy `spec/schemas/` verbatim from that commit. Do not hand-edit individual
31
+ schema files; the standard is owned upstream. The `schema-sync` CI workflow
32
+ (`.github/workflows/schema-sync.yml`) runs `scripts/sync-schemas.sh` on every PR
33
+ and fails on any byte difference from the pinned commit.
34
+
35
+ ## Index
36
+
37
+ `index.json` maps each standard tool's bare `standardName` (taxonomy.md name) to
38
+ the `referenceToolName` the reference implementation (Honua `/mcp`) advertises,
39
+ plus an `implementationStatus` (`implemented` | `known-gap`). The certifier reads
40
+ this index to decide which advertised tools to conformance-check and which
41
+ standard tools to record as known gaps.
@@ -0,0 +1,90 @@
1
+ # Machine-Readable JSON Schemas
2
+
3
+ **Status:** Draft
4
+ **Date:** 2026-06-21
5
+ **Scope:** Implementable JSON Schema (draft 2020-12) bindings for the geospatial MCP standard
6
+
7
+ This directory makes the prose specification **implementable**. It publishes
8
+ [JSON Schema](https://json-schema.org/) (draft 2020-12) documents describing:
9
+
10
+ - each core MCP **tool** `inputSchema` (the argument shape an agent sends on a
11
+ `tools/call`), under [`tools/`](tools/);
12
+ - each core MCP **resource** payload shape (the read-only projection a
13
+ `resources/read` returns), under [`resources/`](resources/).
14
+
15
+ A machine-readable [`index.json`](index.json) maps every standard tool name and
16
+ resource family URI to its schema file, so a harness can resolve a schema from
17
+ a vocabulary key without hard-coding paths.
18
+
19
+ A self-check **conformance fixture tree** lives under
20
+ [`../../conformance/fixtures/`](../../conformance/fixtures/): example tool-call
21
+ inputs and resource payloads that validate against these schemas. An
22
+ implementer can run their JSON Schema validator over the fixtures to confirm the
23
+ schemas load and that the examples conform, before testing their own server.
24
+
25
+ ## Relationship to the prose
26
+
27
+ These schemas are **derived from**, and faithful to, the normative prose:
28
+
29
+ - Tool families and names come from
30
+ [`spec/taxonomy.md` §MCP Tools to Workflow Family Mapping](../taxonomy.md#mcp-tools-to-workflow-family-mapping).
31
+ - Resource families, URI grammar, and inspection fields come from
32
+ [`spec/resources.md`](../resources.md).
33
+ - The canonical concept model (`AnalysisPlan`, `ArtifactRef`, `WorkspaceRef`,
34
+ `GeoprocessingError`, …) comes from
35
+ [`spec/taxonomy.md` §Canonical Concept Model](../taxonomy.md#canonical-concept-model).
36
+
37
+ ### Where the prose is intentionally upstream-owned
38
+
39
+ The prose **defers concrete field spellings** for several canonical objects to
40
+ `honua-server` (`MapPackage`, `AppPackage`, `PublishedService`, `Deployment`
41
+ — see [`resources.md` §2.4 Deferred-Shape Fixtures](../conformance.md#24-deferred-shape-fixtures)
42
+ and the responsibility-level projections in `resources.md`). For those, the
43
+ prose deliberately gives **responsibilities**, not a frozen field table.
44
+
45
+ Per the task of making the standard implementable, where the prose is ambiguous
46
+ or upstream-owned, **these schemas adopt the shape the reference implementation
47
+ (Honua's `/mcp`) already emits**, and mark it as such with a
48
+ `x-honua-reference-shape: true` annotation and a `description` note. This keeps
49
+ the standard constructible today while remaining honest that the concrete field
50
+ names track the reference implementation until the upstream ticket lands.
51
+
52
+ Resources whose concrete shape is still fully deferred upstream (e.g.
53
+ `MapPackage`, `AppPackage`, `PublishedService`, `Deployment`,
54
+ `PublishingResultPackage`) carry **responsibility-level** schemas: the stable
55
+ identifier and `honua://` URI are pinned and required, additional properties are
56
+ permitted (`additionalProperties: true`), and the responsibility fields are
57
+ described but not over-constrained. This matches the conformance rubric, which
58
+ scores those resources on stable identifiers, URI grammar, and
59
+ responsibility-level projection while marking concrete-field assertions
60
+ `not_applicable`.
61
+
62
+ ## Draft and dialect
63
+
64
+ All schemas declare:
65
+
66
+ ```json
67
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
68
+ ```
69
+
70
+ and a stable `$id` of the form
71
+ `https://geospatial-mcp.honua.io/spec/schemas/{tools|resources}/{name}.schema.json`.
72
+
73
+ ## Tool-name prefixing
74
+
75
+ The standard names tools without a vendor prefix (`plan_analysis`,
76
+ `ground_candidates`, …; see `taxonomy.md`). A conformant server MAY advertise
77
+ the tools under a vendor-namespaced name — the reference implementation
78
+ advertises e.g. `honua_plan_analysis`. The `index.json` lists both the bare
79
+ standard name and the reference implementation's advertised name so a harness
80
+ can resolve either. The schemas validate the **argument object**, which is
81
+ prefix-independent.
82
+
83
+ ## Files
84
+
85
+ | File | Describes |
86
+ |---|---|
87
+ | [`index.json`](index.json) | Machine-readable tool/resource → schema map |
88
+ | [`common/geoprocessing-error.schema.json`](common/geoprocessing-error.schema.json) | The canonical `GeoprocessingError` envelope (shared) |
89
+ | [`tools/*.schema.json`](tools/) | One `inputSchema` per core tool family |
90
+ | [`resources/*.schema.json`](resources/) | One payload schema per core resource family |
@@ -0,0 +1,45 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://geospatial-mcp.honua.io/spec/schemas/common/geoprocessing-error.schema.json",
4
+ "title": "GeoprocessingError",
5
+ "description": "Canonical error envelope reused verbatim by every MCP tool and resource failure path. See spec/resources.md §Error Model. No parallel MCP-local error taxonomy is permitted.",
6
+ "type": "object",
7
+ "required": ["kind", "message"],
8
+ "properties": {
9
+ "kind": {
10
+ "type": "string",
11
+ "description": "GeoprocessingErrorKind value.",
12
+ "enum": [
13
+ "ValidationFailed",
14
+ "AuthorizationDenied",
15
+ "UnknownDataset",
16
+ "UnknownProcess",
17
+ "ExecutionFailed",
18
+ "Timeout",
19
+ "Cancelled",
20
+ "OutputBindingFailed"
21
+ ]
22
+ },
23
+ "message": {
24
+ "type": "string",
25
+ "description": "Human-readable summary."
26
+ },
27
+ "stepId": {
28
+ "type": "string",
29
+ "description": "Plan step identifier when applicable."
30
+ },
31
+ "violations": {
32
+ "type": "array",
33
+ "description": "Structured validation failures.",
34
+ "items": {
35
+ "type": "object",
36
+ "required": ["code", "message"],
37
+ "properties": {
38
+ "code": { "type": "string" },
39
+ "message": { "type": "string" },
40
+ "fieldPath": { "type": "string" }
41
+ }
42
+ }
43
+ }
44
+ }
45
+ }
@@ -0,0 +1,79 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://geospatial-mcp.honua.io/spec/schemas/conformance/manifest.schema.json",
4
+ "title": "geospatial-mcp conformance manifest",
5
+ "description": "Static, server-emitted declaration of the MCP tool and resource surface an implementation advertises, used to check coverage of the geospatial-mcp standard vocabulary without running a live scenario harness. A conformant server produces this manifest from its own tools/list and resources/templates/list capability advertisement. See /CONFORMANCE.md and spec/conformance.md.",
6
+ "type": "object",
7
+ "additionalProperties": false,
8
+ "required": ["specDate", "implementation", "tools", "resources"],
9
+ "properties": {
10
+ "specDate": {
11
+ "type": "string",
12
+ "description": "The Date header of the spec/conformance.md document this manifest was produced against (e.g. 2026-04-19), per spec/conformance.md §8 'Conformance spec identifier per run'.",
13
+ "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
14
+ },
15
+ "implementation": {
16
+ "type": "object",
17
+ "description": "Identity of the implementation under test.",
18
+ "additionalProperties": true,
19
+ "required": ["name"],
20
+ "properties": {
21
+ "name": { "type": "string", "description": "Implementation name (e.g. 'Honua /mcp')." },
22
+ "version": { "type": "string", "description": "Optional implementation version or build identifier." },
23
+ "isReferenceImplementation": {
24
+ "type": "boolean",
25
+ "description": "True only for the reference implementation named in /CONFORMANCE.md (Honua). Other implementations omit this or set it false."
26
+ },
27
+ "profiles": {
28
+ "type": "array",
29
+ "description": "Conformance profiles this implementation claims (default ['base'] when omitted). 'base' is the read-only floor; additive profiles such as 'mutation' require the implementation to advertise every tool the index assigns to that profile to reach FULL. See /CONFORMANCE.md §Conformance Profiles.",
30
+ "items": { "type": "string" },
31
+ "uniqueItems": true
32
+ }
33
+ }
34
+ },
35
+ "tools": {
36
+ "type": "array",
37
+ "description": "The tools the implementation advertises that map onto standard vocabulary.",
38
+ "items": {
39
+ "type": "object",
40
+ "additionalProperties": false,
41
+ "required": ["standardName", "advertisedName"],
42
+ "properties": {
43
+ "standardName": {
44
+ "type": "string",
45
+ "description": "The bare taxonomy.md tool name this advertised tool implements (e.g. 'plan_analysis'). MUST match a tools[].standardName in spec/schemas/index.json."
46
+ },
47
+ "advertisedName": {
48
+ "type": "string",
49
+ "description": "The name the server advertises on tools/list (e.g. 'honua_plan_analysis'). MAY be vendor-prefixed."
50
+ },
51
+ "workflowFamily": {
52
+ "type": "string",
53
+ "description": "Optional workflow-family tag the server attaches to the tool, for coverage reporting."
54
+ }
55
+ }
56
+ }
57
+ },
58
+ "resources": {
59
+ "type": "array",
60
+ "description": "The resource families the implementation advertises that map onto standard vocabulary.",
61
+ "items": {
62
+ "type": "object",
63
+ "additionalProperties": false,
64
+ "required": ["family", "uriForm"],
65
+ "properties": {
66
+ "family": {
67
+ "type": "string",
68
+ "description": "Resource family name. MUST match a resources[].family in spec/schemas/index.json."
69
+ },
70
+ "uriForm": {
71
+ "type": "string",
72
+ "description": "The honua:// URI template the server advertises for the family (e.g. honua://results/{result_package_id}).",
73
+ "pattern": "^honua://"
74
+ }
75
+ }
76
+ }
77
+ }
78
+ }
79
+ }