@honua/mcp-server 0.0.10-alpha.0 → 0.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +259 -15
  2. package/certification/geospatial-mcp-schemas/PROVENANCE.md +41 -0
  3. package/certification/geospatial-mcp-schemas/README.md +90 -0
  4. package/certification/geospatial-mcp-schemas/common/geoprocessing-error.schema.json +45 -0
  5. package/certification/geospatial-mcp-schemas/conformance/manifest.schema.json +79 -0
  6. package/certification/geospatial-mcp-schemas/index.json +390 -0
  7. package/certification/geospatial-mcp-schemas/resources/app-package.schema.json +20 -0
  8. package/certification/geospatial-mcp-schemas/resources/app-template.schema.json +15 -0
  9. package/certification/geospatial-mcp-schemas/resources/artifact.schema.json +69 -0
  10. package/certification/geospatial-mcp-schemas/resources/deployment.schema.json +27 -0
  11. package/certification/geospatial-mcp-schemas/resources/map-package.schema.json +21 -0
  12. package/certification/geospatial-mcp-schemas/resources/map-template.schema.json +15 -0
  13. package/certification/geospatial-mcp-schemas/resources/provenance.schema.json +50 -0
  14. package/certification/geospatial-mcp-schemas/resources/published-service.schema.json +21 -0
  15. package/certification/geospatial-mcp-schemas/resources/result-package.schema.json +64 -0
  16. package/certification/geospatial-mcp-schemas/resources/style.schema.json +12 -0
  17. package/certification/geospatial-mcp-schemas/resources/theme.schema.json +12 -0
  18. package/certification/geospatial-mcp-schemas/resources/workspace.schema.json +63 -0
  19. package/certification/geospatial-mcp-schemas/tools/apply_style_preset.schema.json +27 -0
  20. package/certification/geospatial-mcp-schemas/tools/buffer_features.schema.json +60 -0
  21. package/certification/geospatial-mcp-schemas/tools/cancel_job.schema.json +15 -0
  22. package/certification/geospatial-mcp-schemas/tools/clarify_intent.schema.json +69 -0
  23. package/certification/geospatial-mcp-schemas/tools/compose_mixed_protocol_map.schema.json +21 -0
  24. package/certification/geospatial-mcp-schemas/tools/create_app_package.schema.json +33 -0
  25. package/certification/geospatial-mcp-schemas/tools/create_map_package.schema.json +41 -0
  26. package/certification/geospatial-mcp-schemas/tools/edit_features.schema.json +74 -0
  27. package/certification/geospatial-mcp-schemas/tools/execute_plan.schema.json +64 -0
  28. package/certification/geospatial-mcp-schemas/tools/export_dataset.schema.json +60 -0
  29. package/certification/geospatial-mcp-schemas/tools/geocode_address.schema.json +36 -0
  30. package/certification/geospatial-mcp-schemas/tools/geocode_addresses.schema.json +34 -0
  31. package/certification/geospatial-mcp-schemas/tools/get_style.schema.json +40 -0
  32. package/certification/geospatial-mcp-schemas/tools/ground_candidates.schema.json +66 -0
  33. package/certification/geospatial-mcp-schemas/tools/ingest_dataset.schema.json +45 -0
  34. package/certification/geospatial-mcp-schemas/tools/join_features.schema.json +73 -0
  35. package/certification/geospatial-mcp-schemas/tools/list_capabilities.schema.json +21 -0
  36. package/certification/geospatial-mcp-schemas/tools/list_layers.schema.json +14 -0
  37. package/certification/geospatial-mcp-schemas/tools/overlay_features.schema.json +53 -0
  38. package/certification/geospatial-mcp-schemas/tools/plan_analysis.schema.json +19 -0
  39. package/certification/geospatial-mcp-schemas/tools/preview_app_package.schema.json +15 -0
  40. package/certification/geospatial-mcp-schemas/tools/preview_map_package.schema.json +17 -0
  41. package/certification/geospatial-mcp-schemas/tools/preview_package.schema.json +40 -0
  42. package/certification/geospatial-mcp-schemas/tools/propose_operation.schema.json +28 -0
  43. package/certification/geospatial-mcp-schemas/tools/publish_result.schema.json +29 -0
  44. package/certification/geospatial-mcp-schemas/tools/publish_service.schema.json +59 -0
  45. package/certification/geospatial-mcp-schemas/tools/query_features.schema.json +74 -0
  46. package/certification/geospatial-mcp-schemas/tools/refine_map_package.schema.json +22 -0
  47. package/certification/geospatial-mcp-schemas/tools/render_map.schema.json +60 -0
  48. package/certification/geospatial-mcp-schemas/tools/reproject_features.schema.json +49 -0
  49. package/certification/geospatial-mcp-schemas/tools/resolve_entity.schema.json +30 -0
  50. package/certification/geospatial-mcp-schemas/tools/solve_route.schema.json +39 -0
  51. package/certification/geospatial-mcp-schemas/tools/summarize_statistics.schema.json +71 -0
  52. package/certification/geospatial-mcp-schemas/tools/validate_package.schema.json +35 -0
  53. package/certification/geospatial-mcp-schemas/tools/validate_plan.schema.json +61 -0
  54. package/dist/src/capability.d.ts +48 -0
  55. package/dist/src/capability.d.ts.map +1 -0
  56. package/dist/src/capability.js +44 -0
  57. package/dist/src/certification/census-data.d.ts +33 -0
  58. package/dist/src/certification/census-data.d.ts.map +1 -0
  59. package/dist/src/certification/census-data.js +506 -0
  60. package/dist/src/certification/census-fixture-client.d.ts +7 -0
  61. package/dist/src/certification/census-fixture-client.d.ts.map +1 -0
  62. package/dist/src/certification/census-fixture-client.js +226 -0
  63. package/dist/src/certification/certifier.d.ts +125 -0
  64. package/dist/src/certification/certifier.d.ts.map +1 -0
  65. package/dist/src/certification/certifier.js +455 -0
  66. package/dist/src/certification/cli.d.ts +3 -0
  67. package/dist/src/certification/cli.d.ts.map +1 -0
  68. package/dist/src/certification/cli.js +63 -0
  69. package/dist/src/certification/contracts.d.ts +103 -0
  70. package/dist/src/certification/contracts.d.ts.map +1 -0
  71. package/dist/src/certification/contracts.js +245 -0
  72. package/dist/src/certification/fixture-client.d.ts +4 -0
  73. package/dist/src/certification/fixture-client.d.ts.map +1 -0
  74. package/dist/src/certification/fixture-client.js +113 -0
  75. package/dist/src/certification/fixtures.d.ts +45 -0
  76. package/dist/src/certification/fixtures.d.ts.map +1 -0
  77. package/dist/src/certification/fixtures.js +92 -0
  78. package/dist/src/certification/json-schema.d.ts +71 -0
  79. package/dist/src/certification/json-schema.d.ts.map +1 -0
  80. package/dist/src/certification/json-schema.js +172 -0
  81. package/dist/src/certification/lifecycle.d.ts +14 -0
  82. package/dist/src/certification/lifecycle.d.ts.map +1 -0
  83. package/dist/src/certification/lifecycle.js +326 -0
  84. package/dist/src/certification/mock-upstream.d.ts +29 -0
  85. package/dist/src/certification/mock-upstream.d.ts.map +1 -0
  86. package/dist/src/certification/mock-upstream.js +96 -0
  87. package/dist/src/certification/operator-catalog.d.ts +90 -0
  88. package/dist/src/certification/operator-catalog.d.ts.map +1 -0
  89. package/dist/src/certification/operator-catalog.js +650 -0
  90. package/dist/src/certification/report.d.ts +4 -0
  91. package/dist/src/certification/report.d.ts.map +1 -0
  92. package/dist/src/certification/report.js +103 -0
  93. package/dist/src/certification/run.d.ts +28 -0
  94. package/dist/src/certification/run.d.ts.map +1 -0
  95. package/dist/src/certification/run.js +32 -0
  96. package/dist/src/certification/schema-index.d.ts +94 -0
  97. package/dist/src/certification/schema-index.d.ts.map +1 -0
  98. package/dist/src/certification/schema-index.js +109 -0
  99. package/dist/src/certification/target.d.ts +48 -0
  100. package/dist/src/certification/target.d.ts.map +1 -0
  101. package/dist/src/certification/target.js +219 -0
  102. package/dist/src/eval/cli.d.ts +3 -0
  103. package/dist/src/eval/cli.d.ts.map +1 -0
  104. package/dist/src/eval/cli.js +118 -0
  105. package/dist/src/eval/corpus.d.ts +34 -0
  106. package/dist/src/eval/corpus.d.ts.map +1 -0
  107. package/dist/src/eval/corpus.js +150 -0
  108. package/dist/src/eval/drivers/anthropic.d.ts +14 -0
  109. package/dist/src/eval/drivers/anthropic.d.ts.map +1 -0
  110. package/dist/src/eval/drivers/anthropic.js +120 -0
  111. package/dist/src/eval/drivers/bedrock.d.ts +54 -0
  112. package/dist/src/eval/drivers/bedrock.d.ts.map +1 -0
  113. package/dist/src/eval/drivers/bedrock.js +178 -0
  114. package/dist/src/eval/drivers/deterministic.d.ts +17 -0
  115. package/dist/src/eval/drivers/deterministic.d.ts.map +1 -0
  116. package/dist/src/eval/drivers/deterministic.js +64 -0
  117. package/dist/src/eval/drivers/index.d.ts +34 -0
  118. package/dist/src/eval/drivers/index.d.ts.map +1 -0
  119. package/dist/src/eval/drivers/index.js +74 -0
  120. package/dist/src/eval/drivers/openai.d.ts +14 -0
  121. package/dist/src/eval/drivers/openai.d.ts.map +1 -0
  122. package/dist/src/eval/drivers/openai.js +136 -0
  123. package/dist/src/eval/grade.d.ts +14 -0
  124. package/dist/src/eval/grade.d.ts.map +1 -0
  125. package/dist/src/eval/grade.js +75 -0
  126. package/dist/src/eval/northstar-corpus.d.ts +3 -0
  127. package/dist/src/eval/northstar-corpus.d.ts.map +1 -0
  128. package/dist/src/eval/northstar-corpus.js +194 -0
  129. package/dist/src/eval/operator-corpus.d.ts +3 -0
  130. package/dist/src/eval/operator-corpus.d.ts.map +1 -0
  131. package/dist/src/eval/operator-corpus.js +193 -0
  132. package/dist/src/eval/report.d.ts +109 -0
  133. package/dist/src/eval/report.d.ts.map +1 -0
  134. package/dist/src/eval/report.js +133 -0
  135. package/dist/src/eval/runner.d.ts +28 -0
  136. package/dist/src/eval/runner.d.ts.map +1 -0
  137. package/dist/src/eval/runner.js +130 -0
  138. package/dist/src/eval/standalone-corpus.d.ts +3 -0
  139. package/dist/src/eval/standalone-corpus.d.ts.map +1 -0
  140. package/dist/src/eval/standalone-corpus.js +699 -0
  141. package/dist/src/eval/types.d.ts +118 -0
  142. package/dist/src/eval/types.d.ts.map +1 -0
  143. package/dist/src/eval/types.js +10 -0
  144. package/dist/src/helpers.d.ts +11 -0
  145. package/dist/src/helpers.d.ts.map +1 -1
  146. package/dist/src/helpers.js +22 -0
  147. package/dist/src/index.d.ts +31 -0
  148. package/dist/src/index.d.ts.map +1 -1
  149. package/dist/src/index.js +50 -4
  150. package/dist/src/provenance.d.ts +44 -0
  151. package/dist/src/provenance.d.ts.map +1 -0
  152. package/dist/src/provenance.js +37 -0
  153. package/dist/src/proxy.d.ts +46 -0
  154. package/dist/src/proxy.d.ts.map +1 -0
  155. package/dist/src/proxy.js +124 -0
  156. package/dist/src/resources/styles.d.ts +29 -0
  157. package/dist/src/resources/styles.d.ts.map +1 -0
  158. package/dist/src/resources/styles.js +76 -0
  159. package/dist/src/styles.d.ts +112 -0
  160. package/dist/src/styles.d.ts.map +1 -0
  161. package/dist/src/styles.js +144 -0
  162. package/dist/src/tools/apply-style-preset.d.ts +27 -0
  163. package/dist/src/tools/apply-style-preset.d.ts.map +1 -0
  164. package/dist/src/tools/apply-style-preset.js +41 -0
  165. package/dist/src/tools/count-features.d.ts.map +1 -1
  166. package/dist/src/tools/count-features.js +5 -4
  167. package/dist/src/tools/get-extent.d.ts.map +1 -1
  168. package/dist/src/tools/get-extent.js +6 -4
  169. package/dist/src/tools/get-style.d.ts +24 -0
  170. package/dist/src/tools/get-style.d.ts.map +1 -0
  171. package/dist/src/tools/get-style.js +39 -0
  172. package/dist/src/tools/list-services.d.ts.map +1 -1
  173. package/dist/src/tools/list-services.js +16 -1
  174. package/dist/src/tools/query-features.d.ts.map +1 -1
  175. package/dist/src/tools/query-features.js +4 -1
  176. package/package.json +28 -4
@@ -0,0 +1,118 @@
1
+ #!/usr/bin/env node
2
+ import { mkdirSync, writeFileSync } from "node:fs";
3
+ import { resolve } from "node:path";
4
+ import { fileURLToPath } from "node:url";
5
+ import { resolveCorpus } from "./corpus.js";
6
+ import { parseDriverName, selectDrivers } from "./drivers/index.js";
7
+ import { renderMarkdown } from "./report.js";
8
+ import { runEval } from "./runner.js";
9
+ /**
10
+ * Cross-model MCP workflow eval CLI (honua-server #1956).
11
+ *
12
+ * Modes:
13
+ * (default) Run the eval, write artifacts, exit non-zero if the
14
+ * deterministic control did not pass every scenario.
15
+ * --artifact-only Run the eval, write artifacts, ALWAYS exit 0 (evidence
16
+ * upload step; runs with always() in CI).
17
+ * --offline Force the in-memory fixture surface + deterministic driver
18
+ * only, even if API keys / HONUA_MCP_REMOTE_URL are set.
19
+ * --driver <name> Run the deterministic control plus the named live driver(s)
20
+ * (repeatable or comma-separated): anthropic | openai | bedrock.
21
+ * The named driver is run even if its key/opt-in is absent so an
22
+ * explicit request yields an honest driverError, not a silent
23
+ * skip. Without --driver, drivers are auto-resolved from env.
24
+ * --corpus <name> Select the corpus explicitly: analyst | operator | northstar |
25
+ * all. This overrides HONUA_EVAL_CORPUS and is how the `eval:live`
26
+ * script targets the operator surface without relying on shell env
27
+ * (cross-platform). `northstar` targets the three P1-gate
28
+ * workflows. Defaults to the env-driven selection.
29
+ *
30
+ * Corpus selection is env-driven (see `resolveCorpus`): default is the analyst
31
+ * corpus (in-process SDK fixture surface, the CI control); `HONUA_EVAL_CORPUS=operator`
32
+ * targets the honua-server operator MCP surface for live runs; `all` runs both.
33
+ *
34
+ * Offline and deterministic by default: zero model/API calls. Live cross-model
35
+ * runs activate when ANTHROPIC_API_KEY / OPENAI_API_KEY are set (Anthropic/OpenAI
36
+ * APIs) or HONUA_EVAL_BEDROCK is set (Claude via Amazon Bedrock, AWS credential
37
+ * chain), and live remote MCP runs when HONUA_MCP_REMOTE_URL is set. Keys are
38
+ * never hardcoded — they come from the environment.
39
+ */
40
+ const CORPUS_NAMES = ["analyst", "operator", "northstar", "standalone", "all"];
41
+ function parseArgs(argv) {
42
+ let artifactOnly = false;
43
+ let offline = false;
44
+ let outDir = fileURLToPath(new URL("../../../", import.meta.url));
45
+ const drivers = [];
46
+ let corpus;
47
+ for (let i = 0; i < argv.length; i++) {
48
+ const arg = argv[i];
49
+ if (arg === "--artifact-only") {
50
+ artifactOnly = true;
51
+ }
52
+ else if (arg === "--offline") {
53
+ offline = true;
54
+ }
55
+ else if (arg === "--out-dir") {
56
+ const value = argv[++i];
57
+ if (!value) {
58
+ throw new Error("--out-dir requires a directory argument");
59
+ }
60
+ outDir = resolve(value);
61
+ }
62
+ else if (arg === "--corpus") {
63
+ const value = argv[++i];
64
+ if (!value || !CORPUS_NAMES.includes(value)) {
65
+ throw new Error(`--corpus requires one of: ${CORPUS_NAMES.join(", ")}`);
66
+ }
67
+ corpus = value;
68
+ }
69
+ else if (arg === "--driver") {
70
+ const value = argv[++i];
71
+ if (!value) {
72
+ throw new Error("--driver requires a driver name (anthropic | openai | bedrock)");
73
+ }
74
+ for (const name of value.split(",")) {
75
+ const trimmed = name.trim();
76
+ if (trimmed.length > 0) {
77
+ drivers.push(parseDriverName(trimmed));
78
+ }
79
+ }
80
+ }
81
+ }
82
+ return { artifactOnly, offline, outDir, drivers, ...(corpus ? { corpus } : {}) };
83
+ }
84
+ /** Map the `--corpus` alias to the corresponding HONUA_EVAL_CORPUS selection. */
85
+ function corpusForName(name) {
86
+ const selector = name === "analyst" ? "" : name; // analyst is the env default
87
+ return resolveCorpus({ ...process.env, HONUA_EVAL_CORPUS: selector });
88
+ }
89
+ async function main() {
90
+ const { artifactOnly, offline, outDir, drivers, corpus } = parseArgs(process.argv.slice(2));
91
+ const report = await runEval({
92
+ forceOffline: offline,
93
+ ...(drivers.length > 0 ? { drivers: selectDrivers(drivers) } : {}),
94
+ ...(corpus ? { corpus: corpusForName(corpus) } : {}),
95
+ ...(corpus === "standalone" ? { forceStandaloneSurface: true } : {}),
96
+ });
97
+ mkdirSync(outDir, { recursive: true });
98
+ const jsonPath = resolve(outDir, "mcp-eval-results.json");
99
+ const mdPath = resolve(outDir, "mcp-eval-results.md");
100
+ writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
101
+ writeFileSync(mdPath, renderMarkdown(report), "utf8");
102
+ const s = report.summary;
103
+ const status = s.pass ? "PASS" : "FAIL";
104
+ process.stdout.write(`MCP cross-model eval ${status}: ${s.scenarios} scenarios, ${s.modelsEvaluated} models ` +
105
+ `(${s.liveModelsEvaluated} live), surface=${report.surface.backend}.\n`);
106
+ for (const m of report.models) {
107
+ process.stdout.write(` ${m.id}: success ${(m.successRate * 100).toFixed(0)}%, clarify ${(m.clarificationRate * 100).toFixed(0)}%, ` +
108
+ `edit ${(m.editRate * 100).toFixed(0)}% (${m.pass}/${m.scenarios} passed)\n`);
109
+ }
110
+ process.stdout.write(`Artifacts: ${jsonPath}\n ${mdPath}\n`);
111
+ if (!artifactOnly && !s.pass) {
112
+ process.exitCode = 1;
113
+ }
114
+ }
115
+ main().catch((err) => {
116
+ process.stderr.write(`Eval error: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}\n`);
117
+ process.exit(1);
118
+ });
@@ -0,0 +1,34 @@
1
+ import type { Scenario } from "./types.js";
2
+ /**
3
+ * Held-out corpus of GIS workflows for the cross-model eval (honua-server #1956).
4
+ *
5
+ * Each scenario is a natural-language GIS task plus declarative success criteria.
6
+ * The `script` is the "ideal client" tool trajectory the deterministic offline
7
+ * driver follows; live LLM drivers (Claude, GPT) plan their own calls from the
8
+ * prompt and the advertised tool catalog. Criteria are graded identically across
9
+ * all drivers, so the same corpus measures any client against any workflow.
10
+ *
11
+ * Tool/argument shapes target the canonical honua MCP catalog and resolve
12
+ * against the offline fixture backend (service "Parks", layer 0).
13
+ */
14
+ export declare const CORPUS: Scenario[];
15
+ /**
16
+ * Select the corpus to run from the environment (honua-server #1956).
17
+ *
18
+ * The default `analyst` corpus targets the in-process SDK fixture surface and is
19
+ * the CI control. `operator` selects {@link OPERATOR_CORPUS} for live runs against
20
+ * the honua-server operator MCP surface; `northstar` selects {@link NORTHSTAR_CORPUS},
21
+ * the three P1-gate workflows (also live-gated); `all` runs every corpus
22
+ * back-to-back. Selection is case-insensitive and falls back to the analyst corpus
23
+ * for any unset/unknown value so CI behavior is unchanged unless `HONUA_EVAL_CORPUS`
24
+ * is set explicitly.
25
+ */
26
+ export declare function resolveCorpus(env?: NodeJS.ProcessEnv): Scenario[];
27
+ /**
28
+ * Whether a corpus selection targets the PLATFORM-FREE standalone surface (a plain
29
+ * public FeatureServer fixture) rather than the Honua operator/analyst surface.
30
+ * The offline runner uses this to pick the census fixture client so the standalone
31
+ * corpus resolves against real recorded non-Honua data.
32
+ */
33
+ export declare function isStandaloneCorpus(env?: NodeJS.ProcessEnv): boolean;
34
+ //# sourceMappingURL=corpus.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../../../src/eval/corpus.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,MAAM,EAAE,QAAQ,EAkG5B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAgB,aAAa,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,QAAQ,EAAE,CAiB9E;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,OAAO,CAEhF"}
@@ -0,0 +1,150 @@
1
+ import { NORTHSTAR_CORPUS } from "./northstar-corpus.js";
2
+ import { OPERATOR_CORPUS } from "./operator-corpus.js";
3
+ import { STANDALONE_CORPUS } from "./standalone-corpus.js";
4
+ /**
5
+ * Held-out corpus of GIS workflows for the cross-model eval (honua-server #1956).
6
+ *
7
+ * Each scenario is a natural-language GIS task plus declarative success criteria.
8
+ * The `script` is the "ideal client" tool trajectory the deterministic offline
9
+ * driver follows; live LLM drivers (Claude, GPT) plan their own calls from the
10
+ * prompt and the advertised tool catalog. Criteria are graded identically across
11
+ * all drivers, so the same corpus measures any client against any workflow.
12
+ *
13
+ * Tool/argument shapes target the canonical honua MCP catalog and resolve
14
+ * against the offline fixture backend (service "Parks", layer 0).
15
+ */
16
+ export const CORPUS = [
17
+ {
18
+ id: "discover-services",
19
+ title: "Discover available feature services",
20
+ category: "discovery",
21
+ prompt: "I'm new to this server. What feature services are available to query? Just list them.",
22
+ criteria: {
23
+ requiredTools: ["honua_list_services"],
24
+ answerMustInclude: ["Parks"],
25
+ },
26
+ script: [{ tool: "honua_list_services", args: {} }],
27
+ },
28
+ {
29
+ id: "inspect-layer-schema",
30
+ title: "Inspect a layer's schema before querying",
31
+ category: "grounding",
32
+ prompt: "Before I query the Parks service, show me the schema of its first layer (layer 0) — the fields and geometry type.",
33
+ criteria: {
34
+ requiredTools: ["honua_describe_layer"],
35
+ answerMustInclude: ["OBJECTID"],
36
+ },
37
+ script: [{ tool: "honua_describe_layer", args: { serviceId: "Parks", layerId: 0 } }],
38
+ },
39
+ {
40
+ id: "count-then-query",
41
+ title: "Check cardinality, then fetch a sample",
42
+ category: "query",
43
+ prompt: "How many features are in the Parks service layer 0? Then show me a couple of them.",
44
+ criteria: {
45
+ requiredTools: ["honua_count_features", "honua_query_features"],
46
+ expectedToolSequence: ["honua_count_features", "honua_query_features"],
47
+ answerMustInclude: ["Park A"],
48
+ },
49
+ script: [
50
+ { tool: "honua_count_features", args: { serviceId: "Parks", layerId: 0 } },
51
+ { tool: "honua_query_features", args: { serviceId: "Parks", layerId: 0, limit: 2 } },
52
+ ],
53
+ },
54
+ {
55
+ id: "discover-then-query",
56
+ title: "End-to-end discover → query",
57
+ category: "multi-step",
58
+ prompt: "I don't know what's on this server. Find a feature service and show me a few features from one of its layers.",
59
+ criteria: {
60
+ requiredTools: ["honua_list_services", "honua_query_features"],
61
+ expectedToolSequence: ["honua_list_services", "honua_query_features"],
62
+ answerMustInclude: ["Park"],
63
+ },
64
+ script: [
65
+ { tool: "honua_list_services", args: {} },
66
+ { tool: "honua_query_features", args: { serviceId: "Parks", layerId: 0, limit: 2 } },
67
+ ],
68
+ },
69
+ {
70
+ id: "aggregate-statistic",
71
+ title: "Compute an aggregate statistic",
72
+ category: "analysis",
73
+ prompt: "What is the average of the VALUE field for the Parks service, layer 0?",
74
+ criteria: {
75
+ requiredTools: ["honua_statistics"],
76
+ answerMustInclude: ["statistics"],
77
+ },
78
+ script: [
79
+ {
80
+ tool: "honua_statistics",
81
+ args: { serviceId: "Parks", layerId: 0, statisticType: "avg", onField: "VALUE" },
82
+ },
83
+ ],
84
+ },
85
+ {
86
+ id: "spatial-extent",
87
+ title: "Get the spatial extent of a layer",
88
+ category: "query",
89
+ prompt: "What is the bounding box / spatial extent of the Parks service, layer 0?",
90
+ criteria: {
91
+ requiredTools: ["honua_get_extent"],
92
+ answerMustInclude: ["extent"],
93
+ },
94
+ script: [{ tool: "honua_get_extent", args: { serviceId: "Parks", layerId: 0 } }],
95
+ },
96
+ {
97
+ id: "capability-gap",
98
+ title: "Explain a capability gap honestly",
99
+ category: "grounding",
100
+ prompt: "Can I run an attribute query against a WMTS source on this platform? If not, what should I do instead?",
101
+ criteria: {
102
+ requiredTools: ["honua_explain_capability_gap"],
103
+ answerMustInclude: ["explanation"],
104
+ },
105
+ script: [
106
+ {
107
+ tool: "honua_explain_capability_gap",
108
+ args: { capability: "query", protocol: "wmts" },
109
+ },
110
+ ],
111
+ },
112
+ ];
113
+ /**
114
+ * Select the corpus to run from the environment (honua-server #1956).
115
+ *
116
+ * The default `analyst` corpus targets the in-process SDK fixture surface and is
117
+ * the CI control. `operator` selects {@link OPERATOR_CORPUS} for live runs against
118
+ * the honua-server operator MCP surface; `northstar` selects {@link NORTHSTAR_CORPUS},
119
+ * the three P1-gate workflows (also live-gated); `all` runs every corpus
120
+ * back-to-back. Selection is case-insensitive and falls back to the analyst corpus
121
+ * for any unset/unknown value so CI behavior is unchanged unless `HONUA_EVAL_CORPUS`
122
+ * is set explicitly.
123
+ */
124
+ export function resolveCorpus(env = process.env) {
125
+ switch ((env.HONUA_EVAL_CORPUS ?? "").trim().toLowerCase()) {
126
+ case "operator":
127
+ return OPERATOR_CORPUS;
128
+ case "northstar":
129
+ return NORTHSTAR_CORPUS;
130
+ case "standalone":
131
+ return STANDALONE_CORPUS;
132
+ case "all":
133
+ // `all` spans the Honua-surface corpora only. The platform-free standalone
134
+ // corpus targets a DIFFERENT surface (a plain FeatureServer fixture) and is
135
+ // selected explicitly via `standalone`; mixing it here would run its
136
+ // census-specific scenarios against an operator surface.
137
+ return [...CORPUS, ...OPERATOR_CORPUS, ...NORTHSTAR_CORPUS];
138
+ default:
139
+ return CORPUS;
140
+ }
141
+ }
142
+ /**
143
+ * Whether a corpus selection targets the PLATFORM-FREE standalone surface (a plain
144
+ * public FeatureServer fixture) rather than the Honua operator/analyst surface.
145
+ * The offline runner uses this to pick the census fixture client so the standalone
146
+ * corpus resolves against real recorded non-Honua data.
147
+ */
148
+ export function isStandaloneCorpus(env = process.env) {
149
+ return (env.HONUA_EVAL_CORPUS ?? "").trim().toLowerCase() === "standalone";
150
+ }
@@ -0,0 +1,14 @@
1
+ import type { ModelDriver, Scenario, WorkflowContext, WorkflowTranscript } from "../types.js";
2
+ export interface AnthropicDriverOptions {
3
+ apiKey?: string | undefined;
4
+ model?: string;
5
+ }
6
+ export declare class AnthropicDriver implements ModelDriver {
7
+ readonly vendor: "anthropic";
8
+ readonly id: string;
9
+ private readonly apiKey;
10
+ constructor(options?: AnthropicDriverOptions);
11
+ isAvailable(): boolean;
12
+ runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
13
+ }
14
+ //# sourceMappingURL=anthropic.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAkB,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAoC9G,MAAM,WAAW,sBAAsB;IACrC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,eAAgB,YAAW,WAAW;IACjD,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;IACvC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,GAAE,sBAA2B;IAKhD,WAAW,IAAI,OAAO;IAIhB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAoFzF"}
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Claude client driver for the cross-model eval (honua-server #1956).
3
+ *
4
+ * Gated behind ANTHROPIC_API_KEY and a dynamically-imported `@anthropic-ai/sdk`
5
+ * — neither is a dependency of this package, so the offline CI path never needs
6
+ * them. Live runs use the latest Opus model (claude-opus-4-8) with a manual
7
+ * tool-use loop over the advertised MCP catalog. A synthetic `ask_user` tool
8
+ * lets the model signal that it needs clarification, which the harness records
9
+ * separately from success/failure.
10
+ *
11
+ * Live cross-model runs are tracked under honua-io/honua-server#1956.
12
+ */
13
+ const ASK_USER_TOOL = {
14
+ name: "ask_user",
15
+ description: "Call this ONLY if the request is too ambiguous to proceed. Provide the clarifying question you would ask.",
16
+ input_schema: {
17
+ type: "object",
18
+ properties: { question: { type: "string" } },
19
+ required: ["question"],
20
+ additionalProperties: false,
21
+ },
22
+ };
23
+ const SYSTEM_PROMPT = "You are a GIS analyst working through the Honua MCP tools. Complete the user's task by calling the available tools. " +
24
+ "Ground every step in real tool results — do not invent service names, layers, or fields. " +
25
+ "If, and only if, the request is genuinely ambiguous and you cannot proceed, call ask_user. " +
26
+ "When finished, give a concise final answer that references what the tools returned.";
27
+ const MAX_ITERATIONS = 8;
28
+ const DEFAULT_MODEL = "claude-opus-4-8";
29
+ export class AnthropicDriver {
30
+ vendor = "anthropic";
31
+ id;
32
+ apiKey;
33
+ constructor(options = {}) {
34
+ this.id = options.model ?? DEFAULT_MODEL;
35
+ this.apiKey = options.apiKey ?? process.env.ANTHROPIC_API_KEY;
36
+ }
37
+ isAvailable() {
38
+ return typeof this.apiKey === "string" && this.apiKey.length > 0;
39
+ }
40
+ async runWorkflow(scenario, ctx) {
41
+ const steps = [];
42
+ const transcript = {
43
+ scenarioId: scenario.id,
44
+ modelId: this.id,
45
+ steps,
46
+ finalAnswer: "",
47
+ clarificationRequested: false,
48
+ errorCount: 0,
49
+ };
50
+ let Anthropic;
51
+ try {
52
+ // Variable specifier → not statically resolved; keeps @anthropic-ai/sdk
53
+ // an optional, non-bundled dependency for live runs only.
54
+ const specifier = "@anthropic-ai/sdk";
55
+ const mod = (await import(specifier));
56
+ Anthropic = mod.default;
57
+ }
58
+ catch {
59
+ transcript.driverError = "@anthropic-ai/sdk is not installed (required for live Claude eval runs)";
60
+ return transcript;
61
+ }
62
+ if (!this.apiKey) {
63
+ transcript.driverError = "ANTHROPIC_API_KEY is not set";
64
+ return transcript;
65
+ }
66
+ const client = new Anthropic({ apiKey: this.apiKey });
67
+ const tools = [
68
+ ...ctx.tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.inputSchema })),
69
+ ASK_USER_TOOL,
70
+ ];
71
+ const messages = [{ role: "user", content: scenario.prompt }];
72
+ try {
73
+ for (let i = 0; i < MAX_ITERATIONS; i++) {
74
+ const response = await client.messages.create({
75
+ model: this.id,
76
+ max_tokens: 4096,
77
+ system: SYSTEM_PROMPT,
78
+ tools,
79
+ messages,
80
+ });
81
+ const toolUses = response.content.filter((b) => b.type === "tool_use");
82
+ const text = response.content
83
+ .filter((b) => b.type === "text")
84
+ .map((b) => b.text)
85
+ .join("\n");
86
+ if (response.stop_reason !== "tool_use" || toolUses.length === 0) {
87
+ transcript.finalAnswer = text;
88
+ return transcript;
89
+ }
90
+ messages.push({ role: "assistant", content: response.content });
91
+ const toolResults = [];
92
+ for (const use of toolUses) {
93
+ if (use.name === ASK_USER_TOOL.name) {
94
+ transcript.clarificationRequested = true;
95
+ transcript.finalAnswer = text;
96
+ return transcript;
97
+ }
98
+ const result = await ctx.callTool(use.name, (use.input ?? {}));
99
+ steps.push({ tool: use.name, args: (use.input ?? {}), isError: result.isError });
100
+ if (result.isError) {
101
+ transcript.errorCount++;
102
+ }
103
+ toolResults.push({
104
+ type: "tool_result",
105
+ tool_use_id: use.id,
106
+ content: result.text,
107
+ is_error: result.isError,
108
+ });
109
+ }
110
+ messages.push({ role: "user", content: toolResults });
111
+ }
112
+ transcript.driverError = `exceeded ${MAX_ITERATIONS} tool-use iterations without finishing`;
113
+ return transcript;
114
+ }
115
+ catch (err) {
116
+ transcript.driverError = err instanceof Error ? err.message : String(err);
117
+ return transcript;
118
+ }
119
+ }
120
+ }
@@ -0,0 +1,54 @@
1
+ import type { ModelDriver, Scenario, ToolDescriptor, WorkflowContext, WorkflowTranscript } from "../types.js";
2
+ export interface BedrockDriverOptions {
3
+ /** Bedrock model id or inference-profile id. Defaults to Sonnet 4.5. */
4
+ model?: string | undefined;
5
+ /** AWS region. Defaults to AWS_REGION, then us-west-2. */
6
+ region?: string | undefined;
7
+ /** Opt-in flag value (defaults to process.env.HONUA_EVAL_BEDROCK). */
8
+ enabled?: string | undefined;
9
+ }
10
+ export declare class BedrockDriver implements ModelDriver {
11
+ readonly vendor: "bedrock";
12
+ readonly id: string;
13
+ readonly region: string;
14
+ private readonly enabled;
15
+ constructor(options?: BedrockDriverOptions);
16
+ isAvailable(): boolean;
17
+ runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
18
+ }
19
+ /** Map MCP tool descriptors to Bedrock Converse `toolSpec` entries. */
20
+ export declare function toBedrockTools(tools: ToolDescriptor[]): BedrockTool[];
21
+ /** Pull tool-use blocks out of a Converse content array. */
22
+ export declare function extractToolUses(content: BedrockContentBlock[]): BedrockToolUse[];
23
+ /** Concatenate text blocks of a Converse content array. */
24
+ export declare function extractText(content: BedrockContentBlock[]): string;
25
+ /** Build a Converse `toolResult` content block to feed back to the model. */
26
+ export declare function toToolResultBlock(toolUseId: string, text: string, isError: boolean): BedrockContentBlock;
27
+ interface BedrockToolUse {
28
+ toolUseId: string;
29
+ name: string;
30
+ input?: unknown;
31
+ }
32
+ interface BedrockToolResult {
33
+ toolUseId: string;
34
+ content: {
35
+ text: string;
36
+ }[];
37
+ status: "error" | "success";
38
+ }
39
+ interface BedrockContentBlock {
40
+ text?: string;
41
+ toolUse?: BedrockToolUse;
42
+ toolResult?: BedrockToolResult;
43
+ }
44
+ interface BedrockTool {
45
+ toolSpec: {
46
+ name: string;
47
+ description: string;
48
+ inputSchema: {
49
+ json: Record<string, unknown>;
50
+ };
51
+ };
52
+ }
53
+ export {};
54
+ //# sourceMappingURL=bedrock.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/bedrock.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,QAAQ,EACR,cAAc,EAEd,eAAe,EACf,kBAAkB,EACnB,MAAM,aAAa,CAAC;AAgErB,MAAM,WAAW,oBAAoB;IACnC,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,sEAAsE;IACtE,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9B;AAED,qBAAa,aAAc,YAAW,WAAW;IAC/C,QAAQ,CAAC,MAAM,EAAG,SAAS,CAAU;IACrC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;gBAEtB,OAAO,GAAE,oBAAyB;IAO9C,WAAW,IAAI,OAAO;IAIhB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAyEzF;AAUD,uEAAuE;AACvE,wBAAgB,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,CAQrE;AAED,4DAA4D;AAC5D,wBAAgB,eAAe,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,cAAc,EAAE,CAIhF;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,MAAM,CAKlE;AAED,6EAA6E;AAC7E,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,mBAAmB,CAQxG;AAID,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,iBAAiB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5B,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;CAC7B;AACD,UAAU,mBAAmB;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAKD,UAAU,WAAW;IACnB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,CAAC;KAChD,CAAC;CACH"}