@ggui-ai/protocol-conformance 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -14
- package/dist/cli.d.ts +23 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +228 -19
- package/dist/fixtures/index.d.ts +9 -0
- package/dist/fixtures/index.d.ts.map +1 -1
- package/dist/fixtures/index.js +9 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -0
- package/dist/json-equal.d.ts +6 -0
- package/dist/json-equal.d.ts.map +1 -0
- package/dist/json-equal.js +23 -0
- package/dist/refusal-envelope-conformance/cases/refuse-after-fix-caller.json +27 -0
- package/dist/refusal-envelope-conformance/cases/refuse-after-fix-owner-with-balance.json +29 -0
- package/dist/refusal-envelope-conformance/cases/refuse-later.json +27 -0
- package/dist/refusal-envelope-conformance/cases/refuse-never.json +27 -0
- package/dist/refusal-envelope-conformance/cases/refuse-next-period.json +27 -0
- package/dist/refusal-envelope-conformance/cases/refuse-non-render-surface.json +12 -0
- package/dist/refusal-envelope-conformance/index.d.ts +108 -0
- package/dist/refusal-envelope-conformance/index.d.ts.map +1 -0
- package/dist/refusal-envelope-conformance/index.js +119 -0
- package/dist/registry-completeness/index.d.ts +105 -0
- package/dist/registry-completeness/index.d.ts.map +1 -0
- package/dist/registry-completeness/index.js +110 -0
- package/dist/reporter.d.ts.map +1 -1
- package/dist/reporter.js +26 -9
- package/dist/run-conformance.d.ts +38 -0
- package/dist/run-conformance.d.ts.map +1 -1
- package/dist/run-conformance.js +125 -4
- package/dist/transport-refusal-conformance/cases/refuse-deprovisioned-endpoint.json +25 -0
- package/dist/transport-refusal-conformance/cases/refuse-render-only-code.json +11 -0
- package/dist/transport-refusal-conformance/index.d.ts +54 -0
- package/dist/transport-refusal-conformance/index.d.ts.map +1 -0
- package/dist/transport-refusal-conformance/index.js +52 -0
- package/package.json +20 -5
package/dist/reporter.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* `reporter` config.
|
|
15
15
|
*/
|
|
16
16
|
import { fixturesByContract } from './fixtures/index.js';
|
|
17
|
+
import { PURE_FUNCTION_CATALOG_SLUGS } from './run-conformance.js';
|
|
17
18
|
/**
|
|
18
19
|
* Build a {@link ConformanceReporter} that writes the bar-scorecard
|
|
19
20
|
* report. The return value plugs directly into
|
|
@@ -69,21 +70,37 @@ export function createDefaultReporter(options = {}) {
|
|
|
69
70
|
// =============================================================================
|
|
70
71
|
export function formatScorecard(result) {
|
|
71
72
|
// Group fixtures by contract so operators see pass rates per
|
|
72
|
-
// criterion, mirroring `fixturesByContract`'s classification
|
|
73
|
+
// criterion, mirroring `fixturesByContract`'s classification, then
|
|
74
|
+
// the pure-function catalogs, whose rows carry a `<slug>/` prefix
|
|
75
|
+
// instead of living in that map (they are not WS-observable).
|
|
73
76
|
const lines = [];
|
|
74
77
|
for (const [slug, fixtures] of Object.entries(fixturesByContract)) {
|
|
75
78
|
const names = new Set(fixtures.map((f) => f.name));
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
continue;
|
|
82
|
-
const tone = failedCount > 0 ? MARK_FAIL : passedCount > 0 ? MARK_PASS : MARK_SKIP;
|
|
83
|
-
lines.push(` ${tone} ${padRight(slug, 38)} ${passedCount}/${total} pass${failedCount > 0 ? ` · ${failedCount} fail` : ''}${skippedCount > 0 ? ` · ${skippedCount} skip` : ''}`);
|
|
79
|
+
lines.push(...scorecardRow(slug, result, (name) => names.has(name)));
|
|
80
|
+
}
|
|
81
|
+
for (const slug of PURE_FUNCTION_CATALOG_SLUGS) {
|
|
82
|
+
const prefix = `${slug}/`;
|
|
83
|
+
lines.push(...scorecardRow(slug, result, (name) => name.startsWith(prefix)));
|
|
84
84
|
}
|
|
85
85
|
return lines.join('\n');
|
|
86
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* One scorecard line for the rows `belongs` selects, or no line at all
|
|
89
|
+
* when the group graded nothing. A group that is entirely SKIPPED still
|
|
90
|
+
* prints — an obligation nobody graded must be visible, not absent.
|
|
91
|
+
*/
|
|
92
|
+
function scorecardRow(slug, result, belongs) {
|
|
93
|
+
const passedCount = result.passed.filter(belongs).length;
|
|
94
|
+
const failedCount = result.failed.filter((f) => belongs(f.name)).length;
|
|
95
|
+
const skippedCount = result.skipped.filter((s) => belongs(s.name)).length;
|
|
96
|
+
const total = passedCount + failedCount + skippedCount;
|
|
97
|
+
if (total === 0)
|
|
98
|
+
return [];
|
|
99
|
+
const tone = failedCount > 0 ? MARK_FAIL : passedCount > 0 ? MARK_PASS : MARK_SKIP;
|
|
100
|
+
return [
|
|
101
|
+
` ${tone} ${padRight(slug, 38)} ${passedCount}/${total} pass${failedCount > 0 ? ` · ${failedCount} fail` : ''}${skippedCount > 0 ? ` · ${skippedCount} skip` : ''}`,
|
|
102
|
+
];
|
|
103
|
+
}
|
|
87
104
|
export function formatSummary(result) {
|
|
88
105
|
const passed = result.passed.length;
|
|
89
106
|
const failed = result.failed.length;
|
|
@@ -2,6 +2,9 @@ import type { HostContextObservedPayload } from '@ggui-ai/protocol';
|
|
|
2
2
|
import type { ConformanceHost, SetupStep as HostSetupStep } from './conformance-host.js';
|
|
3
3
|
import { type MatchResult } from './match-behavior.js';
|
|
4
4
|
import type { AuthConfig, SessionStateBehavior, StreamUpdateBehavior, VersionMismatchBehavior } from './types.js';
|
|
5
|
+
import { type PreGenerationRefusalInput, type ProjectedRefusalResult } from './refusal-envelope-conformance/index.js';
|
|
6
|
+
import { type RefusalRegistryView } from './registry-completeness/index.js';
|
|
7
|
+
import { type ProjectedTransportRefusal, type TransportRefusalInput } from './transport-refusal-conformance/index.js';
|
|
5
8
|
export interface RunConformanceConfig {
|
|
6
9
|
/**
|
|
7
10
|
* URL of the implementation under test. `http://` / `https://`
|
|
@@ -27,7 +30,42 @@ export interface RunConformanceConfig {
|
|
|
27
30
|
readonly host?: ConformanceHost;
|
|
28
31
|
/** Per-fixture observation window. Default 2000ms. */
|
|
29
32
|
readonly observationTimeoutMs?: number;
|
|
33
|
+
/**
|
|
34
|
+
* The implementation's PRE-GENERATION refusal projector (SPEC §7.1's
|
|
35
|
+
* refused arm). Given a refusal, return the tool result the server
|
|
36
|
+
* would emit for it, or `null` when the code has no envelope on this
|
|
37
|
+
* surface.
|
|
38
|
+
*
|
|
39
|
+
* Supplied ⇒ the `refusal-envelope` catalog is graded as part of this
|
|
40
|
+
* run. Omitted ⇒ its cases are reported SKIPPED with that reason, so
|
|
41
|
+
* an ungraded obligation shows up on the scorecard instead of
|
|
42
|
+
* vanishing from it.
|
|
43
|
+
*/
|
|
44
|
+
readonly refusalProjector?: (refusal: PreGenerationRefusalInput) => ProjectedRefusalResult | null;
|
|
45
|
+
/**
|
|
46
|
+
* The deployment's closed refusal-code registry, keyed by code.
|
|
47
|
+
* Supplied ⇒ the `registry-completeness` pins are graded as part of
|
|
48
|
+
* this run; omitted ⇒ reported SKIPPED, same reasoning as above.
|
|
49
|
+
*/
|
|
50
|
+
readonly refusalRegistry?: RefusalRegistryView;
|
|
51
|
+
/**
|
|
52
|
+
* The deployment's PER-APP ENDPOINT refusal projector (SPEC §7.1's
|
|
53
|
+
* endpoint-level refusal, ggui#825): given a registry refusal, return
|
|
54
|
+
* the HTTP status + JSON-RPC error object the endpoint answers with, or
|
|
55
|
+
* `null` when the code has no transport envelope. Supplied ⇒ the
|
|
56
|
+
* `transport-refusal` catalog is graded; omitted ⇒ SKIPPED, named.
|
|
57
|
+
*/
|
|
58
|
+
readonly transportRefusalProjector?: (refusal: TransportRefusalInput) => ProjectedTransportRefusal | null;
|
|
30
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Row-name prefixes for the PURE-FUNCTION catalogs the runner folds
|
|
62
|
+
* into a run. Prefixed so a catalog row can never collide with a
|
|
63
|
+
* WebSocket fixture name, and so the reporter can group them without a
|
|
64
|
+
* second registry of names.
|
|
65
|
+
*/
|
|
66
|
+
export declare const PURE_FUNCTION_CATALOG_SLUGS: readonly ["refusal-envelope", "registry-completeness", "transport-refusal"];
|
|
67
|
+
/** One member of {@link PURE_FUNCTION_CATALOG_SLUGS}. */
|
|
68
|
+
export type PureFunctionCatalogSlug = (typeof PURE_FUNCTION_CATALOG_SLUGS)[number];
|
|
31
69
|
export interface ConformanceResult {
|
|
32
70
|
readonly passed: readonly string[];
|
|
33
71
|
readonly failed: readonly ConformanceFailure[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-conformance.d.ts","sourceRoot":"","sources":["../src/run-conformance.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-conformance.d.ts","sourceRoot":"","sources":["../src/run-conformance.ts"],"names":[],"mappings":"AA6DA,OAAO,KAAK,EACV,0BAA0B,EAG3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EACV,eAAe,EACf,SAAS,IAAI,aAAa,EAE3B,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAwC,KAAK,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAC7F,OAAO,KAAK,EAEV,UAAU,EAEV,oBAAoB,EACpB,oBAAoB,EAEpB,uBAAuB,EACxB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAGL,KAAK,yBAAyB,EAC9B,KAAK,sBAAsB,EAC5B,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAGL,KAAK,mBAAmB,EACzB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAGL,KAAK,yBAAyB,EAC9B,KAAK,qBAAqB,EAC3B,MAAM,0CAA0C,CAAC;AAMlD,MAAM,WAAW,oBAAoB;IACnC;;;;;;;OAOG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,8DAA8D;IAC9D,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,gEAAgE;IAChE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,qDAAqD;IACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,wDAAwD;IACxD,QAAQ,CAAC,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IACxC;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC;IAChC,sDAAsD;IACtD,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAC1B,OAAO,EAAE,yBAAyB,KAC/B,sBAAsB,GAAG,IAAI,CAAC;IACnC;;;;OAIG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,mBAAmB,CAAC;IAC/C;;;;;;OAMG;IACH,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CACnC,OAAO,EAAE,qBAAqB,KAC3B,yBAAyB,GAAG,IAAI,CAAC;CACvC;AAED;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,6EAI9B,CAAC;AAEX,yDAAyD;AACzD,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,2BAA2B,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnF,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,SAAS,kBAAkB,EAAE,CAAC;IAC/C,QAAQ,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,CAAC;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,aAAa,CAAC,CAAC,OAAO,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAClD,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD;;;;OAIG;IACH,iBAAiB,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,UAAU,CAAC,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI,CAAC;CAC9C;AAMD,wBAAsB,cAAc,CAClC,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,iBAAiB,CAAC,CA+C5B;AA+RD;;;;;;;;;;;;GAYG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAWrD;AAoBD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,gCAAgC;IAC/C,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAC;IACvC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,0BAA0B,CAAC;CAC9C;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,MAAM,qBAAqB,GAC7B;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,GACvD;IACE,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAC;IACvC,QAAQ,CAAC,QAAQ,EAAE,gCAAgC,CAAC;CACrD,GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9B;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,OAAO,GAChB,qBAAqB,CAcvB;AAkOD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,oBAAoB,EAC9B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,eAAe,GAAG,SAAS,GAChC,OAAO,CAAC,WAAW,CAAC,CA8BtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAChD;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,OAAO,GACf,wBAAwB,CAmC1B;AAqED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,aAAa,CA4DhF;AAqHD,YAAY,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,CAAC"}
|
package/dist/run-conformance.js
CHANGED
|
@@ -46,17 +46,37 @@
|
|
|
46
46
|
* (No teardown vocabulary exists in this kit version, so the
|
|
47
47
|
* loop is empty today.)
|
|
48
48
|
* l. Record the outcome via reporter + accumulate in result.
|
|
49
|
-
* 3.
|
|
49
|
+
* 3. Fold in the PURE-FUNCTION catalogs (ggui#786) —
|
|
50
|
+
* `refusal-envelope`, `registry-completeness` and `transport-refusal`. They grade a
|
|
51
|
+
* caller-supplied projector / registry rather than the wire, so
|
|
52
|
+
* they need no transport; when the caller supplies neither, their
|
|
53
|
+
* rows are reported SKIPPED with that reason so an ungraded
|
|
54
|
+
* obligation stays visible on the scorecard.
|
|
55
|
+
* 4. Invoke `reporter.onComplete(result)` + return.
|
|
50
56
|
*
|
|
51
|
-
* Scope v1: pure WS transport. Browser-level
|
|
52
|
-
* failure, props-update) automatically skip via the
|
|
53
|
-
* `unmatchable-on-ws` arm.
|
|
57
|
+
* Scope v1: pure WS transport for the fixture catalog. Browser-level
|
|
58
|
+
* fixtures (bootstrap-failure, props-update) automatically skip via the
|
|
59
|
+
* matcher's `unmatchable-on-ws` arm.
|
|
54
60
|
*/
|
|
55
61
|
import { PROTOCOL_SCHEMA_VERSION } from '@ggui-ai/protocol';
|
|
56
62
|
import { allFixtures, fixturesByContract } from './fixtures/index.js';
|
|
57
63
|
import { isRecord } from '@ggui-ai/protocol';
|
|
58
64
|
import { behaviorIs, deepEqual, matchBehavior } from './match-behavior.js';
|
|
59
65
|
import { openWsTransport } from './ws-transport.js';
|
|
66
|
+
import { refusalEnvelopeCases, runRefusalEnvelopeConformance, } from './refusal-envelope-conformance/index.js';
|
|
67
|
+
import { registryCompletenessPins, runRegistryCompletenessConformance, } from './registry-completeness/index.js';
|
|
68
|
+
import { runTransportRefusalConformance, transportRefusalCases, } from './transport-refusal-conformance/index.js';
|
|
69
|
+
/**
|
|
70
|
+
* Row-name prefixes for the PURE-FUNCTION catalogs the runner folds
|
|
71
|
+
* into a run. Prefixed so a catalog row can never collide with a
|
|
72
|
+
* WebSocket fixture name, and so the reporter can group them without a
|
|
73
|
+
* second registry of names.
|
|
74
|
+
*/
|
|
75
|
+
export const PURE_FUNCTION_CATALOG_SLUGS = [
|
|
76
|
+
'refusal-envelope',
|
|
77
|
+
'registry-completeness',
|
|
78
|
+
'transport-refusal',
|
|
79
|
+
];
|
|
60
80
|
// =============================================================================
|
|
61
81
|
// Runner
|
|
62
82
|
// =============================================================================
|
|
@@ -93,6 +113,7 @@ export async function runConformance(config) {
|
|
|
93
113
|
break;
|
|
94
114
|
}
|
|
95
115
|
}
|
|
116
|
+
runPureFunctionCatalogs(config, reporter, { passed, failed, skipped });
|
|
96
117
|
const result = {
|
|
97
118
|
passed,
|
|
98
119
|
failed,
|
|
@@ -102,6 +123,106 @@ export async function runConformance(config) {
|
|
|
102
123
|
reporter.onComplete?.(result);
|
|
103
124
|
return result;
|
|
104
125
|
}
|
|
126
|
+
/**
|
|
127
|
+
* Fold the pure-function catalogs (ggui#786) into a run.
|
|
128
|
+
*
|
|
129
|
+
* These obligations are not WebSocket-observable — a refusal projection
|
|
130
|
+
* is a pure function and a registry is data — so they are NOT in
|
|
131
|
+
* `fixturesByContract` (see `fixtures/index.ts` for why registering a
|
|
132
|
+
* non-WS case there would be a permanent skip). But leaving them
|
|
133
|
+
* reachable only through a direct import made them invisible to every
|
|
134
|
+
* CLI run: a catalog the runner cannot report is a silent gate
|
|
135
|
+
* (docs/principles/no-silent-block.md).
|
|
136
|
+
*
|
|
137
|
+
* So they run HERE, on the same verdict buckets as the fixtures, and an
|
|
138
|
+
* absent input produces a SKIP row naming what to supply rather than
|
|
139
|
+
* nothing at all.
|
|
140
|
+
*/
|
|
141
|
+
function runPureFunctionCatalogs(config, reporter, buckets) {
|
|
142
|
+
const only = config.only;
|
|
143
|
+
const wanted = (name) => only === undefined || only.length === 0 || only.includes(name);
|
|
144
|
+
const pass = (name) => {
|
|
145
|
+
if (!wanted(name))
|
|
146
|
+
return;
|
|
147
|
+
buckets.passed.push(name);
|
|
148
|
+
reporter.onFixturePass?.(name, 0);
|
|
149
|
+
};
|
|
150
|
+
const skip = (name, reason) => {
|
|
151
|
+
if (!wanted(name))
|
|
152
|
+
return;
|
|
153
|
+
buckets.skipped.push({ name, reason });
|
|
154
|
+
reporter.onFixtureSkip?.(name, reason);
|
|
155
|
+
};
|
|
156
|
+
const fail = (failure) => {
|
|
157
|
+
if (!wanted(failure.name))
|
|
158
|
+
return;
|
|
159
|
+
buckets.failed.push(failure);
|
|
160
|
+
reporter.onFixtureFail?.(failure);
|
|
161
|
+
};
|
|
162
|
+
// ── refusal-envelope ──
|
|
163
|
+
const projector = config.refusalProjector;
|
|
164
|
+
if (projector === undefined) {
|
|
165
|
+
for (const testCase of refusalEnvelopeCases) {
|
|
166
|
+
skip(`refusal-envelope/${testCase.name}`, 'no `refusalProjector` supplied — pass one to runConformance(), or `--projector <module>` on the CLI, to grade SPEC §7.1\u2019s refused arm');
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
const envelope = runRefusalEnvelopeConformance(projector);
|
|
171
|
+
for (const name of envelope.passed)
|
|
172
|
+
pass(`refusal-envelope/${name}`);
|
|
173
|
+
for (const mismatch of envelope.failed) {
|
|
174
|
+
fail({
|
|
175
|
+
name: `refusal-envelope/${mismatch.name}`,
|
|
176
|
+
criterion: 'pre-generation refusal envelope (SPEC §7.1, refused arm)',
|
|
177
|
+
expected: mismatch.expected,
|
|
178
|
+
received: mismatch.actual,
|
|
179
|
+
message: 'the projected refusal envelope does not match the catalog case',
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
// ── transport-refusal (ggui#825) ──
|
|
184
|
+
const endpointProjector = config.transportRefusalProjector;
|
|
185
|
+
if (endpointProjector === undefined) {
|
|
186
|
+
for (const testCase of transportRefusalCases) {
|
|
187
|
+
skip(`transport-refusal/${testCase.name}`, 'no `transportRefusalProjector` supplied — pass one to runConformance(), or `--transport-projector <module>` on the CLI, to grade the endpoint-level refusal (SPEC §7.1, mcp-endpoint surface)');
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
const endpoint = runTransportRefusalConformance(endpointProjector);
|
|
192
|
+
for (const name of endpoint.passed)
|
|
193
|
+
pass(`transport-refusal/${name}`);
|
|
194
|
+
for (const mismatch of endpoint.failed) {
|
|
195
|
+
fail({
|
|
196
|
+
name: `transport-refusal/${mismatch.name}`,
|
|
197
|
+
criterion: 'endpoint-level refusal envelope (SPEC §7.1, mcp-endpoint surface)',
|
|
198
|
+
expected: mismatch.expected,
|
|
199
|
+
received: mismatch.actual,
|
|
200
|
+
message: 'the projected endpoint refusal does not match the catalog case',
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
// ── registry-completeness ──
|
|
205
|
+
const registry = config.refusalRegistry;
|
|
206
|
+
if (registry === undefined) {
|
|
207
|
+
for (const pin of registryCompletenessPins) {
|
|
208
|
+
skip(`registry-completeness/${pin.name}`, 'no `refusalRegistry` supplied — pass the deployment\u2019s refusal-code registry to runConformance(), or `--registry <file.json>` on the CLI, to grade it');
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
const completeness = runRegistryCompletenessConformance(registry);
|
|
213
|
+
for (const name of completeness.passed)
|
|
214
|
+
pass(`registry-completeness/${name}`);
|
|
215
|
+
for (const mismatch of completeness.failed) {
|
|
216
|
+
fail({
|
|
217
|
+
name: `registry-completeness/${mismatch.name}`,
|
|
218
|
+
criterion: 'refusal-registry completeness',
|
|
219
|
+
expected: mismatch.description,
|
|
220
|
+
received: mismatch.violations,
|
|
221
|
+
message: `${String(mismatch.violations.length)} row(s) violate this pin`,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
105
226
|
async function runOneFixture(fixture, config, _reporter) {
|
|
106
227
|
const fixtureStarted = Date.now();
|
|
107
228
|
if (fixture.skipReason !== null) {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "refuse-deprovisioned-endpoint",
|
|
3
|
+
"description": "A deprovisioned app answers EVERY JSON-RPC request on its per-app MCP endpoint (initialize included) with HTTP 403 and a JSON-RPC error whose data.refusal carries the registry projection \u2014 code, message, fix, retry \u2014 so the client reads app_deprovisioned instead of guessing between a deleted app and a bad credential. error.code is -32003 (APP_NOT_FOUND \u2014 the endpoint no longer serves this app) and error.message is App not found; an untyped authorization failure answers -32001 (UNAUTHORIZED) with no data, so a client can tell the two apart by code and can tell neither from a dropped socket by mistake \u2014 -32000 is the SDK client's ConnectionClosed and a first-party server never chooses it (ggui#836).",
|
|
4
|
+
"refusal": {
|
|
5
|
+
"code": "app_deprovisioned",
|
|
6
|
+
"message": "the app record has no owner claim any more \u2014 the app was deprovisioned",
|
|
7
|
+
"fix": "provision the app again; this app id never renders again",
|
|
8
|
+
"retry": "never"
|
|
9
|
+
},
|
|
10
|
+
"expect": {
|
|
11
|
+
"httpStatus": 403,
|
|
12
|
+
"error": {
|
|
13
|
+
"code": -32003,
|
|
14
|
+
"message": "App not found",
|
|
15
|
+
"data": {
|
|
16
|
+
"refusal": {
|
|
17
|
+
"code": "app_deprovisioned",
|
|
18
|
+
"message": "the app record has no owner claim any more \u2014 the app was deprovisioned",
|
|
19
|
+
"fix": "provision the app again; this app id never renders again",
|
|
20
|
+
"retry": "never"
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "refuse-render-only-code",
|
|
3
|
+
"description": "A REGISTERED code whose surfaces exclude mcp-endpoint has no transport envelope: the per-app endpoint's authorization never speaks it, and a projector that types it anyway would put a render-gate state on a surface the registry does not list. The three auth arms (no issuer, foreign issuer, denied subject) stay the bare 403 by the same rule \u2014 naming any of them would say which is true.",
|
|
4
|
+
"refusal": {
|
|
5
|
+
"code": "insufficient_credit",
|
|
6
|
+
"message": "the configured allowance for this app is exhausted",
|
|
7
|
+
"fix": "top up the allowance for this app, then retry",
|
|
8
|
+
"retry": "after-fix"
|
|
9
|
+
},
|
|
10
|
+
"expect": null
|
|
11
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The refusal handed to the projector — the registry projection on the
|
|
3
|
+
* transport surface. Authored here rather than imported from
|
|
4
|
+
* `@ggui-ai/protocol` for the same reason as the render catalog: the kit
|
|
5
|
+
* grades the WIRE, and a stringly input keeps a stale kit from typing
|
|
6
|
+
* its way past a registry change.
|
|
7
|
+
*/
|
|
8
|
+
export interface TransportRefusalInput {
|
|
9
|
+
/** Registered refusal code. */
|
|
10
|
+
readonly code: string;
|
|
11
|
+
/** Precise diagnostic. */
|
|
12
|
+
readonly message: string;
|
|
13
|
+
/** The one recovery step. */
|
|
14
|
+
readonly fix: string;
|
|
15
|
+
/** Retry class (`after-fix` | `next-period` | `later` | `never`). */
|
|
16
|
+
readonly retry: string;
|
|
17
|
+
}
|
|
18
|
+
/** What the endpoint answers: the HTTP status and the JSON-RPC error object. */
|
|
19
|
+
export interface ProjectedTransportRefusal {
|
|
20
|
+
readonly httpStatus: number;
|
|
21
|
+
readonly error: {
|
|
22
|
+
readonly code: number;
|
|
23
|
+
readonly message: string;
|
|
24
|
+
readonly data: {
|
|
25
|
+
readonly refusal: TransportRefusalInput;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/** One transport-refusal case, authored as JSON under `./cases/`. */
|
|
30
|
+
export interface TransportRefusalConformanceCase {
|
|
31
|
+
readonly name: string;
|
|
32
|
+
readonly description: string;
|
|
33
|
+
readonly refusal: TransportRefusalInput;
|
|
34
|
+
/** `null` when the code has no transport envelope (surfaces exclude `mcp-endpoint`). */
|
|
35
|
+
readonly expect: ProjectedTransportRefusal | null;
|
|
36
|
+
}
|
|
37
|
+
export declare const transportRefusalCases: readonly TransportRefusalConformanceCase[];
|
|
38
|
+
export interface TransportRefusalMismatch {
|
|
39
|
+
readonly name: string;
|
|
40
|
+
readonly expected: ProjectedTransportRefusal | null;
|
|
41
|
+
/** What the projector produced — or the message of the error it threw. */
|
|
42
|
+
readonly actual: ProjectedTransportRefusal | null | string;
|
|
43
|
+
}
|
|
44
|
+
export interface TransportRefusalConformanceResult {
|
|
45
|
+
readonly passed: readonly string[];
|
|
46
|
+
readonly failed: readonly TransportRefusalMismatch[];
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Grade a per-app endpoint refusal projector against the catalog. Pure
|
|
50
|
+
* and synchronous: no server, no transport — the deterministic
|
|
51
|
+
* projection is the whole obligation.
|
|
52
|
+
*/
|
|
53
|
+
export declare function runTransportRefusalConformance(project: (refusal: TransportRefusalInput) => ProjectedTransportRefusal | null): TransportRefusalConformanceResult;
|
|
54
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/transport-refusal-conformance/index.ts"],"names":[],"mappings":"AAyBA;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACpC,+BAA+B;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,0BAA0B;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,6BAA6B;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,IAAI,EAAE;YAAE,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;SAAE,CAAC;KAC5D,CAAC;CACH;AAED,qEAAqE;AACrE,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAC;IACxC,wFAAwF;IACxF,QAAQ,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI,CAAC;CACnD;AAED,eAAO,MAAM,qBAAqB,EAAE,SAAS,+BAA+B,EAG3E,CAAC;AAEF,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,yBAAyB,GAAG,IAAI,CAAC;IACpD,0EAA0E;IAC1E,QAAQ,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI,GAAG,MAAM,CAAC;CAC5D;AAED,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,SAAS,wBAAwB,EAAE,CAAC;CACtD;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,CAAC,OAAO,EAAE,qBAAqB,KAAK,yBAAyB,GAAG,IAAI,GAC5E,iCAAiC,CAiBnC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transport-refusal conformance catalog (ggui#825) — grades a deployment's
|
|
3
|
+
* PER-APP ENDPOINT refusal projector: the function that turns a registry
|
|
4
|
+
* refusal into the JSON-RPC error a per-app MCP endpoint answers with when
|
|
5
|
+
* it refuses a request for a typed reason, on ANY method (`initialize`
|
|
6
|
+
* included).
|
|
7
|
+
*
|
|
8
|
+
* The contract it makes observable: a deprovisioned app answers HTTP 403
|
|
9
|
+
* with `{ code: -32003, message: "App not found", data: { refusal } }` where
|
|
10
|
+
* `refusal` is the registry projection without the render-only fields
|
|
11
|
+
* (`code`, `message`, `fix`, `retry`); an authorization failure that is
|
|
12
|
+
* NOT a registry state answers 403 with `-32001` and no `data` — the untyped
|
|
13
|
+
* arms deliberately indistinguishable among themselves, so a client never
|
|
14
|
+
* learns which one it hit (codes ruled in ggui#836).
|
|
15
|
+
*
|
|
16
|
+
* Each case ships as raw JSON under `./cases/`. The kit never imports a
|
|
17
|
+
* concrete server: the adopter supplies the projector, the catalog grades
|
|
18
|
+
* it, and a code whose `surfaces` exclude `mcp-endpoint` must project to
|
|
19
|
+
* `null` (no transport envelope exists for it).
|
|
20
|
+
*/
|
|
21
|
+
import refuseDeprovisionedEndpoint from './cases/refuse-deprovisioned-endpoint.json' with { type: 'json' };
|
|
22
|
+
import refuseRenderOnlyCode from './cases/refuse-render-only-code.json' with { type: 'json' };
|
|
23
|
+
import { jsonEqual } from '../json-equal.js';
|
|
24
|
+
export const transportRefusalCases = [
|
|
25
|
+
refuseDeprovisionedEndpoint,
|
|
26
|
+
refuseRenderOnlyCode,
|
|
27
|
+
];
|
|
28
|
+
/**
|
|
29
|
+
* Grade a per-app endpoint refusal projector against the catalog. Pure
|
|
30
|
+
* and synchronous: no server, no transport — the deterministic
|
|
31
|
+
* projection is the whole obligation.
|
|
32
|
+
*/
|
|
33
|
+
export function runTransportRefusalConformance(project) {
|
|
34
|
+
const passed = [];
|
|
35
|
+
const failed = [];
|
|
36
|
+
for (const testCase of transportRefusalCases) {
|
|
37
|
+
let actual;
|
|
38
|
+
try {
|
|
39
|
+
actual = project(testCase.refusal);
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
actual = error instanceof Error ? error.message : String(error);
|
|
43
|
+
}
|
|
44
|
+
if (typeof actual !== 'string' && jsonEqual(actual, testCase.expect)) {
|
|
45
|
+
passed.push(testCase.name);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
failed.push({ name: testCase.name, expected: testCase.expect, actual });
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return { passed, failed };
|
|
52
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ggui-ai/protocol-conformance",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Conformance test kit for the ggui protocol. Drives a behavioral fixture catalog against a live implementation over WebSocket, a resources/read catalog against a live MCP server, and
|
|
3
|
+
"version": "0.15.0",
|
|
4
|
+
"description": "Conformance test kit for the ggui protocol. Drives a behavioral fixture catalog against a live implementation over WebSocket, a resources/read catalog against a live MCP server, and nine pure-function catalogs (schema, registration, resolution, binding, props-schema, host-helper, refusal-envelope, registry-completeness, transport-refusal) for the gadget, tool-binding, schema-precise-render, host-helper and pre-generation-refusal obligations, reporting a scorecard-style pass / fail / skip. MCP builders adopt this kit to prove their implementation is protocol-conformant.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"ggui",
|
|
@@ -65,11 +65,26 @@
|
|
|
65
65
|
"types": "./dist/host-helper-conformance/index.d.ts",
|
|
66
66
|
"import": "./dist/host-helper-conformance/index.js",
|
|
67
67
|
"default": "./dist/host-helper-conformance/index.js"
|
|
68
|
+
},
|
|
69
|
+
"./refusal-envelope-conformance": {
|
|
70
|
+
"types": "./dist/refusal-envelope-conformance/index.d.ts",
|
|
71
|
+
"import": "./dist/refusal-envelope-conformance/index.js",
|
|
72
|
+
"default": "./dist/refusal-envelope-conformance/index.js"
|
|
73
|
+
},
|
|
74
|
+
"./registry-completeness": {
|
|
75
|
+
"types": "./dist/registry-completeness/index.d.ts",
|
|
76
|
+
"import": "./dist/registry-completeness/index.js",
|
|
77
|
+
"default": "./dist/registry-completeness/index.js"
|
|
78
|
+
},
|
|
79
|
+
"./transport-refusal-conformance": {
|
|
80
|
+
"types": "./dist/transport-refusal-conformance/index.d.ts",
|
|
81
|
+
"import": "./dist/transport-refusal-conformance/index.js",
|
|
82
|
+
"default": "./dist/transport-refusal-conformance/index.js"
|
|
68
83
|
}
|
|
69
84
|
},
|
|
70
85
|
"dependencies": {
|
|
71
86
|
"ws": "^8.20.1",
|
|
72
|
-
"@ggui-ai/protocol": "0.
|
|
87
|
+
"@ggui-ai/protocol": "0.15.0"
|
|
73
88
|
},
|
|
74
89
|
"devDependencies": {
|
|
75
90
|
"@types/node": "^22.0.0",
|
|
@@ -90,11 +105,11 @@
|
|
|
90
105
|
"access": "public"
|
|
91
106
|
},
|
|
92
107
|
"engines": {
|
|
93
|
-
"node": ">=
|
|
108
|
+
"node": ">=22.0.0"
|
|
94
109
|
},
|
|
95
110
|
"author": "ggui contributors <hello@ggui.ai>",
|
|
96
111
|
"scripts": {
|
|
97
|
-
"build": "rm -rf dist.staging && tsc -p tsconfig.build.json --outDir dist.staging && node ../scripts/fix-esm-imports.mjs dist.staging && chmod +x dist.staging
|
|
112
|
+
"build": "rm -rf dist.staging-$$ && tsc -p tsconfig.build.json --outDir dist.staging-$$ && node ../scripts/fix-esm-imports.mjs dist.staging-$$ && chmod +x dist.staging-$$/cli.js && node ../scripts/atomic-swap.mjs dist.staging-$$ dist",
|
|
98
113
|
"dev": "tsc --watch",
|
|
99
114
|
"typecheck": "tsc --noEmit",
|
|
100
115
|
"test": "vitest run --passWithNoTests",
|