@dotsetlabs/bellwether 1.0.3 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +8 -2
  3. package/dist/baseline/accessors.d.ts +1 -1
  4. package/dist/baseline/accessors.js +1 -3
  5. package/dist/baseline/baseline-format.d.ts +287 -0
  6. package/dist/baseline/baseline-format.js +12 -0
  7. package/dist/baseline/comparator.js +249 -11
  8. package/dist/baseline/converter.d.ts +15 -15
  9. package/dist/baseline/converter.js +46 -34
  10. package/dist/baseline/diff.d.ts +1 -1
  11. package/dist/baseline/diff.js +45 -28
  12. package/dist/baseline/error-analyzer.d.ts +1 -1
  13. package/dist/baseline/error-analyzer.js +90 -17
  14. package/dist/baseline/incremental-checker.js +8 -5
  15. package/dist/baseline/index.d.ts +2 -12
  16. package/dist/baseline/index.js +3 -23
  17. package/dist/baseline/performance-tracker.d.ts +0 -1
  18. package/dist/baseline/performance-tracker.js +13 -20
  19. package/dist/baseline/response-fingerprint.js +39 -2
  20. package/dist/baseline/saver.js +41 -10
  21. package/dist/baseline/schema-compare.d.ts +22 -0
  22. package/dist/baseline/schema-compare.js +259 -16
  23. package/dist/baseline/types.d.ts +10 -7
  24. package/dist/cache/response-cache.d.ts +8 -0
  25. package/dist/cache/response-cache.js +110 -0
  26. package/dist/cli/commands/check.js +23 -6
  27. package/dist/cli/commands/explore.js +34 -14
  28. package/dist/cli/index.js +8 -0
  29. package/dist/config/template.js +8 -7
  30. package/dist/config/validator.d.ts +59 -59
  31. package/dist/config/validator.js +245 -90
  32. package/dist/constants/core.d.ts +4 -0
  33. package/dist/constants/core.js +8 -19
  34. package/dist/constants/registry.d.ts +17 -0
  35. package/dist/constants/registry.js +18 -0
  36. package/dist/constants/testing.d.ts +0 -369
  37. package/dist/constants/testing.js +18 -456
  38. package/dist/constants.d.ts +1 -1
  39. package/dist/constants.js +1 -1
  40. package/dist/docs/contract.js +131 -83
  41. package/dist/docs/report.js +8 -5
  42. package/dist/interview/insights.d.ts +17 -0
  43. package/dist/interview/insights.js +52 -0
  44. package/dist/interview/interviewer.js +52 -10
  45. package/dist/interview/prompt-test-generator.d.ts +12 -0
  46. package/dist/interview/prompt-test-generator.js +77 -0
  47. package/dist/interview/resource-test-generator.d.ts +12 -0
  48. package/dist/interview/resource-test-generator.js +20 -0
  49. package/dist/interview/schema-inferrer.js +26 -4
  50. package/dist/interview/schema-test-generator.js +278 -31
  51. package/dist/interview/stateful-test-runner.d.ts +3 -0
  52. package/dist/interview/stateful-test-runner.js +80 -0
  53. package/dist/interview/types.d.ts +12 -0
  54. package/dist/transport/mcp-client.js +1 -1
  55. package/dist/transport/sse-transport.d.ts +7 -3
  56. package/dist/transport/sse-transport.js +157 -67
  57. package/dist/version.js +1 -1
  58. package/man/bellwether.1 +1 -1
  59. package/man/bellwether.1.md +2 -2
  60. package/package.json +1 -1
  61. package/schemas/bellwether-check.schema.json +185 -0
  62. package/schemas/bellwether-explore.schema.json +837 -0
  63. package/scripts/completions/bellwether.bash +10 -4
  64. package/scripts/completions/bellwether.zsh +55 -2
@@ -20,6 +20,75 @@ function validateSecureUrl(url) {
20
20
  throw new Error(`Invalid SSE URL: ${url}`);
21
21
  }
22
22
  }
23
+ /**
24
+ * Minimal SSE parser for streaming responses.
25
+ */
26
+ class SSEParser {
27
+ buffer = '';
28
+ eventName = 'message';
29
+ dataLines = [];
30
+ feed(chunk) {
31
+ const events = [];
32
+ this.buffer += chunk;
33
+ let newlineIndex = this.buffer.indexOf('\n');
34
+ while (newlineIndex !== -1) {
35
+ const rawLine = this.buffer.slice(0, newlineIndex);
36
+ this.buffer = this.buffer.slice(newlineIndex + 1);
37
+ const line = rawLine.replace(/\r$/, '');
38
+ // Empty line signals end of event
39
+ if (line === '') {
40
+ if (this.dataLines.length > 0) {
41
+ events.push({
42
+ event: this.eventName || 'message',
43
+ data: this.dataLines.join('\n'),
44
+ });
45
+ }
46
+ this.eventName = 'message';
47
+ this.dataLines = [];
48
+ newlineIndex = this.buffer.indexOf('\n');
49
+ continue;
50
+ }
51
+ // Comment/heartbeat
52
+ if (line.startsWith(':')) {
53
+ newlineIndex = this.buffer.indexOf('\n');
54
+ continue;
55
+ }
56
+ if (line.startsWith('event:')) {
57
+ this.eventName = line.slice('event:'.length).trim() || 'message';
58
+ }
59
+ else if (line.startsWith('data:')) {
60
+ this.dataLines.push(line.slice('data:'.length).trimStart());
61
+ }
62
+ newlineIndex = this.buffer.indexOf('\n');
63
+ }
64
+ return events;
65
+ }
66
+ flush() {
67
+ const events = [];
68
+ if (this.buffer.length > 0) {
69
+ const line = this.buffer.replace(/\r$/, '');
70
+ this.buffer = '';
71
+ if (line.startsWith(':')) {
72
+ // Ignore comments
73
+ }
74
+ else if (line.startsWith('event:')) {
75
+ this.eventName = line.slice('event:'.length).trim() || 'message';
76
+ }
77
+ else if (line.startsWith('data:')) {
78
+ this.dataLines.push(line.slice('data:'.length).trimStart());
79
+ }
80
+ }
81
+ if (this.dataLines.length > 0) {
82
+ events.push({
83
+ event: this.eventName || 'message',
84
+ data: this.dataLines.join('\n'),
85
+ });
86
+ this.eventName = 'message';
87
+ this.dataLines = [];
88
+ }
89
+ return events;
90
+ }
91
+ }
23
92
  /**
24
93
  * SSETransport connects to MCP servers over HTTP using Server-Sent Events.
25
94
  *
@@ -32,7 +101,7 @@ function validateSecureUrl(url) {
32
101
  * - POST {baseUrl}/message - Endpoint for sending messages
33
102
  */
34
103
  export class SSETransport extends BaseTransport {
35
- eventSource = null;
104
+ streamAbortController = null;
36
105
  abortController = null;
37
106
  connected = false;
38
107
  reconnectAttempts = 0;
@@ -74,61 +143,36 @@ export class SSETransport extends BaseTransport {
74
143
  validateSecureUrl(this.baseUrl);
75
144
  // Reset closing flag on fresh connection
76
145
  this.isClosing = false;
77
- // EventSource is available in browsers natively and in Node.js 18+.
78
- // We use globalThis to check availability at runtime, requiring `any` cast
79
- // because TypeScript's lib.dom.d.ts doesn't type globalThis.EventSource.
80
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
81
- const EventSourceImpl = globalThis.EventSource;
82
- if (!EventSourceImpl) {
83
- throw new Error('EventSource is not available. ' +
84
- 'SSE transport requires Node.js 18+ or a browser environment. ' +
85
- 'For older Node.js versions, consider using streamable-http transport instead.');
146
+ const sseUrl = `${this.baseUrl}/sse`;
147
+ this.log('Connecting to SSE endpoint', { url: sseUrl });
148
+ // Build URL with sessionId as query param for compatibility
149
+ let url = sseUrl;
150
+ if (this.sessionId) {
151
+ const separator = url.includes('?') ? '&' : '?';
152
+ url = `${url}${separator}sessionId=${encodeURIComponent(this.sessionId)}`;
86
153
  }
87
- return new Promise((resolve, reject) => {
88
- const sseUrl = `${this.baseUrl}/sse`;
89
- this.log('Connecting to SSE endpoint', { url: sseUrl });
90
- try {
91
- // Create EventSource - headers need to be passed via URL params or custom implementation
92
- // Note: Standard EventSource doesn't support custom headers
93
- // For authenticated endpoints, session ID should be passed via URL param
94
- let url = sseUrl;
95
- if (this.sessionId) {
96
- const separator = url.includes('?') ? '&' : '?';
97
- url = `${url}${separator}sessionId=${encodeURIComponent(this.sessionId)}`;
98
- }
99
- this.eventSource = new EventSourceImpl(url);
100
- this.eventSource.onopen = () => {
101
- this.log('SSE connection opened');
102
- this.connected = true;
103
- this.reconnectAttempts = 0;
104
- resolve();
105
- };
106
- this.eventSource.onmessage = (event) => {
107
- this.handleSSEMessage(event);
108
- };
109
- // Handle specific event types from MCP SSE protocol
110
- this.eventSource.addEventListener('endpoint', (event) => {
111
- // Server tells us where to send messages
112
- const messageEvent = event;
113
- this.messageEndpoint = messageEvent.data;
114
- this.log('Received message endpoint', { endpoint: this.messageEndpoint ?? '' });
115
- });
116
- this.eventSource.addEventListener('message', (event) => {
117
- this.handleSSEMessage(event);
118
- });
119
- this.eventSource.onerror = (error) => {
120
- this.log('SSE error', { type: error.type });
121
- if (!this.connected) {
122
- // Connection failed on initial connect
123
- reject(new Error('Failed to connect to SSE endpoint'));
124
- return;
125
- }
126
- // Handle reconnection for established connections
127
- this.handleReconnect();
128
- };
129
- }
130
- catch (error) {
131
- reject(error);
154
+ this.streamAbortController = new AbortController();
155
+ const response = await fetch(url, {
156
+ method: 'GET',
157
+ headers: {
158
+ Accept: 'text/event-stream',
159
+ ...this.headers,
160
+ },
161
+ signal: this.streamAbortController.signal,
162
+ });
163
+ if (!response.ok) {
164
+ throw new Error(`Failed to connect to SSE endpoint: HTTP ${response.status}`);
165
+ }
166
+ if (!response.body) {
167
+ throw new Error('SSE response body is empty');
168
+ }
169
+ this.connected = true;
170
+ this.reconnectAttempts = 0;
171
+ // Start streaming in background
172
+ this.readSSEStream(response).catch((error) => {
173
+ this.log('SSE stream error', { error: String(error) });
174
+ if (!this.isClosing) {
175
+ this.handleReconnect();
132
176
  }
133
177
  });
134
178
  }
@@ -142,6 +186,11 @@ export class SSETransport extends BaseTransport {
142
186
  if (!data || data === ':') {
143
187
  return;
144
188
  }
189
+ if (event.event === 'endpoint') {
190
+ this.messageEndpoint = data;
191
+ this.log('Received message endpoint', { endpoint: this.messageEndpoint ?? '' });
192
+ return;
193
+ }
145
194
  this.log('Received SSE message', { data });
146
195
  const message = JSON.parse(data);
147
196
  this.emit('message', message);
@@ -153,6 +202,47 @@ export class SSETransport extends BaseTransport {
153
202
  // Don't emit error for parse failures - just log
154
203
  }
155
204
  }
205
+ /**
206
+ * Stream and parse SSE events from a fetch response.
207
+ */
208
+ async readSSEStream(response) {
209
+ const reader = response.body?.getReader();
210
+ if (!reader) {
211
+ throw new Error('SSE stream reader unavailable');
212
+ }
213
+ const decoder = new TextDecoder();
214
+ const parser = new SSEParser();
215
+ for (;;) {
216
+ const { value, done } = await reader.read();
217
+ if (done) {
218
+ break;
219
+ }
220
+ if (!value) {
221
+ continue;
222
+ }
223
+ const chunk = decoder.decode(value, { stream: true });
224
+ const events = parser.feed(chunk);
225
+ for (const event of events) {
226
+ this.handleSSEMessage(event);
227
+ }
228
+ }
229
+ const tail = decoder.decode();
230
+ if (tail) {
231
+ const tailEvents = parser.feed(tail);
232
+ for (const event of tailEvents) {
233
+ this.handleSSEMessage(event);
234
+ }
235
+ }
236
+ const flushed = parser.flush();
237
+ for (const event of flushed) {
238
+ this.handleSSEMessage(event);
239
+ }
240
+ // Stream ended
241
+ this.connected = false;
242
+ if (!this.isClosing) {
243
+ this.handleReconnect();
244
+ }
245
+ }
156
246
  /**
157
247
  * Handle reconnection after a connection error.
158
248
  *
@@ -161,7 +251,7 @@ export class SSETransport extends BaseTransport {
161
251
  * - Uses capped exponential backoff
162
252
  * - Clears reconnect timer on close
163
253
  * - Checks isClosing flag to prevent reconnection after close()
164
- * - Explicitly closes EventSource on max attempts
254
+ * - Explicitly aborts SSE stream on max attempts
165
255
  */
166
256
  handleReconnect() {
167
257
  // Don't reconnect if we're closing
@@ -173,15 +263,15 @@ export class SSETransport extends BaseTransport {
173
263
  if (this.reconnectAttempts >= this.maxReconnectAttempts) {
174
264
  this.log('Max reconnect attempts reached', { attempts: this.reconnectAttempts });
175
265
  this.connected = false;
176
- // Explicitly close the EventSource to clean up resources
177
- if (this.eventSource) {
266
+ // Explicitly abort the SSE stream to clean up resources
267
+ if (this.streamAbortController) {
178
268
  try {
179
- this.eventSource.close();
269
+ this.streamAbortController.abort();
180
270
  }
181
271
  catch {
182
- // Ignore close errors
272
+ // Ignore abort errors
183
273
  }
184
- this.eventSource = null;
274
+ this.streamAbortController = null;
185
275
  }
186
276
  this.emit('error', new Error(`Max reconnection attempts (${this.maxReconnectAttempts}) exceeded`));
187
277
  this.emit('close');
@@ -269,7 +359,7 @@ export class SSETransport extends BaseTransport {
269
359
  * Close the SSE connection.
270
360
  *
271
361
  * RELIABILITY: Properly cleans up all resources including:
272
- * - EventSource connection
362
+ * - SSE stream connection
273
363
  * - Pending HTTP requests (via abort controller)
274
364
  * - Reconnection timer
275
365
  * - Sets isClosing flag to prevent reconnection attempts
@@ -284,15 +374,15 @@ export class SSETransport extends BaseTransport {
284
374
  clearTimeout(this.reconnectTimer);
285
375
  this.reconnectTimer = null;
286
376
  }
287
- // Close the EventSource connection
288
- if (this.eventSource) {
377
+ // Abort SSE stream
378
+ if (this.streamAbortController) {
289
379
  try {
290
- this.eventSource.close();
380
+ this.streamAbortController.abort();
291
381
  }
292
382
  catch {
293
- // Ignore close errors
383
+ // Ignore abort errors
294
384
  }
295
- this.eventSource = null;
385
+ this.streamAbortController = null;
296
386
  }
297
387
  // Abort any in-flight HTTP requests
298
388
  if (this.abortController) {
package/dist/version.js CHANGED
@@ -30,7 +30,7 @@ function getPackageVersion() {
30
30
  }
31
31
  catch {
32
32
  // Fallback version - should match package.json
33
- return '1.0.3';
33
+ return '2.0.0';
34
34
  }
35
35
  }
36
36
  /**
package/man/bellwether.1 CHANGED
@@ -1,4 +1,4 @@
1
- .TH "BELLWETHER" "1" "2026\-02\-02" "Bellwether 1.0.3" "User Commands"
1
+ .TH "BELLWETHER" "1" "2026\-02\-04" "Bellwether 2.0.0" "User Commands"
2
2
  .SH NAME
3
3
  .PP
4
4
  bellwether \[em] MCP server testing and validation tool
@@ -2,8 +2,8 @@
2
2
  title: BELLWETHER
3
3
  section: 1
4
4
  header: User Commands
5
- footer: Bellwether 1.0.3
6
- date: 2026-02-02
5
+ footer: Bellwether 2.0.0
6
+ date: 2026-02-04
7
7
  ---
8
8
 
9
9
  # NAME
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dotsetlabs/bellwether",
3
- "version": "1.0.3",
3
+ "version": "2.0.0",
4
4
  "description": "The open-source MCP testing tool. Structural drift detection and behavioral documentation for Model Context Protocol servers.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -64,6 +64,30 @@
64
64
  "type": "string"
65
65
  }
66
66
  },
67
+ "semanticInferences": {
68
+ "type": "object",
69
+ "additionalProperties": {
70
+ "type": "array",
71
+ "items": {
72
+ "$ref": "#/$defs/SemanticInference"
73
+ }
74
+ }
75
+ },
76
+ "schemaEvolution": {
77
+ "type": "object",
78
+ "additionalProperties": {
79
+ "$ref": "#/$defs/ResponseSchemaEvolution"
80
+ }
81
+ },
82
+ "errorAnalysisSummaries": {
83
+ "type": "object",
84
+ "additionalProperties": {
85
+ "$ref": "#/$defs/ErrorAnalysisSummary"
86
+ }
87
+ },
88
+ "documentationScore": {
89
+ "$ref": "#/$defs/DocumentationScore"
90
+ },
67
91
  "metadata": {
68
92
  "$ref": "#/$defs/InterviewMetadata"
69
93
  }
@@ -647,6 +671,167 @@
647
671
  "maxChainLength": { "type": "number" }
648
672
  },
649
673
  "additionalProperties": true
674
+ },
675
+ "SemanticInference": {
676
+ "type": "object",
677
+ "required": ["paramName", "inferredType", "confidence", "evidence"],
678
+ "properties": {
679
+ "paramName": { "type": "string" },
680
+ "inferredType": { "type": "string" },
681
+ "confidence": { "type": "number" },
682
+ "evidence": {
683
+ "type": "array",
684
+ "items": { "type": "string" }
685
+ }
686
+ },
687
+ "additionalProperties": true
688
+ },
689
+ "InferredSchema": {
690
+ "type": "object",
691
+ "required": ["type"],
692
+ "properties": {
693
+ "type": { "type": "string" },
694
+ "properties": {
695
+ "type": "object",
696
+ "additionalProperties": { "$ref": "#/$defs/InferredSchema" }
697
+ },
698
+ "items": { "$ref": "#/$defs/InferredSchema" },
699
+ "required": {
700
+ "type": "array",
701
+ "items": { "type": "string" }
702
+ },
703
+ "nullable": { "type": "boolean" },
704
+ "enum": {
705
+ "type": "array",
706
+ "items": {}
707
+ }
708
+ },
709
+ "additionalProperties": true
710
+ },
711
+ "SchemaVersion": {
712
+ "type": "object",
713
+ "required": ["hash", "schema", "observedAt", "sampleCount"],
714
+ "properties": {
715
+ "hash": { "type": "string" },
716
+ "schema": { "$ref": "#/$defs/InferredSchema" },
717
+ "observedAt": { "type": "string", "format": "date-time" },
718
+ "sampleCount": { "type": "number" }
719
+ },
720
+ "additionalProperties": true
721
+ },
722
+ "ResponseSchemaEvolution": {
723
+ "type": "object",
724
+ "required": ["currentHash", "history", "isStable", "stabilityConfidence", "inconsistentFields", "sampleCount"],
725
+ "properties": {
726
+ "currentHash": { "type": "string" },
727
+ "history": {
728
+ "type": "array",
729
+ "items": { "$ref": "#/$defs/SchemaVersion" }
730
+ },
731
+ "isStable": { "type": "boolean" },
732
+ "stabilityConfidence": { "type": "number" },
733
+ "inconsistentFields": {
734
+ "type": "array",
735
+ "items": { "type": "string" }
736
+ },
737
+ "sampleCount": { "type": "number" }
738
+ },
739
+ "additionalProperties": true
740
+ },
741
+ "ErrorPattern": {
742
+ "type": "object",
743
+ "required": ["category", "patternHash", "example", "count"],
744
+ "properties": {
745
+ "category": { "type": "string" },
746
+ "patternHash": { "type": "string" },
747
+ "example": { "type": "string" },
748
+ "count": { "type": "number" }
749
+ },
750
+ "additionalProperties": true
751
+ },
752
+ "EnhancedErrorAnalysis": {
753
+ "type": "object",
754
+ "required": ["pattern", "statusCategory", "rootCause", "remediation", "relatedParameters", "transient", "severity"],
755
+ "properties": {
756
+ "pattern": { "$ref": "#/$defs/ErrorPattern" },
757
+ "httpStatus": { "type": "number" },
758
+ "statusCategory": { "type": "string" },
759
+ "rootCause": { "type": "string" },
760
+ "remediation": { "type": "string" },
761
+ "relatedParameters": {
762
+ "type": "array",
763
+ "items": { "type": "string" }
764
+ },
765
+ "transient": { "type": "boolean" },
766
+ "severity": { "type": "string" },
767
+ "wasExpected": { "type": "boolean" }
768
+ },
769
+ "additionalProperties": true
770
+ },
771
+ "ErrorAnalysisSummary": {
772
+ "type": "object",
773
+ "required": ["tool", "totalErrors", "analyses", "dominantCategory", "transientErrors", "actionableCount"],
774
+ "properties": {
775
+ "tool": { "type": "string" },
776
+ "totalErrors": { "type": "number" },
777
+ "analyses": {
778
+ "type": "array",
779
+ "items": { "$ref": "#/$defs/EnhancedErrorAnalysis" }
780
+ },
781
+ "dominantCategory": { "type": "string" },
782
+ "transientErrors": { "type": "number" },
783
+ "actionableCount": { "type": "number" },
784
+ "remediations": {
785
+ "type": "array",
786
+ "items": { "type": "string" }
787
+ },
788
+ "topRootCauses": {
789
+ "type": "array",
790
+ "items": { "type": "string" }
791
+ },
792
+ "topRemediations": {
793
+ "type": "array",
794
+ "items": { "type": "string" }
795
+ },
796
+ "relatedParameters": {
797
+ "type": "array",
798
+ "items": { "type": "string" }
799
+ },
800
+ "categoryCounts": {
801
+ "type": "object",
802
+ "additionalProperties": { "type": "number" }
803
+ }
804
+ },
805
+ "additionalProperties": true
806
+ },
807
+ "DocumentationComponents": {
808
+ "type": "object",
809
+ "properties": {
810
+ "descriptionCoverage": { "type": "number" },
811
+ "descriptionQuality": { "type": "number" },
812
+ "parameterDocumentation": { "type": "number" },
813
+ "exampleCoverage": { "type": "number" }
814
+ },
815
+ "additionalProperties": true
816
+ },
817
+ "DocumentationScore": {
818
+ "type": "object",
819
+ "required": ["overallScore", "grade", "components", "issues", "suggestions", "toolCount"],
820
+ "properties": {
821
+ "overallScore": { "type": "number" },
822
+ "grade": { "type": "string" },
823
+ "components": { "$ref": "#/$defs/DocumentationComponents" },
824
+ "issues": {
825
+ "type": "array",
826
+ "items": { "type": "object" }
827
+ },
828
+ "suggestions": {
829
+ "type": "array",
830
+ "items": { "type": "string" }
831
+ },
832
+ "toolCount": { "type": "number" }
833
+ },
834
+ "additionalProperties": true
650
835
  }
651
836
  }
652
837
  }