@dotsetlabs/bellwether 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +3 -2
- package/dist/cache/response-cache.d.ts +4 -2
- package/dist/cache/response-cache.js +68 -30
- package/dist/cli/commands/check.js +78 -49
- package/dist/cli/index.js +5 -3
- package/dist/interview/interviewer.js +70 -50
- package/dist/interview/orchestrator.js +49 -22
- package/dist/llm/anthropic.js +49 -16
- package/dist/llm/client.d.ts +2 -0
- package/dist/llm/client.js +61 -0
- package/dist/llm/ollama.js +9 -4
- package/dist/llm/openai.js +34 -23
- package/dist/transport/base-transport.d.ts +1 -1
- package/dist/transport/http-transport.d.ts +2 -2
- package/dist/transport/http-transport.js +26 -6
- package/dist/transport/mcp-client.d.ts +18 -6
- package/dist/transport/mcp-client.js +49 -19
- package/dist/transport/sse-transport.d.ts +1 -1
- package/dist/transport/sse-transport.js +4 -2
- package/dist/transport/stdio-transport.d.ts +1 -1
- package/dist/transport/stdio-transport.js +1 -1
- package/dist/utils/timeout.d.ts +10 -2
- package/dist/utils/timeout.js +9 -5
- package/dist/version.js +1 -1
- package/dist/workflow/executor.js +18 -13
- package/dist/workflow/loader.js +4 -1
- package/dist/workflow/state-tracker.js +22 -18
- package/man/bellwether.1 +204 -0
- package/man/bellwether.1.md +148 -0
- package/package.json +6 -7
|
@@ -252,7 +252,7 @@ export class MCPClient {
|
|
|
252
252
|
return true;
|
|
253
253
|
}
|
|
254
254
|
// Check patterns
|
|
255
|
-
return FILTERED_ENV_PATTERNS.some(pattern => pattern.test(name));
|
|
255
|
+
return FILTERED_ENV_PATTERNS.some((pattern) => pattern.test(name));
|
|
256
256
|
}
|
|
257
257
|
/**
|
|
258
258
|
* Filter sensitive environment variables before passing to subprocess.
|
|
@@ -295,7 +295,9 @@ export class MCPClient {
|
|
|
295
295
|
if (!this.process.stdout || !this.process.stdin) {
|
|
296
296
|
throw new Error('Failed to create stdio streams for server process');
|
|
297
297
|
}
|
|
298
|
-
this.transport = new StdioTransport(this.process.stdout, this.process.stdin, {
|
|
298
|
+
this.transport = new StdioTransport(this.process.stdout, this.process.stdin, {
|
|
299
|
+
debug: this.debug,
|
|
300
|
+
});
|
|
299
301
|
this.transport.on('message', (msg) => {
|
|
300
302
|
this.log('Received:', JSON.stringify(msg));
|
|
301
303
|
this.handleMessage(msg);
|
|
@@ -330,7 +332,7 @@ export class MCPClient {
|
|
|
330
332
|
// Capture stderr for diagnostic messages (limit to 500 chars)
|
|
331
333
|
const currentStderr = this.connectionState.stderrOutput ?? '';
|
|
332
334
|
if (currentStderr.length < 500) {
|
|
333
|
-
this.connectionState.stderrOutput =
|
|
335
|
+
this.connectionState.stderrOutput = `${currentStderr}\n${msg}`.trim().slice(0, 500);
|
|
334
336
|
}
|
|
335
337
|
// Check if stderr contains error indicators that suggest transport issues
|
|
336
338
|
if (this.looksLikeTransportError(msg)) {
|
|
@@ -348,7 +350,8 @@ export class MCPClient {
|
|
|
348
350
|
* @param options - Optional configuration overrides
|
|
349
351
|
*/
|
|
350
352
|
async connectRemote(url, options) {
|
|
351
|
-
const transport = options?.transport ??
|
|
353
|
+
const transport = options?.transport ??
|
|
354
|
+
(this.transportType === 'stdio' ? 'sse' : this.transportType);
|
|
352
355
|
this.transportType = transport;
|
|
353
356
|
// Reset flags for new connection
|
|
354
357
|
this.cleaningUp = false;
|
|
@@ -414,7 +417,7 @@ export class MCPClient {
|
|
|
414
417
|
// npx-based servers often need significant time to download and start
|
|
415
418
|
const delay = Math.max(this.startupDelay, TIMEOUTS.MIN_SERVER_STARTUP_WAIT);
|
|
416
419
|
this.logger.debug({ delay }, 'Waiting for server startup');
|
|
417
|
-
await new Promise(resolve => setTimeout(resolve, delay));
|
|
420
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
418
421
|
// Mark server as ready (startup delay complete)
|
|
419
422
|
// Note: This only means we can *try* to initialize - actual readiness
|
|
420
423
|
// is confirmed by successful initialization response
|
|
@@ -465,6 +468,9 @@ export class MCPClient {
|
|
|
465
468
|
}
|
|
466
469
|
for (const [id, pending] of this.pendingRequests) {
|
|
467
470
|
clearTimeout(pending.timer);
|
|
471
|
+
if (pending.signal && pending.abortListener) {
|
|
472
|
+
pending.signal.removeEventListener('abort', pending.abortListener);
|
|
473
|
+
}
|
|
468
474
|
pending.reject(new Error(`Request ${id} cancelled: ${reason}`));
|
|
469
475
|
}
|
|
470
476
|
this.pendingRequests.clear();
|
|
@@ -472,33 +478,33 @@ export class MCPClient {
|
|
|
472
478
|
/**
|
|
473
479
|
* List all tools available on the server.
|
|
474
480
|
*/
|
|
475
|
-
async listTools() {
|
|
476
|
-
const result = await this.sendRequest('tools/list', {});
|
|
481
|
+
async listTools(options) {
|
|
482
|
+
const result = await this.sendRequest('tools/list', {}, options);
|
|
477
483
|
return result.tools;
|
|
478
484
|
}
|
|
479
485
|
/**
|
|
480
486
|
* List all prompts available on the server.
|
|
481
487
|
*/
|
|
482
|
-
async listPrompts() {
|
|
483
|
-
const result = await this.sendRequest('prompts/list', {});
|
|
488
|
+
async listPrompts(options) {
|
|
489
|
+
const result = await this.sendRequest('prompts/list', {}, options);
|
|
484
490
|
return result.prompts;
|
|
485
491
|
}
|
|
486
492
|
/**
|
|
487
493
|
* List all resources available on the server.
|
|
488
494
|
*/
|
|
489
|
-
async listResources() {
|
|
490
|
-
const result = await this.sendRequest('resources/list', {});
|
|
495
|
+
async listResources(options) {
|
|
496
|
+
const result = await this.sendRequest('resources/list', {}, options);
|
|
491
497
|
return result.resources;
|
|
492
498
|
}
|
|
493
499
|
/**
|
|
494
500
|
* Read a resource from the server by URI.
|
|
495
501
|
*/
|
|
496
|
-
async readResource(uri) {
|
|
502
|
+
async readResource(uri, options) {
|
|
497
503
|
const done = startTiming(this.logger, `readResource:${uri}`);
|
|
498
504
|
try {
|
|
499
505
|
const result = await this.sendRequest('resources/read', {
|
|
500
506
|
uri,
|
|
501
|
-
});
|
|
507
|
+
}, options);
|
|
502
508
|
done();
|
|
503
509
|
return result;
|
|
504
510
|
}
|
|
@@ -510,13 +516,13 @@ export class MCPClient {
|
|
|
510
516
|
/**
|
|
511
517
|
* Get a prompt from the server with the given arguments.
|
|
512
518
|
*/
|
|
513
|
-
async getPrompt(name, args = {}) {
|
|
519
|
+
async getPrompt(name, args = {}, options) {
|
|
514
520
|
const done = startTiming(this.logger, `getPrompt:${name}`);
|
|
515
521
|
try {
|
|
516
522
|
const result = await this.sendRequest('prompts/get', {
|
|
517
523
|
name,
|
|
518
524
|
arguments: args,
|
|
519
|
-
});
|
|
525
|
+
}, options);
|
|
520
526
|
done();
|
|
521
527
|
return result;
|
|
522
528
|
}
|
|
@@ -528,13 +534,13 @@ export class MCPClient {
|
|
|
528
534
|
/**
|
|
529
535
|
* Call a tool on the server.
|
|
530
536
|
*/
|
|
531
|
-
async callTool(name, args = {}) {
|
|
537
|
+
async callTool(name, args = {}, options) {
|
|
532
538
|
const done = startTiming(this.logger, `callTool:${name}`);
|
|
533
539
|
try {
|
|
534
540
|
const result = await this.sendRequest('tools/call', {
|
|
535
541
|
name,
|
|
536
542
|
arguments: args,
|
|
537
|
-
});
|
|
543
|
+
}, options);
|
|
538
544
|
done();
|
|
539
545
|
return result;
|
|
540
546
|
}
|
|
@@ -580,7 +586,7 @@ export class MCPClient {
|
|
|
580
586
|
}
|
|
581
587
|
this.cleanup();
|
|
582
588
|
}
|
|
583
|
-
sendRequest(method, params) {
|
|
589
|
+
sendRequest(method, params, options) {
|
|
584
590
|
return new Promise((resolve, reject) => {
|
|
585
591
|
if (!this.transport) {
|
|
586
592
|
reject(new Error(this.buildConnectionErrorMessage()));
|
|
@@ -594,16 +600,37 @@ export class MCPClient {
|
|
|
594
600
|
params,
|
|
595
601
|
};
|
|
596
602
|
const timer = setTimeout(() => {
|
|
603
|
+
const pending = this.pendingRequests.get(id);
|
|
604
|
+
if (pending?.signal && pending.abortListener) {
|
|
605
|
+
pending.signal.removeEventListener('abort', pending.abortListener);
|
|
606
|
+
}
|
|
597
607
|
this.pendingRequests.delete(id);
|
|
598
608
|
reject(new Error(`Request timeout: ${method}`));
|
|
599
609
|
}, this.timeout);
|
|
610
|
+
let abortListener;
|
|
611
|
+
const signal = options?.signal;
|
|
612
|
+
if (signal) {
|
|
613
|
+
if (signal.aborted) {
|
|
614
|
+
clearTimeout(timer);
|
|
615
|
+
reject(new Error(`Request aborted: ${method}`));
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
618
|
+
abortListener = () => {
|
|
619
|
+
clearTimeout(timer);
|
|
620
|
+
this.pendingRequests.delete(id);
|
|
621
|
+
reject(new Error(`Request aborted: ${method}`));
|
|
622
|
+
};
|
|
623
|
+
signal.addEventListener('abort', abortListener, { once: true });
|
|
624
|
+
}
|
|
600
625
|
this.pendingRequests.set(id, {
|
|
601
626
|
resolve: resolve,
|
|
602
627
|
reject,
|
|
603
628
|
timer,
|
|
629
|
+
signal,
|
|
630
|
+
abortListener,
|
|
604
631
|
});
|
|
605
632
|
this.log('Sending:', JSON.stringify(request));
|
|
606
|
-
this.transport.send(request);
|
|
633
|
+
this.transport.send(request, signal);
|
|
607
634
|
});
|
|
608
635
|
}
|
|
609
636
|
sendNotification(method, params) {
|
|
@@ -628,6 +655,9 @@ export class MCPClient {
|
|
|
628
655
|
if (pending) {
|
|
629
656
|
clearTimeout(pending.timer);
|
|
630
657
|
this.pendingRequests.delete(msg.id);
|
|
658
|
+
if (pending.signal && pending.abortListener) {
|
|
659
|
+
pending.signal.removeEventListener('abort', pending.abortListener);
|
|
660
|
+
}
|
|
631
661
|
const response = msg;
|
|
632
662
|
if (response.error) {
|
|
633
663
|
pending.reject(new Error(`${response.error.message} (code: ${response.error.code})`));
|
|
@@ -69,7 +69,7 @@ export declare class SSETransport extends BaseTransport {
|
|
|
69
69
|
/**
|
|
70
70
|
* Send a JSON-RPC message to the server via HTTP POST.
|
|
71
71
|
*/
|
|
72
|
-
send(message: JSONRPCMessage): void;
|
|
72
|
+
send(message: JSONRPCMessage, _signal?: AbortSignal): void;
|
|
73
73
|
/**
|
|
74
74
|
* Close the SSE connection.
|
|
75
75
|
*
|
|
@@ -147,7 +147,9 @@ export class SSETransport extends BaseTransport {
|
|
|
147
147
|
this.emit('message', message);
|
|
148
148
|
}
|
|
149
149
|
catch (error) {
|
|
150
|
-
this.log('Failed to parse SSE message', {
|
|
150
|
+
this.log('Failed to parse SSE message', {
|
|
151
|
+
error: error instanceof Error ? error.message : String(error),
|
|
152
|
+
});
|
|
151
153
|
// Don't emit error for parse failures - just log
|
|
152
154
|
}
|
|
153
155
|
}
|
|
@@ -216,7 +218,7 @@ export class SSETransport extends BaseTransport {
|
|
|
216
218
|
/**
|
|
217
219
|
* Send a JSON-RPC message to the server via HTTP POST.
|
|
218
220
|
*/
|
|
219
|
-
send(message) {
|
|
221
|
+
send(message, _signal) {
|
|
220
222
|
if (!this.connected) {
|
|
221
223
|
this.emit('error', new Error('Transport not connected'));
|
|
222
224
|
return;
|
|
@@ -37,7 +37,7 @@ export declare class StdioTransport extends BaseTransport {
|
|
|
37
37
|
isConnected(): boolean;
|
|
38
38
|
private setupInputHandler;
|
|
39
39
|
private processBuffer;
|
|
40
|
-
send(message: JSONRPCMessage): void;
|
|
40
|
+
send(message: JSONRPCMessage, _signal?: AbortSignal): void;
|
|
41
41
|
close(): void;
|
|
42
42
|
}
|
|
43
43
|
//# sourceMappingURL=stdio-transport.d.ts.map
|
package/dist/utils/timeout.d.ts
CHANGED
|
@@ -45,6 +45,13 @@ export interface TimeoutConfig {
|
|
|
45
45
|
/** Custom error message */
|
|
46
46
|
errorMessage?: string;
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Optional timeout behavior overrides.
|
|
50
|
+
*/
|
|
51
|
+
export interface TimeoutOptions {
|
|
52
|
+
/** Abort controller to signal when timeout occurs */
|
|
53
|
+
abortController?: AbortController;
|
|
54
|
+
}
|
|
48
55
|
/**
|
|
49
56
|
* Error class for timeout errors.
|
|
50
57
|
*/
|
|
@@ -76,7 +83,7 @@ export declare function checkAborted(signal: AbortSignal | undefined, operationN
|
|
|
76
83
|
* @param operationName - Name of the operation for error messages
|
|
77
84
|
* @returns The promise result or throws TimeoutError
|
|
78
85
|
*/
|
|
79
|
-
export declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, operationName: string): Promise<T>;
|
|
86
|
+
export declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, operationName: string, options?: TimeoutOptions): Promise<T>;
|
|
80
87
|
/**
|
|
81
88
|
* Wrap a promise with a timeout and return a result object instead of throwing.
|
|
82
89
|
*
|
|
@@ -85,7 +92,7 @@ export declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, o
|
|
|
85
92
|
* @param operationName - Name of the operation
|
|
86
93
|
* @returns Object with either result or error
|
|
87
94
|
*/
|
|
88
|
-
export declare function withTimeoutResult<T>(promise: Promise<T>, timeoutMs: number, operationName: string): Promise<{
|
|
95
|
+
export declare function withTimeoutResult<T>(promise: Promise<T>, timeoutMs: number, operationName: string, options?: TimeoutOptions): Promise<{
|
|
89
96
|
success: true;
|
|
90
97
|
result: T;
|
|
91
98
|
} | {
|
|
@@ -114,6 +121,7 @@ export declare function withTimeoutAll<T>(operations: Array<{
|
|
|
114
121
|
promise: Promise<T>;
|
|
115
122
|
timeoutMs: number;
|
|
116
123
|
operationName: string;
|
|
124
|
+
options?: TimeoutOptions;
|
|
117
125
|
}>): Promise<Array<{
|
|
118
126
|
success: true;
|
|
119
127
|
result: T;
|
package/dist/utils/timeout.js
CHANGED
|
@@ -79,11 +79,15 @@ export function checkAborted(signal, operationName) {
|
|
|
79
79
|
* @param operationName - Name of the operation for error messages
|
|
80
80
|
* @returns The promise result or throws TimeoutError
|
|
81
81
|
*/
|
|
82
|
-
export async function withTimeout(promise, timeoutMs, operationName) {
|
|
82
|
+
export async function withTimeout(promise, timeoutMs, operationName, options) {
|
|
83
83
|
let timeoutId;
|
|
84
|
+
const abortController = options?.abortController;
|
|
84
85
|
const timeoutPromise = new Promise((_, reject) => {
|
|
85
86
|
timeoutId = setTimeout(() => {
|
|
86
87
|
logger.warn({ operationName, timeoutMs }, 'Operation timed out');
|
|
88
|
+
if (abortController) {
|
|
89
|
+
abortController.abort(new TimeoutError(operationName, timeoutMs));
|
|
90
|
+
}
|
|
87
91
|
reject(new TimeoutError(operationName, timeoutMs));
|
|
88
92
|
}, timeoutMs);
|
|
89
93
|
});
|
|
@@ -102,9 +106,9 @@ export async function withTimeout(promise, timeoutMs, operationName) {
|
|
|
102
106
|
* @param operationName - Name of the operation
|
|
103
107
|
* @returns Object with either result or error
|
|
104
108
|
*/
|
|
105
|
-
export async function withTimeoutResult(promise, timeoutMs, operationName) {
|
|
109
|
+
export async function withTimeoutResult(promise, timeoutMs, operationName, options) {
|
|
106
110
|
try {
|
|
107
|
-
const result = await withTimeout(promise, timeoutMs, operationName);
|
|
111
|
+
const result = await withTimeout(promise, timeoutMs, operationName, options);
|
|
108
112
|
return { success: true, result };
|
|
109
113
|
}
|
|
110
114
|
catch (error) {
|
|
@@ -140,9 +144,9 @@ export function createTimeoutAbortController(timeoutMs, operationName) {
|
|
|
140
144
|
* @returns Array of results (either success with value or failure with error)
|
|
141
145
|
*/
|
|
142
146
|
export async function withTimeoutAll(operations) {
|
|
143
|
-
return Promise.all(operations.map(async ({ promise, timeoutMs, operationName }) => {
|
|
147
|
+
return Promise.all(operations.map(async ({ promise, timeoutMs, operationName, options }) => {
|
|
144
148
|
try {
|
|
145
|
-
const result = await withTimeout(promise, timeoutMs, operationName);
|
|
149
|
+
const result = await withTimeout(promise, timeoutMs, operationName, options);
|
|
146
150
|
return { success: true, result };
|
|
147
151
|
}
|
|
148
152
|
catch (error) {
|
package/dist/version.js
CHANGED
|
@@ -63,7 +63,7 @@ export class WorkflowExecutor {
|
|
|
63
63
|
emitProgress(workflow, phase, currentStep, startTime, currentStepInfo) {
|
|
64
64
|
if (!this.onProgress)
|
|
65
65
|
return;
|
|
66
|
-
const stepsFailed = this.stepResults.filter(r => !r.success).length;
|
|
66
|
+
const stepsFailed = this.stepResults.filter((r) => !r.success).length;
|
|
67
67
|
this.onProgress({
|
|
68
68
|
phase,
|
|
69
69
|
workflow,
|
|
@@ -118,7 +118,9 @@ export class WorkflowExecutor {
|
|
|
118
118
|
if (requireSuccessfulDeps) {
|
|
119
119
|
const failedDependencies = this.getFailedDependencies(step, i);
|
|
120
120
|
if (failedDependencies.length > 0) {
|
|
121
|
-
const failedStepNames = failedDependencies
|
|
121
|
+
const failedStepNames = failedDependencies
|
|
122
|
+
.map((idx) => `step ${idx + 1} (${workflow.steps[idx]?.tool ?? 'unknown'})`)
|
|
123
|
+
.join(', ');
|
|
122
124
|
this.logger.debug({
|
|
123
125
|
stepIndex: i,
|
|
124
126
|
tool: step.tool,
|
|
@@ -229,7 +231,7 @@ export class WorkflowExecutor {
|
|
|
229
231
|
workflowId: workflow.id,
|
|
230
232
|
success,
|
|
231
233
|
stepsCompleted: this.stepResults.length,
|
|
232
|
-
stepsFailed: this.stepResults.filter(r => !r.success).length,
|
|
234
|
+
stepsFailed: this.stepResults.filter((r) => !r.success).length,
|
|
233
235
|
durationMs,
|
|
234
236
|
}, 'Workflow execution complete');
|
|
235
237
|
done();
|
|
@@ -251,7 +253,7 @@ export class WorkflowExecutor {
|
|
|
251
253
|
async executeStep(step, stepIndex, workflow) {
|
|
252
254
|
const startTime = Date.now();
|
|
253
255
|
// Verify tool exists
|
|
254
|
-
const tool = this.tools.find(t => t.name === step.tool);
|
|
256
|
+
const tool = this.tools.find((t) => t.name === step.tool);
|
|
255
257
|
if (!tool) {
|
|
256
258
|
return {
|
|
257
259
|
step,
|
|
@@ -284,7 +286,8 @@ export class WorkflowExecutor {
|
|
|
284
286
|
let error;
|
|
285
287
|
const stepTimeout = this.options.stepTimeout ?? DEFAULT_OPTIONS.stepTimeout;
|
|
286
288
|
try {
|
|
287
|
-
|
|
289
|
+
const abortController = new AbortController();
|
|
290
|
+
response = await withTimeout(this.client.callTool(step.tool, resolvedArgs, { signal: abortController.signal }), stepTimeout, `Tool call '${step.tool}'`, { abortController });
|
|
288
291
|
if (response.isError) {
|
|
289
292
|
error = this.extractErrorMessage(response);
|
|
290
293
|
}
|
|
@@ -296,7 +299,7 @@ export class WorkflowExecutor {
|
|
|
296
299
|
const assertionResults = step.assertions
|
|
297
300
|
? this.runAssertions(step.assertions, response)
|
|
298
301
|
: undefined;
|
|
299
|
-
const assertionsFailed = assertionResults?.some(r => !r.passed) ?? false;
|
|
302
|
+
const assertionsFailed = assertionResults?.some((r) => !r.passed) ?? false;
|
|
300
303
|
const success = !error && !assertionsFailed;
|
|
301
304
|
// Generate analysis if requested
|
|
302
305
|
let analysis;
|
|
@@ -359,7 +362,7 @@ export class WorkflowExecutor {
|
|
|
359
362
|
if (propertyPath.startsWith('result.') || propertyPath === 'result') {
|
|
360
363
|
// Extract text content from the response
|
|
361
364
|
const content = stepResult.response.content;
|
|
362
|
-
const textContent = content.find(c => c.type === 'text' && c.text !== undefined);
|
|
365
|
+
const textContent = content.find((c) => c.type === 'text' && c.text !== undefined);
|
|
363
366
|
if (!textContent || textContent.text === undefined) {
|
|
364
367
|
throw new Error(`Step ${stepIndex} response has no text content`);
|
|
365
368
|
}
|
|
@@ -418,7 +421,7 @@ export class WorkflowExecutor {
|
|
|
418
421
|
* Run assertions against a step response.
|
|
419
422
|
*/
|
|
420
423
|
runAssertions(assertions, response) {
|
|
421
|
-
return assertions.map(assertion => this.runAssertion(assertion, response));
|
|
424
|
+
return assertions.map((assertion) => this.runAssertion(assertion, response));
|
|
422
425
|
}
|
|
423
426
|
/**
|
|
424
427
|
* Run a single assertion.
|
|
@@ -435,7 +438,7 @@ export class WorkflowExecutor {
|
|
|
435
438
|
let actualValue;
|
|
436
439
|
try {
|
|
437
440
|
// Parse the response content as JSON
|
|
438
|
-
const textContent = response.content.find(c => c.type === 'text' && c.text !== undefined);
|
|
441
|
+
const textContent = response.content.find((c) => c.type === 'text' && c.text !== undefined);
|
|
439
442
|
if (!textContent || textContent.text === undefined) {
|
|
440
443
|
throw new Error('No text content in response');
|
|
441
444
|
}
|
|
@@ -484,14 +487,16 @@ export class WorkflowExecutor {
|
|
|
484
487
|
assertion,
|
|
485
488
|
passed,
|
|
486
489
|
actualValue,
|
|
487
|
-
message: passed
|
|
490
|
+
message: passed
|
|
491
|
+
? undefined
|
|
492
|
+
: (assertion.message ?? `Assertion failed: ${assertion.condition}`),
|
|
488
493
|
};
|
|
489
494
|
}
|
|
490
495
|
/**
|
|
491
496
|
* Extract error message from a tool response.
|
|
492
497
|
*/
|
|
493
498
|
extractErrorMessage(response) {
|
|
494
|
-
const textContent = response.content.find(c => c.type === 'text');
|
|
499
|
+
const textContent = response.content.find((c) => c.type === 'text');
|
|
495
500
|
if (textContent && 'text' in textContent) {
|
|
496
501
|
return String(textContent.text);
|
|
497
502
|
}
|
|
@@ -595,7 +600,7 @@ export class WorkflowExecutor {
|
|
|
595
600
|
if (!this.llm) {
|
|
596
601
|
return success
|
|
597
602
|
? `Workflow "${workflow.name}" completed successfully with ${stepResults.length} steps.`
|
|
598
|
-
: `Workflow "${workflow.name}" failed at step ${stepResults.findIndex(r => !r.success) + 1}.`;
|
|
603
|
+
: `Workflow "${workflow.name}" failed at step ${stepResults.findIndex((r) => !r.success) + 1}.`;
|
|
599
604
|
}
|
|
600
605
|
const prompt = buildWorkflowSummaryPrompt({ workflow, stepResults, success });
|
|
601
606
|
try {
|
|
@@ -604,7 +609,7 @@ export class WorkflowExecutor {
|
|
|
604
609
|
catch {
|
|
605
610
|
return success
|
|
606
611
|
? `Workflow "${workflow.name}" completed successfully with ${stepResults.length} steps.`
|
|
607
|
-
: `Workflow "${workflow.name}" failed at step ${stepResults.findIndex(r => !r.success) + 1}.`;
|
|
612
|
+
: `Workflow "${workflow.name}" failed at step ${stepResults.findIndex((r) => !r.success) + 1}.`;
|
|
608
613
|
}
|
|
609
614
|
}
|
|
610
615
|
}
|
package/dist/workflow/loader.js
CHANGED
|
@@ -6,8 +6,10 @@ import { join } from 'path';
|
|
|
6
6
|
import { parseAllDocuments } from 'yaml';
|
|
7
7
|
import { parseYamlSecure, YAML_SECURITY_LIMITS } from '../utils/yaml-parser.js';
|
|
8
8
|
import { PATHS } from '../constants.js';
|
|
9
|
+
import { getLogger } from '../logging/logger.js';
|
|
9
10
|
/** Default file name for workflow definitions */
|
|
10
11
|
export const DEFAULT_WORKFLOWS_FILE = PATHS.DEFAULT_WORKFLOWS_FILE;
|
|
12
|
+
const logger = getLogger('workflow');
|
|
11
13
|
/**
|
|
12
14
|
* Load workflows from a YAML file.
|
|
13
15
|
* Supports both single-document and multi-document YAML (separated by ---).
|
|
@@ -56,9 +58,10 @@ export function tryLoadDefaultWorkflows(directory) {
|
|
|
56
58
|
try {
|
|
57
59
|
return loadWorkflowsFromFile(path);
|
|
58
60
|
}
|
|
59
|
-
catch {
|
|
61
|
+
catch (error) {
|
|
60
62
|
// If the file exists but is invalid, return null rather than throwing
|
|
61
63
|
// This allows the interview to proceed without workflows
|
|
64
|
+
logger.warn({ path, error: error instanceof Error ? error.message : String(error) }, 'Failed to load default workflow file');
|
|
62
65
|
return null;
|
|
63
66
|
}
|
|
64
67
|
}
|
|
@@ -74,7 +74,7 @@ export class StateTracker {
|
|
|
74
74
|
}
|
|
75
75
|
// Use specified probe tools if provided
|
|
76
76
|
if (this.options.probeTools?.length) {
|
|
77
|
-
this.probeTools = this.options.probeTools.filter(name => this.tools.some(t => t.name === name));
|
|
77
|
+
this.probeTools = this.options.probeTools.filter((name) => this.tools.some((t) => t.name === name));
|
|
78
78
|
}
|
|
79
79
|
this.logger.debug({
|
|
80
80
|
toolCount: this.tools.length,
|
|
@@ -89,9 +89,9 @@ export class StateTracker {
|
|
|
89
89
|
const name = tool.name;
|
|
90
90
|
const description = tool.description ?? '';
|
|
91
91
|
const combined = `${name} ${description}`;
|
|
92
|
-
const isReader = READER_PATTERNS.some(p => p.test(combined));
|
|
93
|
-
const isWriter = WRITER_PATTERNS.some(p => p.test(combined));
|
|
94
|
-
const isProbe = PROBE_PATTERNS.some(p => p.test(combined));
|
|
92
|
+
const isReader = READER_PATTERNS.some((p) => p.test(combined));
|
|
93
|
+
const isWriter = WRITER_PATTERNS.some((p) => p.test(combined));
|
|
94
|
+
const isProbe = PROBE_PATTERNS.some((p) => p.test(combined));
|
|
95
95
|
let role;
|
|
96
96
|
let confidence;
|
|
97
97
|
if (isReader && isWriter) {
|
|
@@ -194,8 +194,9 @@ export class StateTracker {
|
|
|
194
194
|
break;
|
|
195
195
|
}
|
|
196
196
|
try {
|
|
197
|
+
const abortController = new AbortController();
|
|
197
198
|
// Apply timeout to individual probe tool call
|
|
198
|
-
const result = await withTimeout(this.client.callTool(probeName, {}), this.probeTimeout, `Probe tool '${probeName}'
|
|
199
|
+
const result = await withTimeout(this.client.callTool(probeName, {}, { signal: abortController.signal }), this.probeTimeout, `Probe tool '${probeName}'`, { abortController });
|
|
199
200
|
const content = this.extractContent(result);
|
|
200
201
|
stateData[probeName] = content;
|
|
201
202
|
successCount++;
|
|
@@ -240,7 +241,7 @@ export class StateTracker {
|
|
|
240
241
|
* Extract content from a tool call result.
|
|
241
242
|
*/
|
|
242
243
|
extractContent(result) {
|
|
243
|
-
const textContent = result.content.find(c => c.type === 'text' && c.text !== undefined);
|
|
244
|
+
const textContent = result.content.find((c) => c.type === 'text' && c.text !== undefined);
|
|
244
245
|
if (!textContent || textContent.text === undefined) {
|
|
245
246
|
return null;
|
|
246
247
|
}
|
|
@@ -256,7 +257,10 @@ export class StateTracker {
|
|
|
256
257
|
*/
|
|
257
258
|
hashState(data) {
|
|
258
259
|
const json = JSON.stringify(data, null, 0);
|
|
259
|
-
return createHash('sha256')
|
|
260
|
+
return createHash('sha256')
|
|
261
|
+
.update(json)
|
|
262
|
+
.digest('hex')
|
|
263
|
+
.slice(0, DISPLAY_LIMITS.HASH_DISPLAY_LENGTH);
|
|
260
264
|
}
|
|
261
265
|
/**
|
|
262
266
|
* Compare two snapshots and identify changes.
|
|
@@ -351,7 +355,7 @@ export class StateTracker {
|
|
|
351
355
|
for (const stateType of stateTypes) {
|
|
352
356
|
const writers = writerSteps.get(stateType) ?? [];
|
|
353
357
|
// Find most recent writer for this state type
|
|
354
|
-
const recentWriters = writers.filter(w => w < i);
|
|
358
|
+
const recentWriters = writers.filter((w) => w < i);
|
|
355
359
|
if (recentWriters.length > 0) {
|
|
356
360
|
const producerStep = recentWriters[recentWriters.length - 1];
|
|
357
361
|
const producerTool = stepResults[producerStep].step.tool;
|
|
@@ -372,9 +376,9 @@ export class StateTracker {
|
|
|
372
376
|
* Verify dependencies using state snapshots.
|
|
373
377
|
*/
|
|
374
378
|
verifyDependencies(dependencies, _snapshots, changes) {
|
|
375
|
-
return dependencies.map(dep => {
|
|
379
|
+
return dependencies.map((dep) => {
|
|
376
380
|
// Check if the producer step caused any changes
|
|
377
|
-
const producerChanges = changes.filter(c => c.causedByStep === dep.producerStep);
|
|
381
|
+
const producerChanges = changes.filter((c) => c.causedByStep === dep.producerStep);
|
|
378
382
|
const verified = producerChanges.length > 0;
|
|
379
383
|
return {
|
|
380
384
|
...dep,
|
|
@@ -388,19 +392,19 @@ export class StateTracker {
|
|
|
388
392
|
async generateSummary(tracking) {
|
|
389
393
|
const parts = [];
|
|
390
394
|
// Summarize tool roles
|
|
391
|
-
const writers = tracking.toolRoles.filter(t => t.role === 'writer' || t.role === 'both');
|
|
392
|
-
const readers = tracking.toolRoles.filter(t => t.role === 'reader' || t.role === 'both');
|
|
395
|
+
const writers = tracking.toolRoles.filter((t) => t.role === 'writer' || t.role === 'both');
|
|
396
|
+
const readers = tracking.toolRoles.filter((t) => t.role === 'reader' || t.role === 'both');
|
|
393
397
|
if (writers.length > 0) {
|
|
394
|
-
parts.push(`State writers: ${writers.map(t => t.tool).join(', ')}`);
|
|
398
|
+
parts.push(`State writers: ${writers.map((t) => t.tool).join(', ')}`);
|
|
395
399
|
}
|
|
396
400
|
if (readers.length > 0) {
|
|
397
|
-
parts.push(`State readers: ${readers.map(t => t.tool).join(', ')}`);
|
|
401
|
+
parts.push(`State readers: ${readers.map((t) => t.tool).join(', ')}`);
|
|
398
402
|
}
|
|
399
403
|
// Summarize changes
|
|
400
404
|
if (tracking.changes.length > 0) {
|
|
401
|
-
const created = tracking.changes.filter(c => c.type === 'created').length;
|
|
402
|
-
const modified = tracking.changes.filter(c => c.type === 'modified').length;
|
|
403
|
-
const deleted = tracking.changes.filter(c => c.type === 'deleted').length;
|
|
405
|
+
const created = tracking.changes.filter((c) => c.type === 'created').length;
|
|
406
|
+
const modified = tracking.changes.filter((c) => c.type === 'modified').length;
|
|
407
|
+
const deleted = tracking.changes.filter((c) => c.type === 'deleted').length;
|
|
404
408
|
const changeParts = [];
|
|
405
409
|
if (created > 0)
|
|
406
410
|
changeParts.push(`${created} created`);
|
|
@@ -415,7 +419,7 @@ export class StateTracker {
|
|
|
415
419
|
}
|
|
416
420
|
// Summarize dependencies
|
|
417
421
|
if (tracking.dependencies.length > 0) {
|
|
418
|
-
const verified = tracking.dependencies.filter(d => d.verified).length;
|
|
422
|
+
const verified = tracking.dependencies.filter((d) => d.verified).length;
|
|
419
423
|
parts.push(`Dependencies: ${tracking.dependencies.length} inferred (${verified} verified)`);
|
|
420
424
|
}
|
|
421
425
|
return `${parts.join('. ')}.`;
|