@syrin/cli 1.3.2 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +184 -152
- package/dist/cli/commands/config.d.ts +47 -0
- package/dist/cli/commands/config.js +360 -0
- package/dist/cli/commands/dev.d.ts +6 -0
- package/dist/cli/commands/dev.js +67 -15
- package/dist/cli/commands/doctor.js +49 -13
- package/dist/cli/commands/init.d.ts +2 -0
- package/dist/cli/commands/init.js +89 -18
- package/dist/cli/commands/status.d.ts +10 -0
- package/dist/cli/commands/status.js +162 -0
- package/dist/cli/index.js +211 -12
- package/dist/cli/prompts/init-prompt.d.ts +18 -0
- package/dist/cli/prompts/init-prompt.js +159 -99
- package/dist/cli/utils/command-error-handler.js +2 -5
- package/dist/config/env-checker.d.ts +12 -2
- package/dist/config/env-checker.js +88 -38
- package/dist/config/env-templates.d.ts +15 -0
- package/dist/config/env-templates.js +49 -0
- package/dist/config/generator.js +17 -0
- package/dist/config/global-loader.d.ts +50 -0
- package/dist/config/global-loader.js +244 -0
- package/dist/config/loader.d.ts +28 -0
- package/dist/config/loader.js +95 -9
- package/dist/config/merger.d.ts +37 -0
- package/dist/config/merger.js +68 -0
- package/dist/config/schema.d.ts +26 -1
- package/dist/config/schema.js +73 -8
- package/dist/config/types.d.ts +19 -0
- package/dist/config/types.js +26 -1
- package/dist/constants/messages.d.ts +7 -0
- package/dist/constants/messages.js +8 -0
- package/dist/constants/paths.d.ts +6 -0
- package/dist/constants/paths.js +10 -0
- package/dist/events/emitter.js +7 -7
- package/dist/index.js +0 -0
- package/dist/presentation/config-ui.d.ts +34 -0
- package/dist/presentation/config-ui.js +139 -0
- package/dist/presentation/doctor-ui.d.ts +11 -0
- package/dist/presentation/doctor-ui.js +52 -1
- package/dist/presentation/init-ui.d.ts +9 -0
- package/dist/presentation/init-ui.js +33 -0
- package/dist/runtime/analysis/analyser.js +2 -2
- package/dist/runtime/analysis/rules/warnings/w104-generic-description.d.ts +1 -1
- package/dist/runtime/analysis/rules/warnings/w104-generic-description.js +1 -1
- package/dist/runtime/dev/event-mapper.js +19 -3
- package/dist/runtime/dev/session.d.ts +4 -0
- package/dist/runtime/dev/session.js +52 -3
- package/dist/runtime/llm/ollama.js +4 -4
- package/dist/runtime/mcp/client/manager.js +3 -3
- package/dist/runtime/sandbox/executor.js +5 -5
- package/dist/runtime/test/orchestrator.js +4 -4
- package/dist/utils/editor.d.ts +37 -0
- package/dist/utils/editor.js +137 -0
- package/dist/utils/logger.d.ts +24 -6
- package/dist/utils/logger.js +51 -8
- package/package.json +23 -23
- package/dist/runtime/analysis/rules/errors/e001-missing-output-schema.d.ts +0 -22
- package/dist/runtime/analysis/rules/errors/e001-missing-output-schema.js +0 -30
- package/dist/runtime/analysis/rules/errors/e002-underspecified-input.d.ts +0 -24
- package/dist/runtime/analysis/rules/errors/e002-underspecified-input.js +0 -52
- package/dist/runtime/analysis/rules/errors/e003-type-mismatch.d.ts +0 -23
- package/dist/runtime/analysis/rules/errors/e003-type-mismatch.js +0 -73
- package/dist/runtime/analysis/rules/errors/e004-free-text-propagation.d.ts +0 -23
- package/dist/runtime/analysis/rules/errors/e004-free-text-propagation.js +0 -47
- package/dist/runtime/analysis/rules/errors/e005-tool-ambiguity.d.ts +0 -25
- package/dist/runtime/analysis/rules/errors/e005-tool-ambiguity.js +0 -73
- package/dist/runtime/analysis/rules/errors/e006-param-not-in-description.d.ts +0 -22
- package/dist/runtime/analysis/rules/errors/e006-param-not-in-description.js +0 -57
- package/dist/runtime/analysis/rules/errors/e007-output-not-guaranteed.d.ts +0 -23
- package/dist/runtime/analysis/rules/errors/e007-output-not-guaranteed.js +0 -56
- package/dist/runtime/analysis/rules/errors/e008-circular-dependency.d.ts +0 -22
- package/dist/runtime/analysis/rules/errors/e008-circular-dependency.js +0 -84
- package/dist/runtime/analysis/rules/errors/e009-implicit-user-input.d.ts +0 -23
- package/dist/runtime/analysis/rules/errors/e009-implicit-user-input.js +0 -89
- package/dist/runtime/analysis/rules/errors/e010-non-serializable.d.ts +0 -25
- package/dist/runtime/analysis/rules/errors/e010-non-serializable.js +0 -46
- package/dist/runtime/analysis/rules/errors/e011-missing-tool-description.d.ts +0 -24
- package/dist/runtime/analysis/rules/errors/e011-missing-tool-description.js +0 -33
- package/dist/runtime/analysis/rules/errors/e012-side-effect-detected.d.ts +0 -39
- package/dist/runtime/analysis/rules/errors/e012-side-effect-detected.js +0 -40
- package/dist/runtime/analysis/rules/errors/e013-non-deterministic-output.d.ts +0 -37
- package/dist/runtime/analysis/rules/errors/e013-non-deterministic-output.js +0 -34
- package/dist/runtime/analysis/rules/errors/e013-output-explosion.d.ts +0 -39
- package/dist/runtime/analysis/rules/errors/e013-output-explosion.js +0 -36
- package/dist/runtime/analysis/rules/errors/e014-hidden-dependency.d.ts +0 -42
- package/dist/runtime/analysis/rules/errors/e014-hidden-dependency.js +0 -46
- package/dist/runtime/analysis/rules/errors/e014-output-explosion.d.ts +0 -39
- package/dist/runtime/analysis/rules/errors/e014-output-explosion.js +0 -36
- package/dist/runtime/analysis/rules/errors/e015-hidden-dependency.d.ts +0 -42
- package/dist/runtime/analysis/rules/errors/e015-hidden-dependency.js +0 -46
- package/dist/runtime/analysis/rules/errors/e015-unbounded-execution.d.ts +0 -44
- package/dist/runtime/analysis/rules/errors/e015-unbounded-execution.js +0 -66
- package/dist/runtime/analysis/rules/errors/e016-output-validation-failed.d.ts +0 -43
- package/dist/runtime/analysis/rules/errors/e016-output-validation-failed.js +0 -42
- package/dist/runtime/analysis/rules/errors/e016-unbounded-execution.d.ts +0 -44
- package/dist/runtime/analysis/rules/errors/e016-unbounded-execution.js +0 -66
- package/dist/runtime/analysis/rules/errors/e017-input-validation-failed.d.ts +0 -57
- package/dist/runtime/analysis/rules/errors/e017-input-validation-failed.js +0 -80
- package/dist/runtime/analysis/rules/errors/e017-output-validation-failed.d.ts +0 -43
- package/dist/runtime/analysis/rules/errors/e017-output-validation-failed.js +0 -42
- package/dist/runtime/analysis/rules/errors/e018-input-validation-failed.d.ts +0 -57
- package/dist/runtime/analysis/rules/errors/e018-input-validation-failed.js +0 -80
- package/dist/runtime/analysis/rules/errors/e018-tool-execution-failed.d.ts +0 -38
- package/dist/runtime/analysis/rules/errors/e018-tool-execution-failed.js +0 -37
- package/dist/runtime/analysis/rules/errors/e019-tool-execution-failed.d.ts +0 -38
- package/dist/runtime/analysis/rules/errors/e019-tool-execution-failed.js +0 -37
- package/dist/runtime/analysis/rules/errors/e019-unexpected-test-result.d.ts +0 -65
- package/dist/runtime/analysis/rules/errors/e019-unexpected-test-result.js +0 -109
- package/dist/runtime/analysis/rules/errors/e020-unexpected-test-result.d.ts +0 -65
- package/dist/runtime/analysis/rules/errors/e020-unexpected-test-result.js +0 -109
- package/dist/runtime/analysis/rules/warnings/w001-implicit-dependency.d.ts +0 -22
- package/dist/runtime/analysis/rules/warnings/w001-implicit-dependency.js +0 -39
- package/dist/runtime/analysis/rules/warnings/w002-free-text-without-normalization.d.ts +0 -24
- package/dist/runtime/analysis/rules/warnings/w002-free-text-without-normalization.js +0 -40
- package/dist/runtime/analysis/rules/warnings/w003-missing-examples.d.ts +0 -22
- package/dist/runtime/analysis/rules/warnings/w003-missing-examples.js +0 -84
- package/dist/runtime/analysis/rules/warnings/w004-overloaded-responsibility.d.ts +0 -23
- package/dist/runtime/analysis/rules/warnings/w004-overloaded-responsibility.js +0 -96
- package/dist/runtime/analysis/rules/warnings/w005-generic-description.d.ts +0 -53
- package/dist/runtime/analysis/rules/warnings/w005-generic-description.js +0 -108
- package/dist/runtime/analysis/rules/warnings/w006-optional-as-required.d.ts +0 -22
- package/dist/runtime/analysis/rules/warnings/w006-optional-as-required.js +0 -44
- package/dist/runtime/analysis/rules/warnings/w007-broad-output-schema.d.ts +0 -23
- package/dist/runtime/analysis/rules/warnings/w007-broad-output-schema.js +0 -37
- package/dist/runtime/analysis/rules/warnings/w008-multiple-entry-points.d.ts +0 -22
- package/dist/runtime/analysis/rules/warnings/w008-multiple-entry-points.js +0 -97
- package/dist/runtime/analysis/rules/warnings/w009-hidden-side-effects.d.ts +0 -23
- package/dist/runtime/analysis/rules/warnings/w009-hidden-side-effects.js +0 -88
- package/dist/runtime/analysis/rules/warnings/w010-output-not-reusable.d.ts +0 -22
- package/dist/runtime/analysis/rules/warnings/w010-output-not-reusable.js +0 -81
- package/dist/runtime/analysis/rules/warnings/w021-weak-schema.d.ts +0 -40
- package/dist/runtime/analysis/rules/warnings/w021-weak-schema.js +0 -32
- package/dist/runtime/analysis/rules/warnings/w022-high-entropy-output.d.ts +0 -39
- package/dist/runtime/analysis/rules/warnings/w022-high-entropy-output.js +0 -36
- package/dist/runtime/analysis/rules/warnings/w023-unstable-defaults.d.ts +0 -38
- package/dist/runtime/analysis/rules/warnings/w023-unstable-defaults.js +0 -36
- package/dist/runtime/test/dependency-tracker.d.ts +0 -66
- package/dist/runtime/test/dependency-tracker.js +0 -80
- package/dist/runtime/test/formatters.d.ts +0 -18
- package/dist/runtime/test/formatters.js +0 -172
- package/dist/runtime/test/input-generator.d.ts +0 -33
- package/dist/runtime/test/input-generator.js +0 -498
- package/dist/runtime/test/mcp-root-detector.d.ts +0 -31
- package/dist/runtime/test/mcp-root-detector.js +0 -105
- package/dist/runtime/test/retry-tester.d.ts +0 -44
- package/dist/runtime/test/retry-tester.js +0 -103
- package/dist/runtime/test/synthetic-input-generator.d.ts +0 -11
- package/dist/runtime/test/synthetic-input-generator.js +0 -154
- package/dist/runtime/test/test-runner.d.ts +0 -28
- package/dist/runtime/test/test-runner.js +0 -55
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* E020: Unexpected Test Result
|
|
3
|
-
*
|
|
4
|
-
* Condition: Test's actual outcome doesn't match its expectation
|
|
5
|
-
*
|
|
6
|
-
* Why this is fatal:
|
|
7
|
-
* - Test contract is inaccurate
|
|
8
|
-
* - Tool behavior doesn't match declared guarantees
|
|
9
|
-
* - Can cause false positives/negatives in validation
|
|
10
|
-
* - Indicates mismatch between expectations and reality
|
|
11
|
-
*/
|
|
12
|
-
import { BaseRule } from '../base.js';
|
|
13
|
-
import { ERROR_CODES } from '../error-codes.js';
|
|
14
|
-
class E020UnexpectedTestResultRule extends BaseRule {
|
|
15
|
-
id = ERROR_CODES.E020;
|
|
16
|
-
severity = 'error';
|
|
17
|
-
ruleName = 'Unexpected Test Result';
|
|
18
|
-
description = "Test's actual outcome doesn't match its expectation. Tool behavior doesn't match declared guarantees.";
|
|
19
|
-
check(_ctx) {
|
|
20
|
-
// This rule requires behavioral context
|
|
21
|
-
return [];
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Determine if a diagnostic should be created based on behavioral context.
|
|
25
|
-
*/
|
|
26
|
-
shouldCreateDiagnostic(behavioralCtx) {
|
|
27
|
-
// If outcomes don't match, create diagnostic
|
|
28
|
-
if (behavioralCtx.expectedOutcome !== behavioralCtx.actualOutcome) {
|
|
29
|
-
return true;
|
|
30
|
-
}
|
|
31
|
-
// Outcomes match - check if error details also match (for error cases)
|
|
32
|
-
if (behavioralCtx.expectedOutcome === 'error') {
|
|
33
|
-
const expectedType = behavioralCtx.expectedErrorType?.toLowerCase();
|
|
34
|
-
const actualType = behavioralCtx.actualErrorType?.toLowerCase();
|
|
35
|
-
if (expectedType && actualType && expectedType !== actualType) {
|
|
36
|
-
return true; // Type mismatch
|
|
37
|
-
}
|
|
38
|
-
if (expectedType && !actualType) {
|
|
39
|
-
return true; // Expected type but no actual type
|
|
40
|
-
}
|
|
41
|
-
const expectedCode = behavioralCtx.expectedErrorCode;
|
|
42
|
-
const actualCode = behavioralCtx.actualErrorCode;
|
|
43
|
-
if (expectedCode && actualCode && expectedCode !== actualCode) {
|
|
44
|
-
return true; // Code mismatch
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
return false; // Everything matches
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Check with behavioral context (called from test orchestrator).
|
|
51
|
-
*/
|
|
52
|
-
checkWithBehavioralContext(behavioralCtx) {
|
|
53
|
-
const diagnostics = [];
|
|
54
|
-
if (!this.shouldCreateDiagnostic(behavioralCtx)) {
|
|
55
|
-
return diagnostics;
|
|
56
|
-
}
|
|
57
|
-
// If we reach here, there's a mismatch - create diagnostic
|
|
58
|
-
let message = `Test "${behavioralCtx.testName}" in tool "${behavioralCtx.toolName}" expected `;
|
|
59
|
-
// Build expected description
|
|
60
|
-
if (behavioralCtx.expectedOutcome === 'success') {
|
|
61
|
-
message += 'success but got ';
|
|
62
|
-
if (behavioralCtx.actualOutcome === 'error') {
|
|
63
|
-
message += `error (${behavioralCtx.actualErrorCode || behavioralCtx.actualErrorType || 'unknown error'})`;
|
|
64
|
-
}
|
|
65
|
-
else if (behavioralCtx.actualOutcome === 'timeout') {
|
|
66
|
-
message += 'timeout';
|
|
67
|
-
}
|
|
68
|
-
else {
|
|
69
|
-
message += behavioralCtx.actualOutcome;
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
else if (behavioralCtx.expectedOutcome === 'error') {
|
|
73
|
-
message += `error`;
|
|
74
|
-
if (behavioralCtx.expectedErrorType) {
|
|
75
|
-
message += ` of type "${behavioralCtx.expectedErrorType}"`;
|
|
76
|
-
}
|
|
77
|
-
if (behavioralCtx.expectedErrorCode) {
|
|
78
|
-
message += ` (code: ${behavioralCtx.expectedErrorCode})`;
|
|
79
|
-
}
|
|
80
|
-
message += ' but got ';
|
|
81
|
-
if (behavioralCtx.actualOutcome === 'success') {
|
|
82
|
-
message += 'success';
|
|
83
|
-
}
|
|
84
|
-
else {
|
|
85
|
-
message += `${behavioralCtx.actualOutcome} (${behavioralCtx.actualErrorCode || behavioralCtx.actualErrorType || 'unknown error'})`;
|
|
86
|
-
}
|
|
87
|
-
if (behavioralCtx.actualErrorType &&
|
|
88
|
-
behavioralCtx.expectedErrorType &&
|
|
89
|
-
behavioralCtx.actualErrorType.toLowerCase() !==
|
|
90
|
-
behavioralCtx.expectedErrorType.toLowerCase()) {
|
|
91
|
-
message += ' with different type';
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
else {
|
|
95
|
-
message += `${behavioralCtx.expectedOutcome} but got ${behavioralCtx.actualOutcome}`;
|
|
96
|
-
}
|
|
97
|
-
diagnostics.push(this.createDiagnostic(message, behavioralCtx.toolName, undefined, 'Update test expectation to match actual tool behavior, or fix tool implementation to match declared guarantees. Ensure test expectations accurately reflect tool behavior.', {
|
|
98
|
-
testName: behavioralCtx.testName,
|
|
99
|
-
testInput: behavioralCtx.testInput,
|
|
100
|
-
expectedOutcome: behavioralCtx.expectedOutcome,
|
|
101
|
-
actualOutcome: behavioralCtx.actualOutcome,
|
|
102
|
-
expectedError: behavioralCtx.expectedError,
|
|
103
|
-
actualError: behavioralCtx.actualError,
|
|
104
|
-
}));
|
|
105
|
-
return diagnostics;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
export const E020UnexpectedTestResult = new E020UnexpectedTestResultRule();
|
|
109
|
-
//# sourceMappingURL=e020-unexpected-test-result.js.map
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W001: Implicit Tool Dependency
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - High confidence dependency inferred
|
|
6
|
-
* - Not stated explicitly in description
|
|
7
|
-
*
|
|
8
|
-
* Why:
|
|
9
|
-
* - LLM may not chain tools reliably
|
|
10
|
-
*/
|
|
11
|
-
import { BaseRule } from '../base.js';
|
|
12
|
-
import type { AnalysisContext, Diagnostic } from '../../types.js';
|
|
13
|
-
declare class W001ImplicitDependencyRule extends BaseRule {
|
|
14
|
-
readonly id = "W001";
|
|
15
|
-
readonly severity: "warning";
|
|
16
|
-
readonly ruleName = "Implicit Tool Dependency";
|
|
17
|
-
readonly description = "Tool appears to depend on another tool, but the dependency is implicit.";
|
|
18
|
-
check(ctx: AnalysisContext): Diagnostic[];
|
|
19
|
-
}
|
|
20
|
-
export declare const W001ImplicitDependency: W001ImplicitDependencyRule;
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=w001-implicit-dependency.d.ts.map
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W001: Implicit Tool Dependency
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - High confidence dependency inferred
|
|
6
|
-
* - Not stated explicitly in description
|
|
7
|
-
*
|
|
8
|
-
* Why:
|
|
9
|
-
* - LLM may not chain tools reliably
|
|
10
|
-
*/
|
|
11
|
-
import { BaseRule } from '../base.js';
|
|
12
|
-
class W001ImplicitDependencyRule extends BaseRule {
|
|
13
|
-
id = 'W001';
|
|
14
|
-
severity = 'warning';
|
|
15
|
-
ruleName = 'Implicit Tool Dependency';
|
|
16
|
-
description = 'Tool appears to depend on another tool, but the dependency is implicit.';
|
|
17
|
-
check(ctx) {
|
|
18
|
-
const diagnostics = [];
|
|
19
|
-
// Check medium to high confidence dependencies (0.6-0.8)
|
|
20
|
-
const implicitDeps = ctx.dependencies.filter(d => d.confidence >= 0.6 && d.confidence < 0.8);
|
|
21
|
-
for (const dep of implicitDeps) {
|
|
22
|
-
// Find the toTool to check its description
|
|
23
|
-
const toTool = ctx.indexes.toolIndex.get(dep.toTool.toLowerCase());
|
|
24
|
-
if (!toTool) {
|
|
25
|
-
continue;
|
|
26
|
-
}
|
|
27
|
-
const description = toTool.description || '';
|
|
28
|
-
// Check if fromTool is mentioned in description
|
|
29
|
-
const fromToolNameLower = dep.fromTool.toLowerCase();
|
|
30
|
-
const descLower = description.toLowerCase();
|
|
31
|
-
if (!descLower.includes(fromToolNameLower)) {
|
|
32
|
-
diagnostics.push(this.createDiagnostic(`Tool "${dep.toTool}" appears to depend on "${dep.fromTool}", but the dependency is implicit.`, dep.toTool, dep.toField, `Mention "${dep.fromTool}" in the description of "${dep.toTool}" to make the dependency explicit.`));
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
return diagnostics;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
export const W001ImplicitDependency = new W001ImplicitDependencyRule();
|
|
39
|
-
//# sourceMappingURL=w001-implicit-dependency.js.map
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W002: Free-Text Output Without Normalization
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Output is string
|
|
6
|
-
* - Not constrained or normalized
|
|
7
|
-
* - Even if not chained
|
|
8
|
-
*
|
|
9
|
-
* Why:
|
|
10
|
-
* - Hard to reuse
|
|
11
|
-
* - Hard to evolve
|
|
12
|
-
*/
|
|
13
|
-
import { BaseRule } from '../base.js';
|
|
14
|
-
import type { AnalysisContext, Diagnostic } from '../../types.js';
|
|
15
|
-
declare class W002FreeTextWithoutNormalizationRule extends BaseRule {
|
|
16
|
-
readonly id = "W002";
|
|
17
|
-
readonly severity: "warning";
|
|
18
|
-
readonly ruleName = "Free-Text Output Without Normalization";
|
|
19
|
-
readonly description = "Tool returns unconstrained free text. Consider normalizing output.";
|
|
20
|
-
check(ctx: AnalysisContext): Diagnostic[];
|
|
21
|
-
}
|
|
22
|
-
export declare const W002FreeTextWithoutNormalization: W002FreeTextWithoutNormalizationRule;
|
|
23
|
-
export {};
|
|
24
|
-
//# sourceMappingURL=w002-free-text-without-normalization.d.ts.map
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W002: Free-Text Output Without Normalization
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Output is string
|
|
6
|
-
* - Not constrained or normalized
|
|
7
|
-
* - Even if not chained
|
|
8
|
-
*
|
|
9
|
-
* Why:
|
|
10
|
-
* - Hard to reuse
|
|
11
|
-
* - Hard to evolve
|
|
12
|
-
*/
|
|
13
|
-
import { BaseRule } from '../base.js';
|
|
14
|
-
class W002FreeTextWithoutNormalizationRule extends BaseRule {
|
|
15
|
-
id = 'W002';
|
|
16
|
-
severity = 'warning';
|
|
17
|
-
ruleName = 'Free-Text Output Without Normalization';
|
|
18
|
-
description = 'Tool returns unconstrained free text. Consider normalizing output.';
|
|
19
|
-
check(ctx) {
|
|
20
|
-
const diagnostics = [];
|
|
21
|
-
for (const tool of ctx.tools) {
|
|
22
|
-
for (const output of tool.outputs) {
|
|
23
|
-
// Check if output is unconstrained string
|
|
24
|
-
if (output.type !== 'string') {
|
|
25
|
-
continue;
|
|
26
|
-
}
|
|
27
|
-
const hasEnum = Boolean(output.enum && output.enum.length > 0);
|
|
28
|
-
const hasPattern = Boolean(output.pattern);
|
|
29
|
-
const hasDescription = Boolean(output.description && output.description.trim());
|
|
30
|
-
// If it's a string with no constraints, it's unconstrained free text
|
|
31
|
-
if (!hasEnum && !hasPattern && !hasDescription) {
|
|
32
|
-
diagnostics.push(this.createDiagnostic(`Tool "${tool.name}" returns unconstrained free text (field: "${output.name}"). Consider normalizing output.`, tool.name, output.name, `Add constraints to the output: enum values, regex pattern, or a clear description of the expected format.`));
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
return diagnostics;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
export const W002FreeTextWithoutNormalization = new W002FreeTextWithoutNormalizationRule();
|
|
40
|
-
//# sourceMappingURL=w002-free-text-without-normalization.js.map
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W003: Missing Examples for User-Facing Inputs
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Tool takes user-provided input
|
|
6
|
-
* - No examples provided
|
|
7
|
-
*
|
|
8
|
-
* Why:
|
|
9
|
-
* - LLM guessing increases error rate
|
|
10
|
-
*/
|
|
11
|
-
import { BaseRule } from '../base.js';
|
|
12
|
-
import type { AnalysisContext, Diagnostic } from '../../types.js';
|
|
13
|
-
declare class W003MissingExamplesRule extends BaseRule {
|
|
14
|
-
readonly id = "W003";
|
|
15
|
-
readonly severity: "warning";
|
|
16
|
-
readonly ruleName = "Missing Examples for User-Facing Inputs";
|
|
17
|
-
readonly description = "Tool accepts user-provided input but has no examples. LLM accuracy may be reduced.";
|
|
18
|
-
check(ctx: AnalysisContext): Diagnostic[];
|
|
19
|
-
}
|
|
20
|
-
export declare const W003MissingExamples: W003MissingExamplesRule;
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=w003-missing-examples.d.ts.map
|
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W003: Missing Examples for User-Facing Inputs
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Tool takes user-provided input
|
|
6
|
-
* - No examples provided
|
|
7
|
-
*
|
|
8
|
-
* Why:
|
|
9
|
-
* - LLM guessing increases error rate
|
|
10
|
-
*/
|
|
11
|
-
import { BaseRule } from '../base.js';
|
|
12
|
-
import { escapeRegex } from '../../utils.js';
|
|
13
|
-
/**
|
|
14
|
-
* Check if an input field appears to be user-provided.
|
|
15
|
-
* Indicators are treated as plain text (not regex patterns).
|
|
16
|
-
*/
|
|
17
|
-
function isUserFacingInput(inputName, description) {
|
|
18
|
-
const name = inputName.toLowerCase();
|
|
19
|
-
const desc = (description || '').toLowerCase();
|
|
20
|
-
// Keywords that suggest user input (treated as plain text, not regex)
|
|
21
|
-
const userInputIndicators = [
|
|
22
|
-
'user',
|
|
23
|
-
'person',
|
|
24
|
-
'name',
|
|
25
|
-
'email',
|
|
26
|
-
'location',
|
|
27
|
-
'address',
|
|
28
|
-
'preference',
|
|
29
|
-
'query',
|
|
30
|
-
'question',
|
|
31
|
-
'input',
|
|
32
|
-
'text',
|
|
33
|
-
'message',
|
|
34
|
-
];
|
|
35
|
-
// Split into tokens for whole-word matching
|
|
36
|
-
const nameTokens = name
|
|
37
|
-
.split(/[^\w]+|(?<=[a-z])(?=[A-Z])/)
|
|
38
|
-
.filter(t => t.length > 0);
|
|
39
|
-
const descTokens = desc
|
|
40
|
-
.split(/[^\w]+|(?<=[a-z])(?=[A-Z])/)
|
|
41
|
-
.filter(t => t.length > 0);
|
|
42
|
-
const allTokens = new Set([...nameTokens, ...descTokens]);
|
|
43
|
-
// Check for whole-word matches using word boundaries
|
|
44
|
-
// Escape indicators to prevent regex injection
|
|
45
|
-
return userInputIndicators.some(indicator => {
|
|
46
|
-
if (allTokens.has(indicator)) {
|
|
47
|
-
return true;
|
|
48
|
-
}
|
|
49
|
-
const escapedIndicator = escapeRegex(indicator);
|
|
50
|
-
const indicatorRegex = new RegExp(`\\b${escapedIndicator}\\b`, 'i');
|
|
51
|
-
return indicatorRegex.test(name) || indicatorRegex.test(desc);
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
class W003MissingExamplesRule extends BaseRule {
|
|
55
|
-
id = 'W003';
|
|
56
|
-
severity = 'warning';
|
|
57
|
-
ruleName = 'Missing Examples for User-Facing Inputs';
|
|
58
|
-
description = 'Tool accepts user-provided input but has no examples. LLM accuracy may be reduced.';
|
|
59
|
-
check(ctx) {
|
|
60
|
-
const diagnostics = [];
|
|
61
|
-
for (const tool of ctx.tools) {
|
|
62
|
-
for (const input of tool.inputs) {
|
|
63
|
-
// Only check user-facing inputs
|
|
64
|
-
if (!isUserFacingInput(input.name, input.description)) {
|
|
65
|
-
continue;
|
|
66
|
-
}
|
|
67
|
-
// Check if it has an example (treat undefined, empty strings, whitespace-only, and empty arrays/objects as missing)
|
|
68
|
-
const example = input.example;
|
|
69
|
-
const isMissing = example === undefined ||
|
|
70
|
-
(typeof example === 'string' && example.trim().length === 0) ||
|
|
71
|
-
(Array.isArray(example) && example.length === 0) ||
|
|
72
|
-
(typeof example === 'object' &&
|
|
73
|
-
example !== null &&
|
|
74
|
-
Object.keys(example).length === 0);
|
|
75
|
-
if (isMissing) {
|
|
76
|
-
diagnostics.push(this.createDiagnostic(`Tool "${tool.name}" parameter "${input.name}" has no examples. LLM accuracy may be reduced.`, tool.name, input.name, `Add example values to "${input.name}" to help the LLM understand the expected format.`));
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
return diagnostics;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
export const W003MissingExamples = new W003MissingExamplesRule();
|
|
84
|
-
//# sourceMappingURL=w003-missing-examples.js.map
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W004: Overloaded Tool Responsibility
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Tool does multiple conceptual things
|
|
6
|
-
* - Description contains multiple verbs or intents
|
|
7
|
-
*
|
|
8
|
-
* Why:
|
|
9
|
-
* - Tool selection becomes unstable
|
|
10
|
-
* - Hard to compose
|
|
11
|
-
*/
|
|
12
|
-
import { BaseRule } from '../base.js';
|
|
13
|
-
import type { AnalysisContext, Diagnostic } from '../../types.js';
|
|
14
|
-
declare class W004OverloadedResponsibilityRule extends BaseRule {
|
|
15
|
-
readonly id = "W004";
|
|
16
|
-
readonly severity: "warning";
|
|
17
|
-
readonly ruleName = "Overloaded Tool Responsibility";
|
|
18
|
-
readonly description = "Tool appears to handle multiple responsibilities. Tool selection becomes unstable.";
|
|
19
|
-
check(ctx: AnalysisContext): Diagnostic[];
|
|
20
|
-
}
|
|
21
|
-
export declare const W004OverloadedResponsibility: W004OverloadedResponsibilityRule;
|
|
22
|
-
export {};
|
|
23
|
-
//# sourceMappingURL=w004-overloaded-responsibility.d.ts.map
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W004: Overloaded Tool Responsibility
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Tool does multiple conceptual things
|
|
6
|
-
* - Description contains multiple verbs or intents
|
|
7
|
-
*
|
|
8
|
-
* Why:
|
|
9
|
-
* - Tool selection becomes unstable
|
|
10
|
-
* - Hard to compose
|
|
11
|
-
*/
|
|
12
|
-
import { BaseRule } from '../base.js';
|
|
13
|
-
/**
|
|
14
|
-
* Count verbs in a description.
|
|
15
|
-
*/
|
|
16
|
-
function countVerbs(description) {
|
|
17
|
-
const commonVerbs = [
|
|
18
|
-
'get',
|
|
19
|
-
'set',
|
|
20
|
-
'create',
|
|
21
|
-
'delete',
|
|
22
|
-
'update',
|
|
23
|
-
'find',
|
|
24
|
-
'search',
|
|
25
|
-
'fetch',
|
|
26
|
-
'send',
|
|
27
|
-
'receive',
|
|
28
|
-
'process',
|
|
29
|
-
'handle',
|
|
30
|
-
'execute',
|
|
31
|
-
'run',
|
|
32
|
-
'call',
|
|
33
|
-
'invoke',
|
|
34
|
-
'parse',
|
|
35
|
-
'format',
|
|
36
|
-
'validate',
|
|
37
|
-
'transform',
|
|
38
|
-
'convert',
|
|
39
|
-
'merge',
|
|
40
|
-
'split',
|
|
41
|
-
'filter',
|
|
42
|
-
'sort',
|
|
43
|
-
'save',
|
|
44
|
-
'load',
|
|
45
|
-
'read',
|
|
46
|
-
'write',
|
|
47
|
-
];
|
|
48
|
-
const descLower = description.toLowerCase();
|
|
49
|
-
let verbCount = 0;
|
|
50
|
-
for (const verb of commonVerbs) {
|
|
51
|
-
// Check if verb appears as a word (not just as part of another word)
|
|
52
|
-
const regex = new RegExp(`\\b${verb}\\b`, 'i');
|
|
53
|
-
if (regex.test(descLower)) {
|
|
54
|
-
verbCount++;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
return verbCount;
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Count distinct intents (action + object pairs).
|
|
61
|
-
*/
|
|
62
|
-
function countIntents(description) {
|
|
63
|
-
// Simple heuristic: count "and" and "or" which often separate intents
|
|
64
|
-
const connectors = [' and ', ' or ', ', and ', ', or '];
|
|
65
|
-
let intentCount = 1; // Start with 1
|
|
66
|
-
const descLower = description.toLowerCase();
|
|
67
|
-
for (const connector of connectors) {
|
|
68
|
-
const matches = descLower.split(connector);
|
|
69
|
-
if (matches.length > 1) {
|
|
70
|
-
intentCount = Math.max(intentCount, matches.length);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
return intentCount;
|
|
74
|
-
}
|
|
75
|
-
class W004OverloadedResponsibilityRule extends BaseRule {
|
|
76
|
-
id = 'W004';
|
|
77
|
-
severity = 'warning';
|
|
78
|
-
ruleName = 'Overloaded Tool Responsibility';
|
|
79
|
-
description = 'Tool appears to handle multiple responsibilities. Tool selection becomes unstable.';
|
|
80
|
-
check(ctx) {
|
|
81
|
-
const diagnostics = [];
|
|
82
|
-
for (const tool of ctx.tools) {
|
|
83
|
-
const description = tool.description || '';
|
|
84
|
-
// Count verbs and intents
|
|
85
|
-
const verbCount = countVerbs(description);
|
|
86
|
-
const intentCount = countIntents(description);
|
|
87
|
-
// If too many verbs (>3) or multiple intents (>2), it's overloaded
|
|
88
|
-
if (verbCount > 3 || intentCount > 2) {
|
|
89
|
-
diagnostics.push(this.createDiagnostic(`Tool "${tool.name}" appears to handle multiple responsibilities.`, tool.name, undefined, `Split "${tool.name}" into multiple focused tools, each handling a single responsibility.`));
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
return diagnostics;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
export const W004OverloadedResponsibility = new W004OverloadedResponsibilityRule();
|
|
96
|
-
//# sourceMappingURL=w004-overloaded-responsibility.js.map
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W005: Tool Description Too Generic
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Description uses vague verbs:
|
|
6
|
-
* - "get"
|
|
7
|
-
* - "handle"
|
|
8
|
-
* - "process"
|
|
9
|
-
* - No concrete nouns
|
|
10
|
-
*
|
|
11
|
-
* Why:
|
|
12
|
-
* - LLM cannot discriminate tools
|
|
13
|
-
*/
|
|
14
|
-
import { BaseRule } from '../base.js';
|
|
15
|
-
import type { AnalysisContext, Diagnostic } from '../../types.js';
|
|
16
|
-
/**
|
|
17
|
-
* Default list of concrete nouns that make a description specific.
|
|
18
|
-
* This list can be extended or overridden when creating a custom rule instance.
|
|
19
|
-
*
|
|
20
|
-
* @example
|
|
21
|
-
* ```ts
|
|
22
|
-
* import { createW005Rule, DEFAULT_CONCRETE_NOUNS } from './w005-generic-description.js';
|
|
23
|
-
* const customNouns = [...DEFAULT_CONCRETE_NOUNS, 'invoice', 'report'];
|
|
24
|
-
* const customRule = createW005Rule(customNouns);
|
|
25
|
-
* ```
|
|
26
|
-
*/
|
|
27
|
-
export declare const DEFAULT_CONCRETE_NOUNS: readonly ["weather", "location", "user", "file", "data", "email", "message", "order", "payment", "product", "customer", "account", "transaction", "request", "response"];
|
|
28
|
-
declare class W005GenericDescriptionRule extends BaseRule {
|
|
29
|
-
private readonly concreteNouns;
|
|
30
|
-
readonly id = "W005";
|
|
31
|
-
readonly severity: "warning";
|
|
32
|
-
readonly ruleName = "Tool Description Too Generic";
|
|
33
|
-
readonly description = "Description of tool is too generic. LLM cannot discriminate tools.";
|
|
34
|
-
constructor(concreteNouns?: readonly string[]);
|
|
35
|
-
check(ctx: AnalysisContext): Diagnostic[];
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Create a W005 rule instance with custom concrete nouns.
|
|
39
|
-
*
|
|
40
|
-
* @param concreteNouns - Optional list of concrete nouns (defaults to DEFAULT_CONCRETE_NOUNS)
|
|
41
|
-
* @returns A new W005GenericDescriptionRule instance
|
|
42
|
-
*
|
|
43
|
-
* @example
|
|
44
|
-
* ```ts
|
|
45
|
-
* import { createW005Rule, DEFAULT_CONCRETE_NOUNS } from './w005-generic-description.js';
|
|
46
|
-
* const customNouns = [...DEFAULT_CONCRETE_NOUNS, 'invoice', 'report'];
|
|
47
|
-
* const customRule = createW005Rule(customNouns);
|
|
48
|
-
* ```
|
|
49
|
-
*/
|
|
50
|
-
export declare function createW005Rule(concreteNouns?: readonly string[]): W005GenericDescriptionRule;
|
|
51
|
-
export declare const W005GenericDescription: W005GenericDescriptionRule;
|
|
52
|
-
export {};
|
|
53
|
-
//# sourceMappingURL=w005-generic-description.d.ts.map
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W005: Tool Description Too Generic
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Description uses vague verbs:
|
|
6
|
-
* - "get"
|
|
7
|
-
* - "handle"
|
|
8
|
-
* - "process"
|
|
9
|
-
* - No concrete nouns
|
|
10
|
-
*
|
|
11
|
-
* Why:
|
|
12
|
-
* - LLM cannot discriminate tools
|
|
13
|
-
*/
|
|
14
|
-
import { BaseRule } from '../base.js';
|
|
15
|
-
/**
|
|
16
|
-
* Default list of concrete nouns that make a description specific.
|
|
17
|
-
* This list can be extended or overridden when creating a custom rule instance.
|
|
18
|
-
*
|
|
19
|
-
* @example
|
|
20
|
-
* ```ts
|
|
21
|
-
* import { createW005Rule, DEFAULT_CONCRETE_NOUNS } from './w005-generic-description.js';
|
|
22
|
-
* const customNouns = [...DEFAULT_CONCRETE_NOUNS, 'invoice', 'report'];
|
|
23
|
-
* const customRule = createW005Rule(customNouns);
|
|
24
|
-
* ```
|
|
25
|
-
*/
|
|
26
|
-
export const DEFAULT_CONCRETE_NOUNS = [
|
|
27
|
-
'weather',
|
|
28
|
-
'location',
|
|
29
|
-
'user',
|
|
30
|
-
'file',
|
|
31
|
-
'data',
|
|
32
|
-
'email',
|
|
33
|
-
'message',
|
|
34
|
-
'order',
|
|
35
|
-
'payment',
|
|
36
|
-
'product',
|
|
37
|
-
'customer',
|
|
38
|
-
'account',
|
|
39
|
-
'transaction',
|
|
40
|
-
'request',
|
|
41
|
-
'response',
|
|
42
|
-
];
|
|
43
|
-
/**
|
|
44
|
-
* Check if description is too generic.
|
|
45
|
-
*
|
|
46
|
-
* @param description - Tool description to check
|
|
47
|
-
* @param concreteNouns - Optional list of concrete nouns (defaults to DEFAULT_CONCRETE_NOUNS)
|
|
48
|
-
*/
|
|
49
|
-
function isGenericDescription(description, concreteNouns = DEFAULT_CONCRETE_NOUNS) {
|
|
50
|
-
const descLower = description.toLowerCase();
|
|
51
|
-
// Vague verbs
|
|
52
|
-
const vagueVerbs = ['get', 'handle', 'process', 'do', 'make', 'use', 'call'];
|
|
53
|
-
// Check for vague verbs without concrete nouns
|
|
54
|
-
let hasVagueVerb = false;
|
|
55
|
-
for (const verb of vagueVerbs) {
|
|
56
|
-
const regex = new RegExp(`\\b${verb}\\b`, 'i');
|
|
57
|
-
if (regex.test(descLower)) {
|
|
58
|
-
hasVagueVerb = true;
|
|
59
|
-
break;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
if (!hasVagueVerb) {
|
|
63
|
-
return false;
|
|
64
|
-
}
|
|
65
|
-
// Check if description has concrete nouns
|
|
66
|
-
const hasConcreteNoun = concreteNouns.some(noun => descLower.includes(noun));
|
|
67
|
-
// If has vague verb but no concrete noun, it's generic
|
|
68
|
-
return !hasConcreteNoun;
|
|
69
|
-
}
|
|
70
|
-
class W005GenericDescriptionRule extends BaseRule {
|
|
71
|
-
concreteNouns;
|
|
72
|
-
id = 'W005';
|
|
73
|
-
severity = 'warning';
|
|
74
|
-
ruleName = 'Tool Description Too Generic';
|
|
75
|
-
description = 'Description of tool is too generic. LLM cannot discriminate tools.';
|
|
76
|
-
constructor(concreteNouns = DEFAULT_CONCRETE_NOUNS) {
|
|
77
|
-
super();
|
|
78
|
-
this.concreteNouns = concreteNouns;
|
|
79
|
-
}
|
|
80
|
-
check(ctx) {
|
|
81
|
-
const diagnostics = [];
|
|
82
|
-
for (const tool of ctx.tools) {
|
|
83
|
-
const description = tool.description || '';
|
|
84
|
-
if (isGenericDescription(description, this.concreteNouns)) {
|
|
85
|
-
diagnostics.push(this.createDiagnostic(`Description of "${tool.name}" is too generic.`, tool.name, undefined, `Make the description more specific by including concrete nouns and specific actions (e.g., "Get weather data for a location" instead of "Get data").`));
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
return diagnostics;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Create a W005 rule instance with custom concrete nouns.
|
|
93
|
-
*
|
|
94
|
-
* @param concreteNouns - Optional list of concrete nouns (defaults to DEFAULT_CONCRETE_NOUNS)
|
|
95
|
-
* @returns A new W005GenericDescriptionRule instance
|
|
96
|
-
*
|
|
97
|
-
* @example
|
|
98
|
-
* ```ts
|
|
99
|
-
* import { createW005Rule, DEFAULT_CONCRETE_NOUNS } from './w005-generic-description.js';
|
|
100
|
-
* const customNouns = [...DEFAULT_CONCRETE_NOUNS, 'invoice', 'report'];
|
|
101
|
-
* const customRule = createW005Rule(customNouns);
|
|
102
|
-
* ```
|
|
103
|
-
*/
|
|
104
|
-
export function createW005Rule(concreteNouns) {
|
|
105
|
-
return new W005GenericDescriptionRule(concreteNouns);
|
|
106
|
-
}
|
|
107
|
-
export const W005GenericDescription = new W005GenericDescriptionRule();
|
|
108
|
-
//# sourceMappingURL=w005-generic-description.js.map
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W006: Optional Input Used as Required Downstream
|
|
3
|
-
*
|
|
4
|
-
* Condition:
|
|
5
|
-
* - Input marked optional
|
|
6
|
-
* - Treated as required in chaining
|
|
7
|
-
*
|
|
8
|
-
* Why:
|
|
9
|
-
* - Hidden contract violation
|
|
10
|
-
*/
|
|
11
|
-
import { BaseRule } from '../base.js';
|
|
12
|
-
import type { AnalysisContext, Diagnostic } from '../../types.js';
|
|
13
|
-
declare class W006OptionalAsRequiredRule extends BaseRule {
|
|
14
|
-
readonly id = "W006";
|
|
15
|
-
readonly severity: "warning";
|
|
16
|
-
readonly ruleName = "Optional Input Used as Required Downstream";
|
|
17
|
-
readonly description = "Optional input is treated as required downstream. Hidden contract violation.";
|
|
18
|
-
check(ctx: AnalysisContext): Diagnostic[];
|
|
19
|
-
}
|
|
20
|
-
export declare const W006OptionalAsRequired: W006OptionalAsRequiredRule;
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=w006-optional-as-required.d.ts.map
|