@syrin/cli 1.3.1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +36 -0
  2. package/dist/cli/commands/config.d.ts +47 -0
  3. package/dist/cli/commands/config.js +360 -0
  4. package/dist/cli/commands/dev.d.ts +6 -0
  5. package/dist/cli/commands/dev.js +67 -15
  6. package/dist/cli/commands/doctor.js +49 -13
  7. package/dist/cli/commands/init.d.ts +2 -0
  8. package/dist/cli/commands/init.js +89 -18
  9. package/dist/cli/commands/status.d.ts +10 -0
  10. package/dist/cli/commands/status.js +162 -0
  11. package/dist/cli/index.js +211 -12
  12. package/dist/cli/prompts/init-prompt.d.ts +18 -0
  13. package/dist/cli/prompts/init-prompt.js +159 -99
  14. package/dist/cli/utils/command-error-handler.js +2 -5
  15. package/dist/config/env-checker.d.ts +12 -2
  16. package/dist/config/env-checker.js +88 -38
  17. package/dist/config/env-templates.d.ts +15 -0
  18. package/dist/config/env-templates.js +49 -0
  19. package/dist/config/generator.js +17 -0
  20. package/dist/config/global-loader.d.ts +50 -0
  21. package/dist/config/global-loader.js +244 -0
  22. package/dist/config/loader.d.ts +28 -0
  23. package/dist/config/loader.js +95 -9
  24. package/dist/config/merger.d.ts +37 -0
  25. package/dist/config/merger.js +68 -0
  26. package/dist/config/schema.d.ts +26 -1
  27. package/dist/config/schema.js +73 -8
  28. package/dist/config/types.d.ts +19 -0
  29. package/dist/config/types.js +26 -1
  30. package/dist/constants/messages.d.ts +7 -0
  31. package/dist/constants/messages.js +8 -0
  32. package/dist/constants/paths.d.ts +6 -0
  33. package/dist/constants/paths.js +10 -0
  34. package/dist/events/emitter.js +7 -7
  35. package/dist/index.js +0 -0
  36. package/dist/presentation/config-ui.d.ts +34 -0
  37. package/dist/presentation/config-ui.js +139 -0
  38. package/dist/presentation/doctor-ui.d.ts +11 -0
  39. package/dist/presentation/doctor-ui.js +52 -1
  40. package/dist/presentation/init-ui.d.ts +9 -0
  41. package/dist/presentation/init-ui.js +33 -0
  42. package/dist/runtime/analysis/analyser.js +2 -2
  43. package/dist/runtime/analysis/rules/warnings/w104-generic-description.d.ts +1 -1
  44. package/dist/runtime/analysis/rules/warnings/w104-generic-description.js +1 -1
  45. package/dist/runtime/dev/event-mapper.js +19 -3
  46. package/dist/runtime/dev/session.d.ts +4 -0
  47. package/dist/runtime/dev/session.js +52 -3
  48. package/dist/runtime/llm/ollama.js +4 -4
  49. package/dist/runtime/mcp/client/manager.js +3 -3
  50. package/dist/runtime/sandbox/executor.js +5 -5
  51. package/dist/runtime/test/orchestrator.js +4 -4
  52. package/dist/utils/editor.d.ts +37 -0
  53. package/dist/utils/editor.js +137 -0
  54. package/dist/utils/logger.d.ts +24 -6
  55. package/dist/utils/logger.js +51 -8
  56. package/package.json +4 -4
  57. package/dist/runtime/analysis/rules/errors/e001-missing-output-schema.d.ts +0 -22
  58. package/dist/runtime/analysis/rules/errors/e001-missing-output-schema.js +0 -30
  59. package/dist/runtime/analysis/rules/errors/e002-underspecified-input.d.ts +0 -24
  60. package/dist/runtime/analysis/rules/errors/e002-underspecified-input.js +0 -52
  61. package/dist/runtime/analysis/rules/errors/e003-type-mismatch.d.ts +0 -23
  62. package/dist/runtime/analysis/rules/errors/e003-type-mismatch.js +0 -73
  63. package/dist/runtime/analysis/rules/errors/e004-free-text-propagation.d.ts +0 -23
  64. package/dist/runtime/analysis/rules/errors/e004-free-text-propagation.js +0 -47
  65. package/dist/runtime/analysis/rules/errors/e005-tool-ambiguity.d.ts +0 -25
  66. package/dist/runtime/analysis/rules/errors/e005-tool-ambiguity.js +0 -73
  67. package/dist/runtime/analysis/rules/errors/e006-param-not-in-description.d.ts +0 -22
  68. package/dist/runtime/analysis/rules/errors/e006-param-not-in-description.js +0 -57
  69. package/dist/runtime/analysis/rules/errors/e007-output-not-guaranteed.d.ts +0 -23
  70. package/dist/runtime/analysis/rules/errors/e007-output-not-guaranteed.js +0 -56
  71. package/dist/runtime/analysis/rules/errors/e008-circular-dependency.d.ts +0 -22
  72. package/dist/runtime/analysis/rules/errors/e008-circular-dependency.js +0 -84
  73. package/dist/runtime/analysis/rules/errors/e009-implicit-user-input.d.ts +0 -23
  74. package/dist/runtime/analysis/rules/errors/e009-implicit-user-input.js +0 -89
  75. package/dist/runtime/analysis/rules/errors/e010-non-serializable.d.ts +0 -25
  76. package/dist/runtime/analysis/rules/errors/e010-non-serializable.js +0 -46
  77. package/dist/runtime/analysis/rules/errors/e011-missing-tool-description.d.ts +0 -24
  78. package/dist/runtime/analysis/rules/errors/e011-missing-tool-description.js +0 -33
  79. package/dist/runtime/analysis/rules/errors/e012-side-effect-detected.d.ts +0 -39
  80. package/dist/runtime/analysis/rules/errors/e012-side-effect-detected.js +0 -40
  81. package/dist/runtime/analysis/rules/errors/e013-non-deterministic-output.d.ts +0 -37
  82. package/dist/runtime/analysis/rules/errors/e013-non-deterministic-output.js +0 -34
  83. package/dist/runtime/analysis/rules/errors/e013-output-explosion.d.ts +0 -39
  84. package/dist/runtime/analysis/rules/errors/e013-output-explosion.js +0 -36
  85. package/dist/runtime/analysis/rules/errors/e014-hidden-dependency.d.ts +0 -42
  86. package/dist/runtime/analysis/rules/errors/e014-hidden-dependency.js +0 -46
  87. package/dist/runtime/analysis/rules/errors/e014-output-explosion.d.ts +0 -39
  88. package/dist/runtime/analysis/rules/errors/e014-output-explosion.js +0 -36
  89. package/dist/runtime/analysis/rules/errors/e015-hidden-dependency.d.ts +0 -42
  90. package/dist/runtime/analysis/rules/errors/e015-hidden-dependency.js +0 -46
  91. package/dist/runtime/analysis/rules/errors/e015-unbounded-execution.d.ts +0 -44
  92. package/dist/runtime/analysis/rules/errors/e015-unbounded-execution.js +0 -66
  93. package/dist/runtime/analysis/rules/errors/e016-output-validation-failed.d.ts +0 -43
  94. package/dist/runtime/analysis/rules/errors/e016-output-validation-failed.js +0 -42
  95. package/dist/runtime/analysis/rules/errors/e016-unbounded-execution.d.ts +0 -44
  96. package/dist/runtime/analysis/rules/errors/e016-unbounded-execution.js +0 -66
  97. package/dist/runtime/analysis/rules/errors/e017-input-validation-failed.d.ts +0 -57
  98. package/dist/runtime/analysis/rules/errors/e017-input-validation-failed.js +0 -80
  99. package/dist/runtime/analysis/rules/errors/e017-output-validation-failed.d.ts +0 -43
  100. package/dist/runtime/analysis/rules/errors/e017-output-validation-failed.js +0 -42
  101. package/dist/runtime/analysis/rules/errors/e018-input-validation-failed.d.ts +0 -57
  102. package/dist/runtime/analysis/rules/errors/e018-input-validation-failed.js +0 -80
  103. package/dist/runtime/analysis/rules/errors/e018-tool-execution-failed.d.ts +0 -38
  104. package/dist/runtime/analysis/rules/errors/e018-tool-execution-failed.js +0 -37
  105. package/dist/runtime/analysis/rules/errors/e019-tool-execution-failed.d.ts +0 -38
  106. package/dist/runtime/analysis/rules/errors/e019-tool-execution-failed.js +0 -37
  107. package/dist/runtime/analysis/rules/errors/e019-unexpected-test-result.d.ts +0 -65
  108. package/dist/runtime/analysis/rules/errors/e019-unexpected-test-result.js +0 -109
  109. package/dist/runtime/analysis/rules/errors/e020-unexpected-test-result.d.ts +0 -65
  110. package/dist/runtime/analysis/rules/errors/e020-unexpected-test-result.js +0 -109
  111. package/dist/runtime/analysis/rules/warnings/w001-implicit-dependency.d.ts +0 -22
  112. package/dist/runtime/analysis/rules/warnings/w001-implicit-dependency.js +0 -39
  113. package/dist/runtime/analysis/rules/warnings/w002-free-text-without-normalization.d.ts +0 -24
  114. package/dist/runtime/analysis/rules/warnings/w002-free-text-without-normalization.js +0 -40
  115. package/dist/runtime/analysis/rules/warnings/w003-missing-examples.d.ts +0 -22
  116. package/dist/runtime/analysis/rules/warnings/w003-missing-examples.js +0 -84
  117. package/dist/runtime/analysis/rules/warnings/w004-overloaded-responsibility.d.ts +0 -23
  118. package/dist/runtime/analysis/rules/warnings/w004-overloaded-responsibility.js +0 -96
  119. package/dist/runtime/analysis/rules/warnings/w005-generic-description.d.ts +0 -53
  120. package/dist/runtime/analysis/rules/warnings/w005-generic-description.js +0 -108
  121. package/dist/runtime/analysis/rules/warnings/w006-optional-as-required.d.ts +0 -22
  122. package/dist/runtime/analysis/rules/warnings/w006-optional-as-required.js +0 -44
  123. package/dist/runtime/analysis/rules/warnings/w007-broad-output-schema.d.ts +0 -23
  124. package/dist/runtime/analysis/rules/warnings/w007-broad-output-schema.js +0 -37
  125. package/dist/runtime/analysis/rules/warnings/w008-multiple-entry-points.d.ts +0 -22
  126. package/dist/runtime/analysis/rules/warnings/w008-multiple-entry-points.js +0 -97
  127. package/dist/runtime/analysis/rules/warnings/w009-hidden-side-effects.d.ts +0 -23
  128. package/dist/runtime/analysis/rules/warnings/w009-hidden-side-effects.js +0 -88
  129. package/dist/runtime/analysis/rules/warnings/w010-output-not-reusable.d.ts +0 -22
  130. package/dist/runtime/analysis/rules/warnings/w010-output-not-reusable.js +0 -81
  131. package/dist/runtime/analysis/rules/warnings/w021-weak-schema.d.ts +0 -40
  132. package/dist/runtime/analysis/rules/warnings/w021-weak-schema.js +0 -32
  133. package/dist/runtime/analysis/rules/warnings/w022-high-entropy-output.d.ts +0 -39
  134. package/dist/runtime/analysis/rules/warnings/w022-high-entropy-output.js +0 -36
  135. package/dist/runtime/analysis/rules/warnings/w023-unstable-defaults.d.ts +0 -38
  136. package/dist/runtime/analysis/rules/warnings/w023-unstable-defaults.js +0 -36
  137. package/dist/runtime/test/dependency-tracker.d.ts +0 -66
  138. package/dist/runtime/test/dependency-tracker.js +0 -80
  139. package/dist/runtime/test/formatters.d.ts +0 -18
  140. package/dist/runtime/test/formatters.js +0 -172
  141. package/dist/runtime/test/input-generator.d.ts +0 -33
  142. package/dist/runtime/test/input-generator.js +0 -498
  143. package/dist/runtime/test/mcp-root-detector.d.ts +0 -31
  144. package/dist/runtime/test/mcp-root-detector.js +0 -105
  145. package/dist/runtime/test/retry-tester.d.ts +0 -44
  146. package/dist/runtime/test/retry-tester.js +0 -103
  147. package/dist/runtime/test/synthetic-input-generator.d.ts +0 -11
  148. package/dist/runtime/test/synthetic-input-generator.js +0 -154
  149. package/dist/runtime/test/test-runner.d.ts +0 -28
  150. package/dist/runtime/test/test-runner.js +0 -55
@@ -1,65 +0,0 @@
1
- /**
2
- * E019: Unexpected Test Result
3
- *
4
- * Condition: Test's actual outcome doesn't match its expectation
5
- *
6
- * Why this is fatal:
7
- * - Test contract is inaccurate
8
- * - Tool behavior doesn't match declared guarantees
9
- * - Can cause false positives/negatives in validation
10
- * - Indicates mismatch between expectations and reality
11
- */
12
- import { BaseRule } from '../base.js';
13
- import type { AnalysisContext, Diagnostic } from '../../types.js';
14
- /**
15
- * Context for unexpected test result detection.
16
- */
17
- export interface UnexpectedTestResultContext {
18
- /** Tool name */
19
- toolName: string;
20
- /** Test name */
21
- testName: string;
22
- /** Test input */
23
- testInput?: Record<string, unknown>;
24
- /** Expected outcome (success/error/timeout) */
25
- expectedOutcome: 'success' | 'error' | 'timeout';
26
- /** Actual outcome (success/error/timeout) */
27
- actualOutcome: 'success' | 'error' | 'timeout';
28
- /** Expected error code (if expecting error) */
29
- expectedErrorCode?: string;
30
- /** Expected error type (if expecting error) */
31
- expectedErrorType?: string;
32
- /** Actual error code (if actual outcome is error) */
33
- actualErrorCode?: string;
34
- /** Actual error type (if actual outcome is error) */
35
- actualErrorType?: string;
36
- /** Expected error details (if expecting specific error) */
37
- expectedError?: {
38
- type?: string;
39
- details?: Record<string, unknown>;
40
- };
41
- /** Actual error details (if actual outcome is error) */
42
- actualError?: {
43
- code?: string;
44
- type?: string;
45
- message?: string;
46
- };
47
- }
48
- declare class E019UnexpectedTestResultRule extends BaseRule {
49
- readonly id: "E019";
50
- readonly severity: "error";
51
- readonly ruleName = "Unexpected Test Result";
52
- readonly description = "Test's actual outcome doesn't match its expectation. Tool behavior doesn't match declared guarantees.";
53
- check(_ctx: AnalysisContext): Diagnostic[];
54
- /**
55
- * Determine if a diagnostic should be created based on behavioral context.
56
- */
57
- private shouldCreateDiagnostic;
58
- /**
59
- * Check with behavioral context (called from test orchestrator).
60
- */
61
- checkWithBehavioralContext(behavioralCtx: UnexpectedTestResultContext): Diagnostic[];
62
- }
63
- export declare const E019UnexpectedTestResult: E019UnexpectedTestResultRule;
64
- export {};
65
- //# sourceMappingURL=e019-unexpected-test-result.d.ts.map
@@ -1,109 +0,0 @@
1
- /**
2
- * E019: Unexpected Test Result
3
- *
4
- * Condition: Test's actual outcome doesn't match its expectation
5
- *
6
- * Why this is fatal:
7
- * - Test contract is inaccurate
8
- * - Tool behavior doesn't match declared guarantees
9
- * - Can cause false positives/negatives in validation
10
- * - Indicates mismatch between expectations and reality
11
- */
12
- import { BaseRule } from '../base.js';
13
- import { ERROR_CODES } from '../error-codes.js';
14
- class E019UnexpectedTestResultRule extends BaseRule {
15
- id = ERROR_CODES.E019;
16
- severity = 'error';
17
- ruleName = 'Unexpected Test Result';
18
- description = "Test's actual outcome doesn't match its expectation. Tool behavior doesn't match declared guarantees.";
19
- check(_ctx) {
20
- // This rule requires behavioral context
21
- return [];
22
- }
23
- /**
24
- * Determine if a diagnostic should be created based on behavioral context.
25
- */
26
- shouldCreateDiagnostic(behavioralCtx) {
27
- // If outcomes don't match, create diagnostic
28
- if (behavioralCtx.expectedOutcome !== behavioralCtx.actualOutcome) {
29
- return true;
30
- }
31
- // Outcomes match - check if error details also match (for error cases)
32
- if (behavioralCtx.expectedOutcome === 'error') {
33
- const expectedType = behavioralCtx.expectedErrorType?.toLowerCase();
34
- const actualType = behavioralCtx.actualErrorType?.toLowerCase();
35
- if (expectedType && actualType && expectedType !== actualType) {
36
- return true; // Type mismatch
37
- }
38
- if (expectedType && !actualType) {
39
- return true; // Expected type but no actual type
40
- }
41
- const expectedCode = behavioralCtx.expectedErrorCode;
42
- const actualCode = behavioralCtx.actualErrorCode;
43
- if (expectedCode && actualCode && expectedCode !== actualCode) {
44
- return true; // Code mismatch
45
- }
46
- }
47
- return false; // Everything matches
48
- }
49
- /**
50
- * Check with behavioral context (called from test orchestrator).
51
- */
52
- checkWithBehavioralContext(behavioralCtx) {
53
- const diagnostics = [];
54
- if (!this.shouldCreateDiagnostic(behavioralCtx)) {
55
- return diagnostics;
56
- }
57
- // If we reach here, there's a mismatch - create diagnostic
58
- let message = `Test "${behavioralCtx.testName}" in tool "${behavioralCtx.toolName}" expected `;
59
- // Build expected description
60
- if (behavioralCtx.expectedOutcome === 'success') {
61
- message += 'success but got ';
62
- if (behavioralCtx.actualOutcome === 'error') {
63
- message += `error (${behavioralCtx.actualErrorCode || behavioralCtx.actualErrorType || 'unknown error'})`;
64
- }
65
- else if (behavioralCtx.actualOutcome === 'timeout') {
66
- message += 'timeout';
67
- }
68
- else {
69
- message += behavioralCtx.actualOutcome;
70
- }
71
- }
72
- else if (behavioralCtx.expectedOutcome === 'error') {
73
- message += `error`;
74
- if (behavioralCtx.expectedErrorType) {
75
- message += ` of type "${behavioralCtx.expectedErrorType}"`;
76
- }
77
- if (behavioralCtx.expectedErrorCode) {
78
- message += ` (code: ${behavioralCtx.expectedErrorCode})`;
79
- }
80
- message += ' but got ';
81
- if (behavioralCtx.actualOutcome === 'success') {
82
- message += 'success';
83
- }
84
- else {
85
- message += `${behavioralCtx.actualOutcome} (${behavioralCtx.actualErrorCode || behavioralCtx.actualErrorType || 'unknown error'})`;
86
- }
87
- if (behavioralCtx.actualErrorType &&
88
- behavioralCtx.expectedErrorType &&
89
- behavioralCtx.actualErrorType.toLowerCase() !==
90
- behavioralCtx.expectedErrorType.toLowerCase()) {
91
- message += ' with different type';
92
- }
93
- }
94
- else {
95
- message += `${behavioralCtx.expectedOutcome} but got ${behavioralCtx.actualOutcome}`;
96
- }
97
- diagnostics.push(this.createDiagnostic(message, behavioralCtx.toolName, undefined, 'Update test expectation to match actual tool behavior, or fix tool implementation to match declared guarantees. Ensure test expectations accurately reflect tool behavior.', {
98
- testName: behavioralCtx.testName,
99
- testInput: behavioralCtx.testInput,
100
- expectedOutcome: behavioralCtx.expectedOutcome,
101
- actualOutcome: behavioralCtx.actualOutcome,
102
- expectedError: behavioralCtx.expectedError,
103
- actualError: behavioralCtx.actualError,
104
- }));
105
- return diagnostics;
106
- }
107
- }
108
- export const E019UnexpectedTestResult = new E019UnexpectedTestResultRule();
109
- //# sourceMappingURL=e019-unexpected-test-result.js.map
@@ -1,65 +0,0 @@
1
- /**
2
- * E020: Unexpected Test Result
3
- *
4
- * Condition: Test's actual outcome doesn't match its expectation
5
- *
6
- * Why this is fatal:
7
- * - Test contract is inaccurate
8
- * - Tool behavior doesn't match declared guarantees
9
- * - Can cause false positives/negatives in validation
10
- * - Indicates mismatch between expectations and reality
11
- */
12
- import { BaseRule } from '../base.js';
13
- import type { AnalysisContext, Diagnostic } from '../../types.js';
14
- /**
15
- * Context for unexpected test result detection.
16
- */
17
- export interface UnexpectedTestResultContext {
18
- /** Tool name */
19
- toolName: string;
20
- /** Test name */
21
- testName: string;
22
- /** Test input */
23
- testInput?: Record<string, unknown>;
24
- /** Expected outcome (success/error/timeout) */
25
- expectedOutcome: 'success' | 'error' | 'timeout';
26
- /** Actual outcome (success/error/timeout) */
27
- actualOutcome: 'success' | 'error' | 'timeout';
28
- /** Expected error code (if expecting error) */
29
- expectedErrorCode?: string;
30
- /** Expected error type (if expecting error) */
31
- expectedErrorType?: string;
32
- /** Actual error code (if actual outcome is error) */
33
- actualErrorCode?: string;
34
- /** Actual error type (if actual outcome is error) */
35
- actualErrorType?: string;
36
- /** Expected error details (if expecting specific error) */
37
- expectedError?: {
38
- type?: string;
39
- details?: Record<string, unknown>;
40
- };
41
- /** Actual error details (if actual outcome is error) */
42
- actualError?: {
43
- code?: string;
44
- type?: string;
45
- message?: string;
46
- };
47
- }
48
- declare class E020UnexpectedTestResultRule extends BaseRule {
49
- readonly id: "E020";
50
- readonly severity: "error";
51
- readonly ruleName = "Unexpected Test Result";
52
- readonly description = "Test's actual outcome doesn't match its expectation. Tool behavior doesn't match declared guarantees.";
53
- check(_ctx: AnalysisContext): Diagnostic[];
54
- /**
55
- * Determine if a diagnostic should be created based on behavioral context.
56
- */
57
- private shouldCreateDiagnostic;
58
- /**
59
- * Check with behavioral context (called from test orchestrator).
60
- */
61
- checkWithBehavioralContext(behavioralCtx: UnexpectedTestResultContext): Diagnostic[];
62
- }
63
- export declare const E020UnexpectedTestResult: E020UnexpectedTestResultRule;
64
- export {};
65
- //# sourceMappingURL=e020-unexpected-test-result.d.ts.map
@@ -1,109 +0,0 @@
1
- /**
2
- * E020: Unexpected Test Result
3
- *
4
- * Condition: Test's actual outcome doesn't match its expectation
5
- *
6
- * Why this is fatal:
7
- * - Test contract is inaccurate
8
- * - Tool behavior doesn't match declared guarantees
9
- * - Can cause false positives/negatives in validation
10
- * - Indicates mismatch between expectations and reality
11
- */
12
- import { BaseRule } from '../base.js';
13
- import { ERROR_CODES } from '../error-codes.js';
14
- class E020UnexpectedTestResultRule extends BaseRule {
15
- id = ERROR_CODES.E020;
16
- severity = 'error';
17
- ruleName = 'Unexpected Test Result';
18
- description = "Test's actual outcome doesn't match its expectation. Tool behavior doesn't match declared guarantees.";
19
- check(_ctx) {
20
- // This rule requires behavioral context
21
- return [];
22
- }
23
- /**
24
- * Determine if a diagnostic should be created based on behavioral context.
25
- */
26
- shouldCreateDiagnostic(behavioralCtx) {
27
- // If outcomes don't match, create diagnostic
28
- if (behavioralCtx.expectedOutcome !== behavioralCtx.actualOutcome) {
29
- return true;
30
- }
31
- // Outcomes match - check if error details also match (for error cases)
32
- if (behavioralCtx.expectedOutcome === 'error') {
33
- const expectedType = behavioralCtx.expectedErrorType?.toLowerCase();
34
- const actualType = behavioralCtx.actualErrorType?.toLowerCase();
35
- if (expectedType && actualType && expectedType !== actualType) {
36
- return true; // Type mismatch
37
- }
38
- if (expectedType && !actualType) {
39
- return true; // Expected type but no actual type
40
- }
41
- const expectedCode = behavioralCtx.expectedErrorCode;
42
- const actualCode = behavioralCtx.actualErrorCode;
43
- if (expectedCode && actualCode && expectedCode !== actualCode) {
44
- return true; // Code mismatch
45
- }
46
- }
47
- return false; // Everything matches
48
- }
49
- /**
50
- * Check with behavioral context (called from test orchestrator).
51
- */
52
- checkWithBehavioralContext(behavioralCtx) {
53
- const diagnostics = [];
54
- if (!this.shouldCreateDiagnostic(behavioralCtx)) {
55
- return diagnostics;
56
- }
57
- // If we reach here, there's a mismatch - create diagnostic
58
- let message = `Test "${behavioralCtx.testName}" in tool "${behavioralCtx.toolName}" expected `;
59
- // Build expected description
60
- if (behavioralCtx.expectedOutcome === 'success') {
61
- message += 'success but got ';
62
- if (behavioralCtx.actualOutcome === 'error') {
63
- message += `error (${behavioralCtx.actualErrorCode || behavioralCtx.actualErrorType || 'unknown error'})`;
64
- }
65
- else if (behavioralCtx.actualOutcome === 'timeout') {
66
- message += 'timeout';
67
- }
68
- else {
69
- message += behavioralCtx.actualOutcome;
70
- }
71
- }
72
- else if (behavioralCtx.expectedOutcome === 'error') {
73
- message += `error`;
74
- if (behavioralCtx.expectedErrorType) {
75
- message += ` of type "${behavioralCtx.expectedErrorType}"`;
76
- }
77
- if (behavioralCtx.expectedErrorCode) {
78
- message += ` (code: ${behavioralCtx.expectedErrorCode})`;
79
- }
80
- message += ' but got ';
81
- if (behavioralCtx.actualOutcome === 'success') {
82
- message += 'success';
83
- }
84
- else {
85
- message += `${behavioralCtx.actualOutcome} (${behavioralCtx.actualErrorCode || behavioralCtx.actualErrorType || 'unknown error'})`;
86
- }
87
- if (behavioralCtx.actualErrorType &&
88
- behavioralCtx.expectedErrorType &&
89
- behavioralCtx.actualErrorType.toLowerCase() !==
90
- behavioralCtx.expectedErrorType.toLowerCase()) {
91
- message += ' with different type';
92
- }
93
- }
94
- else {
95
- message += `${behavioralCtx.expectedOutcome} but got ${behavioralCtx.actualOutcome}`;
96
- }
97
- diagnostics.push(this.createDiagnostic(message, behavioralCtx.toolName, undefined, 'Update test expectation to match actual tool behavior, or fix tool implementation to match declared guarantees. Ensure test expectations accurately reflect tool behavior.', {
98
- testName: behavioralCtx.testName,
99
- testInput: behavioralCtx.testInput,
100
- expectedOutcome: behavioralCtx.expectedOutcome,
101
- actualOutcome: behavioralCtx.actualOutcome,
102
- expectedError: behavioralCtx.expectedError,
103
- actualError: behavioralCtx.actualError,
104
- }));
105
- return diagnostics;
106
- }
107
- }
108
- export const E020UnexpectedTestResult = new E020UnexpectedTestResultRule();
109
- //# sourceMappingURL=e020-unexpected-test-result.js.map
@@ -1,22 +0,0 @@
1
- /**
2
- * W001: Implicit Tool Dependency
3
- *
4
- * Condition:
5
- * - High confidence dependency inferred
6
- * - Not stated explicitly in description
7
- *
8
- * Why:
9
- * - LLM may not chain tools reliably
10
- */
11
- import { BaseRule } from '../base.js';
12
- import type { AnalysisContext, Diagnostic } from '../../types.js';
13
- declare class W001ImplicitDependencyRule extends BaseRule {
14
- readonly id = "W001";
15
- readonly severity: "warning";
16
- readonly ruleName = "Implicit Tool Dependency";
17
- readonly description = "Tool appears to depend on another tool, but the dependency is implicit.";
18
- check(ctx: AnalysisContext): Diagnostic[];
19
- }
20
- export declare const W001ImplicitDependency: W001ImplicitDependencyRule;
21
- export {};
22
- //# sourceMappingURL=w001-implicit-dependency.d.ts.map
@@ -1,39 +0,0 @@
1
- /**
2
- * W001: Implicit Tool Dependency
3
- *
4
- * Condition:
5
- * - High confidence dependency inferred
6
- * - Not stated explicitly in description
7
- *
8
- * Why:
9
- * - LLM may not chain tools reliably
10
- */
11
- import { BaseRule } from '../base.js';
12
- class W001ImplicitDependencyRule extends BaseRule {
13
- id = 'W001';
14
- severity = 'warning';
15
- ruleName = 'Implicit Tool Dependency';
16
- description = 'Tool appears to depend on another tool, but the dependency is implicit.';
17
- check(ctx) {
18
- const diagnostics = [];
19
- // Check medium to high confidence dependencies (0.6-0.8)
20
- const implicitDeps = ctx.dependencies.filter(d => d.confidence >= 0.6 && d.confidence < 0.8);
21
- for (const dep of implicitDeps) {
22
- // Find the toTool to check its description
23
- const toTool = ctx.indexes.toolIndex.get(dep.toTool.toLowerCase());
24
- if (!toTool) {
25
- continue;
26
- }
27
- const description = toTool.description || '';
28
- // Check if fromTool is mentioned in description
29
- const fromToolNameLower = dep.fromTool.toLowerCase();
30
- const descLower = description.toLowerCase();
31
- if (!descLower.includes(fromToolNameLower)) {
32
- diagnostics.push(this.createDiagnostic(`Tool "${dep.toTool}" appears to depend on "${dep.fromTool}", but the dependency is implicit.`, dep.toTool, dep.toField, `Mention "${dep.fromTool}" in the description of "${dep.toTool}" to make the dependency explicit.`));
33
- }
34
- }
35
- return diagnostics;
36
- }
37
- }
38
- export const W001ImplicitDependency = new W001ImplicitDependencyRule();
39
- //# sourceMappingURL=w001-implicit-dependency.js.map
@@ -1,24 +0,0 @@
1
- /**
2
- * W002: Free-Text Output Without Normalization
3
- *
4
- * Condition:
5
- * - Output is string
6
- * - Not constrained or normalized
7
- * - Even if not chained
8
- *
9
- * Why:
10
- * - Hard to reuse
11
- * - Hard to evolve
12
- */
13
- import { BaseRule } from '../base.js';
14
- import type { AnalysisContext, Diagnostic } from '../../types.js';
15
- declare class W002FreeTextWithoutNormalizationRule extends BaseRule {
16
- readonly id = "W002";
17
- readonly severity: "warning";
18
- readonly ruleName = "Free-Text Output Without Normalization";
19
- readonly description = "Tool returns unconstrained free text. Consider normalizing output.";
20
- check(ctx: AnalysisContext): Diagnostic[];
21
- }
22
- export declare const W002FreeTextWithoutNormalization: W002FreeTextWithoutNormalizationRule;
23
- export {};
24
- //# sourceMappingURL=w002-free-text-without-normalization.d.ts.map
@@ -1,40 +0,0 @@
1
- /**
2
- * W002: Free-Text Output Without Normalization
3
- *
4
- * Condition:
5
- * - Output is string
6
- * - Not constrained or normalized
7
- * - Even if not chained
8
- *
9
- * Why:
10
- * - Hard to reuse
11
- * - Hard to evolve
12
- */
13
- import { BaseRule } from '../base.js';
14
- class W002FreeTextWithoutNormalizationRule extends BaseRule {
15
- id = 'W002';
16
- severity = 'warning';
17
- ruleName = 'Free-Text Output Without Normalization';
18
- description = 'Tool returns unconstrained free text. Consider normalizing output.';
19
- check(ctx) {
20
- const diagnostics = [];
21
- for (const tool of ctx.tools) {
22
- for (const output of tool.outputs) {
23
- // Check if output is unconstrained string
24
- if (output.type !== 'string') {
25
- continue;
26
- }
27
- const hasEnum = Boolean(output.enum && output.enum.length > 0);
28
- const hasPattern = Boolean(output.pattern);
29
- const hasDescription = Boolean(output.description && output.description.trim());
30
- // If it's a string with no constraints, it's unconstrained free text
31
- if (!hasEnum && !hasPattern && !hasDescription) {
32
- diagnostics.push(this.createDiagnostic(`Tool "${tool.name}" returns unconstrained free text (field: "${output.name}"). Consider normalizing output.`, tool.name, output.name, `Add constraints to the output: enum values, regex pattern, or a clear description of the expected format.`));
33
- }
34
- }
35
- }
36
- return diagnostics;
37
- }
38
- }
39
- export const W002FreeTextWithoutNormalization = new W002FreeTextWithoutNormalizationRule();
40
- //# sourceMappingURL=w002-free-text-without-normalization.js.map
@@ -1,22 +0,0 @@
1
- /**
2
- * W003: Missing Examples for User-Facing Inputs
3
- *
4
- * Condition:
5
- * - Tool takes user-provided input
6
- * - No examples provided
7
- *
8
- * Why:
9
- * - LLM guessing increases error rate
10
- */
11
- import { BaseRule } from '../base.js';
12
- import type { AnalysisContext, Diagnostic } from '../../types.js';
13
- declare class W003MissingExamplesRule extends BaseRule {
14
- readonly id = "W003";
15
- readonly severity: "warning";
16
- readonly ruleName = "Missing Examples for User-Facing Inputs";
17
- readonly description = "Tool accepts user-provided input but has no examples. LLM accuracy may be reduced.";
18
- check(ctx: AnalysisContext): Diagnostic[];
19
- }
20
- export declare const W003MissingExamples: W003MissingExamplesRule;
21
- export {};
22
- //# sourceMappingURL=w003-missing-examples.d.ts.map
@@ -1,84 +0,0 @@
1
- /**
2
- * W003: Missing Examples for User-Facing Inputs
3
- *
4
- * Condition:
5
- * - Tool takes user-provided input
6
- * - No examples provided
7
- *
8
- * Why:
9
- * - LLM guessing increases error rate
10
- */
11
- import { BaseRule } from '../base.js';
12
- import { escapeRegex } from '../../utils.js';
13
- /**
14
- * Check if an input field appears to be user-provided.
15
- * Indicators are treated as plain text (not regex patterns).
16
- */
17
- function isUserFacingInput(inputName, description) {
18
- const name = inputName.toLowerCase();
19
- const desc = (description || '').toLowerCase();
20
- // Keywords that suggest user input (treated as plain text, not regex)
21
- const userInputIndicators = [
22
- 'user',
23
- 'person',
24
- 'name',
25
- 'email',
26
- 'location',
27
- 'address',
28
- 'preference',
29
- 'query',
30
- 'question',
31
- 'input',
32
- 'text',
33
- 'message',
34
- ];
35
- // Split into tokens for whole-word matching
36
- const nameTokens = name
37
- .split(/[^\w]+|(?<=[a-z])(?=[A-Z])/)
38
- .filter(t => t.length > 0);
39
- const descTokens = desc
40
- .split(/[^\w]+|(?<=[a-z])(?=[A-Z])/)
41
- .filter(t => t.length > 0);
42
- const allTokens = new Set([...nameTokens, ...descTokens]);
43
- // Check for whole-word matches using word boundaries
44
- // Escape indicators to prevent regex injection
45
- return userInputIndicators.some(indicator => {
46
- if (allTokens.has(indicator)) {
47
- return true;
48
- }
49
- const escapedIndicator = escapeRegex(indicator);
50
- const indicatorRegex = new RegExp(`\\b${escapedIndicator}\\b`, 'i');
51
- return indicatorRegex.test(name) || indicatorRegex.test(desc);
52
- });
53
- }
54
- class W003MissingExamplesRule extends BaseRule {
55
- id = 'W003';
56
- severity = 'warning';
57
- ruleName = 'Missing Examples for User-Facing Inputs';
58
- description = 'Tool accepts user-provided input but has no examples. LLM accuracy may be reduced.';
59
- check(ctx) {
60
- const diagnostics = [];
61
- for (const tool of ctx.tools) {
62
- for (const input of tool.inputs) {
63
- // Only check user-facing inputs
64
- if (!isUserFacingInput(input.name, input.description)) {
65
- continue;
66
- }
67
- // Check if it has an example (treat undefined, empty strings, whitespace-only, and empty arrays/objects as missing)
68
- const example = input.example;
69
- const isMissing = example === undefined ||
70
- (typeof example === 'string' && example.trim().length === 0) ||
71
- (Array.isArray(example) && example.length === 0) ||
72
- (typeof example === 'object' &&
73
- example !== null &&
74
- Object.keys(example).length === 0);
75
- if (isMissing) {
76
- diagnostics.push(this.createDiagnostic(`Tool "${tool.name}" parameter "${input.name}" has no examples. LLM accuracy may be reduced.`, tool.name, input.name, `Add example values to "${input.name}" to help the LLM understand the expected format.`));
77
- }
78
- }
79
- }
80
- return diagnostics;
81
- }
82
- }
83
- export const W003MissingExamples = new W003MissingExamplesRule();
84
- //# sourceMappingURL=w003-missing-examples.js.map
@@ -1,23 +0,0 @@
1
- /**
2
- * W004: Overloaded Tool Responsibility
3
- *
4
- * Condition:
5
- * - Tool does multiple conceptual things
6
- * - Description contains multiple verbs or intents
7
- *
8
- * Why:
9
- * - Tool selection becomes unstable
10
- * - Hard to compose
11
- */
12
- import { BaseRule } from '../base.js';
13
- import type { AnalysisContext, Diagnostic } from '../../types.js';
14
- declare class W004OverloadedResponsibilityRule extends BaseRule {
15
- readonly id = "W004";
16
- readonly severity: "warning";
17
- readonly ruleName = "Overloaded Tool Responsibility";
18
- readonly description = "Tool appears to handle multiple responsibilities. Tool selection becomes unstable.";
19
- check(ctx: AnalysisContext): Diagnostic[];
20
- }
21
- export declare const W004OverloadedResponsibility: W004OverloadedResponsibilityRule;
22
- export {};
23
- //# sourceMappingURL=w004-overloaded-responsibility.d.ts.map