@dotsetlabs/bellwether 1.0.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.md +8 -2
- package/dist/baseline/accessors.d.ts +1 -1
- package/dist/baseline/accessors.js +1 -3
- package/dist/baseline/baseline-format.d.ts +287 -0
- package/dist/baseline/baseline-format.js +12 -0
- package/dist/baseline/comparator.js +249 -11
- package/dist/baseline/converter.d.ts +15 -15
- package/dist/baseline/converter.js +46 -34
- package/dist/baseline/diff.d.ts +1 -1
- package/dist/baseline/diff.js +45 -28
- package/dist/baseline/error-analyzer.d.ts +1 -1
- package/dist/baseline/error-analyzer.js +90 -17
- package/dist/baseline/incremental-checker.js +8 -5
- package/dist/baseline/index.d.ts +2 -12
- package/dist/baseline/index.js +3 -23
- package/dist/baseline/performance-tracker.d.ts +0 -1
- package/dist/baseline/performance-tracker.js +13 -20
- package/dist/baseline/response-fingerprint.js +39 -2
- package/dist/baseline/saver.js +41 -10
- package/dist/baseline/schema-compare.d.ts +22 -0
- package/dist/baseline/schema-compare.js +259 -16
- package/dist/baseline/types.d.ts +10 -7
- package/dist/cache/response-cache.d.ts +8 -0
- package/dist/cache/response-cache.js +110 -0
- package/dist/cli/commands/check.js +23 -6
- package/dist/cli/commands/explore.js +34 -14
- package/dist/cli/index.js +8 -0
- package/dist/config/template.js +8 -7
- package/dist/config/validator.d.ts +59 -59
- package/dist/config/validator.js +245 -90
- package/dist/constants/core.d.ts +4 -0
- package/dist/constants/core.js +8 -19
- package/dist/constants/registry.d.ts +17 -0
- package/dist/constants/registry.js +18 -0
- package/dist/constants/testing.d.ts +0 -369
- package/dist/constants/testing.js +18 -456
- package/dist/constants.d.ts +1 -1
- package/dist/constants.js +1 -1
- package/dist/docs/contract.js +131 -83
- package/dist/docs/report.js +8 -5
- package/dist/interview/insights.d.ts +17 -0
- package/dist/interview/insights.js +52 -0
- package/dist/interview/interviewer.js +52 -10
- package/dist/interview/prompt-test-generator.d.ts +12 -0
- package/dist/interview/prompt-test-generator.js +77 -0
- package/dist/interview/resource-test-generator.d.ts +12 -0
- package/dist/interview/resource-test-generator.js +20 -0
- package/dist/interview/schema-inferrer.js +26 -4
- package/dist/interview/schema-test-generator.js +278 -31
- package/dist/interview/stateful-test-runner.d.ts +3 -0
- package/dist/interview/stateful-test-runner.js +80 -0
- package/dist/interview/types.d.ts +12 -0
- package/dist/transport/mcp-client.js +1 -1
- package/dist/transport/sse-transport.d.ts +7 -3
- package/dist/transport/sse-transport.js +157 -67
- package/dist/version.js +1 -1
- package/man/bellwether.1 +1 -1
- package/man/bellwether.1.md +2 -2
- package/package.json +1 -1
- package/schemas/bellwether-check.schema.json +185 -0
- package/schemas/bellwether-explore.schema.json +837 -0
- package/scripts/completions/bellwether.bash +10 -4
- package/scripts/completions/bellwether.zsh +55 -2
|
@@ -1,42 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Change impact analysis configuration.
|
|
3
|
-
* Used by change-impact-analyzer.ts for semantic breaking change detection.
|
|
4
|
-
*/
|
|
5
|
-
export const CHANGE_IMPACT = {
|
|
6
|
-
/** Risk weights for different schema change types (0-100 scale) */
|
|
7
|
-
RISK_WEIGHTS: {
|
|
8
|
-
parameter_removed: 100,
|
|
9
|
-
parameter_required_added: 90,
|
|
10
|
-
parameter_type_changed: 85,
|
|
11
|
-
enum_value_removed: 80,
|
|
12
|
-
constraint_tightened: 60,
|
|
13
|
-
format_changed: 50,
|
|
14
|
-
constraint_added: 40,
|
|
15
|
-
default_changed: 30,
|
|
16
|
-
constraint_removed: 20,
|
|
17
|
-
parameter_required_removed: 15,
|
|
18
|
-
enum_value_added: 10,
|
|
19
|
-
parameter_added: 10,
|
|
20
|
-
description_changed: 5,
|
|
21
|
-
constraint_relaxed: 5,
|
|
22
|
-
},
|
|
23
|
-
/** Migration complexity thresholds (number of breaking changes) */
|
|
24
|
-
COMPLEXITY_THRESHOLDS: {
|
|
25
|
-
/** 0-1 breaking changes = trivial migration */
|
|
26
|
-
trivial: 1,
|
|
27
|
-
/** 2-3 breaking changes = simple migration */
|
|
28
|
-
simple: 3,
|
|
29
|
-
/** 4-6 breaking changes = moderate migration */
|
|
30
|
-
moderate: 6,
|
|
31
|
-
// 7+ breaking changes = complex migration
|
|
32
|
-
},
|
|
33
|
-
/** Risk score thresholds for severity classification */
|
|
34
|
-
SEVERITY_THRESHOLDS: {
|
|
35
|
-
info: 20,
|
|
36
|
-
warning: 50,
|
|
37
|
-
breaking: 70,
|
|
38
|
-
},
|
|
39
|
-
};
|
|
40
1
|
/**
|
|
41
2
|
* Check command configuration defaults.
|
|
42
3
|
* Used by check.ts and incremental-checker.ts.
|
|
@@ -55,7 +16,7 @@ export const CHECK = {
|
|
|
55
16
|
*/
|
|
56
17
|
export const PERFORMANCE_TRACKING = {
|
|
57
18
|
/** Default regression threshold (10% = tool is 10% slower) */
|
|
58
|
-
DEFAULT_REGRESSION_THRESHOLD: 0.
|
|
19
|
+
DEFAULT_REGRESSION_THRESHOLD: 0.1,
|
|
59
20
|
/** Warning threshold for minor regressions (5%) */
|
|
60
21
|
WARNING_THRESHOLD: 0.05,
|
|
61
22
|
/** Minimum samples required for reliable metrics */
|
|
@@ -123,30 +84,8 @@ export const PERFORMANCE_CONFIDENCE = {
|
|
|
123
84
|
NO_SAMPLES: 'No performance samples collected',
|
|
124
85
|
},
|
|
125
86
|
};
|
|
126
|
-
/**
|
|
127
|
-
* Deprecation lifecycle configuration.
|
|
128
|
-
* Used by deprecation-tracker.ts for tool deprecation management.
|
|
129
|
-
*/
|
|
130
|
-
export const DEPRECATION = {
|
|
131
|
-
/** Default configuration values */
|
|
132
|
-
DEFAULTS: {
|
|
133
|
-
/** Warn when using deprecated tools */
|
|
134
|
-
warnOnUsage: true,
|
|
135
|
-
/** Fail when using tools past their removal date */
|
|
136
|
-
failOnExpired: true,
|
|
137
|
-
/** Default grace period in days after removal date */
|
|
138
|
-
gracePeriodDays: 90,
|
|
139
|
-
},
|
|
140
|
-
/** Days thresholds for warning levels */
|
|
141
|
-
THRESHOLDS: {
|
|
142
|
-
/** Warn about upcoming removal within this many days */
|
|
143
|
-
upcomingRemovalDays: 30,
|
|
144
|
-
/** Critical warning within this many days */
|
|
145
|
-
criticalRemovalDays: 7,
|
|
146
|
-
},
|
|
147
|
-
};
|
|
148
87
|
export const SCHEMA_EVOLUTION = {
|
|
149
|
-
//
|
|
88
|
+
// Schema stability settings (response-schema-tracker.ts and docs)
|
|
150
89
|
/** Default maximum versions to keep per tool */
|
|
151
90
|
DEFAULT_MAX_VERSIONS_PER_TOOL: 50,
|
|
152
91
|
/** Default limit for "most active tools" queries */
|
|
@@ -237,87 +176,6 @@ export const ERROR_ANALYSIS = {
|
|
|
237
176
|
/** Maximum related parameters to extract */
|
|
238
177
|
MAX_RELATED_PARAMETERS: 5,
|
|
239
178
|
};
|
|
240
|
-
/**
|
|
241
|
-
* Migration guide generation configuration.
|
|
242
|
-
* Used by migration-generator.ts for auto-generating migration guides.
|
|
243
|
-
*/
|
|
244
|
-
export const MIGRATION_GUIDE = {
|
|
245
|
-
/** Maximum code examples per migration step */
|
|
246
|
-
MAX_CODE_EXAMPLES_PER_STEP: 3,
|
|
247
|
-
/** Maximum steps in a migration guide */
|
|
248
|
-
MAX_MIGRATION_STEPS: 20,
|
|
249
|
-
/** Minimum changes required to generate a guide */
|
|
250
|
-
MIN_CHANGES_FOR_GUIDE: 1,
|
|
251
|
-
/** Effort estimation thresholds (number of breaking changes) */
|
|
252
|
-
EFFORT_THRESHOLDS: {
|
|
253
|
-
/** 0-1 breaking changes = trivial */
|
|
254
|
-
trivial: 1,
|
|
255
|
-
/** 2-3 breaking changes = minor */
|
|
256
|
-
minor: 3,
|
|
257
|
-
/** 4-6 breaking changes = moderate */
|
|
258
|
-
moderate: 6,
|
|
259
|
-
// 7+ breaking changes = major
|
|
260
|
-
},
|
|
261
|
-
};
|
|
262
|
-
/**
|
|
263
|
-
* Auto-generated test scenario configuration.
|
|
264
|
-
* Used by scenario-generator.ts for generating test scenarios.
|
|
265
|
-
*/
|
|
266
|
-
export const SCENARIO_GENERATION = {
|
|
267
|
-
/** Maximum happy path scenarios per tool */
|
|
268
|
-
MAX_HAPPY_PATH_SCENARIOS: 5,
|
|
269
|
-
/** Maximum edge case scenarios per tool */
|
|
270
|
-
MAX_EDGE_CASE_SCENARIOS: 10,
|
|
271
|
-
/** Maximum error case scenarios per tool */
|
|
272
|
-
MAX_ERROR_CASE_SCENARIOS: 5,
|
|
273
|
-
/** Maximum security test scenarios per tool */
|
|
274
|
-
MAX_SECURITY_SCENARIOS: 5,
|
|
275
|
-
/** Default minimum coverage percentage */
|
|
276
|
-
DEFAULT_MIN_COVERAGE: 80,
|
|
277
|
-
/** Common SQL injection payloads for testing */
|
|
278
|
-
SQL_INJECTION_PAYLOADS: [
|
|
279
|
-
"'; DROP TABLE users; --",
|
|
280
|
-
"1' OR '1'='1",
|
|
281
|
-
"1; SELECT * FROM users",
|
|
282
|
-
],
|
|
283
|
-
/** Common XSS payloads for testing */
|
|
284
|
-
XSS_PAYLOADS: [
|
|
285
|
-
'<script>alert("xss")</script>',
|
|
286
|
-
'"><img src=x onerror=alert(1)>',
|
|
287
|
-
"javascript:alert('xss')",
|
|
288
|
-
],
|
|
289
|
-
/** Common path traversal payloads for testing */
|
|
290
|
-
PATH_TRAVERSAL_PAYLOADS: [
|
|
291
|
-
'../../../etc/passwd',
|
|
292
|
-
'..\\..\\..\\windows\\system32\\config\\sam',
|
|
293
|
-
'/etc/passwd',
|
|
294
|
-
],
|
|
295
|
-
/** Categories of test scenarios */
|
|
296
|
-
CATEGORIES: ['happy_path', 'edge_cases', 'error_handling', 'security'],
|
|
297
|
-
};
|
|
298
|
-
/**
|
|
299
|
-
* PR comment formatting configuration.
|
|
300
|
-
* Used by pr-comment-generator.ts for generating GitHub PR comments.
|
|
301
|
-
*/
|
|
302
|
-
export const PR_COMMENTS = {
|
|
303
|
-
/** Maximum tools to show in detailed section */
|
|
304
|
-
MAX_DETAILED_TOOLS: 10,
|
|
305
|
-
/** Maximum changes to show per tool */
|
|
306
|
-
MAX_CHANGES_PER_TOOL: 5,
|
|
307
|
-
/** Maximum workflows to show in affected section */
|
|
308
|
-
MAX_AFFECTED_WORKFLOWS: 5,
|
|
309
|
-
/** Maximum code examples in migration section */
|
|
310
|
-
MAX_MIGRATION_EXAMPLES: 3,
|
|
311
|
-
/** Truncation length for long values */
|
|
312
|
-
VALUE_TRUNCATE_LENGTH: 50,
|
|
313
|
-
/** Badge colors for different severity levels */
|
|
314
|
-
BADGE_COLORS: {
|
|
315
|
-
breaking: 'red',
|
|
316
|
-
warning: 'orange',
|
|
317
|
-
info: 'blue',
|
|
318
|
-
none: 'green',
|
|
319
|
-
},
|
|
320
|
-
};
|
|
321
179
|
// ==================== Schema Testing (Check Mode) ====================
|
|
322
180
|
/**
|
|
323
181
|
* Schema-based test generation configuration for check mode.
|
|
@@ -372,10 +230,7 @@ export const SCHEMA_TESTING = {
|
|
|
372
230
|
UNDEFINED_STRING: 'undefined',
|
|
373
231
|
},
|
|
374
232
|
/** Invalid enum value to use when testing enum violations */
|
|
375
|
-
INVALID_ENUM_VALUES: [
|
|
376
|
-
'INVALID_ENUM_VALUE_12345',
|
|
377
|
-
'__not_a_valid_option__',
|
|
378
|
-
],
|
|
233
|
+
INVALID_ENUM_VALUES: ['INVALID_ENUM_VALUE_12345', '__not_a_valid_option__'],
|
|
379
234
|
/** Test names for different test categories (used in descriptions) */
|
|
380
235
|
CATEGORY_DESCRIPTIONS: {
|
|
381
236
|
HAPPY_PATH: 'Happy path test',
|
|
@@ -410,9 +265,7 @@ export const OUTCOME_ASSESSMENT = {
|
|
|
410
265
|
* Test categories that expect errors (validation tests).
|
|
411
266
|
* Tools should reject these inputs - rejection counts as success.
|
|
412
267
|
*/
|
|
413
|
-
EXPECTS_ERROR_CATEGORIES: [
|
|
414
|
-
'error_handling',
|
|
415
|
-
],
|
|
268
|
+
EXPECTS_ERROR_CATEGORIES: ['error_handling'],
|
|
416
269
|
/**
|
|
417
270
|
* Test descriptions that indicate error-expectation.
|
|
418
271
|
* Matched case-insensitively against test descriptions.
|
|
@@ -433,17 +286,12 @@ export const OUTCOME_ASSESSMENT = {
|
|
|
433
286
|
* Categories where tests always expect success (happy path).
|
|
434
287
|
* Errors on these tests indicate actual tool problems.
|
|
435
288
|
*/
|
|
436
|
-
EXPECTS_SUCCESS_CATEGORIES: [
|
|
437
|
-
'happy_path',
|
|
438
|
-
],
|
|
289
|
+
EXPECTS_SUCCESS_CATEGORIES: ['happy_path'],
|
|
439
290
|
/**
|
|
440
291
|
* Categories where outcome is unpredictable (edge cases).
|
|
441
292
|
* Either success or error is acceptable.
|
|
442
293
|
*/
|
|
443
|
-
EITHER_OUTCOME_CATEGORIES: [
|
|
444
|
-
'edge_case',
|
|
445
|
-
'boundary',
|
|
446
|
-
],
|
|
294
|
+
EITHER_OUTCOME_CATEGORIES: ['edge_case', 'boundary'],
|
|
447
295
|
/**
|
|
448
296
|
* Reliability metrics calculation.
|
|
449
297
|
* These control how success/failure rates are computed.
|
|
@@ -1061,10 +909,7 @@ export const EXTERNAL_DEPENDENCIES = {
|
|
|
1061
909
|
},
|
|
1062
910
|
anthropic: {
|
|
1063
911
|
name: 'Anthropic',
|
|
1064
|
-
toolPatterns: [
|
|
1065
|
-
/anthropic/i,
|
|
1066
|
-
/claude/i,
|
|
1067
|
-
],
|
|
912
|
+
toolPatterns: [/anthropic/i, /claude/i],
|
|
1068
913
|
errorPatterns: [
|
|
1069
914
|
/anthropic\.com/i,
|
|
1070
915
|
/invalid_api_key/i,
|
|
@@ -1083,11 +928,7 @@ export const EXTERNAL_DEPENDENCIES = {
|
|
|
1083
928
|
},
|
|
1084
929
|
firebase: {
|
|
1085
930
|
name: 'Firebase',
|
|
1086
|
-
toolPatterns: [
|
|
1087
|
-
/firebase/i,
|
|
1088
|
-
/firestore/i,
|
|
1089
|
-
/realtime.*database/i,
|
|
1090
|
-
],
|
|
931
|
+
toolPatterns: [/firebase/i, /firestore/i, /realtime.*database/i],
|
|
1091
932
|
errorPatterns: [
|
|
1092
933
|
/firebase/i,
|
|
1093
934
|
/firestore/i,
|
|
@@ -1136,11 +977,7 @@ export const EXTERNAL_DEPENDENCIES = {
|
|
|
1136
977
|
/sendgrid_email/i, // High confidence: prefixed with sendgrid
|
|
1137
978
|
/sendgrid_send/i, // High confidence: prefixed with sendgrid
|
|
1138
979
|
],
|
|
1139
|
-
errorPatterns: [
|
|
1140
|
-
/sendgrid\.com/i,
|
|
1141
|
-
/api\.sendgrid/i,
|
|
1142
|
-
/INVALID_API_KEY/i,
|
|
1143
|
-
],
|
|
980
|
+
errorPatterns: [/sendgrid\.com/i, /api\.sendgrid/i, /INVALID_API_KEY/i],
|
|
1144
981
|
statusCodes: [401, 403],
|
|
1145
982
|
remediation: 'Configure SendGrid API key (SENDGRID_API_KEY)',
|
|
1146
983
|
credentials: {
|
|
@@ -1331,8 +1168,8 @@ export const CONFIDENCE_INDICATORS = {
|
|
|
1331
1168
|
export const DOCUMENTATION_SCORING = {
|
|
1332
1169
|
/** Component weights for overall score (should sum to 1.0) */
|
|
1333
1170
|
WEIGHTS: {
|
|
1334
|
-
descriptionCoverage: 0.
|
|
1335
|
-
descriptionQuality: 0.
|
|
1171
|
+
descriptionCoverage: 0.3,
|
|
1172
|
+
descriptionQuality: 0.3,
|
|
1336
1173
|
parameterDocumentation: 0.25,
|
|
1337
1174
|
exampleCoverage: 0.15,
|
|
1338
1175
|
},
|
|
@@ -1388,106 +1225,6 @@ export const DOCUMENTATION_SCORING = {
|
|
|
1388
1225
|
/** Threshold for suggesting examples (tools without examples / total tools) */
|
|
1389
1226
|
EXAMPLES_SUGGESTION_THRESHOLD: 0.5,
|
|
1390
1227
|
};
|
|
1391
|
-
// ==================== AI Agent Compatibility Scoring ====================
|
|
1392
|
-
/**
|
|
1393
|
-
* AI Agent Compatibility scoring configuration.
|
|
1394
|
-
* Used by ai-compatibility-scorer.ts for evaluating how well
|
|
1395
|
-
* an MCP server is designed for AI agent consumption.
|
|
1396
|
-
*
|
|
1397
|
-
* Scoring factors:
|
|
1398
|
-
* - Description clarity (20%): LLM understanding
|
|
1399
|
-
* - Parameter naming (15%): Semantic inference
|
|
1400
|
-
* - Error message quality (15%): Actionable errors
|
|
1401
|
-
* - Example completeness (20%): Non-truncated examples
|
|
1402
|
-
* - Workflow documentation (15%): Multi-step guidance
|
|
1403
|
-
* - Response predictability (15%): Schema stability
|
|
1404
|
-
*/
|
|
1405
|
-
export const AI_COMPATIBILITY = {
|
|
1406
|
-
/** Component weights for overall score (should sum to 1.0) */
|
|
1407
|
-
WEIGHTS: {
|
|
1408
|
-
descriptionClarity: 0.20,
|
|
1409
|
-
parameterNaming: 0.15,
|
|
1410
|
-
errorMessageQuality: 0.15,
|
|
1411
|
-
exampleCompleteness: 0.20,
|
|
1412
|
-
workflowDocumentation: 0.15,
|
|
1413
|
-
responsePredictability: 0.15,
|
|
1414
|
-
},
|
|
1415
|
-
/** Grade thresholds (minimum score for each grade) */
|
|
1416
|
-
GRADE_THRESHOLDS: {
|
|
1417
|
-
A: 90,
|
|
1418
|
-
B: 80,
|
|
1419
|
-
C: 70,
|
|
1420
|
-
D: 60,
|
|
1421
|
-
F: 0,
|
|
1422
|
-
},
|
|
1423
|
-
/** Description quality scoring */
|
|
1424
|
-
DESCRIPTION: {
|
|
1425
|
-
/** Minimum acceptable description length */
|
|
1426
|
-
MIN_LENGTH: 50,
|
|
1427
|
-
/** Good description length */
|
|
1428
|
-
GOOD_LENGTH: 100,
|
|
1429
|
-
/** Pattern to detect action verb at start */
|
|
1430
|
-
ACTION_VERB_PATTERN: /^(Get|Create|Update|Delete|List|Search|Find|Fetch|Send|Post|Retrieve|Query|Export|Import|Generate|Calculate|Validate|Check|Convert|Parse|Format|Transform|Add|Remove|Set|Clear|Reset|Initialize|Connect|Disconnect|Start|Stop|Enable|Disable|Sync|Refresh|Load|Save|Upload|Download|Process|Execute|Run|Call|Invoke|Register|Unregister|Subscribe|Unsubscribe|Publish)\s/i,
|
|
1431
|
-
/** Pattern to detect purpose/behavior explanation */
|
|
1432
|
-
PURPOSE_PATTERN: /returns?|provides?|retrieves?|generates?|creates?|enables?|allows?|performs?/i,
|
|
1433
|
-
/** Pattern to detect input/output mentions */
|
|
1434
|
-
IO_PATTERN: /takes?|accepts?|requires?|outputs?|returns?|produces?/i,
|
|
1435
|
-
/** Points for various description qualities */
|
|
1436
|
-
POINTS: {
|
|
1437
|
-
/** Points for minimum length */
|
|
1438
|
-
MIN_LENGTH: 20,
|
|
1439
|
-
/** Points for good length */
|
|
1440
|
-
GOOD_LENGTH: 30,
|
|
1441
|
-
/** Points for action verb */
|
|
1442
|
-
ACTION_VERB: 25,
|
|
1443
|
-
/** Points for purpose explanation */
|
|
1444
|
-
PURPOSE: 25,
|
|
1445
|
-
/** Points for I/O mention */
|
|
1446
|
-
IO_MENTION: 20,
|
|
1447
|
-
},
|
|
1448
|
-
},
|
|
1449
|
-
/** Parameter naming quality scoring */
|
|
1450
|
-
PARAMETER: {
|
|
1451
|
-
/** Generic/bad parameter names to flag */
|
|
1452
|
-
BAD_NAMES: ['data', 'value', 'input', 'output', 'param', 'arg', 'x', 'y', 'n', 'i', 'val', 'obj', 'item', 'thing', 'stuff'],
|
|
1453
|
-
/** Minimum acceptable parameter name length */
|
|
1454
|
-
MIN_NAME_LENGTH: 2,
|
|
1455
|
-
},
|
|
1456
|
-
/** Error message quality scoring */
|
|
1457
|
-
ERROR: {
|
|
1458
|
-
/** Minimum error message length for quality */
|
|
1459
|
-
MIN_MESSAGE_LENGTH: 20,
|
|
1460
|
-
/** Pattern to detect actionable error content */
|
|
1461
|
-
ACTIONABLE_PATTERN: /try|use|provide|specify|check|ensure|make sure|should|must|need|require|expected|format|valid/i,
|
|
1462
|
-
/** Pattern to detect remediation hints */
|
|
1463
|
-
REMEDIATION_PATTERN: /example|e\.g\.|such as|instead|correct|fix|solution|hint/i,
|
|
1464
|
-
/** Default score when no errors observed */
|
|
1465
|
-
DEFAULT_SCORE: 70,
|
|
1466
|
-
},
|
|
1467
|
-
/** Example completeness scoring */
|
|
1468
|
-
EXAMPLE: {
|
|
1469
|
-
/** Weight for coverage in score */
|
|
1470
|
-
COVERAGE_WEIGHT: 0.6,
|
|
1471
|
-
/** Weight for quality (non-truncated) in score */
|
|
1472
|
-
QUALITY_WEIGHT: 0.4,
|
|
1473
|
-
},
|
|
1474
|
-
/** Workflow documentation scoring */
|
|
1475
|
-
WORKFLOW: {
|
|
1476
|
-
/** Pattern to detect sequence hints */
|
|
1477
|
-
SEQUENCE_PATTERN: /first|then|after|before|next|followed by|prior to|subsequently|finally|once|when/i,
|
|
1478
|
-
/** Pattern to detect dependency hints */
|
|
1479
|
-
DEPENDENCY_PATTERN: /requires?|needs?|depends? on|must have|expects?|assumes?|prerequisite/i,
|
|
1480
|
-
},
|
|
1481
|
-
/** Response predictability scoring */
|
|
1482
|
-
RESPONSE: {
|
|
1483
|
-
/** Default score when no evolution data */
|
|
1484
|
-
DEFAULT_SCORE: 80,
|
|
1485
|
-
},
|
|
1486
|
-
/** Maximum recommendations to generate */
|
|
1487
|
-
MAX_RECOMMENDATIONS: 5,
|
|
1488
|
-
/** Score threshold below which to recommend improvements */
|
|
1489
|
-
RECOMMENDATION_THRESHOLD: 80,
|
|
1490
|
-
};
|
|
1491
1228
|
// ==================== Contract Testing ====================
|
|
1492
1229
|
/**
|
|
1493
1230
|
* Contract-as-code testing configuration.
|
|
@@ -1540,92 +1277,6 @@ export const CONTRACT_TESTING = {
|
|
|
1540
1277
|
TIMEOUT: 1000,
|
|
1541
1278
|
},
|
|
1542
1279
|
};
|
|
1543
|
-
// ==================== Regression Risk Scoring ====================
|
|
1544
|
-
/**
|
|
1545
|
-
* Regression risk scoring configuration.
|
|
1546
|
-
* Used by risk-scorer.ts for prioritizing fixes based on
|
|
1547
|
-
* weighted risk factors.
|
|
1548
|
-
*/
|
|
1549
|
-
export const REGRESSION_RISK = {
|
|
1550
|
-
/** Risk factor weights (should sum to 1.0) */
|
|
1551
|
-
WEIGHTS: {
|
|
1552
|
-
/** Weight for breaking change severity */
|
|
1553
|
-
breakingChangeSeverity: 0.35,
|
|
1554
|
-
/** Weight for affected tool importance */
|
|
1555
|
-
toolImportance: 0.25,
|
|
1556
|
-
/** Weight for error rate delta */
|
|
1557
|
-
errorRateDelta: 0.15,
|
|
1558
|
-
/** Weight for performance regression */
|
|
1559
|
-
performanceRegression: 0.15,
|
|
1560
|
-
/** Weight for security posture changes */
|
|
1561
|
-
securityPosture: 0.10,
|
|
1562
|
-
},
|
|
1563
|
-
/** Risk level thresholds (minimum score for each level) */
|
|
1564
|
-
LEVEL_THRESHOLDS: {
|
|
1565
|
-
critical: 80,
|
|
1566
|
-
high: 60,
|
|
1567
|
-
medium: 40,
|
|
1568
|
-
low: 20,
|
|
1569
|
-
info: 0,
|
|
1570
|
-
},
|
|
1571
|
-
/** Breaking change severity scores */
|
|
1572
|
-
BREAKING_SCORES: {
|
|
1573
|
-
/** Score for removed tool */
|
|
1574
|
-
toolRemoved: 100,
|
|
1575
|
-
/** Score for removed required parameter */
|
|
1576
|
-
requiredParamRemoved: 90,
|
|
1577
|
-
/** Score for type change */
|
|
1578
|
-
typeChanged: 80,
|
|
1579
|
-
/** Score for removed enum value */
|
|
1580
|
-
enumValueRemoved: 70,
|
|
1581
|
-
/** Score for tightened constraint */
|
|
1582
|
-
constraintTightened: 50,
|
|
1583
|
-
/** Score for added required parameter */
|
|
1584
|
-
requiredParamAdded: 40,
|
|
1585
|
-
},
|
|
1586
|
-
/** Tool importance indicators (patterns in descriptions) */
|
|
1587
|
-
IMPORTANCE_PATTERNS: {
|
|
1588
|
-
/** Patterns indicating high-frequency tools */
|
|
1589
|
-
highFrequency: [/primary|main|core|essential|critical|frequently/i],
|
|
1590
|
-
/** Patterns indicating low-frequency tools */
|
|
1591
|
-
lowFrequency: [/rarely|admin|debug|internal|deprecated/i],
|
|
1592
|
-
},
|
|
1593
|
-
/** Error rate change thresholds */
|
|
1594
|
-
ERROR_RATE: {
|
|
1595
|
-
/** Threshold for significant increase (%) */
|
|
1596
|
-
SIGNIFICANT_INCREASE: 10,
|
|
1597
|
-
/** Threshold for critical increase (%) */
|
|
1598
|
-
CRITICAL_INCREASE: 25,
|
|
1599
|
-
/** Base score for error rate calculation */
|
|
1600
|
-
BASE_SCORE: 50,
|
|
1601
|
-
},
|
|
1602
|
-
/** Performance regression scoring */
|
|
1603
|
-
PERFORMANCE: {
|
|
1604
|
-
/** Threshold for minor regression (%) */
|
|
1605
|
-
MINOR_REGRESSION: 10,
|
|
1606
|
-
/** Threshold for major regression (%) */
|
|
1607
|
-
MAJOR_REGRESSION: 25,
|
|
1608
|
-
/** Threshold for critical regression (%) */
|
|
1609
|
-
CRITICAL_REGRESSION: 50,
|
|
1610
|
-
/** Scores for each threshold */
|
|
1611
|
-
SCORES: {
|
|
1612
|
-
minor: 30,
|
|
1613
|
-
major: 60,
|
|
1614
|
-
critical: 90,
|
|
1615
|
-
},
|
|
1616
|
-
},
|
|
1617
|
-
/** Security change scoring */
|
|
1618
|
-
SECURITY: {
|
|
1619
|
-
/** Score for new vulnerability */
|
|
1620
|
-
NEW_VULNERABILITY: 100,
|
|
1621
|
-
/** Score for resolved vulnerability */
|
|
1622
|
-
RESOLVED_VULNERABILITY: -20,
|
|
1623
|
-
/** Score for severity increase */
|
|
1624
|
-
SEVERITY_INCREASE: 50,
|
|
1625
|
-
},
|
|
1626
|
-
/** Maximum recommendations to include */
|
|
1627
|
-
MAX_RECOMMENDATIONS: 5,
|
|
1628
|
-
};
|
|
1629
1280
|
// ==================== Smart Value Generation ====================
|
|
1630
1281
|
/**
|
|
1631
1282
|
* Smart value generation configuration for check mode.
|
|
@@ -1638,11 +1289,7 @@ export const SMART_VALUE_GENERATION = {
|
|
|
1638
1289
|
/** Geographic coordinate patterns and defaults */
|
|
1639
1290
|
COORDINATES: {
|
|
1640
1291
|
/** Patterns that indicate latitude fields */
|
|
1641
|
-
LATITUDE_PATTERNS: [
|
|
1642
|
-
/^lat(itude)?$/i,
|
|
1643
|
-
/_lat$/i,
|
|
1644
|
-
/lat_/i,
|
|
1645
|
-
],
|
|
1292
|
+
LATITUDE_PATTERNS: [/^lat(itude)?$/i, /_lat$/i, /lat_/i],
|
|
1646
1293
|
/** Patterns that indicate longitude fields */
|
|
1647
1294
|
LONGITUDE_PATTERNS: [
|
|
1648
1295
|
/^lon(g|gitude)?$/i,
|
|
@@ -1696,17 +1343,9 @@ export const SMART_VALUE_GENERATION = {
|
|
|
1696
1343
|
/** Enhanced ID field patterns and defaults */
|
|
1697
1344
|
IDENTIFIERS: {
|
|
1698
1345
|
/** Patterns that indicate UUID format */
|
|
1699
|
-
UUID_PATTERNS: [
|
|
1700
|
-
/uuid$/i,
|
|
1701
|
-
/guid$/i,
|
|
1702
|
-
/format.*uuid/i,
|
|
1703
|
-
],
|
|
1346
|
+
UUID_PATTERNS: [/uuid$/i, /guid$/i, /format.*uuid/i],
|
|
1704
1347
|
/** Patterns that indicate numeric IDs */
|
|
1705
|
-
NUMERIC_ID_PATTERNS: [
|
|
1706
|
-
/^\d+$/,
|
|
1707
|
-
/numeric.*id/i,
|
|
1708
|
-
/integer.*id/i,
|
|
1709
|
-
],
|
|
1348
|
+
NUMERIC_ID_PATTERNS: [/^\d+$/, /numeric.*id/i, /integer.*id/i],
|
|
1710
1349
|
/** Default ID values for different formats */
|
|
1711
1350
|
DEFAULTS: {
|
|
1712
1351
|
uuid: '550e8400-e29b-41d4-a716-446655440000',
|
|
@@ -1718,18 +1357,9 @@ export const SMART_VALUE_GENERATION = {
|
|
|
1718
1357
|
/** Resource name patterns */
|
|
1719
1358
|
RESOURCE_NAMES: {
|
|
1720
1359
|
/** Patterns that indicate file names */
|
|
1721
|
-
FILE_PATTERNS: [
|
|
1722
|
-
/^file(name)?$/i,
|
|
1723
|
-
/_file$/i,
|
|
1724
|
-
/^filename$/i,
|
|
1725
|
-
],
|
|
1360
|
+
FILE_PATTERNS: [/^file(name)?$/i, /_file$/i, /^filename$/i],
|
|
1726
1361
|
/** Patterns that indicate directory/path */
|
|
1727
|
-
PATH_PATTERNS: [
|
|
1728
|
-
/^path$/i,
|
|
1729
|
-
/^dir(ectory)?$/i,
|
|
1730
|
-
/_path$/i,
|
|
1731
|
-
/_dir$/i,
|
|
1732
|
-
],
|
|
1362
|
+
PATH_PATTERNS: [/^path$/i, /^dir(ectory)?$/i, /_path$/i, /_dir$/i],
|
|
1733
1363
|
/** Default values */
|
|
1734
1364
|
DEFAULTS: {
|
|
1735
1365
|
filename: 'example.txt',
|
|
@@ -1740,12 +1370,7 @@ export const SMART_VALUE_GENERATION = {
|
|
|
1740
1370
|
/** Account/user patterns */
|
|
1741
1371
|
ACCOUNT: {
|
|
1742
1372
|
/** Patterns for account identifiers */
|
|
1743
|
-
PATTERNS: [
|
|
1744
|
-
/^account/i,
|
|
1745
|
-
/^user/i,
|
|
1746
|
-
/_account$/i,
|
|
1747
|
-
/_user$/i,
|
|
1748
|
-
],
|
|
1373
|
+
PATTERNS: [/^account/i, /^user/i, /_account$/i, /_user$/i],
|
|
1749
1374
|
/** Default values */
|
|
1750
1375
|
DEFAULTS: {
|
|
1751
1376
|
accountId: 'acct_123456789',
|
|
@@ -1763,12 +1388,7 @@ export const SMART_VALUE_GENERATION = {
|
|
|
1763
1388
|
/^per_page$/i,
|
|
1764
1389
|
],
|
|
1765
1390
|
/** Patterns for offset/page fields */
|
|
1766
|
-
OFFSET_PATTERNS: [
|
|
1767
|
-
/^offset$/i,
|
|
1768
|
-
/^skip$/i,
|
|
1769
|
-
/^page$/i,
|
|
1770
|
-
/^start$/i,
|
|
1771
|
-
],
|
|
1391
|
+
OFFSET_PATTERNS: [/^offset$/i, /^skip$/i, /^page$/i, /^start$/i],
|
|
1772
1392
|
/** Default values */
|
|
1773
1393
|
DEFAULTS: {
|
|
1774
1394
|
limit: 10,
|
|
@@ -1777,62 +1397,4 @@ export const SMART_VALUE_GENERATION = {
|
|
|
1777
1397
|
},
|
|
1778
1398
|
},
|
|
1779
1399
|
};
|
|
1780
|
-
// ==================== Intelligent Test Pruning ====================
|
|
1781
|
-
/**
|
|
1782
|
-
* Intelligent test pruning configuration.
|
|
1783
|
-
* Used to skip unnecessary tests based on tool characteristics
|
|
1784
|
-
* and testing history.
|
|
1785
|
-
*/
|
|
1786
|
-
export const TEST_PRUNING = {
|
|
1787
|
-
/** Test categories that can be pruned */
|
|
1788
|
-
CATEGORIES: {
|
|
1789
|
-
boundary: 'boundary',
|
|
1790
|
-
enum: 'enum',
|
|
1791
|
-
optionalCombinations: 'optional_combinations',
|
|
1792
|
-
errorHandling: 'error_handling',
|
|
1793
|
-
happyPath: 'happy_path',
|
|
1794
|
-
security: 'security',
|
|
1795
|
-
semantic: 'semantic',
|
|
1796
|
-
},
|
|
1797
|
-
/** Categories that should always run */
|
|
1798
|
-
ALWAYS_RUN: ['happy_path', 'error_handling'],
|
|
1799
|
-
/** Tool prioritization weights */
|
|
1800
|
-
PRIORITY_WEIGHTS: {
|
|
1801
|
-
/** Weight for previous error history */
|
|
1802
|
-
errorHistory: 0.30,
|
|
1803
|
-
/** Weight for external dependencies */
|
|
1804
|
-
externalDependency: 0.25,
|
|
1805
|
-
/** Weight for schema complexity */
|
|
1806
|
-
schemaComplexity: 0.20,
|
|
1807
|
-
/** Weight for time since last test */
|
|
1808
|
-
timeSinceLastTest: 0.15,
|
|
1809
|
-
/** Weight for change frequency */
|
|
1810
|
-
changeFrequency: 0.10,
|
|
1811
|
-
},
|
|
1812
|
-
/** Schema complexity thresholds */
|
|
1813
|
-
SCHEMA_COMPLEXITY: {
|
|
1814
|
-
/** Number of parameters for "complex" classification */
|
|
1815
|
-
HIGH_PARAM_COUNT: 10,
|
|
1816
|
-
/** Number of nested levels for "complex" classification */
|
|
1817
|
-
HIGH_NESTING_DEPTH: 3,
|
|
1818
|
-
/** Number of required params for priority boost */
|
|
1819
|
-
MANY_REQUIRED_PARAMS: 5,
|
|
1820
|
-
},
|
|
1821
|
-
/** Historical success thresholds */
|
|
1822
|
-
SUCCESS_HISTORY: {
|
|
1823
|
-
/** Success rate threshold to reduce testing (%) */
|
|
1824
|
-
HIGH_SUCCESS_THRESHOLD: 95,
|
|
1825
|
-
/** Number of consecutive successes to consider stable */
|
|
1826
|
-
STABLE_RUN_COUNT: 5,
|
|
1827
|
-
},
|
|
1828
|
-
/** Time-based thresholds */
|
|
1829
|
-
TIME_THRESHOLDS: {
|
|
1830
|
-
/** Hours since last test to increase priority */
|
|
1831
|
-
STALE_HOURS: 168, // 1 week
|
|
1832
|
-
/** Hours since last test for maximum priority */
|
|
1833
|
-
VERY_STALE_HOURS: 720, // 30 days
|
|
1834
|
-
},
|
|
1835
|
-
/** Maximum tests to skip per tool (safety limit) */
|
|
1836
|
-
MAX_SKIPPED_CATEGORIES_PER_TOOL: 3,
|
|
1837
|
-
};
|
|
1838
1400
|
//# sourceMappingURL=testing.js.map
|
package/dist/constants.d.ts
CHANGED
package/dist/constants.js
CHANGED