@dotsetlabs/bellwether 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +291 -0
- package/LICENSE +21 -0
- package/README.md +739 -0
- package/dist/auth/credentials.d.ts +64 -0
- package/dist/auth/credentials.js +218 -0
- package/dist/auth/index.d.ts +6 -0
- package/dist/auth/index.js +6 -0
- package/dist/auth/keychain.d.ts +64 -0
- package/dist/auth/keychain.js +268 -0
- package/dist/baseline/ab-testing.d.ts +80 -0
- package/dist/baseline/ab-testing.js +236 -0
- package/dist/baseline/ai-compatibility-scorer.d.ts +95 -0
- package/dist/baseline/ai-compatibility-scorer.js +606 -0
- package/dist/baseline/calibration.d.ts +77 -0
- package/dist/baseline/calibration.js +136 -0
- package/dist/baseline/category-matching.d.ts +85 -0
- package/dist/baseline/category-matching.js +289 -0
- package/dist/baseline/change-impact-analyzer.d.ts +98 -0
- package/dist/baseline/change-impact-analyzer.js +592 -0
- package/dist/baseline/comparator.d.ts +64 -0
- package/dist/baseline/comparator.js +916 -0
- package/dist/baseline/confidence.d.ts +55 -0
- package/dist/baseline/confidence.js +122 -0
- package/dist/baseline/converter.d.ts +61 -0
- package/dist/baseline/converter.js +585 -0
- package/dist/baseline/dependency-analyzer.d.ts +89 -0
- package/dist/baseline/dependency-analyzer.js +567 -0
- package/dist/baseline/deprecation-tracker.d.ts +133 -0
- package/dist/baseline/deprecation-tracker.js +322 -0
- package/dist/baseline/diff.d.ts +55 -0
- package/dist/baseline/diff.js +1584 -0
- package/dist/baseline/documentation-scorer.d.ts +205 -0
- package/dist/baseline/documentation-scorer.js +466 -0
- package/dist/baseline/embeddings.d.ts +118 -0
- package/dist/baseline/embeddings.js +251 -0
- package/dist/baseline/error-analyzer.d.ts +198 -0
- package/dist/baseline/error-analyzer.js +721 -0
- package/dist/baseline/evaluation/evaluator.d.ts +42 -0
- package/dist/baseline/evaluation/evaluator.js +323 -0
- package/dist/baseline/evaluation/expanded-dataset.d.ts +45 -0
- package/dist/baseline/evaluation/expanded-dataset.js +1164 -0
- package/dist/baseline/evaluation/golden-dataset.d.ts +58 -0
- package/dist/baseline/evaluation/golden-dataset.js +717 -0
- package/dist/baseline/evaluation/index.d.ts +15 -0
- package/dist/baseline/evaluation/index.js +15 -0
- package/dist/baseline/evaluation/types.d.ts +186 -0
- package/dist/baseline/evaluation/types.js +8 -0
- package/dist/baseline/external-dependency-detector.d.ts +181 -0
- package/dist/baseline/external-dependency-detector.js +524 -0
- package/dist/baseline/golden-output.d.ts +162 -0
- package/dist/baseline/golden-output.js +636 -0
- package/dist/baseline/health-scorer.d.ts +174 -0
- package/dist/baseline/health-scorer.js +451 -0
- package/dist/baseline/incremental-checker.d.ts +97 -0
- package/dist/baseline/incremental-checker.js +174 -0
- package/dist/baseline/index.d.ts +31 -0
- package/dist/baseline/index.js +42 -0
- package/dist/baseline/migration-generator.d.ts +137 -0
- package/dist/baseline/migration-generator.js +554 -0
- package/dist/baseline/migrations.d.ts +60 -0
- package/dist/baseline/migrations.js +197 -0
- package/dist/baseline/performance-tracker.d.ts +214 -0
- package/dist/baseline/performance-tracker.js +577 -0
- package/dist/baseline/pr-comment-generator.d.ts +117 -0
- package/dist/baseline/pr-comment-generator.js +546 -0
- package/dist/baseline/response-fingerprint.d.ts +127 -0
- package/dist/baseline/response-fingerprint.js +728 -0
- package/dist/baseline/response-schema-tracker.d.ts +129 -0
- package/dist/baseline/response-schema-tracker.js +420 -0
- package/dist/baseline/risk-scorer.d.ts +54 -0
- package/dist/baseline/risk-scorer.js +434 -0
- package/dist/baseline/saver.d.ts +89 -0
- package/dist/baseline/saver.js +554 -0
- package/dist/baseline/scenario-generator.d.ts +151 -0
- package/dist/baseline/scenario-generator.js +905 -0
- package/dist/baseline/schema-compare.d.ts +86 -0
- package/dist/baseline/schema-compare.js +557 -0
- package/dist/baseline/schema-evolution.d.ts +189 -0
- package/dist/baseline/schema-evolution.js +467 -0
- package/dist/baseline/semantic.d.ts +203 -0
- package/dist/baseline/semantic.js +908 -0
- package/dist/baseline/synonyms.d.ts +60 -0
- package/dist/baseline/synonyms.js +386 -0
- package/dist/baseline/telemetry.d.ts +165 -0
- package/dist/baseline/telemetry.js +294 -0
- package/dist/baseline/test-pruner.d.ts +120 -0
- package/dist/baseline/test-pruner.js +387 -0
- package/dist/baseline/types.d.ts +449 -0
- package/dist/baseline/types.js +5 -0
- package/dist/baseline/version.d.ts +138 -0
- package/dist/baseline/version.js +206 -0
- package/dist/cache/index.d.ts +5 -0
- package/dist/cache/index.js +5 -0
- package/dist/cache/response-cache.d.ts +151 -0
- package/dist/cache/response-cache.js +287 -0
- package/dist/ci/index.d.ts +60 -0
- package/dist/ci/index.js +342 -0
- package/dist/cli/commands/auth.d.ts +12 -0
- package/dist/cli/commands/auth.js +352 -0
- package/dist/cli/commands/badge.d.ts +3 -0
- package/dist/cli/commands/badge.js +74 -0
- package/dist/cli/commands/baseline-accept.d.ts +15 -0
- package/dist/cli/commands/baseline-accept.js +178 -0
- package/dist/cli/commands/baseline-migrate.d.ts +12 -0
- package/dist/cli/commands/baseline-migrate.js +164 -0
- package/dist/cli/commands/baseline.d.ts +14 -0
- package/dist/cli/commands/baseline.js +449 -0
- package/dist/cli/commands/beta.d.ts +10 -0
- package/dist/cli/commands/beta.js +231 -0
- package/dist/cli/commands/check.d.ts +11 -0
- package/dist/cli/commands/check.js +820 -0
- package/dist/cli/commands/cloud/badge.d.ts +3 -0
- package/dist/cli/commands/cloud/badge.js +74 -0
- package/dist/cli/commands/cloud/diff.d.ts +6 -0
- package/dist/cli/commands/cloud/diff.js +79 -0
- package/dist/cli/commands/cloud/history.d.ts +6 -0
- package/dist/cli/commands/cloud/history.js +102 -0
- package/dist/cli/commands/cloud/link.d.ts +9 -0
- package/dist/cli/commands/cloud/link.js +119 -0
- package/dist/cli/commands/cloud/login.d.ts +7 -0
- package/dist/cli/commands/cloud/login.js +499 -0
- package/dist/cli/commands/cloud/projects.d.ts +6 -0
- package/dist/cli/commands/cloud/projects.js +44 -0
- package/dist/cli/commands/cloud/shared.d.ts +7 -0
- package/dist/cli/commands/cloud/shared.js +42 -0
- package/dist/cli/commands/cloud/teams.d.ts +8 -0
- package/dist/cli/commands/cloud/teams.js +169 -0
- package/dist/cli/commands/cloud/upload.d.ts +8 -0
- package/dist/cli/commands/cloud/upload.js +181 -0
- package/dist/cli/commands/contract.d.ts +11 -0
- package/dist/cli/commands/contract.js +280 -0
- package/dist/cli/commands/discover.d.ts +3 -0
- package/dist/cli/commands/discover.js +82 -0
- package/dist/cli/commands/eval.d.ts +9 -0
- package/dist/cli/commands/eval.js +187 -0
- package/dist/cli/commands/explore.d.ts +11 -0
- package/dist/cli/commands/explore.js +437 -0
- package/dist/cli/commands/feedback.d.ts +9 -0
- package/dist/cli/commands/feedback.js +174 -0
- package/dist/cli/commands/golden.d.ts +12 -0
- package/dist/cli/commands/golden.js +407 -0
- package/dist/cli/commands/history.d.ts +10 -0
- package/dist/cli/commands/history.js +202 -0
- package/dist/cli/commands/init.d.ts +9 -0
- package/dist/cli/commands/init.js +219 -0
- package/dist/cli/commands/interview.d.ts +3 -0
- package/dist/cli/commands/interview.js +903 -0
- package/dist/cli/commands/link.d.ts +10 -0
- package/dist/cli/commands/link.js +169 -0
- package/dist/cli/commands/login.d.ts +7 -0
- package/dist/cli/commands/login.js +499 -0
- package/dist/cli/commands/preset.d.ts +33 -0
- package/dist/cli/commands/preset.js +297 -0
- package/dist/cli/commands/profile.d.ts +33 -0
- package/dist/cli/commands/profile.js +286 -0
- package/dist/cli/commands/registry.d.ts +11 -0
- package/dist/cli/commands/registry.js +146 -0
- package/dist/cli/commands/shared.d.ts +79 -0
- package/dist/cli/commands/shared.js +196 -0
- package/dist/cli/commands/teams.d.ts +8 -0
- package/dist/cli/commands/teams.js +169 -0
- package/dist/cli/commands/test.d.ts +9 -0
- package/dist/cli/commands/test.js +500 -0
- package/dist/cli/commands/upload.d.ts +8 -0
- package/dist/cli/commands/upload.js +223 -0
- package/dist/cli/commands/validate-config.d.ts +6 -0
- package/dist/cli/commands/validate-config.js +35 -0
- package/dist/cli/commands/verify.d.ts +11 -0
- package/dist/cli/commands/verify.js +283 -0
- package/dist/cli/commands/watch.d.ts +12 -0
- package/dist/cli/commands/watch.js +253 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.js +178 -0
- package/dist/cli/interactive.d.ts +47 -0
- package/dist/cli/interactive.js +216 -0
- package/dist/cli/output/terminal-reporter.d.ts +19 -0
- package/dist/cli/output/terminal-reporter.js +104 -0
- package/dist/cli/output.d.ts +226 -0
- package/dist/cli/output.js +438 -0
- package/dist/cli/utils/env.d.ts +5 -0
- package/dist/cli/utils/env.js +14 -0
- package/dist/cli/utils/progress.d.ts +59 -0
- package/dist/cli/utils/progress.js +206 -0
- package/dist/cli/utils/server-context.d.ts +10 -0
- package/dist/cli/utils/server-context.js +36 -0
- package/dist/cloud/auth.d.ts +144 -0
- package/dist/cloud/auth.js +374 -0
- package/dist/cloud/client.d.ts +24 -0
- package/dist/cloud/client.js +65 -0
- package/dist/cloud/http-client.d.ts +38 -0
- package/dist/cloud/http-client.js +215 -0
- package/dist/cloud/index.d.ts +23 -0
- package/dist/cloud/index.js +25 -0
- package/dist/cloud/mock-client.d.ts +107 -0
- package/dist/cloud/mock-client.js +545 -0
- package/dist/cloud/types.d.ts +515 -0
- package/dist/cloud/types.js +15 -0
- package/dist/config/defaults.d.ts +160 -0
- package/dist/config/defaults.js +169 -0
- package/dist/config/loader.d.ts +24 -0
- package/dist/config/loader.js +122 -0
- package/dist/config/template.d.ts +42 -0
- package/dist/config/template.js +647 -0
- package/dist/config/validator.d.ts +2112 -0
- package/dist/config/validator.js +658 -0
- package/dist/constants/cloud.d.ts +107 -0
- package/dist/constants/cloud.js +110 -0
- package/dist/constants/core.d.ts +521 -0
- package/dist/constants/core.js +556 -0
- package/dist/constants/testing.d.ts +1283 -0
- package/dist/constants/testing.js +1568 -0
- package/dist/constants.d.ts +10 -0
- package/dist/constants.js +10 -0
- package/dist/contract/index.d.ts +6 -0
- package/dist/contract/index.js +5 -0
- package/dist/contract/validator.d.ts +177 -0
- package/dist/contract/validator.js +574 -0
- package/dist/cost/index.d.ts +6 -0
- package/dist/cost/index.js +5 -0
- package/dist/cost/tracker.d.ts +134 -0
- package/dist/cost/tracker.js +313 -0
- package/dist/discovery/discovery.d.ts +16 -0
- package/dist/discovery/discovery.js +173 -0
- package/dist/discovery/types.d.ts +51 -0
- package/dist/discovery/types.js +2 -0
- package/dist/docs/agents.d.ts +3 -0
- package/dist/docs/agents.js +995 -0
- package/dist/docs/contract.d.ts +51 -0
- package/dist/docs/contract.js +1681 -0
- package/dist/docs/generator.d.ts +4 -0
- package/dist/docs/generator.js +4 -0
- package/dist/docs/html-reporter.d.ts +9 -0
- package/dist/docs/html-reporter.js +757 -0
- package/dist/docs/index.d.ts +10 -0
- package/dist/docs/index.js +11 -0
- package/dist/docs/junit-reporter.d.ts +18 -0
- package/dist/docs/junit-reporter.js +210 -0
- package/dist/docs/report.d.ts +14 -0
- package/dist/docs/report.js +44 -0
- package/dist/docs/sarif-reporter.d.ts +19 -0
- package/dist/docs/sarif-reporter.js +335 -0
- package/dist/docs/shared.d.ts +35 -0
- package/dist/docs/shared.js +162 -0
- package/dist/docs/templates.d.ts +12 -0
- package/dist/docs/templates.js +76 -0
- package/dist/errors/index.d.ts +6 -0
- package/dist/errors/index.js +6 -0
- package/dist/errors/retry.d.ts +92 -0
- package/dist/errors/retry.js +323 -0
- package/dist/errors/types.d.ts +321 -0
- package/dist/errors/types.js +584 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.js +32 -0
- package/dist/interview/dependency-resolver.d.ts +11 -0
- package/dist/interview/dependency-resolver.js +32 -0
- package/dist/interview/interviewer.d.ts +232 -0
- package/dist/interview/interviewer.js +1939 -0
- package/dist/interview/mock-response-generator.d.ts +7 -0
- package/dist/interview/mock-response-generator.js +102 -0
- package/dist/interview/orchestrator.d.ts +237 -0
- package/dist/interview/orchestrator.js +1296 -0
- package/dist/interview/rate-limiter.d.ts +15 -0
- package/dist/interview/rate-limiter.js +55 -0
- package/dist/interview/response-validator.d.ts +10 -0
- package/dist/interview/response-validator.js +132 -0
- package/dist/interview/schema-inferrer.d.ts +8 -0
- package/dist/interview/schema-inferrer.js +71 -0
- package/dist/interview/schema-test-generator.d.ts +71 -0
- package/dist/interview/schema-test-generator.js +834 -0
- package/dist/interview/smart-value-generator.d.ts +155 -0
- package/dist/interview/smart-value-generator.js +554 -0
- package/dist/interview/stateful-test-runner.d.ts +19 -0
- package/dist/interview/stateful-test-runner.js +106 -0
- package/dist/interview/types.d.ts +561 -0
- package/dist/interview/types.js +2 -0
- package/dist/llm/anthropic.d.ts +41 -0
- package/dist/llm/anthropic.js +355 -0
- package/dist/llm/client.d.ts +123 -0
- package/dist/llm/client.js +42 -0
- package/dist/llm/factory.d.ts +38 -0
- package/dist/llm/factory.js +145 -0
- package/dist/llm/fallback.d.ts +140 -0
- package/dist/llm/fallback.js +379 -0
- package/dist/llm/index.d.ts +18 -0
- package/dist/llm/index.js +15 -0
- package/dist/llm/ollama.d.ts +37 -0
- package/dist/llm/ollama.js +330 -0
- package/dist/llm/openai.d.ts +25 -0
- package/dist/llm/openai.js +320 -0
- package/dist/llm/token-budget.d.ts +161 -0
- package/dist/llm/token-budget.js +395 -0
- package/dist/logging/logger.d.ts +70 -0
- package/dist/logging/logger.js +130 -0
- package/dist/metrics/collector.d.ts +106 -0
- package/dist/metrics/collector.js +547 -0
- package/dist/metrics/index.d.ts +7 -0
- package/dist/metrics/index.js +7 -0
- package/dist/metrics/prometheus.d.ts +20 -0
- package/dist/metrics/prometheus.js +241 -0
- package/dist/metrics/types.d.ts +209 -0
- package/dist/metrics/types.js +5 -0
- package/dist/persona/builtins.d.ts +54 -0
- package/dist/persona/builtins.js +219 -0
- package/dist/persona/index.d.ts +8 -0
- package/dist/persona/index.js +8 -0
- package/dist/persona/loader.d.ts +30 -0
- package/dist/persona/loader.js +190 -0
- package/dist/persona/types.d.ts +144 -0
- package/dist/persona/types.js +5 -0
- package/dist/persona/validation.d.ts +94 -0
- package/dist/persona/validation.js +332 -0
- package/dist/prompts/index.d.ts +5 -0
- package/dist/prompts/index.js +5 -0
- package/dist/prompts/templates.d.ts +180 -0
- package/dist/prompts/templates.js +431 -0
- package/dist/registry/client.d.ts +49 -0
- package/dist/registry/client.js +191 -0
- package/dist/registry/index.d.ts +7 -0
- package/dist/registry/index.js +6 -0
- package/dist/registry/types.d.ts +140 -0
- package/dist/registry/types.js +6 -0
- package/dist/scenarios/evaluator.d.ts +43 -0
- package/dist/scenarios/evaluator.js +206 -0
- package/dist/scenarios/index.d.ts +10 -0
- package/dist/scenarios/index.js +9 -0
- package/dist/scenarios/loader.d.ts +20 -0
- package/dist/scenarios/loader.js +285 -0
- package/dist/scenarios/types.d.ts +153 -0
- package/dist/scenarios/types.js +8 -0
- package/dist/security/index.d.ts +17 -0
- package/dist/security/index.js +18 -0
- package/dist/security/payloads.d.ts +61 -0
- package/dist/security/payloads.js +268 -0
- package/dist/security/security-tester.d.ts +42 -0
- package/dist/security/security-tester.js +582 -0
- package/dist/security/types.d.ts +166 -0
- package/dist/security/types.js +8 -0
- package/dist/transport/base-transport.d.ts +59 -0
- package/dist/transport/base-transport.js +38 -0
- package/dist/transport/http-transport.d.ts +67 -0
- package/dist/transport/http-transport.js +238 -0
- package/dist/transport/mcp-client.d.ts +141 -0
- package/dist/transport/mcp-client.js +496 -0
- package/dist/transport/sse-transport.d.ts +88 -0
- package/dist/transport/sse-transport.js +316 -0
- package/dist/transport/stdio-transport.d.ts +43 -0
- package/dist/transport/stdio-transport.js +238 -0
- package/dist/transport/types.d.ts +125 -0
- package/dist/transport/types.js +16 -0
- package/dist/utils/concurrency.d.ts +123 -0
- package/dist/utils/concurrency.js +213 -0
- package/dist/utils/formatters.d.ts +16 -0
- package/dist/utils/formatters.js +37 -0
- package/dist/utils/index.d.ts +8 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/jsonpath.d.ts +87 -0
- package/dist/utils/jsonpath.js +326 -0
- package/dist/utils/markdown.d.ts +113 -0
- package/dist/utils/markdown.js +265 -0
- package/dist/utils/network.d.ts +14 -0
- package/dist/utils/network.js +17 -0
- package/dist/utils/sanitize.d.ts +92 -0
- package/dist/utils/sanitize.js +191 -0
- package/dist/utils/semantic.d.ts +194 -0
- package/dist/utils/semantic.js +1051 -0
- package/dist/utils/smart-truncate.d.ts +94 -0
- package/dist/utils/smart-truncate.js +361 -0
- package/dist/utils/timeout.d.ts +153 -0
- package/dist/utils/timeout.js +205 -0
- package/dist/utils/yaml-parser.d.ts +58 -0
- package/dist/utils/yaml-parser.js +86 -0
- package/dist/validation/index.d.ts +32 -0
- package/dist/validation/index.js +32 -0
- package/dist/validation/semantic-test-generator.d.ts +50 -0
- package/dist/validation/semantic-test-generator.js +176 -0
- package/dist/validation/semantic-types.d.ts +66 -0
- package/dist/validation/semantic-types.js +94 -0
- package/dist/validation/semantic-validator.d.ts +38 -0
- package/dist/validation/semantic-validator.js +340 -0
- package/dist/verification/index.d.ts +6 -0
- package/dist/verification/index.js +5 -0
- package/dist/verification/types.d.ts +133 -0
- package/dist/verification/types.js +5 -0
- package/dist/verification/verifier.d.ts +30 -0
- package/dist/verification/verifier.js +309 -0
- package/dist/version.d.ts +19 -0
- package/dist/version.js +48 -0
- package/dist/workflow/auto-generator.d.ts +27 -0
- package/dist/workflow/auto-generator.js +513 -0
- package/dist/workflow/discovery.d.ts +40 -0
- package/dist/workflow/discovery.js +195 -0
- package/dist/workflow/executor.d.ts +82 -0
- package/dist/workflow/executor.js +611 -0
- package/dist/workflow/index.d.ts +10 -0
- package/dist/workflow/index.js +10 -0
- package/dist/workflow/loader.d.ts +24 -0
- package/dist/workflow/loader.js +194 -0
- package/dist/workflow/state-tracker.d.ts +98 -0
- package/dist/workflow/state-tracker.js +424 -0
- package/dist/workflow/types.d.ts +337 -0
- package/dist/workflow/types.js +5 -0
- package/package.json +94 -0
- package/schemas/bellwether-check.schema.json +651 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cost tracking and estimation for LLM API usage.
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Token usage information.
|
|
6
|
+
*/
|
|
7
|
+
export interface TokenUsage {
|
|
8
|
+
inputTokens: number;
|
|
9
|
+
outputTokens: number;
|
|
10
|
+
totalTokens: number;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Cost estimation result.
|
|
14
|
+
*/
|
|
15
|
+
export interface CostEstimate {
|
|
16
|
+
usage: TokenUsage;
|
|
17
|
+
costUSD: number;
|
|
18
|
+
model: string;
|
|
19
|
+
breakdown: {
|
|
20
|
+
inputCost: number;
|
|
21
|
+
outputCost: number;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Cumulative cost tracker for a session.
|
|
26
|
+
*/
|
|
27
|
+
export declare class CostTracker {
|
|
28
|
+
private model;
|
|
29
|
+
private totalInputTokens;
|
|
30
|
+
private totalOutputTokens;
|
|
31
|
+
private callCount;
|
|
32
|
+
constructor(model: string);
|
|
33
|
+
/**
|
|
34
|
+
* Add token usage from an API call.
|
|
35
|
+
*/
|
|
36
|
+
addUsage(inputTokens: number, outputTokens: number): void;
|
|
37
|
+
/**
|
|
38
|
+
* Get total usage so far.
|
|
39
|
+
*/
|
|
40
|
+
getUsage(): TokenUsage;
|
|
41
|
+
/**
|
|
42
|
+
* Get total cost estimate.
|
|
43
|
+
*/
|
|
44
|
+
getCost(): CostEstimate;
|
|
45
|
+
/**
|
|
46
|
+
* Get call count.
|
|
47
|
+
*/
|
|
48
|
+
getCallCount(): number;
|
|
49
|
+
/**
|
|
50
|
+
* Format cost summary for display.
|
|
51
|
+
*/
|
|
52
|
+
formatSummary(): string;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Estimate cost for an interview before running.
|
|
56
|
+
*/
|
|
57
|
+
export declare function estimateInterviewCost(model: string, toolCount: number, questionsPerTool: number, personas?: number): CostEstimate;
|
|
58
|
+
/**
|
|
59
|
+
* Format a pre-interview cost estimate.
|
|
60
|
+
*/
|
|
61
|
+
export declare function formatCostEstimate(estimate: CostEstimate): string;
|
|
62
|
+
/**
|
|
63
|
+
* Get pricing info for a model.
|
|
64
|
+
*/
|
|
65
|
+
export declare function getModelPricing(model: string): {
|
|
66
|
+
input: number;
|
|
67
|
+
output: number;
|
|
68
|
+
} | null;
|
|
69
|
+
/**
|
|
70
|
+
* Time estimation result for an interview.
|
|
71
|
+
*/
|
|
72
|
+
export interface InterviewTimeEstimate {
|
|
73
|
+
/** Estimated duration in seconds */
|
|
74
|
+
durationSeconds: number;
|
|
75
|
+
/** Estimated duration in minutes (rounded) */
|
|
76
|
+
durationMinutes: number;
|
|
77
|
+
/** Whether estimate assumes parallel execution */
|
|
78
|
+
isParallel: boolean;
|
|
79
|
+
/** Whether using a local model (slower) */
|
|
80
|
+
isLocalModel: boolean;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Check if a provider uses local inference (slower than cloud APIs).
|
|
84
|
+
*/
|
|
85
|
+
export declare function isLocalProvider(provider: string): boolean;
|
|
86
|
+
/**
|
|
87
|
+
* Estimate interview time based on tool count and configuration.
|
|
88
|
+
*/
|
|
89
|
+
export declare function estimateInterviewTime(toolCount: number, questionsPerTool: number, personas: number, parallelPersonas?: boolean, provider?: string, promptCount?: number, resourceCount?: number, checkMode?: boolean): InterviewTimeEstimate;
|
|
90
|
+
/**
|
|
91
|
+
* Format a combined cost and time estimate for display.
|
|
92
|
+
*/
|
|
93
|
+
export declare function formatCostAndTimeEstimate(cost: CostEstimate, time: InterviewTimeEstimate): string;
|
|
94
|
+
/**
|
|
95
|
+
* Optimization suggestion for reducing cost or time.
|
|
96
|
+
*/
|
|
97
|
+
export interface OptimizationSuggestion {
|
|
98
|
+
/** Flag to use (e.g., "--ci") */
|
|
99
|
+
flag: string;
|
|
100
|
+
/** Human-readable description */
|
|
101
|
+
description: string;
|
|
102
|
+
/** Estimated savings (e.g., "~80% cheaper") */
|
|
103
|
+
estimatedSavings: string;
|
|
104
|
+
/** Priority for sorting suggestions */
|
|
105
|
+
priority: 'high' | 'medium' | 'low';
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Context for generating optimization suggestions.
|
|
109
|
+
*/
|
|
110
|
+
export interface OptimizationContext {
|
|
111
|
+
/** Estimated cost in USD */
|
|
112
|
+
estimatedCost: number;
|
|
113
|
+
/** Number of tools discovered */
|
|
114
|
+
toolCount: number;
|
|
115
|
+
/** Number of personas being used */
|
|
116
|
+
personaCount: number;
|
|
117
|
+
/** Whether using parallel personas */
|
|
118
|
+
isParallelPersonas: boolean;
|
|
119
|
+
/** Whether using a premium model (--quality flag) */
|
|
120
|
+
isPremiumModel: boolean;
|
|
121
|
+
/** Whether using CI preset */
|
|
122
|
+
isUsingCiPreset: boolean;
|
|
123
|
+
/** Whether scenarios file exists */
|
|
124
|
+
hasScenariosFile: boolean;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Generate optimization suggestions based on the interview context.
|
|
128
|
+
*/
|
|
129
|
+
export declare function suggestOptimizations(context: OptimizationContext): OptimizationSuggestion[];
|
|
130
|
+
/**
|
|
131
|
+
* Format optimization suggestions for CLI display.
|
|
132
|
+
*/
|
|
133
|
+
export declare function formatOptimizationSuggestions(suggestions: OptimizationSuggestion[], maxSuggestions?: number): string;
|
|
134
|
+
//# sourceMappingURL=tracker.d.ts.map
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cost tracking and estimation for LLM API usage.
|
|
3
|
+
*/
|
|
4
|
+
import { FORMATTING, COST_THRESHOLDS, TIME_ESTIMATION, } from '../constants.js';
|
|
5
|
+
/**
|
|
6
|
+
* Pricing per 1M tokens (input/output) for various models.
|
|
7
|
+
* Prices as of January 2026.
|
|
8
|
+
*/
|
|
9
|
+
const MODEL_PRICING = {
|
|
10
|
+
// OpenAI models - GPT-5 series (latest)
|
|
11
|
+
'gpt-5.2': { input: 1.75, output: 14.00 },
|
|
12
|
+
'gpt-5.1': { input: 1.25, output: 10.00 },
|
|
13
|
+
'gpt-5': { input: 1.25, output: 10.00 },
|
|
14
|
+
'gpt-5-mini': { input: 0.25, output: 2.00 },
|
|
15
|
+
'gpt-5-nano': { input: 0.05, output: 0.40 },
|
|
16
|
+
// OpenAI models - GPT-4.1 series (legacy)
|
|
17
|
+
'gpt-4.1': { input: 2.00, output: 8.00 },
|
|
18
|
+
'gpt-4.1-mini': { input: 0.40, output: 1.60 },
|
|
19
|
+
'gpt-4.1-nano': { input: 0.10, output: 0.40 },
|
|
20
|
+
// OpenAI models - GPT-4o series (legacy)
|
|
21
|
+
'gpt-4o': { input: 2.50, output: 10.00 },
|
|
22
|
+
'gpt-4o-mini': { input: 0.15, output: 0.60 },
|
|
23
|
+
'gpt-4-turbo': { input: 10.00, output: 30.00 },
|
|
24
|
+
'gpt-4': { input: 30.00, output: 60.00 },
|
|
25
|
+
'gpt-3.5-turbo': { input: 0.50, output: 1.50 },
|
|
26
|
+
// Anthropic models - Claude 4.5 series (latest)
|
|
27
|
+
'claude-opus-4-5': { input: 5.00, output: 25.00 },
|
|
28
|
+
'claude-opus-4-5-20251101': { input: 5.00, output: 25.00 },
|
|
29
|
+
'claude-sonnet-4-5': { input: 3.00, output: 15.00 },
|
|
30
|
+
'claude-sonnet-4-5-20250929': { input: 3.00, output: 15.00 },
|
|
31
|
+
'claude-haiku-4-5': { input: 1.00, output: 5.00 },
|
|
32
|
+
'claude-haiku-4-5-20251001': { input: 1.00, output: 5.00 },
|
|
33
|
+
// Anthropic models - Claude 4 series (legacy)
|
|
34
|
+
'claude-opus-4-20250514': { input: 15.00, output: 75.00 },
|
|
35
|
+
'claude-sonnet-4-20250514': { input: 3.00, output: 15.00 },
|
|
36
|
+
// Anthropic models - Claude 3.x series (legacy)
|
|
37
|
+
'claude-3-5-sonnet-20241022': { input: 3.00, output: 15.00 },
|
|
38
|
+
'claude-3-5-haiku-20241022': { input: 0.80, output: 4.00 },
|
|
39
|
+
'claude-3-opus-20240229': { input: 15.00, output: 75.00 },
|
|
40
|
+
'claude-3-sonnet-20240229': { input: 3.00, output: 15.00 },
|
|
41
|
+
'claude-3-haiku-20240307': { input: 0.25, output: 1.25 },
|
|
42
|
+
// Ollama (local, no cost)
|
|
43
|
+
'llama3.2': { input: 0, output: 0 },
|
|
44
|
+
'llama3.1': { input: 0, output: 0 },
|
|
45
|
+
'mixtral': { input: 0, output: 0 },
|
|
46
|
+
'codellama': { input: 0, output: 0 },
|
|
47
|
+
'gemma3': { input: 0, output: 0 },
|
|
48
|
+
'gemma2': { input: 0, output: 0 },
|
|
49
|
+
'qwen3': { input: 0, output: 0 },
|
|
50
|
+
'qwen3:8b': { input: 0, output: 0 },
|
|
51
|
+
'qwen3:4b': { input: 0, output: 0 },
|
|
52
|
+
'qwen2.5': { input: 0, output: 0 },
|
|
53
|
+
'phi3': { input: 0, output: 0 },
|
|
54
|
+
'mistral': { input: 0, output: 0 },
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Cumulative cost tracker for a session.
|
|
58
|
+
*/
|
|
59
|
+
export class CostTracker {
|
|
60
|
+
model;
|
|
61
|
+
totalInputTokens = 0;
|
|
62
|
+
totalOutputTokens = 0;
|
|
63
|
+
callCount = 0;
|
|
64
|
+
constructor(model) {
|
|
65
|
+
this.model = model;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Add token usage from an API call.
|
|
69
|
+
*/
|
|
70
|
+
addUsage(inputTokens, outputTokens) {
|
|
71
|
+
this.totalInputTokens += inputTokens;
|
|
72
|
+
this.totalOutputTokens += outputTokens;
|
|
73
|
+
this.callCount++;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Get total usage so far.
|
|
77
|
+
*/
|
|
78
|
+
getUsage() {
|
|
79
|
+
return {
|
|
80
|
+
inputTokens: this.totalInputTokens,
|
|
81
|
+
outputTokens: this.totalOutputTokens,
|
|
82
|
+
totalTokens: this.totalInputTokens + this.totalOutputTokens,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Get total cost estimate.
|
|
87
|
+
*/
|
|
88
|
+
getCost() {
|
|
89
|
+
const usage = this.getUsage();
|
|
90
|
+
const pricing = MODEL_PRICING[this.model] ?? { input: 0, output: 0 };
|
|
91
|
+
const inputCost = (usage.inputTokens / 1_000_000) * pricing.input;
|
|
92
|
+
const outputCost = (usage.outputTokens / 1_000_000) * pricing.output;
|
|
93
|
+
return {
|
|
94
|
+
usage,
|
|
95
|
+
costUSD: inputCost + outputCost,
|
|
96
|
+
model: this.model,
|
|
97
|
+
breakdown: {
|
|
98
|
+
inputCost,
|
|
99
|
+
outputCost,
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Get call count.
|
|
105
|
+
*/
|
|
106
|
+
getCallCount() {
|
|
107
|
+
return this.callCount;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Format cost summary for display.
|
|
111
|
+
*/
|
|
112
|
+
formatSummary() {
|
|
113
|
+
const cost = this.getCost();
|
|
114
|
+
const usage = cost.usage;
|
|
115
|
+
const lines = [
|
|
116
|
+
`LLM Usage Summary:`,
|
|
117
|
+
` Model: ${cost.model}`,
|
|
118
|
+
` API Calls: ${this.callCount}`,
|
|
119
|
+
` Tokens: ${usage.totalTokens.toLocaleString()} (${usage.inputTokens.toLocaleString()} in, ${usage.outputTokens.toLocaleString()} out)`,
|
|
120
|
+
];
|
|
121
|
+
if (cost.costUSD > 0) {
|
|
122
|
+
lines.push(` Estimated Cost: $${cost.costUSD.toFixed(FORMATTING.PRICE_PRECISION)}`);
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
lines.push(` Estimated Cost: Free (local model)`);
|
|
126
|
+
}
|
|
127
|
+
return lines.join('\n');
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Estimate cost for an interview before running.
|
|
132
|
+
*/
|
|
133
|
+
export function estimateInterviewCost(model, toolCount, questionsPerTool, personas = 3) {
|
|
134
|
+
// Average estimates based on typical interview patterns
|
|
135
|
+
const avgInputPerQuestion = 500; // tokens
|
|
136
|
+
const avgOutputPerQuestion = 300; // tokens
|
|
137
|
+
const totalQuestions = toolCount * questionsPerTool * personas;
|
|
138
|
+
// Add overhead for system prompts and tool schemas
|
|
139
|
+
const schemaOverhead = toolCount * 200; // tokens per tool schema
|
|
140
|
+
const inputTokens = (totalQuestions * avgInputPerQuestion) + schemaOverhead;
|
|
141
|
+
const outputTokens = totalQuestions * avgOutputPerQuestion;
|
|
142
|
+
const pricing = MODEL_PRICING[model] ?? { input: 0, output: 0 };
|
|
143
|
+
const inputCost = (inputTokens / 1_000_000) * pricing.input;
|
|
144
|
+
const outputCost = (outputTokens / 1_000_000) * pricing.output;
|
|
145
|
+
return {
|
|
146
|
+
usage: {
|
|
147
|
+
inputTokens,
|
|
148
|
+
outputTokens,
|
|
149
|
+
totalTokens: inputTokens + outputTokens,
|
|
150
|
+
},
|
|
151
|
+
costUSD: inputCost + outputCost,
|
|
152
|
+
model,
|
|
153
|
+
breakdown: {
|
|
154
|
+
inputCost,
|
|
155
|
+
outputCost,
|
|
156
|
+
},
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Format a pre-interview cost estimate.
|
|
161
|
+
*/
|
|
162
|
+
export function formatCostEstimate(estimate) {
|
|
163
|
+
const lines = [
|
|
164
|
+
`Cost Estimate for Interview:`,
|
|
165
|
+
` Model: ${estimate.model}`,
|
|
166
|
+
` Estimated Tokens: ~${estimate.usage.totalTokens.toLocaleString()}`,
|
|
167
|
+
];
|
|
168
|
+
if (estimate.costUSD > 0) {
|
|
169
|
+
lines.push(` Estimated Cost: ~$${estimate.costUSD.toFixed(FORMATTING.PRICE_PRECISION)}`);
|
|
170
|
+
}
|
|
171
|
+
else {
|
|
172
|
+
lines.push(` Estimated Cost: Free (local model)`);
|
|
173
|
+
}
|
|
174
|
+
return lines.join('\n');
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Get pricing info for a model.
|
|
178
|
+
*/
|
|
179
|
+
export function getModelPricing(model) {
|
|
180
|
+
return MODEL_PRICING[model] ?? null;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Check if a provider uses local inference (slower than cloud APIs).
|
|
184
|
+
*/
|
|
185
|
+
export function isLocalProvider(provider) {
|
|
186
|
+
return provider === 'ollama';
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Estimate interview time based on tool count and configuration.
|
|
190
|
+
*/
|
|
191
|
+
export function estimateInterviewTime(toolCount, questionsPerTool, personas, parallelPersonas = false, provider = 'openai', promptCount = 0, resourceCount = 0, checkMode = false) {
|
|
192
|
+
const isLocal = isLocalProvider(provider);
|
|
193
|
+
// Fast CI mode: skips all LLM calls, only executes tool calls
|
|
194
|
+
if (checkMode) {
|
|
195
|
+
// Just tool execution + network overhead
|
|
196
|
+
const totalItems = toolCount + promptCount + resourceCount;
|
|
197
|
+
const executionTime = totalItems * 0.5; // ~0.5s per item for direct calls
|
|
198
|
+
const fixedOverhead = TIME_ESTIMATION.DISCOVERY_OVERHEAD_SECONDS + 5; // Discovery + minimal synthesis
|
|
199
|
+
const totalSeconds = executionTime + fixedOverhead;
|
|
200
|
+
return {
|
|
201
|
+
durationSeconds: Math.round(totalSeconds),
|
|
202
|
+
durationMinutes: Math.max(1, Math.round(totalSeconds / 60)),
|
|
203
|
+
isParallel: parallelPersonas,
|
|
204
|
+
isLocalModel: isLocal,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
const totalQuestions = toolCount * questionsPerTool * personas;
|
|
208
|
+
// Base time: seconds per question + overhead per tool
|
|
209
|
+
const questionTime = totalQuestions * TIME_ESTIMATION.SECONDS_PER_QUESTION;
|
|
210
|
+
const toolOverhead = toolCount * TIME_ESTIMATION.SECONDS_PER_TOOL_OVERHEAD;
|
|
211
|
+
// Add time for prompts and resources
|
|
212
|
+
const promptTime = promptCount * TIME_ESTIMATION.SECONDS_PER_PROMPT;
|
|
213
|
+
const resourceTime = resourceCount * TIME_ESTIMATION.SECONDS_PER_RESOURCE;
|
|
214
|
+
// Fixed overhead for discovery and synthesis
|
|
215
|
+
const fixedOverhead = TIME_ESTIMATION.DISCOVERY_OVERHEAD_SECONDS +
|
|
216
|
+
TIME_ESTIMATION.SYNTHESIS_OVERHEAD_SECONDS;
|
|
217
|
+
let totalSeconds = questionTime + toolOverhead + promptTime + resourceTime + fixedOverhead;
|
|
218
|
+
// Apply parallel efficiency if running personas in parallel
|
|
219
|
+
if (parallelPersonas && personas > 1) {
|
|
220
|
+
// Parallel execution reduces the question time portion
|
|
221
|
+
const parallelFactor = 1 - (1 - 1 / personas) * TIME_ESTIMATION.PARALLEL_EFFICIENCY;
|
|
222
|
+
totalSeconds =
|
|
223
|
+
questionTime * parallelFactor + toolOverhead + promptTime + resourceTime + fixedOverhead;
|
|
224
|
+
}
|
|
225
|
+
// Local models (Ollama) are significantly slower than cloud APIs
|
|
226
|
+
if (isLocal) {
|
|
227
|
+
totalSeconds *= TIME_ESTIMATION.LOCAL_MODEL_MULTIPLIER;
|
|
228
|
+
}
|
|
229
|
+
return {
|
|
230
|
+
durationSeconds: Math.round(totalSeconds),
|
|
231
|
+
durationMinutes: Math.max(1, Math.round(totalSeconds / 60)),
|
|
232
|
+
isParallel: parallelPersonas,
|
|
233
|
+
isLocalModel: isLocal,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Format a combined cost and time estimate for display.
|
|
238
|
+
*/
|
|
239
|
+
export function formatCostAndTimeEstimate(cost, time) {
|
|
240
|
+
const costStr = cost.costUSD > 0
|
|
241
|
+
? `~$${cost.costUSD.toFixed(2)}`
|
|
242
|
+
: 'Free (local)';
|
|
243
|
+
const timeStr = `~${time.durationMinutes} min`;
|
|
244
|
+
// Add note for local models being slower
|
|
245
|
+
const localNote = time.isLocalModel ? ' (local models are slower)' : '';
|
|
246
|
+
return `Estimated: ${costStr} | ${timeStr}${localNote}`;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Generate optimization suggestions based on the interview context.
|
|
250
|
+
*/
|
|
251
|
+
export function suggestOptimizations(context) {
|
|
252
|
+
const suggestions = [];
|
|
253
|
+
// Suggest --ci if cost > threshold and not already using ci preset
|
|
254
|
+
if (context.estimatedCost > COST_THRESHOLDS.SUGGEST_CI_THRESHOLD &&
|
|
255
|
+
!context.isUsingCiPreset) {
|
|
256
|
+
suggestions.push({
|
|
257
|
+
flag: '--ci',
|
|
258
|
+
description: 'Use CI mode for fast, cheap runs',
|
|
259
|
+
estimatedSavings: '~80% cheaper',
|
|
260
|
+
priority: 'high',
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
// Suggest --scenarios-only if cost > threshold and has scenarios file
|
|
264
|
+
if (context.estimatedCost > COST_THRESHOLDS.SUGGEST_SCENARIOS_ONLY_THRESHOLD &&
|
|
265
|
+
context.hasScenariosFile) {
|
|
266
|
+
suggestions.push({
|
|
267
|
+
flag: '--scenarios-only',
|
|
268
|
+
description: 'Run only custom scenarios (no LLM)',
|
|
269
|
+
estimatedSavings: 'Free',
|
|
270
|
+
priority: 'high',
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
// Suggest --parallel-personas if many tools and multiple personas
|
|
274
|
+
if (context.toolCount >= COST_THRESHOLDS.PARALLEL_PERSONAS_TOOL_THRESHOLD &&
|
|
275
|
+
context.personaCount > 1 &&
|
|
276
|
+
!context.isParallelPersonas) {
|
|
277
|
+
suggestions.push({
|
|
278
|
+
flag: '--parallel-personas',
|
|
279
|
+
description: 'Run personas in parallel',
|
|
280
|
+
estimatedSavings: '~50% faster',
|
|
281
|
+
priority: 'medium',
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
// Suggest removing --quality if premium model + many tools
|
|
285
|
+
if (context.isPremiumModel &&
|
|
286
|
+
context.toolCount >= COST_THRESHOLDS.QUALITY_TOOL_THRESHOLD) {
|
|
287
|
+
suggestions.push({
|
|
288
|
+
flag: 'remove --quality',
|
|
289
|
+
description: 'Use budget model for large codebases',
|
|
290
|
+
estimatedSavings: '~60% cheaper',
|
|
291
|
+
priority: 'medium',
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
// Sort by priority
|
|
295
|
+
const priorityOrder = { high: 0, medium: 1, low: 2 };
|
|
296
|
+
suggestions.sort((a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]);
|
|
297
|
+
return suggestions;
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Format optimization suggestions for CLI display.
|
|
301
|
+
*/
|
|
302
|
+
export function formatOptimizationSuggestions(suggestions, maxSuggestions = 3) {
|
|
303
|
+
if (suggestions.length === 0) {
|
|
304
|
+
return '';
|
|
305
|
+
}
|
|
306
|
+
const lines = ['Optimization suggestions:'];
|
|
307
|
+
const toShow = suggestions.slice(0, maxSuggestions);
|
|
308
|
+
for (const suggestion of toShow) {
|
|
309
|
+
lines.push(` ${suggestion.flag} - ${suggestion.description} (${suggestion.estimatedSavings})`);
|
|
310
|
+
}
|
|
311
|
+
return lines.join('\n');
|
|
312
|
+
}
|
|
313
|
+
//# sourceMappingURL=tracker.js.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { MCPClient } from '../transport/mcp-client.js';
|
|
2
|
+
import type { DiscoveryResult, ToolDetail } from './types.js';
|
|
3
|
+
import type { MCPTool } from '../transport/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Discover an MCP server's capabilities by connecting and querying it.
|
|
6
|
+
*/
|
|
7
|
+
export declare function discover(client: MCPClient, command: string, args: string[]): Promise<DiscoveryResult>;
|
|
8
|
+
/**
|
|
9
|
+
* Parse tool information into a more detailed structure.
|
|
10
|
+
*/
|
|
11
|
+
export declare function parseToolDetail(tool: MCPTool): ToolDetail;
|
|
12
|
+
/**
|
|
13
|
+
* Generate a summary of discovered capabilities.
|
|
14
|
+
*/
|
|
15
|
+
export declare function summarizeDiscovery(result: DiscoveryResult): string;
|
|
16
|
+
//# sourceMappingURL=discovery.d.ts.map
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { getLogger } from '../logging/logger.js';
|
|
2
|
+
import { DISPLAY_LIMITS } from '../constants.js';
|
|
3
|
+
const logger = getLogger('discovery');
|
|
4
|
+
/**
|
|
5
|
+
* Discover an MCP server's capabilities by connecting and querying it.
|
|
6
|
+
*/
|
|
7
|
+
export async function discover(client, command, args) {
|
|
8
|
+
// Initialize connection
|
|
9
|
+
const initResult = await client.initialize();
|
|
10
|
+
// Discover tools
|
|
11
|
+
let tools = [];
|
|
12
|
+
if (initResult.capabilities.tools) {
|
|
13
|
+
try {
|
|
14
|
+
tools = await client.listTools();
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
logger.error({ error }, 'Failed to list tools');
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
// Discover prompts
|
|
21
|
+
let prompts = [];
|
|
22
|
+
if (initResult.capabilities.prompts) {
|
|
23
|
+
try {
|
|
24
|
+
prompts = await client.listPrompts();
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
logger.error({ error }, 'Failed to list prompts');
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
// Discover resources
|
|
31
|
+
let resources = [];
|
|
32
|
+
if (initResult.capabilities.resources) {
|
|
33
|
+
try {
|
|
34
|
+
resources = await client.listResources();
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
logger.error({ error }, 'Failed to list resources');
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
serverInfo: initResult.serverInfo,
|
|
42
|
+
protocolVersion: initResult.protocolVersion,
|
|
43
|
+
capabilities: initResult.capabilities,
|
|
44
|
+
tools,
|
|
45
|
+
prompts,
|
|
46
|
+
resources,
|
|
47
|
+
timestamp: new Date(),
|
|
48
|
+
serverCommand: command,
|
|
49
|
+
serverArgs: args,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Parse tool information into a more detailed structure.
|
|
54
|
+
*/
|
|
55
|
+
export function parseToolDetail(tool) {
|
|
56
|
+
const schema = tool.inputSchema;
|
|
57
|
+
const requiredParams = schema?.required ?? [];
|
|
58
|
+
const optionalParams = [];
|
|
59
|
+
if (schema?.properties) {
|
|
60
|
+
for (const param of Object.keys(schema.properties)) {
|
|
61
|
+
if (!requiredParams.includes(param)) {
|
|
62
|
+
optionalParams.push(param);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
name: tool.name,
|
|
68
|
+
description: tool.description ?? 'No description provided',
|
|
69
|
+
inputSchema: schema ?? null,
|
|
70
|
+
requiredParams,
|
|
71
|
+
optionalParams,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Generate a summary of discovered capabilities.
|
|
76
|
+
*/
|
|
77
|
+
export function summarizeDiscovery(result) {
|
|
78
|
+
const lines = [];
|
|
79
|
+
// Header with box drawing
|
|
80
|
+
lines.push('┌─────────────────────────────────────────────────────────────────┐');
|
|
81
|
+
lines.push(`│ ${result.serverInfo.name} v${result.serverInfo.version}`.padEnd(66) + '│');
|
|
82
|
+
lines.push('└─────────────────────────────────────────────────────────────────┘');
|
|
83
|
+
lines.push('');
|
|
84
|
+
// Server info
|
|
85
|
+
lines.push(`Protocol Version: ${result.protocolVersion}`);
|
|
86
|
+
lines.push(`Server Command: ${result.serverCommand} ${result.serverArgs.join(' ')}`);
|
|
87
|
+
lines.push('');
|
|
88
|
+
// Capabilities overview
|
|
89
|
+
lines.push('CAPABILITIES');
|
|
90
|
+
lines.push('────────────');
|
|
91
|
+
const caps = [];
|
|
92
|
+
if (result.capabilities.tools)
|
|
93
|
+
caps.push(`${result.tools.length} Tools`);
|
|
94
|
+
if (result.capabilities.prompts)
|
|
95
|
+
caps.push(`${result.prompts.length} Prompts`);
|
|
96
|
+
if (result.capabilities.resources)
|
|
97
|
+
caps.push(`${(result.resources ?? []).length} Resources`);
|
|
98
|
+
if (result.capabilities.logging)
|
|
99
|
+
caps.push('Logging');
|
|
100
|
+
lines.push(caps.join(' • ') || 'None discovered');
|
|
101
|
+
lines.push('');
|
|
102
|
+
// Tools section
|
|
103
|
+
if (result.tools.length > 0) {
|
|
104
|
+
lines.push('TOOLS');
|
|
105
|
+
lines.push('─────');
|
|
106
|
+
for (const tool of result.tools) {
|
|
107
|
+
const detail = parseToolDetail(tool);
|
|
108
|
+
const requiredStr = detail.requiredParams.length > 0
|
|
109
|
+
? detail.requiredParams.join(', ')
|
|
110
|
+
: '';
|
|
111
|
+
const optionalStr = detail.optionalParams.length > 0
|
|
112
|
+
? detail.optionalParams.map(p => `${p}?`).join(', ')
|
|
113
|
+
: '';
|
|
114
|
+
const allParams = [requiredStr, optionalStr].filter(Boolean).join(', ');
|
|
115
|
+
lines.push(` ${tool.name}(${allParams})`);
|
|
116
|
+
if (tool.description) {
|
|
117
|
+
// Truncate long descriptions
|
|
118
|
+
const desc = tool.description.length > DISPLAY_LIMITS.DESCRIPTION_MAX_LENGTH
|
|
119
|
+
? tool.description.substring(0, DISPLAY_LIMITS.DESCRIPTION_TRUNCATE_AT) + '...'
|
|
120
|
+
: tool.description;
|
|
121
|
+
lines.push(` └─ ${desc}`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
lines.push('');
|
|
125
|
+
}
|
|
126
|
+
// Prompts section
|
|
127
|
+
if (result.prompts.length > 0) {
|
|
128
|
+
lines.push('PROMPTS');
|
|
129
|
+
lines.push('───────');
|
|
130
|
+
for (const prompt of result.prompts) {
|
|
131
|
+
const args = prompt.arguments?.map(a => {
|
|
132
|
+
return a.required ? a.name : `${a.name}?`;
|
|
133
|
+
}).join(', ') ?? '';
|
|
134
|
+
lines.push(` ${prompt.name}(${args})`);
|
|
135
|
+
if (prompt.description) {
|
|
136
|
+
const desc = prompt.description.length > DISPLAY_LIMITS.DESCRIPTION_MAX_LENGTH
|
|
137
|
+
? prompt.description.substring(0, DISPLAY_LIMITS.DESCRIPTION_TRUNCATE_AT) + '...'
|
|
138
|
+
: prompt.description;
|
|
139
|
+
lines.push(` └─ ${desc}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
lines.push('');
|
|
143
|
+
}
|
|
144
|
+
// Resources section
|
|
145
|
+
const resources = result.resources ?? [];
|
|
146
|
+
if (resources.length > 0) {
|
|
147
|
+
lines.push('RESOURCES');
|
|
148
|
+
lines.push('─────────');
|
|
149
|
+
for (const resource of resources) {
|
|
150
|
+
const mimeType = resource.mimeType ? ` [${resource.mimeType}]` : '';
|
|
151
|
+
lines.push(` ${resource.name}${mimeType}`);
|
|
152
|
+
lines.push(` URI: ${resource.uri}`);
|
|
153
|
+
if (resource.description) {
|
|
154
|
+
const desc = resource.description.length > DISPLAY_LIMITS.DESCRIPTION_MAX_LENGTH
|
|
155
|
+
? resource.description.substring(0, DISPLAY_LIMITS.DESCRIPTION_TRUNCATE_AT) + '...'
|
|
156
|
+
: resource.description;
|
|
157
|
+
lines.push(` └─ ${desc}`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
lines.push('');
|
|
161
|
+
}
|
|
162
|
+
// Quick start hint
|
|
163
|
+
lines.push('QUICK START');
|
|
164
|
+
lines.push('───────────');
|
|
165
|
+
lines.push(` bellwether check ${result.serverCommand} ${result.serverArgs.join(' ')}`);
|
|
166
|
+
lines.push(` bellwether explore ${result.serverCommand} ${result.serverArgs.join(' ')}`);
|
|
167
|
+
lines.push('');
|
|
168
|
+
lines.push(' Commands:');
|
|
169
|
+
lines.push(' check Schema validation and drift detection (free, fast)');
|
|
170
|
+
lines.push(' explore LLM-powered behavioral exploration');
|
|
171
|
+
return lines.join('\n');
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=discovery.js.map
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { MCPTool, MCPPrompt, MCPResource, MCPServerInfo, MCPServerCapabilities } from '../transport/types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Result of discovering an MCP server's capabilities.
|
|
4
|
+
*/
|
|
5
|
+
export interface DiscoveryResult {
|
|
6
|
+
/** Server identification info */
|
|
7
|
+
serverInfo: MCPServerInfo;
|
|
8
|
+
/** Protocol version negotiated */
|
|
9
|
+
protocolVersion: string;
|
|
10
|
+
/** Server capabilities */
|
|
11
|
+
capabilities: MCPServerCapabilities;
|
|
12
|
+
/** List of available tools */
|
|
13
|
+
tools: MCPTool[];
|
|
14
|
+
/** List of available prompts */
|
|
15
|
+
prompts: MCPPrompt[];
|
|
16
|
+
/** List of available resources */
|
|
17
|
+
resources: MCPResource[];
|
|
18
|
+
/** Timestamp of discovery */
|
|
19
|
+
timestamp: Date;
|
|
20
|
+
/** Command used to start the server */
|
|
21
|
+
serverCommand: string;
|
|
22
|
+
/** Arguments passed to the server */
|
|
23
|
+
serverArgs: string[];
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Detailed tool information with parsed schema.
|
|
27
|
+
*/
|
|
28
|
+
export interface ToolDetail {
|
|
29
|
+
name: string;
|
|
30
|
+
description: string;
|
|
31
|
+
inputSchema: ToolInputSchema | null;
|
|
32
|
+
requiredParams: string[];
|
|
33
|
+
optionalParams: string[];
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Parsed JSON Schema for tool inputs.
|
|
37
|
+
*/
|
|
38
|
+
export interface ToolInputSchema {
|
|
39
|
+
type: string;
|
|
40
|
+
properties?: Record<string, PropertySchema>;
|
|
41
|
+
required?: string[];
|
|
42
|
+
additionalProperties?: boolean;
|
|
43
|
+
}
|
|
44
|
+
export interface PropertySchema {
|
|
45
|
+
type: string;
|
|
46
|
+
description?: string;
|
|
47
|
+
enum?: string[];
|
|
48
|
+
default?: unknown;
|
|
49
|
+
items?: PropertySchema;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=types.d.ts.map
|