codeprobe 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +406 -0
- package/dist/cli.d.ts +7 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +104 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/ab.d.ts +7 -0
- package/dist/commands/ab.d.ts.map +1 -0
- package/dist/commands/ab.js +230 -0
- package/dist/commands/ab.js.map +1 -0
- package/dist/commands/agents.d.ts +10 -0
- package/dist/commands/agents.d.ts.map +1 -0
- package/dist/commands/agents.js +326 -0
- package/dist/commands/agents.js.map +1 -0
- package/dist/commands/autotest.d.ts +10 -0
- package/dist/commands/autotest.d.ts.map +1 -0
- package/dist/commands/autotest.js +408 -0
- package/dist/commands/autotest.js.map +1 -0
- package/dist/commands/benchmark.d.ts +6 -0
- package/dist/commands/benchmark.d.ts.map +1 -0
- package/dist/commands/benchmark.js +215 -0
- package/dist/commands/benchmark.js.map +1 -0
- package/dist/commands/check.d.ts +10 -0
- package/dist/commands/check.d.ts.map +1 -0
- package/dist/commands/check.js +333 -0
- package/dist/commands/check.js.map +1 -0
- package/dist/commands/context.d.ts +16 -0
- package/dist/commands/context.d.ts.map +1 -0
- package/dist/commands/context.js +219 -0
- package/dist/commands/context.js.map +1 -0
- package/dist/commands/cost.d.ts +9 -0
- package/dist/commands/cost.d.ts.map +1 -0
- package/dist/commands/cost.js +142 -0
- package/dist/commands/cost.js.map +1 -0
- package/dist/commands/dashboard.d.ts +10 -0
- package/dist/commands/dashboard.d.ts.map +1 -0
- package/dist/commands/dashboard.js +462 -0
- package/dist/commands/dashboard.js.map +1 -0
- package/dist/commands/diff.d.ts +6 -0
- package/dist/commands/diff.d.ts.map +1 -0
- package/dist/commands/diff.js +118 -0
- package/dist/commands/diff.js.map +1 -0
- package/dist/commands/doctor.d.ts +12 -0
- package/dist/commands/doctor.d.ts.map +1 -0
- package/dist/commands/doctor.js +203 -0
- package/dist/commands/doctor.js.map +1 -0
- package/dist/commands/explain.d.ts +7 -0
- package/dist/commands/explain.d.ts.map +1 -0
- package/dist/commands/explain.js +164 -0
- package/dist/commands/explain.js.map +1 -0
- package/dist/commands/flaky.d.ts +10 -0
- package/dist/commands/flaky.d.ts.map +1 -0
- package/dist/commands/flaky.js +141 -0
- package/dist/commands/flaky.js.map +1 -0
- package/dist/commands/generateClaudeMd.d.ts +11 -0
- package/dist/commands/generateClaudeMd.d.ts.map +1 -0
- package/dist/commands/generateClaudeMd.js +278 -0
- package/dist/commands/generateClaudeMd.js.map +1 -0
- package/dist/commands/generateRules.d.ts +11 -0
- package/dist/commands/generateRules.d.ts.map +1 -0
- package/dist/commands/generateRules.js +413 -0
- package/dist/commands/generateRules.js.map +1 -0
- package/dist/commands/heatmap.d.ts +7 -0
- package/dist/commands/heatmap.d.ts.map +1 -0
- package/dist/commands/heatmap.js +117 -0
- package/dist/commands/heatmap.js.map +1 -0
- package/dist/commands/history.d.ts +13 -0
- package/dist/commands/history.d.ts.map +1 -0
- package/dist/commands/history.js +113 -0
- package/dist/commands/history.js.map +1 -0
- package/dist/commands/hooks.d.ts +9 -0
- package/dist/commands/hooks.d.ts.map +1 -0
- package/dist/commands/hooks.js +199 -0
- package/dist/commands/hooks.js.map +1 -0
- package/dist/commands/improve.d.ts +7 -0
- package/dist/commands/improve.d.ts.map +1 -0
- package/dist/commands/improve.js +192 -0
- package/dist/commands/improve.js.map +1 -0
- package/dist/commands/init.d.ts +9 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +270 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/installHook.d.ts +9 -0
- package/dist/commands/installHook.d.ts.map +1 -0
- package/dist/commands/installHook.js +78 -0
- package/dist/commands/installHook.js.map +1 -0
- package/dist/commands/lint.d.ts +6 -0
- package/dist/commands/lint.d.ts.map +1 -0
- package/dist/commands/lint.js +237 -0
- package/dist/commands/lint.js.map +1 -0
- package/dist/commands/map.d.ts +9 -0
- package/dist/commands/map.d.ts.map +1 -0
- package/dist/commands/map.js +114 -0
- package/dist/commands/map.js.map +1 -0
- package/dist/commands/mcp.d.ts +6 -0
- package/dist/commands/mcp.d.ts.map +1 -0
- package/dist/commands/mcp.js +151 -0
- package/dist/commands/mcp.js.map +1 -0
- package/dist/commands/models.d.ts +9 -0
- package/dist/commands/models.d.ts.map +1 -0
- package/dist/commands/models.js +89 -0
- package/dist/commands/models.js.map +1 -0
- package/dist/commands/pack.d.ts +10 -0
- package/dist/commands/pack.d.ts.map +1 -0
- package/dist/commands/pack.js +248 -0
- package/dist/commands/pack.js.map +1 -0
- package/dist/commands/recommend.d.ts +10 -0
- package/dist/commands/recommend.d.ts.map +1 -0
- package/dist/commands/recommend.js +472 -0
- package/dist/commands/recommend.js.map +1 -0
- package/dist/commands/regression.d.ts +10 -0
- package/dist/commands/regression.d.ts.map +1 -0
- package/dist/commands/regression.js +212 -0
- package/dist/commands/regression.js.map +1 -0
- package/dist/commands/repl.d.ts +9 -0
- package/dist/commands/repl.d.ts.map +1 -0
- package/dist/commands/repl.js +245 -0
- package/dist/commands/repl.js.map +1 -0
- package/dist/commands/scan.d.ts +10 -0
- package/dist/commands/scan.d.ts.map +1 -0
- package/dist/commands/scan.js +352 -0
- package/dist/commands/scan.js.map +1 -0
- package/dist/commands/score.d.ts +10 -0
- package/dist/commands/score.d.ts.map +1 -0
- package/dist/commands/score.js +192 -0
- package/dist/commands/score.js.map +1 -0
- package/dist/commands/security.d.ts +10 -0
- package/dist/commands/security.d.ts.map +1 -0
- package/dist/commands/security.js +211 -0
- package/dist/commands/security.js.map +1 -0
- package/dist/commands/simulate.d.ts +7 -0
- package/dist/commands/simulate.d.ts.map +1 -0
- package/dist/commands/simulate.js +149 -0
- package/dist/commands/simulate.js.map +1 -0
- package/dist/commands/summary.d.ts +9 -0
- package/dist/commands/summary.d.ts.map +1 -0
- package/dist/commands/summary.js +271 -0
- package/dist/commands/summary.js.map +1 -0
- package/dist/commands/test.d.ts +9 -0
- package/dist/commands/test.d.ts.map +1 -0
- package/dist/commands/test.js +219 -0
- package/dist/commands/test.js.map +1 -0
- package/dist/commands/ui.d.ts +8 -0
- package/dist/commands/ui.d.ts.map +1 -0
- package/dist/commands/ui.js +222 -0
- package/dist/commands/ui.js.map +1 -0
- package/dist/commands/validate.d.ts +7 -0
- package/dist/commands/validate.d.ts.map +1 -0
- package/dist/commands/validate.js +254 -0
- package/dist/commands/validate.js.map +1 -0
- package/dist/commands/workflow.d.ts +39 -0
- package/dist/commands/workflow.d.ts.map +1 -0
- package/dist/commands/workflow.js +309 -0
- package/dist/commands/workflow.js.map +1 -0
- package/dist/core/__tests__/contextAnalyzer.test.d.ts +2 -0
- package/dist/core/__tests__/contextAnalyzer.test.d.ts.map +1 -0
- package/dist/core/__tests__/contextAnalyzer.test.js +48 -0
- package/dist/core/__tests__/contextAnalyzer.test.js.map +1 -0
- package/dist/core/__tests__/promptLinter.test.d.ts +2 -0
- package/dist/core/__tests__/promptLinter.test.d.ts.map +1 -0
- package/dist/core/__tests__/promptLinter.test.js +74 -0
- package/dist/core/__tests__/promptLinter.test.js.map +1 -0
- package/dist/core/__tests__/promptRunner.test.d.ts +2 -0
- package/dist/core/__tests__/promptRunner.test.d.ts.map +1 -0
- package/dist/core/__tests__/promptRunner.test.js +84 -0
- package/dist/core/__tests__/promptRunner.test.js.map +1 -0
- package/dist/core/__tests__/securityScanner.test.d.ts +2 -0
- package/dist/core/__tests__/securityScanner.test.d.ts.map +1 -0
- package/dist/core/__tests__/securityScanner.test.js +39 -0
- package/dist/core/__tests__/securityScanner.test.js.map +1 -0
- package/dist/core/agentTracer.d.ts +21 -0
- package/dist/core/agentTracer.d.ts.map +1 -0
- package/dist/core/agentTracer.js +355 -0
- package/dist/core/agentTracer.js.map +1 -0
- package/dist/core/anthropicClient.d.ts +26 -0
- package/dist/core/anthropicClient.d.ts.map +1 -0
- package/dist/core/anthropicClient.js +62 -0
- package/dist/core/anthropicClient.js.map +1 -0
- package/dist/core/benchmarkRunner.d.ts +25 -0
- package/dist/core/benchmarkRunner.d.ts.map +1 -0
- package/dist/core/benchmarkRunner.js +182 -0
- package/dist/core/benchmarkRunner.js.map +1 -0
- package/dist/core/contextAnalyzer.d.ts +19 -0
- package/dist/core/contextAnalyzer.d.ts.map +1 -0
- package/dist/core/contextAnalyzer.js +221 -0
- package/dist/core/contextAnalyzer.js.map +1 -0
- package/dist/core/contextPacker.d.ts +26 -0
- package/dist/core/contextPacker.d.ts.map +1 -0
- package/dist/core/contextPacker.js +358 -0
- package/dist/core/contextPacker.js.map +1 -0
- package/dist/core/datasetRunner.d.ts +10 -0
- package/dist/core/datasetRunner.d.ts.map +1 -0
- package/dist/core/datasetRunner.js +130 -0
- package/dist/core/datasetRunner.js.map +1 -0
- package/dist/core/doctorRunner.d.ts +24 -0
- package/dist/core/doctorRunner.d.ts.map +1 -0
- package/dist/core/doctorRunner.js +278 -0
- package/dist/core/doctorRunner.js.map +1 -0
- package/dist/core/hookScanner.d.ts +24 -0
- package/dist/core/hookScanner.d.ts.map +1 -0
- package/dist/core/hookScanner.js +226 -0
- package/dist/core/hookScanner.js.map +1 -0
- package/dist/core/mcpScanner.d.ts +22 -0
- package/dist/core/mcpScanner.d.ts.map +1 -0
- package/dist/core/mcpScanner.js +290 -0
- package/dist/core/mcpScanner.js.map +1 -0
- package/dist/core/modelRegistry.d.ts +35 -0
- package/dist/core/modelRegistry.d.ts.map +1 -0
- package/dist/core/modelRegistry.js +97 -0
- package/dist/core/modelRegistry.js.map +1 -0
- package/dist/core/promptDiff.d.ts +25 -0
- package/dist/core/promptDiff.d.ts.map +1 -0
- package/dist/core/promptDiff.js +130 -0
- package/dist/core/promptDiff.js.map +1 -0
- package/dist/core/promptExplainer.d.ts +17 -0
- package/dist/core/promptExplainer.d.ts.map +1 -0
- package/dist/core/promptExplainer.js +334 -0
- package/dist/core/promptExplainer.js.map +1 -0
- package/dist/core/promptImprover.d.ts +19 -0
- package/dist/core/promptImprover.d.ts.map +1 -0
- package/dist/core/promptImprover.js +260 -0
- package/dist/core/promptImprover.js.map +1 -0
- package/dist/core/promptLinter.d.ts +24 -0
- package/dist/core/promptLinter.d.ts.map +1 -0
- package/dist/core/promptLinter.js +319 -0
- package/dist/core/promptLinter.js.map +1 -0
- package/dist/core/promptRunner.d.ts +31 -0
- package/dist/core/promptRunner.d.ts.map +1 -0
- package/dist/core/promptRunner.js +427 -0
- package/dist/core/promptRunner.js.map +1 -0
- package/dist/core/providers/anthropic.d.ts +10 -0
- package/dist/core/providers/anthropic.d.ts.map +1 -0
- package/dist/core/providers/anthropic.js +26 -0
- package/dist/core/providers/anthropic.js.map +1 -0
- package/dist/core/providers/base.d.ts +22 -0
- package/dist/core/providers/base.d.ts.map +1 -0
- package/dist/core/providers/base.js +2 -0
- package/dist/core/providers/base.js.map +1 -0
- package/dist/core/providers/factory.d.ts +7 -0
- package/dist/core/providers/factory.d.ts.map +1 -0
- package/dist/core/providers/factory.js +42 -0
- package/dist/core/providers/factory.js.map +1 -0
- package/dist/core/providers/google.d.ts +10 -0
- package/dist/core/providers/google.d.ts.map +1 -0
- package/dist/core/providers/google.js +47 -0
- package/dist/core/providers/google.js.map +1 -0
- package/dist/core/providers/openai.d.ts +19 -0
- package/dist/core/providers/openai.d.ts.map +1 -0
- package/dist/core/providers/openai.js +54 -0
- package/dist/core/providers/openai.js.map +1 -0
- package/dist/core/regressionRunner.d.ts +11 -0
- package/dist/core/regressionRunner.d.ts.map +1 -0
- package/dist/core/regressionRunner.js +116 -0
- package/dist/core/regressionRunner.js.map +1 -0
- package/dist/core/repositorySimulator.d.ts +17 -0
- package/dist/core/repositorySimulator.d.ts.map +1 -0
- package/dist/core/repositorySimulator.js +104 -0
- package/dist/core/repositorySimulator.js.map +1 -0
- package/dist/core/scorer.d.ts +30 -0
- package/dist/core/scorer.d.ts.map +1 -0
- package/dist/core/scorer.js +317 -0
- package/dist/core/scorer.js.map +1 -0
- package/dist/core/securityScanner.d.ts +23 -0
- package/dist/core/securityScanner.d.ts.map +1 -0
- package/dist/core/securityScanner.js +216 -0
- package/dist/core/securityScanner.js.map +1 -0
- package/dist/core/skillValidator.d.ts +41 -0
- package/dist/core/skillValidator.d.ts.map +1 -0
- package/dist/core/skillValidator.js +235 -0
- package/dist/core/skillValidator.js.map +1 -0
- package/dist/core/testHistory.d.ts +44 -0
- package/dist/core/testHistory.d.ts.map +1 -0
- package/dist/core/testHistory.js +91 -0
- package/dist/core/testHistory.js.map +1 -0
- package/dist/tokenizers/claudeTokenizer.d.ts +26 -0
- package/dist/tokenizers/claudeTokenizer.d.ts.map +1 -0
- package/dist/tokenizers/claudeTokenizer.js +83 -0
- package/dist/tokenizers/claudeTokenizer.js.map +1 -0
- package/dist/types/agent.d.ts +26 -0
- package/dist/types/agent.d.ts.map +1 -0
- package/dist/types/agent.js +5 -0
- package/dist/types/agent.js.map +1 -0
- package/dist/types/config.d.ts +30 -0
- package/dist/types/config.d.ts.map +1 -0
- package/dist/types/config.js +5 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/context.d.ts +77 -0
- package/dist/types/context.d.ts.map +1 -0
- package/dist/types/context.js +5 -0
- package/dist/types/context.js.map +1 -0
- package/dist/types/dataset.d.ts +26 -0
- package/dist/types/dataset.d.ts.map +1 -0
- package/dist/types/dataset.js +5 -0
- package/dist/types/dataset.js.map +1 -0
- package/dist/types/diagnostics.d.ts +31 -0
- package/dist/types/diagnostics.d.ts.map +1 -0
- package/dist/types/diagnostics.js +5 -0
- package/dist/types/diagnostics.js.map +1 -0
- package/dist/types/prompt.d.ts +53 -0
- package/dist/types/prompt.d.ts.map +1 -0
- package/dist/types/prompt.js +5 -0
- package/dist/types/prompt.js.map +1 -0
- package/dist/types/results.d.ts +42 -0
- package/dist/types/results.d.ts.map +1 -0
- package/dist/types/results.js +5 -0
- package/dist/types/results.js.map +1 -0
- package/dist/ui/dashboard.d.ts +57 -0
- package/dist/ui/dashboard.d.ts.map +1 -0
- package/dist/ui/dashboard.js +644 -0
- package/dist/ui/dashboard.js.map +1 -0
- package/dist/utils/__tests__/hashing.test.d.ts +2 -0
- package/dist/utils/__tests__/hashing.test.d.ts.map +1 -0
- package/dist/utils/__tests__/hashing.test.js +28 -0
- package/dist/utils/__tests__/hashing.test.js.map +1 -0
- package/dist/utils/__tests__/output.test.d.ts +2 -0
- package/dist/utils/__tests__/output.test.d.ts.map +1 -0
- package/dist/utils/__tests__/output.test.js +62 -0
- package/dist/utils/__tests__/output.test.js.map +1 -0
- package/dist/utils/cache.d.ts +29 -0
- package/dist/utils/cache.d.ts.map +1 -0
- package/dist/utils/cache.js +87 -0
- package/dist/utils/cache.js.map +1 -0
- package/dist/utils/config.d.ts +15 -0
- package/dist/utils/config.d.ts.map +1 -0
- package/dist/utils/config.js +61 -0
- package/dist/utils/config.js.map +1 -0
- package/dist/utils/errors.d.ts +43 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +83 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/fs.d.ts +44 -0
- package/dist/utils/fs.d.ts.map +1 -0
- package/dist/utils/fs.js +119 -0
- package/dist/utils/fs.js.map +1 -0
- package/dist/utils/hashing.d.ts +13 -0
- package/dist/utils/hashing.d.ts.map +1 -0
- package/dist/utils/hashing.js +18 -0
- package/dist/utils/hashing.js.map +1 -0
- package/dist/utils/logger.d.ts +32 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +76 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/output.d.ts +34 -0
- package/dist/utils/output.d.ts.map +1 -0
- package/dist/utils/output.js +99 -0
- package/dist/utils/output.js.map +1 -0
- package/dist/utils/paths.d.ts +33 -0
- package/dist/utils/paths.d.ts.map +1 -0
- package/dist/utils/paths.js +51 -0
- package/dist/utils/paths.js.map +1 -0
- package/dist/utils/spinner.d.ts +23 -0
- package/dist/utils/spinner.d.ts.map +1 -0
- package/dist/utils/spinner.js +79 -0
- package/dist/utils/spinner.js.map +1 -0
- package/package.json +60 -0
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `codeprobe ab <prompt-a> <prompt-b>` — A/B test two prompt specs against the
|
|
3
|
+
* same test inputs and compare results side by side.
|
|
4
|
+
*/
|
|
5
|
+
import { parsePromptSpec, runSingleTest, evaluateAssertions } from '../core/promptRunner.js';
|
|
6
|
+
import { resolvePath } from '../utils/paths.js';
|
|
7
|
+
import { formatTable } from '../utils/output.js';
|
|
8
|
+
import { setLogLevel } from '../utils/logger.js';
|
|
9
|
+
import { scoreOutput as canonicalScoreOutput } from '../core/scorer.js';
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
// Scorer — wraps the canonical scorer to produce a numeric score
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
function scoreOutput(output, spec, test) {
|
|
14
|
+
const result = canonicalScoreOutput(output, spec, test);
|
|
15
|
+
return result.overall;
|
|
16
|
+
}
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// Core logic
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
async function runAbTest(specA, specB, runs, options) {
|
|
21
|
+
// Use tests from A, falling back to B's tests if A has none
|
|
22
|
+
const tests = (specA.tests && specA.tests.length > 0)
|
|
23
|
+
? specA.tests
|
|
24
|
+
: (specB.tests ?? []);
|
|
25
|
+
if (tests.length === 0) {
|
|
26
|
+
throw new Error('Neither prompt spec defines tests. At least one spec must include a "tests" section.');
|
|
27
|
+
}
|
|
28
|
+
const rows = [];
|
|
29
|
+
for (const test of tests) {
|
|
30
|
+
let totalScoreA = 0;
|
|
31
|
+
let totalScoreB = 0;
|
|
32
|
+
let lastAssertionsA = [];
|
|
33
|
+
let lastAssertionsB = [];
|
|
34
|
+
let lastOutputA = '';
|
|
35
|
+
let lastOutputB = '';
|
|
36
|
+
for (let run = 0; run < runs; run++) {
|
|
37
|
+
// Run with prompt A
|
|
38
|
+
const resultA = await runSingleTest(specA, test, options);
|
|
39
|
+
const expectA = test.expect;
|
|
40
|
+
const assertionsA = expectA
|
|
41
|
+
? evaluateAssertions(resultA.output, expectA)
|
|
42
|
+
: [];
|
|
43
|
+
totalScoreA += scoreOutput(resultA.output, specA, test);
|
|
44
|
+
lastAssertionsA = assertionsA;
|
|
45
|
+
lastOutputA = resultA.output;
|
|
46
|
+
// Run with prompt B
|
|
47
|
+
const resultB = await runSingleTest(specB, test, options);
|
|
48
|
+
const expectB = test.expect;
|
|
49
|
+
const assertionsB = expectB
|
|
50
|
+
? evaluateAssertions(resultB.output, expectB)
|
|
51
|
+
: [];
|
|
52
|
+
totalScoreB += scoreOutput(resultB.output, specB, test);
|
|
53
|
+
lastAssertionsB = assertionsB;
|
|
54
|
+
lastOutputB = resultB.output;
|
|
55
|
+
}
|
|
56
|
+
const avgScoreA = Math.round(totalScoreA / runs);
|
|
57
|
+
const avgScoreB = Math.round(totalScoreB / runs);
|
|
58
|
+
const delta = avgScoreB - avgScoreA;
|
|
59
|
+
const winner = delta > 0 ? 'B' : delta < 0 ? 'A' : 'tie';
|
|
60
|
+
rows.push({
|
|
61
|
+
testName: test.name,
|
|
62
|
+
scoreA: avgScoreA,
|
|
63
|
+
scoreB: avgScoreB,
|
|
64
|
+
winner,
|
|
65
|
+
delta,
|
|
66
|
+
assertionsA: lastAssertionsA,
|
|
67
|
+
assertionsB: lastAssertionsB,
|
|
68
|
+
outputA: lastOutputA,
|
|
69
|
+
outputB: lastOutputB,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
const winsA = rows.filter((r) => r.winner === 'A').length;
|
|
73
|
+
const winsB = rows.filter((r) => r.winner === 'B').length;
|
|
74
|
+
const ties = rows.filter((r) => r.winner === 'tie').length;
|
|
75
|
+
const avgA = rows.length > 0
|
|
76
|
+
? rows.reduce((sum, r) => sum + r.scoreA, 0) / rows.length
|
|
77
|
+
: 0;
|
|
78
|
+
const avgB = rows.length > 0
|
|
79
|
+
? rows.reduce((sum, r) => sum + r.scoreB, 0) / rows.length
|
|
80
|
+
: 0;
|
|
81
|
+
const avgDiff = avgB - avgA;
|
|
82
|
+
let recommendation;
|
|
83
|
+
if (Math.abs(avgDiff) < 0.5) {
|
|
84
|
+
recommendation = 'Results are effectively tied — no clear winner';
|
|
85
|
+
}
|
|
86
|
+
else if (avgDiff > 0) {
|
|
87
|
+
recommendation = `Prompt B scores higher on average (+${avgDiff.toFixed(1)})`;
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
recommendation = `Prompt A scores higher on average (+${Math.abs(avgDiff).toFixed(1)})`;
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
promptA: specA.name,
|
|
94
|
+
promptB: specB.name,
|
|
95
|
+
tests: rows,
|
|
96
|
+
winsA,
|
|
97
|
+
winsB,
|
|
98
|
+
ties,
|
|
99
|
+
avgA,
|
|
100
|
+
avgB,
|
|
101
|
+
recommendation,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
// ---------------------------------------------------------------------------
|
|
105
|
+
// Command registration
|
|
106
|
+
// ---------------------------------------------------------------------------
|
|
107
|
+
export function registerAbCommand(program) {
|
|
108
|
+
program
|
|
109
|
+
.command('ab <prompt-a> <prompt-b>')
|
|
110
|
+
.description('A/B test two prompt specs — compare scores side by side')
|
|
111
|
+
.option('--json', 'Output as JSON')
|
|
112
|
+
.option('-v, --verbose', 'Show detailed per-test comparison')
|
|
113
|
+
.option('--runs <n>', 'Runs per test', '1')
|
|
114
|
+
.option('--mode <mode>', 'Execution mode: mock or live', 'mock')
|
|
115
|
+
.action(async (pathA, pathB, options) => {
|
|
116
|
+
if (options.json) {
|
|
117
|
+
setLogLevel('silent');
|
|
118
|
+
}
|
|
119
|
+
const chalk = (await import('chalk')).default;
|
|
120
|
+
const resolvedA = resolvePath(pathA);
|
|
121
|
+
const resolvedB = resolvePath(pathB);
|
|
122
|
+
const runs = Math.max(1, parseInt(options.runs, 10) || 1);
|
|
123
|
+
const runOpts = {
|
|
124
|
+
mode: options.mode === 'live' ? 'live' : 'mock',
|
|
125
|
+
verbose: options.verbose,
|
|
126
|
+
};
|
|
127
|
+
// Parse both specs
|
|
128
|
+
let specA;
|
|
129
|
+
let specB;
|
|
130
|
+
try {
|
|
131
|
+
specA = await parsePromptSpec(resolvedA);
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
135
|
+
console.error(chalk.red(`Error parsing prompt A: ${msg}`));
|
|
136
|
+
process.exitCode = 1;
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
try {
|
|
140
|
+
specB = await parsePromptSpec(resolvedB);
|
|
141
|
+
}
|
|
142
|
+
catch (err) {
|
|
143
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
144
|
+
console.error(chalk.red(`Error parsing prompt B: ${msg}`));
|
|
145
|
+
process.exitCode = 1;
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
// Run the A/B comparison
|
|
149
|
+
const result = await runAbTest(specA, specB, runs, runOpts);
|
|
150
|
+
// JSON output
|
|
151
|
+
if (options.json) {
|
|
152
|
+
console.log(JSON.stringify(result, null, 2));
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
// Pretty output
|
|
156
|
+
console.log('');
|
|
157
|
+
console.log(chalk.bold('A/B Comparison'));
|
|
158
|
+
console.log('');
|
|
159
|
+
console.log(` Prompt A: ${chalk.cyan(result.promptA)}`);
|
|
160
|
+
console.log(` Prompt B: ${chalk.cyan(result.promptB)}`);
|
|
161
|
+
if (runs > 1) {
|
|
162
|
+
console.log(chalk.dim(` Runs per test: ${runs}`));
|
|
163
|
+
}
|
|
164
|
+
console.log('');
|
|
165
|
+
// Build comparison table
|
|
166
|
+
const tableRows = result.tests.map((row) => {
|
|
167
|
+
const winnerLabel = row.winner === 'tie'
|
|
168
|
+
? chalk.dim('tie')
|
|
169
|
+
: row.winner === 'A'
|
|
170
|
+
? chalk.green('A')
|
|
171
|
+
: chalk.green('B');
|
|
172
|
+
const deltaStr = row.delta === 0
|
|
173
|
+
? chalk.dim('0')
|
|
174
|
+
: row.delta > 0
|
|
175
|
+
? chalk.green(`+${row.delta}`)
|
|
176
|
+
: chalk.red(`${row.delta}`);
|
|
177
|
+
return [
|
|
178
|
+
row.testName,
|
|
179
|
+
String(row.scoreA),
|
|
180
|
+
String(row.scoreB),
|
|
181
|
+
winnerLabel,
|
|
182
|
+
deltaStr,
|
|
183
|
+
];
|
|
184
|
+
});
|
|
185
|
+
const table = formatTable(['Test', 'A Score', 'B Score', 'Winner', 'Delta'], tableRows);
|
|
186
|
+
for (const line of table.split('\n')) {
|
|
187
|
+
console.log(` ${line}`);
|
|
188
|
+
}
|
|
189
|
+
// Verbose: per-test details
|
|
190
|
+
if (options.verbose) {
|
|
191
|
+
console.log('');
|
|
192
|
+
console.log(chalk.bold(' Per-Test Details'));
|
|
193
|
+
for (const row of result.tests) {
|
|
194
|
+
console.log('');
|
|
195
|
+
console.log(chalk.bold(` ${row.testName}`));
|
|
196
|
+
console.log(chalk.dim(` Output A (${row.outputA.length} chars):`));
|
|
197
|
+
console.log(` ${row.outputA.slice(0, 200).replace(/\n/g, '\n ')}`);
|
|
198
|
+
console.log(chalk.dim(` Output B (${row.outputB.length} chars):`));
|
|
199
|
+
console.log(` ${row.outputB.slice(0, 200).replace(/\n/g, '\n ')}`);
|
|
200
|
+
if (row.assertionsA.length > 0) {
|
|
201
|
+
console.log(chalk.dim(' Assertions A:'));
|
|
202
|
+
for (const a of row.assertionsA) {
|
|
203
|
+
const icon = a.passed ? chalk.green('PASS') : chalk.red('FAIL');
|
|
204
|
+
console.log(` ${icon} [${a.type}] expected: ${a.expected}`);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (row.assertionsB.length > 0) {
|
|
208
|
+
console.log(chalk.dim(' Assertions B:'));
|
|
209
|
+
for (const a of row.assertionsB) {
|
|
210
|
+
const icon = a.passed ? chalk.green('PASS') : chalk.red('FAIL');
|
|
211
|
+
console.log(` ${icon} [${a.type}] expected: ${a.expected}`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
// Summary
|
|
217
|
+
console.log('');
|
|
218
|
+
console.log(chalk.bold(' Summary:'));
|
|
219
|
+
console.log(` Prompt A wins: ${result.winsA} test${result.winsA !== 1 ? 's' : ''}`);
|
|
220
|
+
console.log(` Prompt B wins: ${result.winsB} test${result.winsB !== 1 ? 's' : ''}`);
|
|
221
|
+
if (result.ties > 0) {
|
|
222
|
+
console.log(` Ties: ${result.ties} test${result.ties !== 1 ? 's' : ''}`);
|
|
223
|
+
}
|
|
224
|
+
console.log(` Average A: ${result.avgA.toFixed(1)} Average B: ${result.avgB.toFixed(1)}`);
|
|
225
|
+
console.log('');
|
|
226
|
+
console.log(` Recommendation: ${result.recommendation}`);
|
|
227
|
+
console.log('');
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
//# sourceMappingURL=ab.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ab.js","sourceRoot":"","sources":["../../src/commands/ab.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGjD,OAAO,EAAE,WAAW,IAAI,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAExE,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,SAAS,WAAW,CAAC,MAAc,EAAE,IAAgB,EAAE,IAAgB;IACrE,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AA8BD,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,KAAK,UAAU,SAAS,CACtB,KAAiB,EACjB,KAAiB,EACjB,IAAY,EACZ,OAAmB;IAEnB,4DAA4D;IAC5D,MAAM,KAAK,GAAiB,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACjE,CAAC,CAAC,KAAK,CAAC,KAAK;QACb,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAExB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,eAAe,GAAsB,EAAE,CAAC;QAC5C,IAAI,eAAe,GAAsB,EAAE,CAAC;QAC5C,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;YACpC,oBAAoB;YACpB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;YAC5B,MAAM,WAAW,GAAG,OAAO;gBACzB,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;gBAC7C,CAAC,CAAC,EAAE,CAAC;YACP,WAAW,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACxD,eAAe,GAAG,WAAW,CAAC;YAC9B,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAE7B,oBAAoB;YACpB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;YAC5B,MAAM,WAAW,GAAG,OAAO;gBACzB,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;gBAC7C,CAAC,CAAC,EAAE,CAAC;YACP,WAAW,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACxD,eAAe,GAAG,WAAW,CAAC;YAC9B,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,GAAG,SAAS,CAAC;QACpC,MAAM,MAAM,GACV,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC;YACR,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,SAAS;YACjB,MAAM;YACN,KAAK;YACL,WAAW,EAAE,eAAe;YAC5B,WAAW,EAAE,eAAe;YAC5B,OAAO,EAAE,WAAW;YACpB,OAAO,EAAE,WAAW;SACrB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;QAC1B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;QAC1D,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;QAC1B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;QAC1D,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;IAC5B,IAAI,cAAsB,CAAC;IAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC;QAC5B,cAAc,GAAG,gDAAgD,CAAC;IACpE,CAAC;SAAM,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACvB,cAAc,GAAG,uCAAuC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAChF,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,uCAAuC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC1F,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,IAAI;QACnB,OAAO,EAAE,KAAK,CAAC,IAAI;QACnB,KAAK,EAAE,IAAI;QACX,KAAK;QACL,KAAK;QACL,IAAI;QACJ,IAAI;QACJ,IAAI;QACJ,cAAc;KACf,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,UAAU,iBAAiB,CAAC,OAAgB;IAChD,OAAO;SACJ,OAAO,CAAC,0BAA0B,CAAC;SACnC,WAAW,CAAC,yDAAyD,CAAC;SACtE,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;SAClC,MAAM,CAAC,eAAe,EAAE,mCAAmC,CAAC;SAC5D,MAAM,CAAC,YAAY,EAAE,eAAe,EAAE,GAAG,CAAC;SAC1C,MAAM,CAAC,eAAe,EAAE,8BAA8B,EAAE,MAAM,CAAC;SAC/D,MAAM,CAAC,KAAK,EACX,KAAa,EACb,KAAa,EACb,OAKC,EACD,EAAE;QACF,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,WAAW,CAAC,QAAQ,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;QAE9C,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAE1D,MAAM,OAAO,GAAe;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC/C,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC;QAEF,mBAAmB;QACnB,IAAI,KAAiB,CAAC;QACtB,IAAI,KAAiB,CAAC;QACtB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC,CAAC;YAC3D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC,CAAC;YAC3D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,yBAAyB;QACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAE5D,cAAc;QACd,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,gBAAgB;QAChB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACzD,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEhB,yBAAyB;QACzB,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACzC,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,KAAK,KAAK;gBACtC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;gBAClB,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG;oBAClB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;oBAClB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,KAAK,CAAC;gBAC9B,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAChB,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC;oBACb,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBAC9B,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAEhC,OAAO;gBACL,GAAG,CAAC,QAAQ;gBACZ,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClB,WAAW;gBACX,QAAQ;aACT,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,WAAW,CACvB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,EACjD,SAAS,CACV,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,4BAA4B;QAC5B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC9C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC/B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC;gBAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC;gBAE7E,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC;oBAC5C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;wBAChC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;wBAChE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC,IAAI,eAAe,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACnE,CAAC;gBACH,CAAC;gBACD,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC;oBAC5C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;wBAChC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;wBAChE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC,IAAI,eAAe,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACnE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,UAAU;QACV,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,KAAK,QAAQ,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,KAAK,QAAQ,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,IAAI,QAAQ,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `codeprobe agents [path]` — Scan for AI coding tool assets.
|
|
3
|
+
*
|
|
4
|
+
* Detects configuration and workflow files for Claude Code, Cursor,
|
|
5
|
+
* Windsurf, GitHub Copilot, Aider, Continue.dev, Cline, OpenAI Codex,
|
|
6
|
+
* and other AI coding tools.
|
|
7
|
+
*/
|
|
8
|
+
import { Command } from 'commander';
|
|
9
|
+
export declare function registerAgentsCommand(program: Command): void;
|
|
10
|
+
//# sourceMappingURL=agents.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agents.d.ts","sourceRoot":"","sources":["../../src/commands/agents.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA+QpC,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAwE5D"}
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `codeprobe agents [path]` — Scan for AI coding tool assets.
|
|
3
|
+
*
|
|
4
|
+
* Detects configuration and workflow files for Claude Code, Cursor,
|
|
5
|
+
* Windsurf, GitHub Copilot, Aider, Continue.dev, Cline, OpenAI Codex,
|
|
6
|
+
* and other AI coding tools.
|
|
7
|
+
*/
|
|
8
|
+
import { resolvePath } from '../utils/paths.js';
|
|
9
|
+
import { walkDirectory, getRelativePath } from '../utils/fs.js';
|
|
10
|
+
import { readFile } from 'node:fs/promises';
|
|
11
|
+
import { setLogLevel } from '../utils/logger.js';
|
|
12
|
+
const DEFAULT_IGNORE_DIRS = new Set([
|
|
13
|
+
'node_modules', '.git', 'dist', 'build', 'coverage',
|
|
14
|
+
'.cache', '.turbo',
|
|
15
|
+
]);
|
|
16
|
+
/** Patterns for identifying Claude assets. */
|
|
17
|
+
const ASSET_PATTERNS = [
|
|
18
|
+
{
|
|
19
|
+
namePattern: /^CLAUDE\.md$/i,
|
|
20
|
+
type: 'claude-config',
|
|
21
|
+
confidence: 'high',
|
|
22
|
+
reason: 'Claude Code configuration file',
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
namePattern: /^\.claude\/?/,
|
|
26
|
+
type: 'claude-config',
|
|
27
|
+
confidence: 'high',
|
|
28
|
+
reason: 'Claude configuration directory',
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
namePattern: /\.prompt\.(ya?ml|json)$/i,
|
|
32
|
+
type: 'prompt-spec',
|
|
33
|
+
confidence: 'high',
|
|
34
|
+
reason: 'Prompt specification file',
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
namePattern: /\.skill\.(ya?ml|md)$/i,
|
|
38
|
+
type: 'skill',
|
|
39
|
+
confidence: 'high',
|
|
40
|
+
reason: 'Skill definition file',
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
namePattern: /mcp\.json$/i,
|
|
44
|
+
type: 'mcp-config',
|
|
45
|
+
confidence: 'high',
|
|
46
|
+
reason: 'MCP configuration file',
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
namePattern: /\.mcp\.(ya?ml|json)$/i,
|
|
50
|
+
type: 'mcp-config',
|
|
51
|
+
confidence: 'high',
|
|
52
|
+
reason: 'MCP configuration file',
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
namePattern: /hooks?\.(ya?ml|json|ts|js)$/i,
|
|
56
|
+
type: 'hook',
|
|
57
|
+
confidence: 'medium',
|
|
58
|
+
reason: 'Possible hook configuration',
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
namePattern: /codeprobe\.config\.(ya?ml|json)$/i,
|
|
62
|
+
type: 'context-file',
|
|
63
|
+
confidence: 'high',
|
|
64
|
+
reason: 'codeprobe configuration file',
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
namePattern: /AGENTS?\.md$/i,
|
|
68
|
+
type: 'agent',
|
|
69
|
+
confidence: 'medium',
|
|
70
|
+
reason: 'Agent instructions file',
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
namePattern: /CONTEXT\.md$/i,
|
|
74
|
+
type: 'context-file',
|
|
75
|
+
confidence: 'medium',
|
|
76
|
+
reason: 'Context documentation file',
|
|
77
|
+
},
|
|
78
|
+
// ── Cursor ──
|
|
79
|
+
{
|
|
80
|
+
namePattern: /^\.cursorrules$/,
|
|
81
|
+
type: 'cursor-config',
|
|
82
|
+
confidence: 'high',
|
|
83
|
+
reason: 'Cursor rules configuration file',
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
namePattern: /^\.cursor[/\\]rules[/\\]/,
|
|
87
|
+
type: 'cursor-config',
|
|
88
|
+
confidence: 'high',
|
|
89
|
+
reason: 'Cursor rule definition',
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
namePattern: /^\.cursor[/\\]mcp\.json$/,
|
|
93
|
+
type: 'mcp-config',
|
|
94
|
+
confidence: 'high',
|
|
95
|
+
reason: 'MCP configuration file for Cursor',
|
|
96
|
+
},
|
|
97
|
+
// ── Windsurf ──
|
|
98
|
+
{
|
|
99
|
+
namePattern: /^\.windsurfrules$/,
|
|
100
|
+
type: 'windsurf-config',
|
|
101
|
+
confidence: 'high',
|
|
102
|
+
reason: 'Windsurf rules configuration file',
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
namePattern: /^\.windsurf[/\\]rules[/\\]/,
|
|
106
|
+
type: 'windsurf-config',
|
|
107
|
+
confidence: 'high',
|
|
108
|
+
reason: 'Windsurf rule definition',
|
|
109
|
+
},
|
|
110
|
+
// ── GitHub Copilot ──
|
|
111
|
+
{
|
|
112
|
+
namePattern: /^\.github[/\\]copilot-instructions\.md$/,
|
|
113
|
+
type: 'copilot-config',
|
|
114
|
+
confidence: 'high',
|
|
115
|
+
reason: 'GitHub Copilot custom instructions',
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
namePattern: /^\.copilot[/\\]/,
|
|
119
|
+
type: 'copilot-config',
|
|
120
|
+
confidence: 'medium',
|
|
121
|
+
reason: 'GitHub Copilot configuration',
|
|
122
|
+
},
|
|
123
|
+
// ── Aider ──
|
|
124
|
+
{
|
|
125
|
+
namePattern: /^\.aider\.conf\.yml$/,
|
|
126
|
+
type: 'aider-config',
|
|
127
|
+
confidence: 'high',
|
|
128
|
+
reason: 'Aider configuration file',
|
|
129
|
+
},
|
|
130
|
+
{
|
|
131
|
+
namePattern: /^\.aiderignore$/,
|
|
132
|
+
type: 'aider-config',
|
|
133
|
+
confidence: 'high',
|
|
134
|
+
reason: 'Aider ignore file',
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
namePattern: /^\.aider\.model\.settings\.yml$/,
|
|
138
|
+
type: 'aider-config',
|
|
139
|
+
confidence: 'high',
|
|
140
|
+
reason: 'Aider model settings file',
|
|
141
|
+
},
|
|
142
|
+
// ── Continue.dev ──
|
|
143
|
+
{
|
|
144
|
+
namePattern: /^\.continue[/\\]config\.json$/,
|
|
145
|
+
type: 'continue-config',
|
|
146
|
+
confidence: 'high',
|
|
147
|
+
reason: 'Continue.dev configuration file',
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
namePattern: /^\.continuerules$/,
|
|
151
|
+
type: 'continue-config',
|
|
152
|
+
confidence: 'high',
|
|
153
|
+
reason: 'Continue.dev rules file',
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
namePattern: /^\.continue[/\\]/,
|
|
157
|
+
type: 'continue-config',
|
|
158
|
+
confidence: 'medium',
|
|
159
|
+
reason: 'Continue.dev configuration',
|
|
160
|
+
},
|
|
161
|
+
// ── Cline ──
|
|
162
|
+
{
|
|
163
|
+
namePattern: /^\.clinerules$/,
|
|
164
|
+
type: 'cline-config',
|
|
165
|
+
confidence: 'high',
|
|
166
|
+
reason: 'Cline rules configuration file',
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
namePattern: /^\.cline[/\\]/,
|
|
170
|
+
type: 'cline-config',
|
|
171
|
+
confidence: 'medium',
|
|
172
|
+
reason: 'Cline configuration',
|
|
173
|
+
},
|
|
174
|
+
// ── OpenAI Codex CLI ──
|
|
175
|
+
{
|
|
176
|
+
namePattern: /^[Cc][Oo][Dd][Ee][Xx]\.md$/,
|
|
177
|
+
type: 'codex-config',
|
|
178
|
+
confidence: 'high',
|
|
179
|
+
reason: 'OpenAI Codex CLI instruction file',
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
namePattern: /^\.codex[/\\]/,
|
|
183
|
+
type: 'codex-config',
|
|
184
|
+
confidence: 'medium',
|
|
185
|
+
reason: 'OpenAI Codex configuration',
|
|
186
|
+
},
|
|
187
|
+
// ── General AI ──
|
|
188
|
+
{
|
|
189
|
+
namePattern: /^AI\.md$/,
|
|
190
|
+
type: 'context-file',
|
|
191
|
+
confidence: 'medium',
|
|
192
|
+
reason: 'AI instructions file',
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
namePattern: /^\.ai[/\\]/,
|
|
196
|
+
type: 'context-file',
|
|
197
|
+
confidence: 'medium',
|
|
198
|
+
reason: 'AI configuration directory',
|
|
199
|
+
},
|
|
200
|
+
// ── Agentic workflow ──
|
|
201
|
+
{
|
|
202
|
+
namePattern: /^tasks[/\\]todo\.md$/i,
|
|
203
|
+
type: 'agentic-workflow',
|
|
204
|
+
confidence: 'high',
|
|
205
|
+
reason: 'Agentic workflow task list',
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
namePattern: /^tasks[/\\]lessons\.md$/i,
|
|
209
|
+
type: 'agentic-workflow',
|
|
210
|
+
confidence: 'high',
|
|
211
|
+
reason: 'Agentic workflow lessons learned',
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
namePattern: /^PLAN\.md$/,
|
|
215
|
+
type: 'agentic-workflow',
|
|
216
|
+
confidence: 'medium',
|
|
217
|
+
reason: 'Agentic workflow plan file',
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
namePattern: /^plans[/\\].*\.md$/i,
|
|
221
|
+
type: 'agentic-workflow',
|
|
222
|
+
confidence: 'medium',
|
|
223
|
+
reason: 'Agentic workflow plan',
|
|
224
|
+
},
|
|
225
|
+
];
|
|
226
|
+
/**
|
|
227
|
+
* Scan a directory for Claude-related assets.
|
|
228
|
+
*/
|
|
229
|
+
async function agentTracer(rootPath) {
|
|
230
|
+
const entries = await walkDirectory(rootPath, { ignoreDirs: DEFAULT_IGNORE_DIRS });
|
|
231
|
+
const assets = [];
|
|
232
|
+
for (const entry of entries) {
|
|
233
|
+
const relPath = getRelativePath(rootPath, entry.path);
|
|
234
|
+
const fileName = entry.path.split('/').pop() ?? '';
|
|
235
|
+
for (const pattern of ASSET_PATTERNS) {
|
|
236
|
+
if (pattern.namePattern.test(fileName) || pattern.namePattern.test(relPath)) {
|
|
237
|
+
let metadata;
|
|
238
|
+
// Try to extract metadata from the file
|
|
239
|
+
if (entry.isFile && entry.size < 50_000) {
|
|
240
|
+
try {
|
|
241
|
+
const content = await readFile(entry.path, 'utf-8');
|
|
242
|
+
metadata = {
|
|
243
|
+
sizeBytes: entry.size,
|
|
244
|
+
lineCount: content.split('\n').length,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
catch {
|
|
248
|
+
// Skip metadata extraction
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
assets.push({
|
|
252
|
+
path: relPath,
|
|
253
|
+
type: pattern.type,
|
|
254
|
+
confidence: pattern.confidence,
|
|
255
|
+
reason: pattern.reason,
|
|
256
|
+
metadata,
|
|
257
|
+
});
|
|
258
|
+
break; // Only match the first pattern per file
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return assets;
|
|
263
|
+
}
|
|
264
|
+
export function registerAgentsCommand(program) {
|
|
265
|
+
program
|
|
266
|
+
.command('agents [path]')
|
|
267
|
+
.description('Scan for AI coding tool assets — Claude, Cursor, Windsurf, Copilot, and more')
|
|
268
|
+
.option('--json', 'Output findings as JSON')
|
|
269
|
+
.action(async (pathArg, options) => {
|
|
270
|
+
if (options.json) {
|
|
271
|
+
setLogLevel('silent');
|
|
272
|
+
}
|
|
273
|
+
const chalk = (await import('chalk')).default;
|
|
274
|
+
const targetPath = resolvePath(pathArg ?? '.');
|
|
275
|
+
const assets = await agentTracer(targetPath);
|
|
276
|
+
if (options.json) {
|
|
277
|
+
console.log(JSON.stringify(assets, null, 2));
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
if (assets.length === 0) {
|
|
281
|
+
console.log(chalk.dim('\nNo AI coding tool assets found.'));
|
|
282
|
+
console.log(chalk.dim('Run `codeprobe init` to create starter files.\n'));
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
console.log(chalk.bold(`\nAI Tool Assets (${assets.length} found)`));
|
|
286
|
+
console.log('');
|
|
287
|
+
// Group by type
|
|
288
|
+
const grouped = new Map();
|
|
289
|
+
for (const asset of assets) {
|
|
290
|
+
const list = grouped.get(asset.type) ?? [];
|
|
291
|
+
list.push(asset);
|
|
292
|
+
grouped.set(asset.type, list);
|
|
293
|
+
}
|
|
294
|
+
const typeLabels = {
|
|
295
|
+
'claude-config': 'Claude Code',
|
|
296
|
+
'cursor-config': 'Cursor',
|
|
297
|
+
'windsurf-config': 'Windsurf',
|
|
298
|
+
'copilot-config': 'GitHub Copilot',
|
|
299
|
+
'aider-config': 'Aider',
|
|
300
|
+
'continue-config': 'Continue.dev',
|
|
301
|
+
'cline-config': 'Cline',
|
|
302
|
+
'codex-config': 'OpenAI Codex',
|
|
303
|
+
'agent': 'Agents',
|
|
304
|
+
'skill': 'Skills',
|
|
305
|
+
'hook': 'Hooks',
|
|
306
|
+
'mcp-config': 'MCP Configs',
|
|
307
|
+
'prompt-spec': 'Prompt Specs',
|
|
308
|
+
'context-file': 'Context Files',
|
|
309
|
+
'agentic-workflow': 'Agentic Workflow',
|
|
310
|
+
'other': 'Other',
|
|
311
|
+
};
|
|
312
|
+
for (const [type, items] of grouped) {
|
|
313
|
+
console.log(chalk.bold(` ${typeLabels[type] ?? type}`));
|
|
314
|
+
for (const item of items) {
|
|
315
|
+
const confColor = item.confidence === 'high'
|
|
316
|
+
? chalk.green
|
|
317
|
+
: item.confidence === 'medium'
|
|
318
|
+
? chalk.yellow
|
|
319
|
+
: chalk.dim;
|
|
320
|
+
console.log(` ${item.path} ${confColor(`[${item.confidence}]`)} ${chalk.dim(item.reason)}`);
|
|
321
|
+
}
|
|
322
|
+
console.log('');
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
//# sourceMappingURL=agents.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agents.js","sourceRoot":"","sources":["../../src/commands/agents.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU;IACnD,QAAQ,EAAE,QAAQ;CACnB,CAAC,CAAC;AAEH,8CAA8C;AAC9C,MAAM,cAAc,GAKf;IACH;QACE,WAAW,EAAE,eAAe;QAC5B,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,gCAAgC;KACzC;IACD;QACE,WAAW,EAAE,cAAc;QAC3B,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,gCAAgC;KACzC;IACD;QACE,WAAW,EAAE,0BAA0B;QACvC,IAAI,EAAE,aAAa;QACnB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,2BAA2B;KACpC;IACD;QACE,WAAW,EAAE,uBAAuB;QACpC,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,uBAAuB;KAChC;IACD;QACE,WAAW,EAAE,aAAa;QAC1B,IAAI,EAAE,YAAY;QAClB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,wBAAwB;KACjC;IACD;QACE,WAAW,EAAE,uBAAuB;QACpC,IAAI,EAAE,YAAY;QAClB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,wBAAwB;KACjC;IACD;QACE,WAAW,EAAE,8BAA8B;QAC3C,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,6BAA6B;KACtC;IACD;QACE,WAAW,EAAE,mCAAmC;QAChD,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,8BAA8B;KACvC;IACD;QACE,WAAW,EAAE,eAAe;QAC5B,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,yBAAyB;KAClC;IACD;QACE,WAAW,EAAE,eAAe;QAC5B,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,4BAA4B;KACrC;IACD,eAAe;IACf;QACE,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,iCAAiC;KAC1C;IACD;QACE,WAAW,EAAE,0BAA0B;QACvC,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,wBAAwB;KACjC;IACD;QACE,WAAW,EAAE,0BAA0B;QACvC,IAAI,EAAE,YAAY;QAClB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,mCAAmC;KAC5C;IACD,iBAAiB;IACjB;QACE,WAAW,EAAE,mBAAmB;QAChC,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,mCAAmC;KAC5C;IACD;QACE,WAAW,EAAE,4BAA4B;QACzC,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,0BAA0B;KACnC;IACD,uBAAuB;IACvB;QACE,WAAW,EAAE,yCAAyC;QACtD,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,oCAAoC;KAC7C;IACD;QACE,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,8BAA8B;KACvC;IACD,cAAc;IACd;QACE,WAAW,EAAE,sBAAsB;QACnC,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,0BAA0B;KACnC;IACD;QACE,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,mBAAmB;KAC5B;IACD;QACE,WAAW,EAAE,iCAAiC;QAC9C,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,2BAA2B;KACpC;IACD,qBAAqB;IACrB;QACE,WAAW,EAAE,+BAA+B;QAC5C,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,iCAAiC;KAC1C;IACD;QACE,WAAW,EAAE,mBAAmB;QAChC,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,yBAAyB;KAClC;IACD;QACE,WAAW,EAAE,kBAAkB;QAC/B,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,4BAA4B;KACrC;IACD,cAAc;IACd;QACE,WAAW,EAAE,gBAAgB;QAC7B,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,gCAAgC;KACzC;IACD;QACE,WAAW,EAAE,eAAe;QAC5B,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,qBAAqB;KAC9B;IACD,yBAAyB;IACzB;QACE,WAAW,EAAE,4BAA4B;QACzC,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,mCAAmC;KAC5C;IACD;QACE,WAAW,EAAE,eAAe;QAC5B,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,4BAA4B;KACrC;IACD,mBAAmB;IACnB;QACE,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,sBAAsB;KAC/B;IACD;QACE,WAAW,EAAE,YAAY;QACzB,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,4BAA4B;KACrC;IACD,yBAAyB;IACzB;QACE,WAAW,EAAE,uBAAuB;QACpC,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,4BAA4B;KACrC;IACD;QACE,WAAW,EAAE,0BAA0B;QACvC,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,kCAAkC;KAC3C;IACD;QACE,WAAW,EAAE,YAAY;QACzB,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,4BAA4B;KACrC;IACD;QACE,WAAW,EAAE,qBAAqB;QAClC,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,uBAAuB;KAChC;CACF,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,QAAgB;IACzC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACnF,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAEnD,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;YACrC,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5E,IAAI,QAA6C,CAAC;gBAElD,wCAAwC;gBACxC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,CAAC;oBACxC,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;wBACpD,QAAQ,GAAG;4BACT,SAAS,EAAE,KAAK,CAAC,IAAI;4BACrB,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;yBACtC,CAAC;oBACJ,CAAC;oBAAC,MAAM,CAAC;wBACP,2BAA2B;oBAC7B,CAAC;gBACH,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,QAAQ;iBACT,CAAC,CAAC;gBACH,MAAM,CAAC,wCAAwC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,OAAO;SACJ,OAAO,CAAC,eAAe,CAAC;SACxB,WAAW,CAAC,8EAA8E,CAAC;SAC3F,MAAM,CAAC,QAAQ,EAAE,yBAAyB,CAAC;SAC3C,MAAM,CAAC,KAAK,EACX,OAA2B,EAC3B,OAA2B,EAC3B,EAAE;QACF,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,WAAW,CAAC,QAAQ,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;QAC9C,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;QAE7C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC,CAAC;YAC5D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC,CAAC;YAC1E,OAAO;QACT,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEhB,gBAAgB;QAChB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkC,CAAC;QAC1D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAoC;YAClD,eAAe,EAAE,aAAa;YAC9B,eAAe,EAAE,QAAQ;YACzB,iBAAiB,EAAE,UAAU;YAC7B,gBAAgB,EAAE,gBAAgB;YAClC,cAAc,EAAE,OAAO;YACvB,iBAAiB,EAAE,cAAc;YACjC,cAAc,EAAE,OAAO;YACvB,cAAc,EAAE,cAAc;YAC9B,OAAO,EAAE,QAAQ;YACjB,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,aAAa;YAC3B,aAAa,EAAE,cAAc;YAC7B,cAAc,EAAE,eAAe;YAC/B,kBAAkB,EAAE,kBAAkB;YACtC,OAAO,EAAE,OAAO;SACjB,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;YACzD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,KAAK,MAAM;oBAC1C,CAAC,CAAC,KAAK,CAAC,KAAK;oBACb,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,QAAQ;wBAC5B,CAAC,CAAC,KAAK,CAAC,MAAM;wBACd,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `codeprobe autotest <prompt-file>` — Auto-generate diverse test cases
|
|
3
|
+
* for a prompt spec by analyzing its structure offline (no API calls).
|
|
4
|
+
*
|
|
5
|
+
* Generates edge cases, format validation, boundary tests, negative tests,
|
|
6
|
+
* language tests, and injection resistance tests.
|
|
7
|
+
*/
|
|
8
|
+
import { Command } from 'commander';
|
|
9
|
+
export declare function registerAutotestCommand(program: Command): void;
|
|
10
|
+
//# sourceMappingURL=autotest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"autotest.d.ts","sourceRoot":"","sources":["../../src/commands/autotest.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoYpC,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAiH9D"}
|