claudecode-omc 4.7.4 → 4.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +50 -0
- package/agents/test-engineer.md +74 -0
- package/bridge/cli.cjs +9335 -117
- package/dist/cli/index.js +201 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/testing/analyzers/complexity.d.ts +18 -0
- package/dist/testing/analyzers/complexity.d.ts.map +1 -0
- package/dist/testing/analyzers/complexity.js +121 -0
- package/dist/testing/analyzers/complexity.js.map +1 -0
- package/dist/testing/analyzers/coverage.d.ts +13 -0
- package/dist/testing/analyzers/coverage.d.ts.map +1 -0
- package/dist/testing/analyzers/coverage.js +99 -0
- package/dist/testing/analyzers/coverage.js.map +1 -0
- package/dist/testing/analyzers/quality-scorer.d.ts +8 -0
- package/dist/testing/analyzers/quality-scorer.d.ts.map +1 -0
- package/dist/testing/analyzers/quality-scorer.js +128 -0
- package/dist/testing/analyzers/quality-scorer.js.map +1 -0
- package/dist/testing/analyzers/types.d.ts +56 -0
- package/dist/testing/analyzers/types.d.ts.map +1 -0
- package/dist/testing/analyzers/types.js +2 -0
- package/dist/testing/analyzers/types.js.map +1 -0
- package/dist/testing/cli/agent-integration.d.ts +20 -0
- package/dist/testing/cli/agent-integration.d.ts.map +1 -0
- package/dist/testing/cli/agent-integration.js +60 -0
- package/dist/testing/cli/agent-integration.js.map +1 -0
- package/dist/testing/cli/commands.d.ts +100 -0
- package/dist/testing/cli/commands.d.ts.map +1 -0
- package/dist/testing/cli/commands.js +250 -0
- package/dist/testing/cli/commands.js.map +1 -0
- package/dist/testing/cli/ultraqa-integration.d.ts +13 -0
- package/dist/testing/cli/ultraqa-integration.d.ts.map +1 -0
- package/dist/testing/cli/ultraqa-integration.js +68 -0
- package/dist/testing/cli/ultraqa-integration.js.map +1 -0
- package/dist/testing/detectors/go.d.ts +3 -0
- package/dist/testing/detectors/go.d.ts.map +1 -0
- package/dist/testing/detectors/go.js +38 -0
- package/dist/testing/detectors/go.js.map +1 -0
- package/dist/testing/detectors/index.d.ts +8 -0
- package/dist/testing/detectors/index.d.ts.map +1 -0
- package/dist/testing/detectors/index.js +46 -0
- package/dist/testing/detectors/index.js.map +1 -0
- package/dist/testing/detectors/package-json.d.ts +3 -0
- package/dist/testing/detectors/package-json.d.ts.map +1 -0
- package/dist/testing/detectors/package-json.js +52 -0
- package/dist/testing/detectors/package-json.js.map +1 -0
- package/dist/testing/detectors/python.d.ts +3 -0
- package/dist/testing/detectors/python.d.ts.map +1 -0
- package/dist/testing/detectors/python.js +37 -0
- package/dist/testing/detectors/python.js.map +1 -0
- package/dist/testing/detectors/rust.d.ts +3 -0
- package/dist/testing/detectors/rust.d.ts.map +1 -0
- package/dist/testing/detectors/rust.js +39 -0
- package/dist/testing/detectors/rust.js.map +1 -0
- package/dist/testing/generators/contract.d.ts +14 -0
- package/dist/testing/generators/contract.d.ts.map +1 -0
- package/dist/testing/generators/contract.js +163 -0
- package/dist/testing/generators/contract.js.map +1 -0
- package/dist/testing/generators/e2e.d.ts +34 -0
- package/dist/testing/generators/e2e.d.ts.map +1 -0
- package/dist/testing/generators/e2e.js +74 -0
- package/dist/testing/generators/e2e.js.map +1 -0
- package/dist/testing/generators/go.d.ts +12 -0
- package/dist/testing/generators/go.d.ts.map +1 -0
- package/dist/testing/generators/go.js +144 -0
- package/dist/testing/generators/go.js.map +1 -0
- package/dist/testing/generators/nodejs.d.ts +12 -0
- package/dist/testing/generators/nodejs.d.ts.map +1 -0
- package/dist/testing/generators/nodejs.js +37 -0
- package/dist/testing/generators/nodejs.js.map +1 -0
- package/dist/testing/generators/python.d.ts +12 -0
- package/dist/testing/generators/python.d.ts.map +1 -0
- package/dist/testing/generators/python.js +163 -0
- package/dist/testing/generators/python.js.map +1 -0
- package/dist/testing/generators/react.d.ts +12 -0
- package/dist/testing/generators/react.d.ts.map +1 -0
- package/dist/testing/generators/react.js +31 -0
- package/dist/testing/generators/react.js.map +1 -0
- package/dist/testing/generators/rust.d.ts +11 -0
- package/dist/testing/generators/rust.d.ts.map +1 -0
- package/dist/testing/generators/rust.js +138 -0
- package/dist/testing/generators/rust.js.map +1 -0
- package/dist/testing/index.d.ts +6 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +11 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/integrations/autopilot.d.ts +42 -0
- package/dist/testing/integrations/autopilot.d.ts.map +1 -0
- package/dist/testing/integrations/autopilot.js +55 -0
- package/dist/testing/integrations/autopilot.js.map +1 -0
- package/dist/testing/integrations/cicd.d.ts +26 -0
- package/dist/testing/integrations/cicd.d.ts.map +1 -0
- package/dist/testing/integrations/cicd.js +162 -0
- package/dist/testing/integrations/cicd.js.map +1 -0
- package/dist/testing/integrations/giskard/behavioral-tests.d.ts +4 -0
- package/dist/testing/integrations/giskard/behavioral-tests.d.ts.map +1 -0
- package/dist/testing/integrations/giskard/behavioral-tests.js +66 -0
- package/dist/testing/integrations/giskard/behavioral-tests.js.map +1 -0
- package/dist/testing/integrations/giskard/types.d.ts +35 -0
- package/dist/testing/integrations/giskard/types.d.ts.map +1 -0
- package/dist/testing/integrations/giskard/types.js +2 -0
- package/dist/testing/integrations/giskard/types.js.map +1 -0
- package/dist/testing/integrations/promptfoo/config-generator.d.ts +5 -0
- package/dist/testing/integrations/promptfoo/config-generator.d.ts.map +1 -0
- package/dist/testing/integrations/promptfoo/config-generator.js +44 -0
- package/dist/testing/integrations/promptfoo/config-generator.js.map +1 -0
- package/dist/testing/integrations/promptfoo/types.d.ts +36 -0
- package/dist/testing/integrations/promptfoo/types.d.ts.map +1 -0
- package/dist/testing/integrations/promptfoo/types.js +2 -0
- package/dist/testing/integrations/promptfoo/types.js.map +1 -0
- package/dist/testing/integrations/ralph.d.ts +65 -0
- package/dist/testing/integrations/ralph.d.ts.map +1 -0
- package/dist/testing/integrations/ralph.js +69 -0
- package/dist/testing/integrations/ralph.js.map +1 -0
- package/dist/testing/performance/cache-manager.d.ts +16 -0
- package/dist/testing/performance/cache-manager.d.ts.map +1 -0
- package/dist/testing/performance/cache-manager.js +39 -0
- package/dist/testing/performance/cache-manager.js.map +1 -0
- package/dist/testing/performance/parallel-generator.d.ts +23 -0
- package/dist/testing/performance/parallel-generator.d.ts.map +1 -0
- package/dist/testing/performance/parallel-generator.js +31 -0
- package/dist/testing/performance/parallel-generator.js.map +1 -0
- package/dist/testing/types.d.ts +23 -0
- package/dist/testing/types.d.ts.map +1 -0
- package/dist/testing/types.js +2 -0
- package/dist/testing/types.js.map +1 -0
- package/docs/2026-03-06-llm-testing-system-phase1.md +0 -0
- package/docs/plans/2026-03-06-llm-testing-system-design.md +311 -0
- package/docs/plans/2026-03-06-llm-testing-system-phase1.md +1268 -0
- package/docs/plans/2026-03-06-llm-testing-system-phase2.md +3053 -0
- package/docs/plans/2026-03-06-llm-testing-system-phase3.md +1830 -0
- package/docs/testing/PHASE2.md +266 -0
- package/docs/testing/PHASE3.md +601 -0
- package/docs/testing/README.md +634 -0
- package/package.json +1 -1
- package/skills/test-gen/skill.md +531 -0
- package/skills/ultraqa.md +58 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CI/CD Integration - GitHub Actions Workflow Generator
|
|
3
|
+
*/
|
|
4
|
+
export async function generateGitHubActionsWorkflow(config) {
|
|
5
|
+
const { language, languages, testCommand, testCommands, nodeVersion = '20', pythonVersion = '3.11', goVersion = '1.21', rustVersion = 'stable', coverage = false, coverageProvider = 'codecov', artifacts = false, artifactPath = 'test-results/', matrix, } = config;
|
|
6
|
+
const runsOn = matrix?.os ? '$' + '{{ matrix.os }}' : 'ubuntu-latest';
|
|
7
|
+
let workflow = `name: Test
|
|
8
|
+
|
|
9
|
+
on:
|
|
10
|
+
push:
|
|
11
|
+
branches: [main, dev]
|
|
12
|
+
pull_request:
|
|
13
|
+
branches: [main, dev]
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
test:
|
|
17
|
+
runs-on: ${runsOn}
|
|
18
|
+
`;
|
|
19
|
+
// Add matrix strategy if specified
|
|
20
|
+
if (matrix) {
|
|
21
|
+
workflow += ` strategy:
|
|
22
|
+
matrix:
|
|
23
|
+
`;
|
|
24
|
+
if (matrix.nodeVersion) {
|
|
25
|
+
workflow += ` node-version: [${matrix.nodeVersion.join(', ')}]\n`;
|
|
26
|
+
}
|
|
27
|
+
if (matrix.pythonVersion) {
|
|
28
|
+
workflow += ` python-version: [${matrix.pythonVersion.join(', ')}]\n`;
|
|
29
|
+
}
|
|
30
|
+
if (matrix.goVersion) {
|
|
31
|
+
workflow += ` go-version: [${matrix.goVersion.join(', ')}]\n`;
|
|
32
|
+
}
|
|
33
|
+
if (matrix.os) {
|
|
34
|
+
workflow += ` os: [${matrix.os.join(', ')}]\n`;
|
|
35
|
+
}
|
|
36
|
+
if (matrix.language) {
|
|
37
|
+
workflow += ` language: [${matrix.language.join(', ')}]\n`;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
workflow += ` steps:
|
|
41
|
+
- uses: actions/checkout@v4
|
|
42
|
+
|
|
43
|
+
`;
|
|
44
|
+
// Handle multi-language setup
|
|
45
|
+
if (languages && matrix?.language) {
|
|
46
|
+
// Multi-language matrix build
|
|
47
|
+
const matrixLang = '$' + '{{ matrix.language }}';
|
|
48
|
+
workflow += ` - name: Setup Node.js
|
|
49
|
+
if: matrix.language == 'nodejs'
|
|
50
|
+
uses: actions/setup-node@v4
|
|
51
|
+
with:
|
|
52
|
+
node-version: ${nodeVersion}
|
|
53
|
+
cache: 'pnpm'
|
|
54
|
+
|
|
55
|
+
- name: Setup Python
|
|
56
|
+
if: matrix.language == 'python'
|
|
57
|
+
uses: actions/setup-python@v5
|
|
58
|
+
with:
|
|
59
|
+
python-version: ${pythonVersion}
|
|
60
|
+
|
|
61
|
+
- name: Setup Go
|
|
62
|
+
if: matrix.language == 'go'
|
|
63
|
+
uses: actions/setup-go@v5
|
|
64
|
+
with:
|
|
65
|
+
go-version: ${goVersion}
|
|
66
|
+
|
|
67
|
+
- name: Install Node.js dependencies
|
|
68
|
+
if: matrix.language == 'nodejs'
|
|
69
|
+
run: pnpm install
|
|
70
|
+
|
|
71
|
+
- name: Install Python dependencies
|
|
72
|
+
if: matrix.language == 'python'
|
|
73
|
+
run: pip install -r requirements.txt
|
|
74
|
+
|
|
75
|
+
- name: Run tests
|
|
76
|
+
run: |
|
|
77
|
+
if [ "${matrixLang}" == "nodejs" ]; then
|
|
78
|
+
${testCommands?.nodejs || 'pnpm test'}
|
|
79
|
+
elif [ "${matrixLang}" == "python" ]; then
|
|
80
|
+
${testCommands?.python || 'pytest'}
|
|
81
|
+
elif [ "${matrixLang}" == "go" ]; then
|
|
82
|
+
${testCommands?.go || 'go test ./...'}
|
|
83
|
+
fi
|
|
84
|
+
`;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
// Single language setup
|
|
88
|
+
if (language === 'nodejs' || (!language && testCommand?.includes('pnpm'))) {
|
|
89
|
+
const nodeVer = matrix?.nodeVersion ? '$' + '{{ matrix.node-version }}' : nodeVersion;
|
|
90
|
+
workflow += ` - name: Setup Node.js
|
|
91
|
+
uses: actions/setup-node@v4
|
|
92
|
+
with:
|
|
93
|
+
node-version: ${nodeVer}
|
|
94
|
+
cache: 'pnpm'
|
|
95
|
+
|
|
96
|
+
- name: Install dependencies
|
|
97
|
+
run: pnpm install
|
|
98
|
+
|
|
99
|
+
- name: Run tests
|
|
100
|
+
run: ${testCommand || 'pnpm test'}
|
|
101
|
+
`;
|
|
102
|
+
}
|
|
103
|
+
else if (language === 'python') {
|
|
104
|
+
const pyVer = matrix?.pythonVersion ? '$' + '{{ matrix.python-version }}' : pythonVersion;
|
|
105
|
+
workflow += ` - name: Setup Python
|
|
106
|
+
uses: actions/setup-python@v5
|
|
107
|
+
with:
|
|
108
|
+
python-version: ${pyVer}
|
|
109
|
+
|
|
110
|
+
- name: Install dependencies
|
|
111
|
+
run: pip install -r requirements.txt
|
|
112
|
+
|
|
113
|
+
- name: Run tests
|
|
114
|
+
run: ${testCommand || 'pytest'}
|
|
115
|
+
`;
|
|
116
|
+
}
|
|
117
|
+
else if (language === 'go') {
|
|
118
|
+
const goVer = matrix?.goVersion ? '$' + '{{ matrix.go-version }}' : goVersion;
|
|
119
|
+
workflow += ` - name: Setup Go
|
|
120
|
+
uses: actions/setup-go@v5
|
|
121
|
+
with:
|
|
122
|
+
go-version: ${goVer}
|
|
123
|
+
|
|
124
|
+
- name: Run tests
|
|
125
|
+
run: ${testCommand || 'go test ./...'}
|
|
126
|
+
`;
|
|
127
|
+
}
|
|
128
|
+
else if (language === 'rust') {
|
|
129
|
+
workflow += ` - name: Setup Rust
|
|
130
|
+
uses: actions-rust-lang/setup-rust-toolchain@v1
|
|
131
|
+
with:
|
|
132
|
+
toolchain: ${rustVersion}
|
|
133
|
+
|
|
134
|
+
- name: Run tests
|
|
135
|
+
run: ${testCommand || 'cargo test'}
|
|
136
|
+
`;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
// Add coverage reporting
|
|
140
|
+
if (coverage) {
|
|
141
|
+
const token = '$' + '{{ secrets.CODECOV_TOKEN }}';
|
|
142
|
+
workflow += `
|
|
143
|
+
- name: Upload coverage
|
|
144
|
+
uses: ${coverageProvider}/codecov-action@v4
|
|
145
|
+
with:
|
|
146
|
+
token: ${token}
|
|
147
|
+
`;
|
|
148
|
+
}
|
|
149
|
+
// Add test artifacts
|
|
150
|
+
if (artifacts) {
|
|
151
|
+
workflow += `
|
|
152
|
+
- name: Upload test results
|
|
153
|
+
if: always()
|
|
154
|
+
uses: actions/upload-artifact@v4
|
|
155
|
+
with:
|
|
156
|
+
name: test-results
|
|
157
|
+
path: ${artifactPath}
|
|
158
|
+
`;
|
|
159
|
+
}
|
|
160
|
+
return workflow;
|
|
161
|
+
}
|
|
162
|
+
//# sourceMappingURL=cicd.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cicd.js","sourceRoot":"","sources":["../../../src/testing/integrations/cicd.ts"],"names":[],"mappings":"AAAA;;GAEG;AAwBH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,MAAsB;IAEtB,MAAM,EACJ,QAAQ,EACR,SAAS,EACT,WAAW,EACX,YAAY,EACZ,WAAW,GAAG,IAAI,EAClB,aAAa,GAAG,MAAM,EACtB,SAAS,GAAG,MAAM,EAClB,WAAW,GAAG,QAAQ,EACtB,QAAQ,GAAG,KAAK,EAChB,gBAAgB,GAAG,SAAS,EAC5B,SAAS,GAAG,KAAK,EACjB,YAAY,GAAG,eAAe,EAC9B,MAAM,GACP,GAAG,MAAM,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC;IAEtE,IAAI,QAAQ,GAAG;;;;;;;;;;eAUF,MAAM;CACpB,CAAC;IAEA,mCAAmC;IACnC,IAAI,MAAM,EAAE,CAAC;QACX,QAAQ,IAAI;;CAEf,CAAC;QACE,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,QAAQ,IAAI,0BAA0B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QAC3E,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,QAAQ,IAAI,4BAA4B,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/E,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,QAAQ,IAAI,wBAAwB,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QACvE,CAAC;QACD,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YACd,QAAQ,IAAI,gBAAgB,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QACxD,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,QAAQ,IAAI,sBAAsB,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QACpE,CAAC;IACH,CAAC;IAED,QAAQ,IAAI;;;CAGb,CAAC;IAEA,8BAA8B;IAC9B,IAAI,SAAS,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC;QAClC,8BAA8B;QAC9B,MAAM,UAAU,GAAG,GAAG,GAAG,uBAAuB,CAAC;QACjD,QAAQ,IAAI;;;;0BAIU,WAAW;;;;;;;4BAOT,aAAa;;;;;;wBAMjB,SAAS;;;;;;;;;;;;kBAYf,UAAU;cACd,YAAY,EAAE,MAAM,IAAI,WAAW;oBAC7B,UAAU;cAChB,YAAY,EAAE,MAAM,IAAI,QAAQ;oBAC1B,UAAU;cAChB,YAAY,EAAE,EAAE,IAAI,eAAe;;CAEhD,CAAC;IACA,CAAC;SAAM,CAAC;QACN,wBAAwB;QACxB,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YAC1E,MAAM,OAAO,GAAG,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,2BAA2B,CAAC,CAAC,CAAC,WAAW,CAAC;YACtF,QAAQ,IAAI;;;0BAGQ,OAAO;;;;;;;eAOlB,WAAW,IAAI,WAAW;CACxC,CAAC;QACE,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC,CAAC,aAAa,CAAC;YAC1F,QAAQ,IAAI;;;4BAGU,KAAK;;;;;;eAMlB,WAAW,IAAI,QAAQ;CACrC,CAAC;QACE,CAAC;aAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,yBAAyB,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9E,QAAQ,IAAI;;;wBAGM,KAAK;;;eAGd,WAAW,IAAI,eAAe;CAC5C,CAAC;QACE,CAAC;aAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC/B,QAAQ,IAAI;;;uBAGK,WAAW;;;eAGnB,WAAW,IAAI,YAAY;CACzC,CAAC;QACE,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,GAAG,GAAG,6BAA6B,CAAC;QAClD,QAAQ,IAAI;;gBAEA,gBAAgB;;mBAEb,KAAK;CACvB,CAAC;IACA,CAAC;IAED,qBAAqB;IACrB,IAAI,SAAS,EAAE,CAAC;QACd,QAAQ,IAAI;;;;;;kBAME,YAAY;CAC7B,CAAC;IACA,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { PerturbationTestSuite, RobustnessTestSuite, GeneratePerturbationOptions, GenerateRobustnessOptions } from './types.js';
|
|
2
|
+
export declare function generatePerturbationTests(options: GeneratePerturbationOptions): Promise<PerturbationTestSuite>;
|
|
3
|
+
export declare function generateRobustnessTests(options: GenerateRobustnessOptions): Promise<RobustnessTestSuite>;
|
|
4
|
+
//# sourceMappingURL=behavioral-tests.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"behavioral-tests.d.ts","sourceRoot":"","sources":["../../../../src/testing/integrations/giskard/behavioral-tests.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,qBAAqB,EAErB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EAC1B,MAAM,YAAY,CAAC;AAEpB,wBAAsB,yBAAyB,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAqBpH;AAkBD,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAoB9G"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export async function generatePerturbationTests(options) {
|
|
2
|
+
const { testCases, perturbations } = options;
|
|
3
|
+
const tests = [];
|
|
4
|
+
for (const testCase of testCases) {
|
|
5
|
+
for (const perturbationType of perturbations) {
|
|
6
|
+
const perturbed = applyPerturbation(testCase.input, perturbationType);
|
|
7
|
+
tests.push({
|
|
8
|
+
original: testCase.input,
|
|
9
|
+
perturbed,
|
|
10
|
+
perturbationType,
|
|
11
|
+
expectedBehavior: `should still classify as ${testCase.expectedOutput}`,
|
|
12
|
+
expectedOutput: testCase.expectedOutput,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return {
|
|
17
|
+
tests,
|
|
18
|
+
totalTests: tests.length,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function applyPerturbation(text, type) {
|
|
22
|
+
switch (type) {
|
|
23
|
+
case 'typo':
|
|
24
|
+
// Simple typo: swap two adjacent characters
|
|
25
|
+
const pos = Math.floor(Math.random() * (text.length - 1));
|
|
26
|
+
return text.slice(0, pos) + text[pos + 1] + text[pos] + text.slice(pos + 2);
|
|
27
|
+
case 'negation':
|
|
28
|
+
return `not ${text}`;
|
|
29
|
+
case 'synonym':
|
|
30
|
+
// Simple synonym replacement (in real implementation, use NLP library)
|
|
31
|
+
return text.replace(/hello/gi, 'hi').replace(/goodbye/gi, 'bye');
|
|
32
|
+
default:
|
|
33
|
+
return text;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export async function generateRobustnessTests(options) {
|
|
37
|
+
const { modelEndpoint, testInputs, robustnessChecks } = options;
|
|
38
|
+
const checks = [];
|
|
39
|
+
for (const checkType of robustnessChecks) {
|
|
40
|
+
const testCases = testInputs.map((input) => ({
|
|
41
|
+
input: applyRobustnessTransform(input, checkType),
|
|
42
|
+
expected: input, // Expected to behave same as original
|
|
43
|
+
}));
|
|
44
|
+
checks.push({
|
|
45
|
+
type: checkType,
|
|
46
|
+
testCases,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
checks,
|
|
51
|
+
totalChecks: checks.length,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function applyRobustnessTransform(text, type) {
|
|
55
|
+
switch (type) {
|
|
56
|
+
case 'case-sensitivity':
|
|
57
|
+
return text.toUpperCase();
|
|
58
|
+
case 'whitespace':
|
|
59
|
+
return ` ${text} `;
|
|
60
|
+
case 'special-chars':
|
|
61
|
+
return `${text}!!!`;
|
|
62
|
+
default:
|
|
63
|
+
return text;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=behavioral-tests.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"behavioral-tests.js","sourceRoot":"","sources":["../../../../src/testing/integrations/giskard/behavioral-tests.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAoC;IAClF,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,KAAK,GAAuB,EAAE,CAAC;IAErC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,gBAAgB,IAAI,aAAa,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,iBAAiB,CAAC,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;YACtE,KAAK,CAAC,IAAI,CAAC;gBACT,QAAQ,EAAE,QAAQ,CAAC,KAAK;gBACxB,SAAS;gBACT,gBAAgB;gBAChB,gBAAgB,EAAE,4BAA4B,QAAQ,CAAC,cAAc,EAAE;gBACvE,cAAc,EAAE,QAAQ,CAAC,cAAc;aACxC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,UAAU,EAAE,KAAK,CAAC,MAAM;KACzB,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,IAAY;IACnD,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,MAAM;YACT,4CAA4C;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAC9E,KAAK,UAAU;YACb,OAAO,OAAO,IAAI,EAAE,CAAC;QACvB,KAAK,SAAS;YACZ,uEAAuE;YACvE,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACnE;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAkC;IAC9E,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC;IAChE,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC;YACnD,KAAK,EAAE,wBAAwB,CAAC,KAAK,EAAE,SAAS,CAAC;YACjD,QAAQ,EAAE,KAAK,EAAE,sCAAsC;SACxD,CAAC,CAAC,CAAC;QAEJ,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,SAAgB;YACtB,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,MAAM;QACN,WAAW,EAAE,MAAM,CAAC,MAAM;KAC3B,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAY,EAAE,IAAY;IAC1D,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,kBAAkB;YACrB,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC5B,KAAK,YAAY;YACf,OAAO,KAAK,IAAI,IAAI,CAAC;QACvB,KAAK,eAAe;YAClB,OAAO,GAAG,IAAI,KAAK,CAAC;QACtB;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export interface PerturbationTest {
|
|
2
|
+
original: string;
|
|
3
|
+
perturbed: string;
|
|
4
|
+
perturbationType: 'typo' | 'negation' | 'synonym' | 'paraphrase';
|
|
5
|
+
expectedBehavior: string;
|
|
6
|
+
expectedOutput?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface PerturbationTestSuite {
|
|
9
|
+
tests: PerturbationTest[];
|
|
10
|
+
totalTests: number;
|
|
11
|
+
}
|
|
12
|
+
export interface RobustnessCheck {
|
|
13
|
+
type: 'case-sensitivity' | 'whitespace' | 'special-chars' | 'unicode';
|
|
14
|
+
testCases: Array<{
|
|
15
|
+
input: string;
|
|
16
|
+
expected: string;
|
|
17
|
+
}>;
|
|
18
|
+
}
|
|
19
|
+
export interface RobustnessTestSuite {
|
|
20
|
+
checks: RobustnessCheck[];
|
|
21
|
+
totalChecks: number;
|
|
22
|
+
}
|
|
23
|
+
export interface GeneratePerturbationOptions {
|
|
24
|
+
testCases: Array<{
|
|
25
|
+
input: string;
|
|
26
|
+
expectedOutput: string;
|
|
27
|
+
}>;
|
|
28
|
+
perturbations: Array<'typo' | 'negation' | 'synonym' | 'paraphrase'>;
|
|
29
|
+
}
|
|
30
|
+
export interface GenerateRobustnessOptions {
|
|
31
|
+
modelEndpoint: string;
|
|
32
|
+
testInputs: string[];
|
|
33
|
+
robustnessChecks: Array<'case-sensitivity' | 'whitespace' | 'special-chars' | 'unicode'>;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/testing/integrations/giskard/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,GAAG,YAAY,CAAC;IACjE,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,kBAAkB,GAAG,YAAY,GAAG,eAAe,GAAG,SAAS,CAAC;IACtE,SAAS,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,2BAA2B;IAC1C,SAAS,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,aAAa,EAAE,KAAK,CAAC,MAAM,GAAG,UAAU,GAAG,SAAS,GAAG,YAAY,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,gBAAgB,EAAE,KAAK,CAAC,kBAAkB,GAAG,YAAY,GAAG,eAAe,GAAG,SAAS,CAAC,CAAC;CAC1F"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/testing/integrations/giskard/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { PromptfooConfig, PromptfooTestCase, GenerateConfigOptions, GenerateTestCasesOptions } from './types.js';
|
|
2
|
+
export declare function generatePromptfooConfig(options: GenerateConfigOptions): Promise<PromptfooConfig>;
|
|
3
|
+
export declare function generateTestCases(options: GenerateTestCasesOptions): Promise<PromptfooTestCase[]>;
|
|
4
|
+
export declare function savePromptfooConfig(config: PromptfooConfig, outputPath: string): Promise<void>;
|
|
5
|
+
//# sourceMappingURL=config-generator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-generator.d.ts","sourceRoot":"","sources":["../../../../src/testing/integrations/promptfoo/config-generator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,eAAe,EACf,iBAAiB,EACjB,qBAAqB,EACrB,wBAAwB,EACzB,MAAM,YAAY,CAAC;AAEpB,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAwBtG;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAevG;AAED,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpG"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { writeFile } from 'fs/promises';
|
|
2
|
+
import * as yaml from 'yaml';
|
|
3
|
+
export async function generatePromptfooConfig(options) {
|
|
4
|
+
const { promptFile, testCases, provider, outputPath } = options;
|
|
5
|
+
const tests = testCases.map((tc) => {
|
|
6
|
+
const [assertType, assertValue] = tc.expected.split(':');
|
|
7
|
+
return {
|
|
8
|
+
vars: { input: tc.input },
|
|
9
|
+
assert: [
|
|
10
|
+
{
|
|
11
|
+
type: assertType,
|
|
12
|
+
value: assertValue,
|
|
13
|
+
},
|
|
14
|
+
],
|
|
15
|
+
};
|
|
16
|
+
});
|
|
17
|
+
const config = {
|
|
18
|
+
prompts: [`file://${promptFile}`],
|
|
19
|
+
providers: [provider],
|
|
20
|
+
tests,
|
|
21
|
+
outputPath: outputPath || './promptfoo-results.json',
|
|
22
|
+
};
|
|
23
|
+
return config;
|
|
24
|
+
}
|
|
25
|
+
export async function generateTestCases(options) {
|
|
26
|
+
const { codeExamples, assertionType } = options;
|
|
27
|
+
return codeExamples.map((example) => ({
|
|
28
|
+
vars: {
|
|
29
|
+
code: example.code,
|
|
30
|
+
language: example.language,
|
|
31
|
+
},
|
|
32
|
+
assert: [
|
|
33
|
+
{
|
|
34
|
+
type: assertionType,
|
|
35
|
+
value: example.language === 'javascript' ? 'function' : 'def',
|
|
36
|
+
},
|
|
37
|
+
],
|
|
38
|
+
}));
|
|
39
|
+
}
|
|
40
|
+
export async function savePromptfooConfig(config, outputPath) {
|
|
41
|
+
const yamlContent = yaml.stringify(config);
|
|
42
|
+
await writeFile(outputPath, yamlContent, 'utf-8');
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=config-generator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-generator.js","sourceRoot":"","sources":["../../../../src/testing/integrations/promptfoo/config-generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAQ7B,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAA8B;IAC1E,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAEhE,MAAM,KAAK,GAAwB,SAAS,CAAC,GAAG,CAAC,CAAC,EAAuC,EAAE,EAAE;QAC3F,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzD,OAAO;YACL,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE;YACzB,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,UAAiB;oBACvB,KAAK,EAAE,WAAW;iBACnB;aACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAoB;QAC9B,OAAO,EAAE,CAAC,UAAU,UAAU,EAAE,CAAC;QACjC,SAAS,EAAE,CAAC,QAAQ,CAAC;QACrB,KAAK;QACL,UAAU,EAAE,UAAU,IAAI,0BAA0B;KACrD,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAiC;IACvE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAEhD,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,OAA2C,EAAE,EAAE,CAAC,CAAC;QACxE,IAAI,EAAE;YACJ,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B;QACD,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,OAAO,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK;aAC9D;SACF;KACF,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,UAAkB;IACnF,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,SAAS,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;AACpD,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export interface PromptfooConfig {
|
|
2
|
+
prompts: string[];
|
|
3
|
+
providers: string[];
|
|
4
|
+
tests: PromptfooTestCase[];
|
|
5
|
+
defaultTest?: {
|
|
6
|
+
assert?: PromptfooAssertion[];
|
|
7
|
+
};
|
|
8
|
+
outputPath?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface PromptfooTestCase {
|
|
11
|
+
vars?: Record<string, any>;
|
|
12
|
+
assert?: PromptfooAssertion[];
|
|
13
|
+
description?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface PromptfooAssertion {
|
|
16
|
+
type: 'equals' | 'contains' | 'not-contains' | 'regex' | 'javascript' | 'llm-rubric';
|
|
17
|
+
value?: string;
|
|
18
|
+
threshold?: number;
|
|
19
|
+
}
|
|
20
|
+
export interface GenerateConfigOptions {
|
|
21
|
+
promptFile: string;
|
|
22
|
+
testCases: Array<{
|
|
23
|
+
input: string;
|
|
24
|
+
expected: string;
|
|
25
|
+
}>;
|
|
26
|
+
provider: string;
|
|
27
|
+
outputPath?: string;
|
|
28
|
+
}
|
|
29
|
+
export interface GenerateTestCasesOptions {
|
|
30
|
+
codeExamples: Array<{
|
|
31
|
+
code: string;
|
|
32
|
+
language: string;
|
|
33
|
+
}>;
|
|
34
|
+
assertionType: 'contains' | 'equals' | 'regex';
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/testing/integrations/promptfoo/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE;QACZ,MAAM,CAAC,EAAE,kBAAkB,EAAE,CAAC;KAC/B,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,QAAQ,GAAG,UAAU,GAAG,cAAc,GAAG,OAAO,GAAG,YAAY,GAAG,YAAY,CAAC;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtD,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,wBAAwB;IACvC,YAAY,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxD,aAAa,EAAE,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;CAChD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/testing/integrations/promptfoo/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ralph Mode Testing Integration
|
|
3
|
+
*
|
|
4
|
+
* Integrates automated test generation and execution into Ralph's verify cycle.
|
|
5
|
+
* Automatically generates tests for modified code and runs them during verification.
|
|
6
|
+
*/
|
|
7
|
+
interface RalphState {
|
|
8
|
+
currentIteration: number;
|
|
9
|
+
modifiedFiles: string[];
|
|
10
|
+
verifyPhase: 'pre-verify' | 'post-verify' | 'fix';
|
|
11
|
+
}
|
|
12
|
+
interface RalphTestingConfig {
|
|
13
|
+
testingEnabled: boolean;
|
|
14
|
+
testGenerationPhase: string;
|
|
15
|
+
filesToTest: string[];
|
|
16
|
+
}
|
|
17
|
+
interface GenerateTestsForIterationOptions {
|
|
18
|
+
modifiedFiles: string[];
|
|
19
|
+
iterationNumber: number;
|
|
20
|
+
previousTestResults: {
|
|
21
|
+
passed: number;
|
|
22
|
+
failed: number;
|
|
23
|
+
};
|
|
24
|
+
coverageThreshold?: number;
|
|
25
|
+
}
|
|
26
|
+
interface IterationTestResult {
|
|
27
|
+
generatedTests: Array<{
|
|
28
|
+
file: string;
|
|
29
|
+
testFile: string;
|
|
30
|
+
}>;
|
|
31
|
+
shouldRunTests: boolean;
|
|
32
|
+
coverageCheck?: {
|
|
33
|
+
enabled: boolean;
|
|
34
|
+
threshold: number;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
interface TestExecutionResult {
|
|
38
|
+
passed: number;
|
|
39
|
+
failed: number;
|
|
40
|
+
coverage?: number;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Integrates testing into Ralph's verification cycle
|
|
44
|
+
* Called at the start of Ralph's verify phase
|
|
45
|
+
*/
|
|
46
|
+
export declare function integrateWithRalph(state: RalphState): Promise<RalphTestingConfig>;
|
|
47
|
+
/**
|
|
48
|
+
* Generates tests for the current Ralph iteration
|
|
49
|
+
* Analyzes modified files and determines which tests to generate
|
|
50
|
+
*/
|
|
51
|
+
export declare function generateTestsForIteration(options: GenerateTestsForIterationOptions): Promise<IterationTestResult>;
|
|
52
|
+
/**
|
|
53
|
+
* Runs tests during Ralph's verification cycle
|
|
54
|
+
* Executes generated tests and returns results
|
|
55
|
+
*/
|
|
56
|
+
export declare function runTestsInRalphCycle(testFiles: string[]): Promise<TestExecutionResult>;
|
|
57
|
+
/**
|
|
58
|
+
* Checks if coverage thresholds are met
|
|
59
|
+
*/
|
|
60
|
+
export declare function checkCoverageThreshold(coverage: number, threshold: number): Promise<{
|
|
61
|
+
met: boolean;
|
|
62
|
+
message: string;
|
|
63
|
+
}>;
|
|
64
|
+
export {};
|
|
65
|
+
//# sourceMappingURL=ralph.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ralph.d.ts","sourceRoot":"","sources":["../../../src/testing/integrations/ralph.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,UAAU,UAAU;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,YAAY,GAAG,aAAa,GAAG,KAAK,CAAC;CACnD;AAED,UAAU,kBAAkB;IAC1B,cAAc,EAAE,OAAO,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,UAAU,gCAAgC;IACxC,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACxD,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,UAAU,mBAAmB;IAC3B,cAAc,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1D,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAMvF;AAED;;;GAGG;AACH,wBAAsB,yBAAyB,CAC7C,OAAO,EAAE,gCAAgC,GACxC,OAAO,CAAC,mBAAmB,CAAC,CA4B9B;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAS5F;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,GAAG,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAO5C"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ralph Mode Testing Integration
|
|
3
|
+
*
|
|
4
|
+
* Integrates automated test generation and execution into Ralph's verify cycle.
|
|
5
|
+
* Automatically generates tests for modified code and runs them during verification.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Integrates testing into Ralph's verification cycle
|
|
9
|
+
* Called at the start of Ralph's verify phase
|
|
10
|
+
*/
|
|
11
|
+
export async function integrateWithRalph(state) {
|
|
12
|
+
return {
|
|
13
|
+
testingEnabled: true,
|
|
14
|
+
testGenerationPhase: state.verifyPhase,
|
|
15
|
+
filesToTest: state.modifiedFiles,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Generates tests for the current Ralph iteration
|
|
20
|
+
* Analyzes modified files and determines which tests to generate
|
|
21
|
+
*/
|
|
22
|
+
export async function generateTestsForIteration(options) {
|
|
23
|
+
const { modifiedFiles, iterationNumber, previousTestResults, coverageThreshold } = options;
|
|
24
|
+
if (modifiedFiles.length === 0) {
|
|
25
|
+
return {
|
|
26
|
+
generatedTests: [],
|
|
27
|
+
shouldRunTests: false,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const generatedTests = modifiedFiles.map((file) => ({
|
|
31
|
+
file,
|
|
32
|
+
testFile: file.replace(/\.(ts|js|py|go)$/, '.test.$1'),
|
|
33
|
+
}));
|
|
34
|
+
const result = {
|
|
35
|
+
generatedTests,
|
|
36
|
+
shouldRunTests: true,
|
|
37
|
+
};
|
|
38
|
+
if (coverageThreshold !== undefined) {
|
|
39
|
+
result.coverageCheck = {
|
|
40
|
+
enabled: true,
|
|
41
|
+
threshold: coverageThreshold,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
return result;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Runs tests during Ralph's verification cycle
|
|
48
|
+
* Executes generated tests and returns results
|
|
49
|
+
*/
|
|
50
|
+
export async function runTestsInRalphCycle(testFiles) {
|
|
51
|
+
// Integration point for running tests during Ralph cycle
|
|
52
|
+
// This will be called by Ralph's verify phase
|
|
53
|
+
// In a real implementation, this would execute the test runner
|
|
54
|
+
return {
|
|
55
|
+
passed: testFiles.length,
|
|
56
|
+
failed: 0,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Checks if coverage thresholds are met
|
|
61
|
+
*/
|
|
62
|
+
export async function checkCoverageThreshold(coverage, threshold) {
|
|
63
|
+
const met = coverage >= threshold;
|
|
64
|
+
const message = met
|
|
65
|
+
? `Coverage ${coverage}% meets threshold ${threshold}%`
|
|
66
|
+
: `Coverage ${coverage}% below threshold ${threshold}%`;
|
|
67
|
+
return { met, message };
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=ralph.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ralph.js","sourceRoot":"","sources":["../../../src/testing/integrations/ralph.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAoCH;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAiB;IACxD,OAAO;QACL,cAAc,EAAE,IAAI;QACpB,mBAAmB,EAAE,KAAK,CAAC,WAAW;QACtC,WAAW,EAAE,KAAK,CAAC,aAAa;KACjC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,OAAyC;IAEzC,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;IAE3F,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO;YACL,cAAc,EAAE,EAAE;YAClB,cAAc,EAAE,KAAK;SACtB,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClD,IAAI;QACJ,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,UAAU,CAAC;KACvD,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAwB;QAClC,cAAc;QACd,cAAc,EAAE,IAAI;KACrB,CAAC;IAEF,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,aAAa,GAAG;YACrB,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,iBAAiB;SAC7B,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,SAAmB;IAC5D,yDAAyD;IACzD,8CAA8C;IAC9C,+DAA+D;IAE/D,OAAO;QACL,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,MAAM,EAAE,CAAC;KACV,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,QAAgB,EAChB,SAAiB;IAEjB,MAAM,GAAG,GAAG,QAAQ,IAAI,SAAS,CAAC;IAClC,MAAM,OAAO,GAAG,GAAG;QACjB,CAAC,CAAC,YAAY,QAAQ,qBAAqB,SAAS,GAAG;QACvD,CAAC,CAAC,YAAY,QAAQ,qBAAqB,SAAS,GAAG,CAAC;IAE1D,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
interface CacheOptions {
|
|
2
|
+
ttl: number;
|
|
3
|
+
}
|
|
4
|
+
export declare class CacheManager {
|
|
5
|
+
private cache;
|
|
6
|
+
private ttl;
|
|
7
|
+
constructor(options: CacheOptions);
|
|
8
|
+
set<T>(key: string, value: T): Promise<void>;
|
|
9
|
+
get<T>(key: string): Promise<T | null>;
|
|
10
|
+
clear(): Promise<void>;
|
|
11
|
+
delete(key: string): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
export declare function getCachedAnalysis<T>(key: string): Promise<T | null>;
|
|
14
|
+
export declare function setCachedAnalysis<T>(key: string, value: T): Promise<void>;
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=cache-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-manager.d.ts","sourceRoot":"","sources":["../../../src/testing/performance/cache-manager.ts"],"names":[],"mappings":"AAAA,UAAU,YAAY;IACpB,GAAG,EAAE,MAAM,CAAC;CACb;AAOD,qBAAa,YAAY;IACvB,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,GAAG,CAAS;gBAER,OAAO,EAAE,YAAY;IAK3B,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAK5C,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAgBtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAGzC;AAKD,wBAAsB,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAEzE;AAED,wBAAsB,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/E"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export class CacheManager {
|
|
2
|
+
cache;
|
|
3
|
+
ttl;
|
|
4
|
+
constructor(options) {
|
|
5
|
+
this.cache = new Map();
|
|
6
|
+
this.ttl = options.ttl * 1000; // Convert to milliseconds
|
|
7
|
+
}
|
|
8
|
+
async set(key, value) {
|
|
9
|
+
const expiresAt = Date.now() + this.ttl;
|
|
10
|
+
this.cache.set(key, { value, expiresAt });
|
|
11
|
+
}
|
|
12
|
+
async get(key) {
|
|
13
|
+
const entry = this.cache.get(key);
|
|
14
|
+
if (!entry) {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
// Check if expired
|
|
18
|
+
if (Date.now() > entry.expiresAt) {
|
|
19
|
+
this.cache.delete(key);
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
return entry.value;
|
|
23
|
+
}
|
|
24
|
+
async clear() {
|
|
25
|
+
this.cache.clear();
|
|
26
|
+
}
|
|
27
|
+
async delete(key) {
|
|
28
|
+
this.cache.delete(key);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
// Global cache instance
|
|
32
|
+
const globalCache = new CacheManager({ ttl: 3600 });
|
|
33
|
+
export async function getCachedAnalysis(key) {
|
|
34
|
+
return globalCache.get(key);
|
|
35
|
+
}
|
|
36
|
+
export async function setCachedAnalysis(key, value) {
|
|
37
|
+
return globalCache.set(key, value);
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=cache-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-manager.js","sourceRoot":"","sources":["../../../src/testing/performance/cache-manager.ts"],"names":[],"mappings":"AASA,MAAM,OAAO,YAAY;IACf,KAAK,CAA+B;IACpC,GAAG,CAAS;IAEpB,YAAY,OAAqB;QAC/B,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,0BAA0B;IAC3D,CAAC;IAED,KAAK,CAAC,GAAG,CAAI,GAAW,EAAE,KAAQ;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,GAAG,CAAI,GAAW;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC,KAAU,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;CACF;AAED,wBAAwB;AACxB,MAAM,WAAW,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;AAEpD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAI,GAAW;IACpD,OAAO,WAAW,CAAC,GAAG,CAAI,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAI,GAAW,EAAE,KAAQ;IAC9D,OAAO,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
interface ParallelGeneratorOptions {
|
|
2
|
+
maxConcurrency: number;
|
|
3
|
+
}
|
|
4
|
+
interface GenerateTestsOptions {
|
|
5
|
+
files: string[];
|
|
6
|
+
maxConcurrency: number;
|
|
7
|
+
}
|
|
8
|
+
interface GenerationResult {
|
|
9
|
+
generatedTests: Array<{
|
|
10
|
+
file: string;
|
|
11
|
+
success: boolean;
|
|
12
|
+
}>;
|
|
13
|
+
totalTime: number;
|
|
14
|
+
maxConcurrentTasks: number;
|
|
15
|
+
}
|
|
16
|
+
export declare class ParallelGenerator {
|
|
17
|
+
private maxConcurrency;
|
|
18
|
+
constructor(options: ParallelGeneratorOptions);
|
|
19
|
+
generate(files: string[]): Promise<GenerationResult>;
|
|
20
|
+
}
|
|
21
|
+
export declare function generateTestsInParallel(options: GenerateTestsOptions): Promise<GenerationResult>;
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=parallel-generator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-generator.d.ts","sourceRoot":"","sources":["../../../src/testing/performance/parallel-generator.ts"],"names":[],"mappings":"AAAA,UAAU,wBAAwB;IAChC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,oBAAoB;IAC5B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,gBAAgB;IACxB,cAAc,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,cAAc,CAAS;gBAEnB,OAAO,EAAE,wBAAwB;IAIvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAyB3D;AAED,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,gBAAgB,CAAC,CAG3B"}
|