@memberjunction/testing-cli 0.0.1 → 2.119.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +486 -29
- package/dist/commands/compare.d.ts +24 -0
- package/dist/commands/compare.d.ts.map +1 -0
- package/dist/commands/compare.js +50 -0
- package/dist/commands/compare.js.map +1 -0
- package/dist/commands/history.d.ts +23 -0
- package/dist/commands/history.d.ts.map +1 -0
- package/dist/commands/history.js +47 -0
- package/dist/commands/history.js.map +1 -0
- package/dist/commands/list.d.ts +39 -0
- package/dist/commands/list.d.ts.map +1 -0
- package/dist/commands/list.js +177 -0
- package/dist/commands/list.js.map +1 -0
- package/dist/commands/report.d.ts +22 -0
- package/dist/commands/report.d.ts.map +1 -0
- package/dist/commands/report.js +46 -0
- package/dist/commands/report.js.map +1 -0
- package/dist/commands/run.d.ts +21 -0
- package/dist/commands/run.d.ts.map +1 -0
- package/dist/commands/run.js +139 -0
- package/dist/commands/run.js.map +1 -0
- package/dist/commands/suite.d.ts +21 -0
- package/dist/commands/suite.d.ts.map +1 -0
- package/dist/commands/suite.js +105 -0
- package/dist/commands/suite.js.map +1 -0
- package/dist/commands/validate.d.ts +33 -0
- package/dist/commands/validate.d.ts.map +1 -0
- package/dist/commands/validate.js +266 -0
- package/dist/commands/validate.js.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +50 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/mj-provider.d.ts +10 -0
- package/dist/lib/mj-provider.d.ts.map +1 -0
- package/dist/lib/mj-provider.js +205 -0
- package/dist/lib/mj-provider.js.map +1 -0
- package/dist/types.d.ts +103 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +7 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/config-loader.d.ts +42 -0
- package/dist/utils/config-loader.d.ts.map +1 -0
- package/dist/utils/config-loader.js +70 -0
- package/dist/utils/config-loader.js.map +1 -0
- package/dist/utils/output-formatter.d.ts +64 -0
- package/dist/utils/output-formatter.d.ts.map +1 -0
- package/dist/utils/output-formatter.js +280 -0
- package/dist/utils/output-formatter.js.map +1 -0
- package/dist/utils/spinner-manager.d.ts +43 -0
- package/dist/utils/spinner-manager.d.ts.map +1 -0
- package/dist/utils/spinner-manager.js +89 -0
- package/dist/utils/spinner-manager.js.map +1 -0
- package/package.json +38 -7
package/README.md
CHANGED
|
@@ -1,45 +1,502 @@
|
|
|
1
|
-
#
|
|
1
|
+
# MemberJunction Testing CLI
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Command-line interface for the MemberJunction Testing Framework. Provides a thin CLI layer on top of the Testing Engine, enabling test execution, management, and reporting from the command line.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## Architecture
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
The CLI is designed as a thin wrapper around the Testing Engine, ensuring feature parity between CLI and API:
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
```
|
|
10
|
+
MJCLI (mj command)
|
|
11
|
+
↓
|
|
12
|
+
Testing CLI Commands (mj test *)
|
|
13
|
+
↓
|
|
14
|
+
Testing Engine (actual logic)
|
|
15
|
+
↓
|
|
16
|
+
Database / Entities
|
|
17
|
+
```
|
|
10
18
|
|
|
11
|
-
This
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
19
|
+
This architecture allows:
|
|
20
|
+
- **CLI and API parity** - Same engine powers both interfaces
|
|
21
|
+
- **Thin CLI layer** - Commands focus on user interaction, not business logic
|
|
22
|
+
- **Testability** - Engine can be tested independently
|
|
23
|
+
- **Flexibility** - Easy to add GraphQL/REST APIs later
|
|
15
24
|
|
|
16
|
-
##
|
|
25
|
+
## Integration with MJCLI
|
|
17
26
|
|
|
18
|
-
|
|
27
|
+
The Testing CLI integrates with the main MJCLI package following the same pattern as MetadataSync:
|
|
19
28
|
|
|
20
|
-
|
|
29
|
+
**In MJCLI package** (`packages/MJCLI/src/commands/test/`):
|
|
30
|
+
```typescript
|
|
31
|
+
// Thin command that delegates to Testing CLI
|
|
32
|
+
import { RunCommand } from '@memberjunction/testing-cli';
|
|
21
33
|
|
|
22
|
-
|
|
34
|
+
export default class TestRun extends Command {
|
|
35
|
+
async run() {
|
|
36
|
+
// Minimal setup, delegate to testing CLI
|
|
37
|
+
await RunCommand.execute(this.parse);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
```
|
|
23
41
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
42
|
+
**In Testing CLI package** (`packages/TestingFramework/CLI/src/commands/`):
|
|
43
|
+
```typescript
|
|
44
|
+
// Full implementation
|
|
45
|
+
export class RunCommand {
|
|
46
|
+
async execute(args, flags) {
|
|
47
|
+
// Use Testing Engine for actual work
|
|
48
|
+
const engine = new TestEngine();
|
|
49
|
+
const result = await engine.runTest(...);
|
|
50
|
+
// Format output for CLI
|
|
51
|
+
this.displayResults(result);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
```
|
|
28
55
|
|
|
29
|
-
##
|
|
56
|
+
## Commands
|
|
30
57
|
|
|
31
|
-
|
|
32
|
-
- Contains no executable code
|
|
33
|
-
- Provides no functionality
|
|
34
|
-
- Should not be installed as a dependency
|
|
35
|
-
- Exists only for administrative purposes
|
|
58
|
+
### `mj test run`
|
|
36
59
|
|
|
37
|
-
|
|
60
|
+
Run a single test or suite of tests.
|
|
38
61
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
62
|
+
```bash
|
|
63
|
+
# Run specific test by ID
|
|
64
|
+
mj test run <test-id>
|
|
42
65
|
|
|
43
|
-
|
|
66
|
+
# Run test by name
|
|
67
|
+
mj test run --name="Active Members Count"
|
|
44
68
|
|
|
45
|
-
|
|
69
|
+
# Run test suite
|
|
70
|
+
mj test run --suite=<suite-id>
|
|
71
|
+
|
|
72
|
+
# Run with filtering
|
|
73
|
+
mj test run --tag=smoke
|
|
74
|
+
mj test run --category=agent-eval
|
|
75
|
+
mj test run --difficulty=easy
|
|
76
|
+
|
|
77
|
+
# Run all tests
|
|
78
|
+
mj test run --all
|
|
79
|
+
|
|
80
|
+
# Dry run (validate without executing)
|
|
81
|
+
mj test run <test-id> --dry-run
|
|
82
|
+
|
|
83
|
+
# Specify environment
|
|
84
|
+
mj test run <test-id> --environment=staging
|
|
85
|
+
|
|
86
|
+
# Output formats
|
|
87
|
+
mj test run <test-id> --format=json
|
|
88
|
+
mj test run <test-id> --format=markdown
|
|
89
|
+
mj test run <test-id> --format=console (default)
|
|
90
|
+
|
|
91
|
+
# Save results to file
|
|
92
|
+
mj test run <test-id> --output=results.json
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**Example Output:**
|
|
96
|
+
```
|
|
97
|
+
[TEST_START] Active Members Count
|
|
98
|
+
[TYPE] Agent Eval
|
|
99
|
+
[CONTEXT] {"testId": "abc-123", "environment": "staging"}
|
|
100
|
+
|
|
101
|
+
[STEP 1/3] Execute Agent
|
|
102
|
+
[INPUT] "How many active members do we have?"
|
|
103
|
+
[AGENT] Skip Analytics Agent
|
|
104
|
+
[DURATION] 1.8s
|
|
105
|
+
✓ Agent executed successfully
|
|
106
|
+
|
|
107
|
+
[STEP 2/3] Run Oracles
|
|
108
|
+
[ORACLE] trace-no-errors: PASSED ✓
|
|
109
|
+
[ORACLE] sql-validate: COUNT(*) = 402 (expected 380-420) ✓
|
|
110
|
+
[ORACLE] schema-validate: Output matches schema ✓
|
|
111
|
+
|
|
112
|
+
[STEP 3/3] Calculate Score
|
|
113
|
+
[PASSED_CHECKS] 3/3
|
|
114
|
+
[SCORE] 1.0000 (100%)
|
|
115
|
+
[COST] $0.0042 USD
|
|
116
|
+
|
|
117
|
+
[TEST_PASS] Active Members Count
|
|
118
|
+
[DURATION] 1.8s
|
|
119
|
+
[TRACE_ID] Agent Run: abc-xyz-123
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### `mj test suite`
|
|
123
|
+
|
|
124
|
+
Run a complete test suite.
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
# Run suite by ID
|
|
128
|
+
mj test suite <suite-id>
|
|
129
|
+
|
|
130
|
+
# Run suite by name
|
|
131
|
+
mj test suite --name="Agent Quality Suite"
|
|
132
|
+
|
|
133
|
+
# Parallel execution
|
|
134
|
+
mj test suite <suite-id> --parallel
|
|
135
|
+
|
|
136
|
+
# Fail fast (stop on first failure)
|
|
137
|
+
mj test suite <suite-id> --fail-fast
|
|
138
|
+
|
|
139
|
+
# Specific sequence
|
|
140
|
+
mj test suite <suite-id> --sequence=1,3,5
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
**Example Output:**
|
|
144
|
+
```
|
|
145
|
+
[SUITE_START] Agent Quality Suite
|
|
146
|
+
[TESTS] 15 tests queued
|
|
147
|
+
|
|
148
|
+
[1/15] Active Members Count
|
|
149
|
+
✓ PASSED (1.8s, score: 1.0000)
|
|
150
|
+
|
|
151
|
+
[2/15] Revenue Year-to-Date
|
|
152
|
+
✓ PASSED (2.1s, score: 0.9500)
|
|
153
|
+
|
|
154
|
+
[3/15] Complex Aggregation
|
|
155
|
+
✗ FAILED (3.2s, score: 0.6000)
|
|
156
|
+
- Oracle 'llm-judge' failed: Component quality below threshold
|
|
157
|
+
|
|
158
|
+
...
|
|
159
|
+
|
|
160
|
+
[SUITE_COMPLETE] Agent Quality Suite
|
|
161
|
+
[SUMMARY] 13/15 passed (86.7%)
|
|
162
|
+
[DURATION] 28.4s
|
|
163
|
+
[COST] $0.12 USD
|
|
164
|
+
[FAILURES] 2 tests failed - see details above
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### `mj test list`
|
|
168
|
+
|
|
169
|
+
List available tests, suites, and types.
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
# List all tests
|
|
173
|
+
mj test list
|
|
174
|
+
|
|
175
|
+
# List by type
|
|
176
|
+
mj test list --type=agent-eval
|
|
177
|
+
|
|
178
|
+
# List test suites
|
|
179
|
+
mj test list --suites
|
|
180
|
+
|
|
181
|
+
# List test types
|
|
182
|
+
mj test list --types
|
|
183
|
+
|
|
184
|
+
# Verbose output (show configuration)
|
|
185
|
+
mj test list --verbose
|
|
186
|
+
|
|
187
|
+
# Filter by tag
|
|
188
|
+
mj test list --tag=smoke
|
|
189
|
+
|
|
190
|
+
# Filter by status
|
|
191
|
+
mj test list --status=active
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
**Example Output:**
|
|
195
|
+
```
|
|
196
|
+
Available Tests (42):
|
|
197
|
+
|
|
198
|
+
Agent Evals (28):
|
|
199
|
+
- active-members-count [easy] Tags: membership, kpi
|
|
200
|
+
- revenue-ytd [easy] Tags: financial, kpi
|
|
201
|
+
- trend-analysis [medium] Tags: analytics, trends
|
|
202
|
+
- complex-aggregation [hard] Tags: advanced, performance
|
|
203
|
+
|
|
204
|
+
Workflow Scenarios (8):
|
|
205
|
+
- customer-onboarding [medium] Tags: workflow, customer
|
|
206
|
+
- invoice-processing [hard] Tags: financial, workflow
|
|
207
|
+
|
|
208
|
+
Test Suites (5):
|
|
209
|
+
- smoke-tests 15 tests
|
|
210
|
+
- regression-suite 42 tests
|
|
211
|
+
- nightly-full 87 tests
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### `mj test validate`
|
|
215
|
+
|
|
216
|
+
Validate test definitions without executing.
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
# Validate specific test
|
|
220
|
+
mj test validate <test-id>
|
|
221
|
+
|
|
222
|
+
# Validate all tests
|
|
223
|
+
mj test validate --all
|
|
224
|
+
|
|
225
|
+
# Validate by type
|
|
226
|
+
mj test validate --type=agent-eval
|
|
227
|
+
|
|
228
|
+
# Save validation report
|
|
229
|
+
mj test validate --all --save-report
|
|
230
|
+
|
|
231
|
+
# Specify output file
|
|
232
|
+
mj test validate --all --output=validation-report.md
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
**Example Output:**
|
|
236
|
+
```
|
|
237
|
+
Validating Tests...
|
|
238
|
+
|
|
239
|
+
✓ active-members-count
|
|
240
|
+
- Configuration valid
|
|
241
|
+
- Oracles registered: trace-no-errors, sql-validate, schema-validate
|
|
242
|
+
- Expected outcomes defined
|
|
243
|
+
- No issues found
|
|
244
|
+
|
|
245
|
+
✗ complex-aggregation
|
|
246
|
+
- Configuration valid
|
|
247
|
+
- Oracles registered: trace-no-errors, llm-judge
|
|
248
|
+
⚠ Warning: Missing schema validation
|
|
249
|
+
⚠ Warning: LLM judge rubric not found: 'component-quality-v2'
|
|
250
|
+
|
|
251
|
+
✓ revenue-ytd
|
|
252
|
+
- Configuration valid
|
|
253
|
+
- Oracles registered: trace-no-errors, sql-validate
|
|
254
|
+
- Expected outcomes defined
|
|
255
|
+
- No issues found
|
|
256
|
+
|
|
257
|
+
[SUMMARY] 2/3 tests valid, 1 with warnings
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
### `mj test report`
|
|
261
|
+
|
|
262
|
+
Generate test run reports.
|
|
263
|
+
|
|
264
|
+
```bash
|
|
265
|
+
# Generate report for recent runs
|
|
266
|
+
mj test report --suite=<suite-id>
|
|
267
|
+
|
|
268
|
+
# Date range
|
|
269
|
+
mj test report --from=2025-01-01 --to=2025-01-31
|
|
270
|
+
|
|
271
|
+
# Specific tests
|
|
272
|
+
mj test report --test=<test-id>
|
|
273
|
+
|
|
274
|
+
# Output formats
|
|
275
|
+
mj test report --format=markdown
|
|
276
|
+
mj test report --format=json
|
|
277
|
+
mj test report --format=html
|
|
278
|
+
|
|
279
|
+
# Save to file
|
|
280
|
+
mj test report --suite=<suite-id> --output=report.md
|
|
281
|
+
|
|
282
|
+
# Include cost analysis
|
|
283
|
+
mj test report --suite=<suite-id> --include-costs
|
|
284
|
+
|
|
285
|
+
# Include trends
|
|
286
|
+
mj test report --suite=<suite-id> --include-trends
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
**Example Markdown Output:**
|
|
290
|
+
```markdown
|
|
291
|
+
# Test Report: Agent Quality Suite
|
|
292
|
+
Generated: 2025-01-09 14:30:00
|
|
293
|
+
|
|
294
|
+
## Summary
|
|
295
|
+
- **Suite:** Agent Quality Suite
|
|
296
|
+
- **Period:** Last 30 days
|
|
297
|
+
- **Total Runs:** 42
|
|
298
|
+
- **Success Rate:** 95.2%
|
|
299
|
+
- **Total Cost:** $5.28 USD
|
|
300
|
+
- **Avg Duration:** 28.4s
|
|
301
|
+
|
|
302
|
+
## Test Results
|
|
303
|
+
| Test Name | Runs | Pass Rate | Avg Score | Avg Cost | Trend |
|
|
304
|
+
|------------------------|------|-----------|-----------|----------|-------|
|
|
305
|
+
| Active Members Count | 42 | 100% | 1.0000 | $0.004 | → |
|
|
306
|
+
| Revenue YTD | 42 | 97.6% | 0.9850 | $0.005 | ↗ |
|
|
307
|
+
| Complex Aggregation | 42 | 85.7% | 0.8200 | $0.012 | ↘ |
|
|
308
|
+
|
|
309
|
+
## Failures
|
|
310
|
+
2 failures in the last 30 days:
|
|
311
|
+
|
|
312
|
+
### Complex Aggregation (2025-01-08)
|
|
313
|
+
- **Score:** 0.6000
|
|
314
|
+
- **Reason:** LLM judge failed - component quality below threshold
|
|
315
|
+
- **Cost:** $0.012
|
|
316
|
+
- **Trace:** [View Agent Run](link)
|
|
317
|
+
|
|
318
|
+
## Cost Analysis
|
|
319
|
+
- **Total Cost:** $5.28
|
|
320
|
+
- **Most Expensive Test:** Complex Aggregation ($0.50)
|
|
321
|
+
- **Cost Trend:** +12% vs previous month
|
|
322
|
+
|
|
323
|
+
## Recommendations
|
|
324
|
+
- Complex Aggregation: Review LLM judge rubric threshold
|
|
325
|
+
- Consider caching results for Active Members Count (no failures)
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
### `mj test history`
|
|
329
|
+
|
|
330
|
+
View test execution history.
|
|
331
|
+
|
|
332
|
+
```bash
|
|
333
|
+
# History for specific test
|
|
334
|
+
mj test history <test-id>
|
|
335
|
+
|
|
336
|
+
# Recent runs
|
|
337
|
+
mj test history --recent=10
|
|
338
|
+
|
|
339
|
+
# By date range
|
|
340
|
+
mj test history --from=2025-01-01
|
|
341
|
+
|
|
342
|
+
# Filter by status
|
|
343
|
+
mj test history --status=failed
|
|
344
|
+
|
|
345
|
+
# Show details
|
|
346
|
+
mj test history <test-id> --verbose
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
### `mj test compare`
|
|
350
|
+
|
|
351
|
+
Compare test runs to detect regressions.
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
# Compare two specific runs
|
|
355
|
+
mj test compare <run-id-1> <run-id-2>
|
|
356
|
+
|
|
357
|
+
# Compare versions
|
|
358
|
+
mj test compare --version=2.1.0 --version=2.2.0
|
|
359
|
+
|
|
360
|
+
# Compare git commits
|
|
361
|
+
mj test compare --commit=abc123 --commit=def456
|
|
362
|
+
|
|
363
|
+
# Show only differences
|
|
364
|
+
mj test compare <run-id-1> <run-id-2> --diff-only
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
**Example Output:**
|
|
368
|
+
```
|
|
369
|
+
Comparing Test Runs:
|
|
370
|
+
Run 1: 2025-01-01 (v2.1.0, commit abc123)
|
|
371
|
+
Run 2: 2025-01-09 (v2.2.0, commit def456)
|
|
372
|
+
|
|
373
|
+
Test Results:
|
|
374
|
+
Active Members Count: SAME (1.0000 → 1.0000)
|
|
375
|
+
Revenue YTD: SAME (0.9850 → 0.9850)
|
|
376
|
+
Complex Aggregation: WORSE (0.9200 → 0.6000) ⚠
|
|
377
|
+
Trend Analysis: BETTER (0.7500 → 0.9100) ↗
|
|
378
|
+
|
|
379
|
+
Performance:
|
|
380
|
+
Avg Duration: BETTER (28.4s → 24.1s) ↗
|
|
381
|
+
Total Cost: WORSE ($0.12 → $0.18) ⚠
|
|
382
|
+
|
|
383
|
+
Summary: 1 regression detected in Complex Aggregation
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
## Output Formats
|
|
387
|
+
|
|
388
|
+
### Console (Default)
|
|
389
|
+
Human-readable, color-coded output with progress indicators and emojis.
|
|
390
|
+
|
|
391
|
+
### JSON
|
|
392
|
+
Machine-readable for CI/CD integration:
|
|
393
|
+
```json
|
|
394
|
+
{
|
|
395
|
+
"testId": "abc-123",
|
|
396
|
+
"status": "passed",
|
|
397
|
+
"score": 1.0,
|
|
398
|
+
"duration": 1.8,
|
|
399
|
+
"cost": 0.0042,
|
|
400
|
+
"oracleResults": [
|
|
401
|
+
{
|
|
402
|
+
"type": "trace-no-errors",
|
|
403
|
+
"passed": true,
|
|
404
|
+
"score": 1.0,
|
|
405
|
+
"message": "All 3 steps completed without errors"
|
|
406
|
+
}
|
|
407
|
+
]
|
|
408
|
+
}
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
### Markdown
|
|
412
|
+
Formatted reports for documentation:
|
|
413
|
+
```markdown
|
|
414
|
+
# Test Run: Active Members Count
|
|
415
|
+
**Status:** PASSED ✓
|
|
416
|
+
**Score:** 1.0000 (100%)
|
|
417
|
+
**Duration:** 1.8s
|
|
418
|
+
**Cost:** $0.0042
|
|
419
|
+
|
|
420
|
+
## Oracle Results
|
|
421
|
+
- ✓ trace-no-errors: All 3 steps completed without errors
|
|
422
|
+
- ✓ sql-validate: COUNT(*) = 402 (expected 380-420)
|
|
423
|
+
- ✓ schema-validate: Output matches schema
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
## CI/CD Integration
|
|
427
|
+
|
|
428
|
+
### GitHub Actions Example
|
|
429
|
+
```yaml
|
|
430
|
+
name: Test Suite
|
|
431
|
+
on: [push, pull_request]
|
|
432
|
+
|
|
433
|
+
jobs:
|
|
434
|
+
test-suite:
|
|
435
|
+
runs-on: ubuntu-latest
|
|
436
|
+
steps:
|
|
437
|
+
- uses: actions/checkout@v3
|
|
438
|
+
|
|
439
|
+
- name: Run Smoke Tests
|
|
440
|
+
run: mj test suite smoke-tests --fail-fast --format=json > results.json
|
|
441
|
+
|
|
442
|
+
- name: Check Results
|
|
443
|
+
run: |
|
|
444
|
+
if ! jq -e '.passRate >= 0.95' results.json; then
|
|
445
|
+
echo "Test pass rate below 95%"
|
|
446
|
+
exit 1
|
|
447
|
+
fi
|
|
448
|
+
|
|
449
|
+
- name: Upload Results
|
|
450
|
+
uses: actions/upload-artifact@v3
|
|
451
|
+
with:
|
|
452
|
+
name: test-results
|
|
453
|
+
path: results.json
|
|
454
|
+
```
|
|
455
|
+
|
|
456
|
+
## Configuration
|
|
457
|
+
|
|
458
|
+
The CLI uses the standard MJ configuration file (`mj.config.cjs`):
|
|
459
|
+
|
|
460
|
+
```javascript
|
|
461
|
+
module.exports = {
|
|
462
|
+
// Standard MJ config
|
|
463
|
+
databaseSettings: { ... },
|
|
464
|
+
|
|
465
|
+
// Testing framework config
|
|
466
|
+
testing: {
|
|
467
|
+
defaultEnvironment: 'dev',
|
|
468
|
+
defaultFormat: 'console',
|
|
469
|
+
failFast: false,
|
|
470
|
+
parallel: false,
|
|
471
|
+
maxParallelTests: 5,
|
|
472
|
+
timeout: 300000 // 5 minutes
|
|
473
|
+
}
|
|
474
|
+
};
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
## Error Handling
|
|
478
|
+
|
|
479
|
+
The CLI provides clear error messages with actionable guidance:
|
|
480
|
+
|
|
481
|
+
```
|
|
482
|
+
✗ Error: Test 'complex-aggregation' failed validation
|
|
483
|
+
|
|
484
|
+
Issues found:
|
|
485
|
+
- LLM judge rubric not found: 'component-quality-v2'
|
|
486
|
+
- Missing required oracle: 'schema-validate'
|
|
487
|
+
|
|
488
|
+
Suggestions:
|
|
489
|
+
1. Create rubric 'component-quality-v2' or update test config
|
|
490
|
+
2. Add schema-validate oracle to test configuration
|
|
491
|
+
|
|
492
|
+
Run 'mj test validate complex-aggregation --verbose' for details
|
|
493
|
+
```
|
|
494
|
+
|
|
495
|
+
## Future Enhancements
|
|
496
|
+
|
|
497
|
+
- [ ] Interactive mode for test creation
|
|
498
|
+
- [ ] Watch mode (re-run on file changes)
|
|
499
|
+
- [ ] Test coverage reports
|
|
500
|
+
- [ ] Parallel suite execution
|
|
501
|
+
- [ ] Result streaming for long-running tests
|
|
502
|
+
- [ ] Integration with MJ UI for clickable trace links
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Compare command implementation
|
|
3
|
+
* @module @memberjunction/testing-cli
|
|
4
|
+
*/
|
|
5
|
+
import { UserInfo } from '@memberjunction/core';
|
|
6
|
+
import { CompareFlags } from '../types';
|
|
7
|
+
/**
|
|
8
|
+
* Compare command - Compare test runs to detect regressions
|
|
9
|
+
*
|
|
10
|
+
* Note: This is a placeholder implementation. Full comparison requires
|
|
11
|
+
* tracking test runs with version/commit metadata.
|
|
12
|
+
*/
|
|
13
|
+
export declare class CompareCommand {
|
|
14
|
+
/**
|
|
15
|
+
* Execute the compare command
|
|
16
|
+
*
|
|
17
|
+
* @param runId1 - First run ID (optional)
|
|
18
|
+
* @param runId2 - Second run ID (optional)
|
|
19
|
+
* @param flags - Command flags
|
|
20
|
+
* @param contextUser - Optional user context (will be fetched if not provided)
|
|
21
|
+
*/
|
|
22
|
+
execute(runId1: string | undefined, runId2: string | undefined, flags: CompareFlags, contextUser?: UserInfo): Promise<void>;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=compare.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/commands/compare.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAGxC;;;;;GAKG;AACH,qBAAa,cAAc;IACvB;;;;;;;OAOG;IACG,OAAO,CACT,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,YAAY,EACnB,WAAW,CAAC,EAAE,QAAQ,GACvB,OAAO,CAAC,IAAI,CAAC;CAyBnB"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @fileoverview Compare command implementation
|
|
4
|
+
* @module @memberjunction/testing-cli
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.CompareCommand = void 0;
|
|
8
|
+
const output_formatter_1 = require("../utils/output-formatter");
|
|
9
|
+
/**
|
|
10
|
+
* Compare command - Compare test runs to detect regressions
|
|
11
|
+
*
|
|
12
|
+
* Note: This is a placeholder implementation. Full comparison requires
|
|
13
|
+
* tracking test runs with version/commit metadata.
|
|
14
|
+
*/
|
|
15
|
+
class CompareCommand {
|
|
16
|
+
/**
|
|
17
|
+
* Execute the compare command
|
|
18
|
+
*
|
|
19
|
+
* @param runId1 - First run ID (optional)
|
|
20
|
+
* @param runId2 - Second run ID (optional)
|
|
21
|
+
* @param flags - Command flags
|
|
22
|
+
* @param contextUser - Optional user context (will be fetched if not provided)
|
|
23
|
+
*/
|
|
24
|
+
async execute(runId1, runId2, flags, contextUser) {
|
|
25
|
+
try {
|
|
26
|
+
console.log(output_formatter_1.OutputFormatter.formatInfo('Compare command not yet implemented'));
|
|
27
|
+
console.log('\nPlanned features:');
|
|
28
|
+
console.log(' - Compare two specific test runs by ID');
|
|
29
|
+
console.log(' - Compare runs by version or git commit');
|
|
30
|
+
console.log(' - Detect regressions in scores or pass rates');
|
|
31
|
+
console.log(' - Show performance and cost differences');
|
|
32
|
+
console.log(' - Filter with --diff-only to show only changes');
|
|
33
|
+
console.log('\nRequires:');
|
|
34
|
+
console.log(' - Test Run Results with version/commit metadata');
|
|
35
|
+
console.log(' - Comparison algorithms for detecting regressions');
|
|
36
|
+
// TODO: Implement full comparison
|
|
37
|
+
// - Load two test runs from database
|
|
38
|
+
// - Compare scores, pass/fail status, duration, cost
|
|
39
|
+
// - Identify regressions (score decreased, new failures)
|
|
40
|
+
// - Display side-by-side comparison
|
|
41
|
+
// - Exit with non-zero if regressions detected
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
console.error(output_formatter_1.OutputFormatter.formatError('Failed to compare test runs', error));
|
|
45
|
+
process.exit(1);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
exports.CompareCommand = CompareCommand;
|
|
50
|
+
//# sourceMappingURL=compare.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/commands/compare.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAIH,gEAA4D;AAE5D;;;;;GAKG;AACH,MAAa,cAAc;IACvB;;;;;;;OAOG;IACH,KAAK,CAAC,OAAO,CACT,MAA0B,EAC1B,MAA0B,EAC1B,KAAmB,EACnB,WAAsB;QAEtB,IAAI,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,kCAAe,CAAC,UAAU,CAAC,qCAAqC,CAAC,CAAC,CAAC;YAC/E,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;YAChE,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;YAEnE,kCAAkC;YAClC,qCAAqC;YACrC,qDAAqD;YACrD,yDAAyD;YACzD,oCAAoC;YACpC,+CAA+C;QAEnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,kCAAe,CAAC,WAAW,CAAC,6BAA6B,EAAE,KAAc,CAAC,CAAC,CAAC;YAC1F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACL,CAAC;CACJ;AAvCD,wCAuCC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview History command implementation
|
|
3
|
+
* @module @memberjunction/testing-cli
|
|
4
|
+
*/
|
|
5
|
+
import { UserInfo } from '@memberjunction/core';
|
|
6
|
+
import { HistoryFlags } from '../types';
|
|
7
|
+
/**
|
|
8
|
+
* History command - View test execution history
|
|
9
|
+
*
|
|
10
|
+
* Note: This is a placeholder implementation. Full history tracking requires
|
|
11
|
+
* querying Test Run Results entities from the database.
|
|
12
|
+
*/
|
|
13
|
+
export declare class HistoryCommand {
|
|
14
|
+
/**
|
|
15
|
+
* Execute the history command
|
|
16
|
+
*
|
|
17
|
+
* @param testId - Optional test ID to show history for
|
|
18
|
+
* @param flags - Command flags
|
|
19
|
+
* @param contextUser - Optional user context (will be fetched if not provided)
|
|
20
|
+
*/
|
|
21
|
+
execute(testId: string | undefined, flags: HistoryFlags, contextUser?: UserInfo): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=history.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../src/commands/history.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAGxC;;;;;GAKG;AACH,qBAAa,cAAc;IACvB;;;;;;OAMG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;CAuBxG"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @fileoverview History command implementation
|
|
4
|
+
* @module @memberjunction/testing-cli
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.HistoryCommand = void 0;
|
|
8
|
+
const output_formatter_1 = require("../utils/output-formatter");
|
|
9
|
+
/**
|
|
10
|
+
* History command - View test execution history
|
|
11
|
+
*
|
|
12
|
+
* Note: This is a placeholder implementation. Full history tracking requires
|
|
13
|
+
* querying Test Run Results entities from the database.
|
|
14
|
+
*/
|
|
15
|
+
class HistoryCommand {
|
|
16
|
+
/**
|
|
17
|
+
* Execute the history command
|
|
18
|
+
*
|
|
19
|
+
* @param testId - Optional test ID to show history for
|
|
20
|
+
* @param flags - Command flags
|
|
21
|
+
* @param contextUser - Optional user context (will be fetched if not provided)
|
|
22
|
+
*/
|
|
23
|
+
async execute(testId, flags, contextUser) {
|
|
24
|
+
try {
|
|
25
|
+
console.log(output_formatter_1.OutputFormatter.formatInfo('History command not yet implemented'));
|
|
26
|
+
console.log('\nPlanned features:');
|
|
27
|
+
console.log(' - View execution history for specific tests');
|
|
28
|
+
console.log(' - Filter by date range and status');
|
|
29
|
+
console.log(' - Show recent runs with --recent=N');
|
|
30
|
+
console.log(' - Display detailed results with --verbose');
|
|
31
|
+
console.log('\nRequires:');
|
|
32
|
+
console.log(' - Test Run Results entity tracking');
|
|
33
|
+
console.log(' - Persistent storage of test execution results');
|
|
34
|
+
// TODO: Implement full history
|
|
35
|
+
// - Query Test Run Results for specific test
|
|
36
|
+
// - Apply date range and status filters
|
|
37
|
+
// - Display in tabular format with timestamps
|
|
38
|
+
// - Show detailed oracle results in verbose mode
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
console.error(output_formatter_1.OutputFormatter.formatError('Failed to show history', error));
|
|
42
|
+
process.exit(1);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
exports.HistoryCommand = HistoryCommand;
|
|
47
|
+
//# sourceMappingURL=history.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/commands/history.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAIH,gEAA4D;AAE5D;;;;;GAKG;AACH,MAAa,cAAc;IACvB;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,MAA0B,EAAE,KAAmB,EAAE,WAAsB;QACjF,IAAI,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,kCAAe,CAAC,UAAU,CAAC,qCAAqC,CAAC,CAAC,CAAC;YAC/E,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;YAC7D,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;YAEhE,+BAA+B;YAC/B,6CAA6C;YAC7C,wCAAwC;YACxC,8CAA8C;YAC9C,iDAAiD;QAErD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,kCAAe,CAAC,WAAW,CAAC,wBAAwB,EAAE,KAAc,CAAC,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACL,CAAC;CACJ;AA/BD,wCA+BC"}
|