@drift-ci/cli 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +28 -0
- package/dist/commands/baseline.d.ts +26 -0
- package/dist/commands/baseline.d.ts.map +1 -0
- package/dist/commands/baseline.js +227 -0
- package/dist/commands/baseline.js.map +1 -0
- package/dist/commands/compare.d.ts +16 -0
- package/dist/commands/compare.d.ts.map +1 -0
- package/dist/commands/compare.js +109 -0
- package/dist/commands/compare.js.map +1 -0
- package/dist/commands/config.d.ts +11 -0
- package/dist/commands/config.d.ts.map +1 -0
- package/dist/commands/config.js +78 -0
- package/dist/commands/config.js.map +1 -0
- package/dist/commands/init.d.ts +10 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +80 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/run.d.ts +16 -0
- package/dist/commands/run.d.ts.map +1 -0
- package/dist/commands/run.js +132 -0
- package/dist/commands/run.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/reporters/index.d.ts +11 -0
- package/dist/reporters/index.d.ts.map +1 -0
- package/dist/reporters/index.js +16 -0
- package/dist/reporters/index.js.map +1 -0
- package/dist/reporters/ink.d.ts +23 -0
- package/dist/reporters/ink.d.ts.map +1 -0
- package/dist/reporters/ink.js +89 -0
- package/dist/reporters/ink.js.map +1 -0
- package/dist/reporters/interface.d.ts +18 -0
- package/dist/reporters/interface.d.ts.map +1 -0
- package/dist/reporters/interface.js +2 -0
- package/dist/reporters/interface.js.map +1 -0
- package/dist/reporters/json.d.ts +36 -0
- package/dist/reporters/json.d.ts.map +1 -0
- package/dist/reporters/json.js +37 -0
- package/dist/reporters/json.js.map +1 -0
- package/dist/reporters/text.d.ts +14 -0
- package/dist/reporters/text.d.ts.map +1 -0
- package/dist/reporters/text.js +94 -0
- package/dist/reporters/text.js.map +1 -0
- package/package.json +52 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
export class TextReporter {
|
|
2
|
+
out;
|
|
3
|
+
constructor(opts = {}) {
|
|
4
|
+
this.out = opts.out ?? process.stdout;
|
|
5
|
+
}
|
|
6
|
+
onRunStart(_ctx) { }
|
|
7
|
+
onCaseComplete(_result) { }
|
|
8
|
+
onRunEnd(ctx) {
|
|
9
|
+
renderSummary(ctx, (line) => this.out.write(line + '\n'));
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export function renderSummary({ suite, run, deltas, loaded }, emit) {
|
|
13
|
+
emit('');
|
|
14
|
+
emit(`Suite: ${suite.name} (${suite.id})`);
|
|
15
|
+
emit(`Provider: ${run.provider}`);
|
|
16
|
+
emit(`Run: ${run.id}`);
|
|
17
|
+
emit(`Cases: ${run.summary.passed}/${run.summary.total} passed, ` +
|
|
18
|
+
`${run.summary.transient} transient, ` +
|
|
19
|
+
`${run.summary.evaluatorErrors} evaluator errors, ` +
|
|
20
|
+
`${run.summary.failed} config errors, ` +
|
|
21
|
+
`${run.summary.regressions} regressions`);
|
|
22
|
+
emit(`Avg score: ${run.summary.avgScore.toFixed(3)}`);
|
|
23
|
+
emit(`Avg latency: ${run.summary.avgLatencyMs.toFixed(1)} ms`);
|
|
24
|
+
if (loaded.upgradedInMemory) {
|
|
25
|
+
emit(`Config: auto-upgraded in memory (persist by bumping version)`);
|
|
26
|
+
}
|
|
27
|
+
const regressionSet = new Set(deltas?.regressions ?? []);
|
|
28
|
+
const improvementSet = new Set(deltas?.improvements ?? []);
|
|
29
|
+
const missingSet = new Set(deltas?.missingBaselines ?? []);
|
|
30
|
+
const staleSuiteSet = new Set(deltas?.staleBaselines ?? []);
|
|
31
|
+
const staleJudgeSet = new Set(deltas?.staleJudges ?? []);
|
|
32
|
+
const noScoreSet = new Set(deltas?.noScore ?? []);
|
|
33
|
+
emit('');
|
|
34
|
+
emit('Per case:');
|
|
35
|
+
for (const c of run.cases) {
|
|
36
|
+
emit(formatCaseLine(c, deltas, {
|
|
37
|
+
regressionSet,
|
|
38
|
+
improvementSet,
|
|
39
|
+
missingSet,
|
|
40
|
+
staleSuiteSet,
|
|
41
|
+
staleJudgeSet,
|
|
42
|
+
noScoreSet,
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
if (deltas) {
|
|
46
|
+
if (deltas.missingBaselines.length > 0) {
|
|
47
|
+
emit('');
|
|
48
|
+
emit(`\u2139 ${deltas.missingBaselines.length} case(s) have no baseline yet. ` +
|
|
49
|
+
`Run: drift-ci baseline init`);
|
|
50
|
+
}
|
|
51
|
+
if (deltas.staleBaselines.length > 0) {
|
|
52
|
+
emit('');
|
|
53
|
+
emit(`\u26A0 ${deltas.staleBaselines.length} baseline(s) are stale (suite definition changed). ` +
|
|
54
|
+
`Run: drift-ci baseline accept --cases <id>`);
|
|
55
|
+
}
|
|
56
|
+
if (deltas.staleJudges.length > 0) {
|
|
57
|
+
emit('');
|
|
58
|
+
emit(`\u26A0 ${deltas.staleJudges.length} baseline(s) used a different judge. ` +
|
|
59
|
+
`Scores are not directly comparable.`);
|
|
60
|
+
}
|
|
61
|
+
if (deltas.regressions.length > 0) {
|
|
62
|
+
emit('');
|
|
63
|
+
emit(`\u2717 ${deltas.regressions.length} regression(s) detected. ` +
|
|
64
|
+
`Review the diff, then: drift-ci baseline accept --cases <id> if intentional.`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function formatCaseLine(c, deltas, sets) {
|
|
69
|
+
let marker = c.status === 'pass' ? '\u2713' : '\u2717';
|
|
70
|
+
const flags = [];
|
|
71
|
+
if (sets.regressionSet.has(c.caseId)) {
|
|
72
|
+
marker = '\u2717';
|
|
73
|
+
flags.push('REGRESSION');
|
|
74
|
+
}
|
|
75
|
+
else if (sets.improvementSet.has(c.caseId)) {
|
|
76
|
+
flags.push('improved');
|
|
77
|
+
}
|
|
78
|
+
if (sets.missingSet.has(c.caseId))
|
|
79
|
+
flags.push('no baseline');
|
|
80
|
+
if (sets.staleSuiteSet.has(c.caseId))
|
|
81
|
+
flags.push('stale-suite');
|
|
82
|
+
if (sets.staleJudgeSet.has(c.caseId))
|
|
83
|
+
flags.push('stale-judge');
|
|
84
|
+
if (sets.noScoreSet.has(c.caseId))
|
|
85
|
+
flags.push('no-score');
|
|
86
|
+
const scoreText = Number.isNaN(c.score) ? '\u2014' : c.score.toFixed(3);
|
|
87
|
+
const delta = deltas?.deltas[c.caseId];
|
|
88
|
+
const deltaText = delta === undefined || delta === 0
|
|
89
|
+
? ''
|
|
90
|
+
: ` \u0394${delta > 0 ? '+' : ''}${delta.toFixed(3)}`;
|
|
91
|
+
const flagText = flags.length > 0 ? ` [${flags.join(', ')}]` : '';
|
|
92
|
+
return ` ${marker} ${c.caseId.padEnd(24)} ${c.status.padEnd(20)} score=${scoreText}${deltaText}${flagText}`;
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=text.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/reporters/text.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,YAAY;IACN,GAAG,CAAwB;IAE5C,YAAY,OAA4B,EAAE;QACxC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;IACxC,CAAC;IAED,UAAU,CAAC,IAAqB,IAAS,CAAC;IAE1C,cAAc,CAAC,OAAmB,IAAS,CAAC;IAE5C,QAAQ,CAAC,GAAkB;QACzB,aAAa,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IAC5D,CAAC;CACF;AAED,MAAM,UAAU,aAAa,CAC3B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAiB,EAC7C,IAA4B;IAE5B,IAAI,CAAC,EAAE,CAAC,CAAC;IACT,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,aAAa,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAClC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5B,IAAI,CACF,aAAa,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,WAAW;QAC7D,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,cAAc;QACtC,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,qBAAqB;QACnD,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,kBAAkB;QACvC,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,cAAc,CAC3C,CAAC;IACF,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC5B,IAAI,CAAC,mEAAmE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,gBAAgB,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC;IAElD,IAAI,CAAC,EAAE,CAAC,CAAC;IACT,IAAI,CAAC,WAAW,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE;YAC7B,aAAa;YACb,cAAc;YACd,UAAU;YACV,aAAa;YACb,aAAa;YACb,UAAU;SACX,CAAC,CAAC,CAAC;IACN,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,gBAAgB,CAAC,MAAM,iCAAiC;gBACvE,6BAA6B,CAChC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,cAAc,CAAC,MAAM,qDAAqD;gBACzF,4CAA4C,CAC/C,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,WAAW,CAAC,MAAM,uCAAuC;gBACxE,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,WAAW,CAAC,MAAM,2BAA2B;gBAC5D,8EAA8E,CACjF,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAWD,SAAS,cAAc,CACrB,CAAa,EACb,MAA+B,EAC/B,IAAc;IAEd,IAAI,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,QAAQ,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;SAAM,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC7D,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,SAAS,GACb,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,CAAC;QAChC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,UAAU,SAAS,GAAG,SAAS,GAAG,QAAQ,EAAE,CAAC;AAC/G,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@drift-ci/cli",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Behaviour regression testing for LLM applications",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"homepage": "https://github.com/Drift-CI/drift-ci#readme",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/Drift-CI/drift-ci.git",
|
|
10
|
+
"directory": "packages/cli"
|
|
11
|
+
},
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/Drift-CI/drift-ci/issues"
|
|
14
|
+
},
|
|
15
|
+
"type": "module",
|
|
16
|
+
"bin": {
|
|
17
|
+
"drift-ci": "./dist/index.js"
|
|
18
|
+
},
|
|
19
|
+
"main": "./dist/index.js",
|
|
20
|
+
"dependencies": {
|
|
21
|
+
"better-sqlite3": "^11.10.0",
|
|
22
|
+
"commander": "^12.1.0",
|
|
23
|
+
"ink": "^5.2.1",
|
|
24
|
+
"js-yaml": "^4.1.1",
|
|
25
|
+
"react": "^18.3.1",
|
|
26
|
+
"@drift-ci/core": "1.0.0"
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"@types/js-yaml": "^4.0.9",
|
|
30
|
+
"@types/node": "^20.14.0",
|
|
31
|
+
"@types/react": "^18.3.28",
|
|
32
|
+
"@vitest/coverage-v8": "^1.6.1",
|
|
33
|
+
"ink-testing-library": "^4.0.0",
|
|
34
|
+
"rimraf": "^6.0.0",
|
|
35
|
+
"typescript": "^5.5.0",
|
|
36
|
+
"vitest": "^1.6.0"
|
|
37
|
+
},
|
|
38
|
+
"files": [
|
|
39
|
+
"dist",
|
|
40
|
+
"README.md"
|
|
41
|
+
],
|
|
42
|
+
"engines": {
|
|
43
|
+
"node": ">=20.0.0"
|
|
44
|
+
},
|
|
45
|
+
"scripts": {
|
|
46
|
+
"build": "tsc -p tsconfig.json && node scripts/add-shebang.mjs",
|
|
47
|
+
"dev": "tsc -p tsconfig.json --watch",
|
|
48
|
+
"test": "vitest run",
|
|
49
|
+
"test:coverage": "vitest run --coverage",
|
|
50
|
+
"clean": "rimraf dist"
|
|
51
|
+
}
|
|
52
|
+
}
|