@drift-ci/cli 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +28 -0
  3. package/dist/commands/baseline.d.ts +26 -0
  4. package/dist/commands/baseline.d.ts.map +1 -0
  5. package/dist/commands/baseline.js +227 -0
  6. package/dist/commands/baseline.js.map +1 -0
  7. package/dist/commands/compare.d.ts +16 -0
  8. package/dist/commands/compare.d.ts.map +1 -0
  9. package/dist/commands/compare.js +109 -0
  10. package/dist/commands/compare.js.map +1 -0
  11. package/dist/commands/config.d.ts +11 -0
  12. package/dist/commands/config.d.ts.map +1 -0
  13. package/dist/commands/config.js +78 -0
  14. package/dist/commands/config.js.map +1 -0
  15. package/dist/commands/init.d.ts +10 -0
  16. package/dist/commands/init.d.ts.map +1 -0
  17. package/dist/commands/init.js +80 -0
  18. package/dist/commands/init.js.map +1 -0
  19. package/dist/commands/run.d.ts +16 -0
  20. package/dist/commands/run.d.ts.map +1 -0
  21. package/dist/commands/run.js +132 -0
  22. package/dist/commands/run.js.map +1 -0
  23. package/dist/index.d.ts +2 -0
  24. package/dist/index.d.ts.map +1 -0
  25. package/dist/index.js +30 -0
  26. package/dist/index.js.map +1 -0
  27. package/dist/reporters/index.d.ts +11 -0
  28. package/dist/reporters/index.d.ts.map +1 -0
  29. package/dist/reporters/index.js +16 -0
  30. package/dist/reporters/index.js.map +1 -0
  31. package/dist/reporters/ink.d.ts +23 -0
  32. package/dist/reporters/ink.d.ts.map +1 -0
  33. package/dist/reporters/ink.js +89 -0
  34. package/dist/reporters/ink.js.map +1 -0
  35. package/dist/reporters/interface.d.ts +18 -0
  36. package/dist/reporters/interface.d.ts.map +1 -0
  37. package/dist/reporters/interface.js +2 -0
  38. package/dist/reporters/interface.js.map +1 -0
  39. package/dist/reporters/json.d.ts +36 -0
  40. package/dist/reporters/json.d.ts.map +1 -0
  41. package/dist/reporters/json.js +37 -0
  42. package/dist/reporters/json.js.map +1 -0
  43. package/dist/reporters/text.d.ts +14 -0
  44. package/dist/reporters/text.d.ts.map +1 -0
  45. package/dist/reporters/text.js +94 -0
  46. package/dist/reporters/text.js.map +1 -0
  47. package/package.json +52 -0
@@ -0,0 +1,94 @@
1
+ export class TextReporter {
2
+ out;
3
+ constructor(opts = {}) {
4
+ this.out = opts.out ?? process.stdout;
5
+ }
6
+ onRunStart(_ctx) { }
7
+ onCaseComplete(_result) { }
8
+ onRunEnd(ctx) {
9
+ renderSummary(ctx, (line) => this.out.write(line + '\n'));
10
+ }
11
+ }
12
+ export function renderSummary({ suite, run, deltas, loaded }, emit) {
13
+ emit('');
14
+ emit(`Suite: ${suite.name} (${suite.id})`);
15
+ emit(`Provider: ${run.provider}`);
16
+ emit(`Run: ${run.id}`);
17
+ emit(`Cases: ${run.summary.passed}/${run.summary.total} passed, ` +
18
+ `${run.summary.transient} transient, ` +
19
+ `${run.summary.evaluatorErrors} evaluator errors, ` +
20
+ `${run.summary.failed} config errors, ` +
21
+ `${run.summary.regressions} regressions`);
22
+ emit(`Avg score: ${run.summary.avgScore.toFixed(3)}`);
23
+ emit(`Avg latency: ${run.summary.avgLatencyMs.toFixed(1)} ms`);
24
+ if (loaded.upgradedInMemory) {
25
+ emit(`Config: auto-upgraded in memory (persist by bumping version)`);
26
+ }
27
+ const regressionSet = new Set(deltas?.regressions ?? []);
28
+ const improvementSet = new Set(deltas?.improvements ?? []);
29
+ const missingSet = new Set(deltas?.missingBaselines ?? []);
30
+ const staleSuiteSet = new Set(deltas?.staleBaselines ?? []);
31
+ const staleJudgeSet = new Set(deltas?.staleJudges ?? []);
32
+ const noScoreSet = new Set(deltas?.noScore ?? []);
33
+ emit('');
34
+ emit('Per case:');
35
+ for (const c of run.cases) {
36
+ emit(formatCaseLine(c, deltas, {
37
+ regressionSet,
38
+ improvementSet,
39
+ missingSet,
40
+ staleSuiteSet,
41
+ staleJudgeSet,
42
+ noScoreSet,
43
+ }));
44
+ }
45
+ if (deltas) {
46
+ if (deltas.missingBaselines.length > 0) {
47
+ emit('');
48
+ emit(`\u2139 ${deltas.missingBaselines.length} case(s) have no baseline yet. ` +
49
+ `Run: drift-ci baseline init`);
50
+ }
51
+ if (deltas.staleBaselines.length > 0) {
52
+ emit('');
53
+ emit(`\u26A0 ${deltas.staleBaselines.length} baseline(s) are stale (suite definition changed). ` +
54
+ `Run: drift-ci baseline accept --cases <id>`);
55
+ }
56
+ if (deltas.staleJudges.length > 0) {
57
+ emit('');
58
+ emit(`\u26A0 ${deltas.staleJudges.length} baseline(s) used a different judge. ` +
59
+ `Scores are not directly comparable.`);
60
+ }
61
+ if (deltas.regressions.length > 0) {
62
+ emit('');
63
+ emit(`\u2717 ${deltas.regressions.length} regression(s) detected. ` +
64
+ `Review the diff, then: drift-ci baseline accept --cases <id> if intentional.`);
65
+ }
66
+ }
67
+ }
68
+ function formatCaseLine(c, deltas, sets) {
69
+ let marker = c.status === 'pass' ? '\u2713' : '\u2717';
70
+ const flags = [];
71
+ if (sets.regressionSet.has(c.caseId)) {
72
+ marker = '\u2717';
73
+ flags.push('REGRESSION');
74
+ }
75
+ else if (sets.improvementSet.has(c.caseId)) {
76
+ flags.push('improved');
77
+ }
78
+ if (sets.missingSet.has(c.caseId))
79
+ flags.push('no baseline');
80
+ if (sets.staleSuiteSet.has(c.caseId))
81
+ flags.push('stale-suite');
82
+ if (sets.staleJudgeSet.has(c.caseId))
83
+ flags.push('stale-judge');
84
+ if (sets.noScoreSet.has(c.caseId))
85
+ flags.push('no-score');
86
+ const scoreText = Number.isNaN(c.score) ? '\u2014' : c.score.toFixed(3);
87
+ const delta = deltas?.deltas[c.caseId];
88
+ const deltaText = delta === undefined || delta === 0
89
+ ? ''
90
+ : ` \u0394${delta > 0 ? '+' : ''}${delta.toFixed(3)}`;
91
+ const flagText = flags.length > 0 ? ` [${flags.join(', ')}]` : '';
92
+ return ` ${marker} ${c.caseId.padEnd(24)} ${c.status.padEnd(20)} score=${scoreText}${deltaText}${flagText}`;
93
+ }
94
+ //# sourceMappingURL=text.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/reporters/text.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,YAAY;IACN,GAAG,CAAwB;IAE5C,YAAY,OAA4B,EAAE;QACxC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;IACxC,CAAC;IAED,UAAU,CAAC,IAAqB,IAAS,CAAC;IAE1C,cAAc,CAAC,OAAmB,IAAS,CAAC;IAE5C,QAAQ,CAAC,GAAkB;QACzB,aAAa,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IAC5D,CAAC;CACF;AAED,MAAM,UAAU,aAAa,CAC3B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAiB,EAC7C,IAA4B;IAE5B,IAAI,CAAC,EAAE,CAAC,CAAC;IACT,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,aAAa,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAClC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5B,IAAI,CACF,aAAa,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,WAAW;QAC7D,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,cAAc;QACtC,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,qBAAqB;QACnD,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,kBAAkB;QACvC,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,cAAc,CAC3C,CAAC;IACF,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC5B,IAAI,CAAC,mEAAmE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,gBAAgB,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC;IAElD,IAAI,CAAC,EAAE,CAAC,CAAC;IACT,IAAI,CAAC,WAAW,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE;YAC7B,aAAa;YACb,cAAc;YACd,UAAU;YACV,aAAa;YACb,aAAa;YACb,UAAU;SACX,CAAC,CAAC,CAAC;IACN,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,gBAAgB,CAAC,MAAM,iCAAiC;gBACvE,6BAA6B,CAChC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,cAAc,CAAC,MAAM,qDAAqD;gBACzF,4CAA4C,CAC/C,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,WAAW,CAAC,MAAM,uCAAuC;gBACxE,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,EAAE,CAAC,CAAC;YACT,IAAI,CACF,UAAU,MAAM,CAAC,WAAW,CAAC,MAAM,2BAA2B;gBAC5D,8EAA8E,CACjF,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAWD,SAAS,cAAc,CACrB,CAAa,EACb,MAA+B,EAC/B,IAAc;IAEd,IAAI,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,QAAQ,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;SAAM,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC7D,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,SAAS,GACb,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,CAAC;QAChC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,UAAU,SAAS,GAAG,SAAS,GAAG,QAAQ,EAAE,CAAC;AAC/G,CAAC"}
package/package.json ADDED
@@ -0,0 +1,52 @@
1
+ {
2
+ "name": "@drift-ci/cli",
3
+ "version": "1.0.0",
4
+ "description": "Behaviour regression testing for LLM applications",
5
+ "license": "MIT",
6
+ "homepage": "https://github.com/Drift-CI/drift-ci#readme",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/Drift-CI/drift-ci.git",
10
+ "directory": "packages/cli"
11
+ },
12
+ "bugs": {
13
+ "url": "https://github.com/Drift-CI/drift-ci/issues"
14
+ },
15
+ "type": "module",
16
+ "bin": {
17
+ "drift-ci": "./dist/index.js"
18
+ },
19
+ "main": "./dist/index.js",
20
+ "dependencies": {
21
+ "better-sqlite3": "^11.10.0",
22
+ "commander": "^12.1.0",
23
+ "ink": "^5.2.1",
24
+ "js-yaml": "^4.1.1",
25
+ "react": "^18.3.1",
26
+ "@drift-ci/core": "1.0.0"
27
+ },
28
+ "devDependencies": {
29
+ "@types/js-yaml": "^4.0.9",
30
+ "@types/node": "^20.14.0",
31
+ "@types/react": "^18.3.28",
32
+ "@vitest/coverage-v8": "^1.6.1",
33
+ "ink-testing-library": "^4.0.0",
34
+ "rimraf": "^6.0.0",
35
+ "typescript": "^5.5.0",
36
+ "vitest": "^1.6.0"
37
+ },
38
+ "files": [
39
+ "dist",
40
+ "README.md"
41
+ ],
42
+ "engines": {
43
+ "node": ">=20.0.0"
44
+ },
45
+ "scripts": {
46
+ "build": "tsc -p tsconfig.json && node scripts/add-shebang.mjs",
47
+ "dev": "tsc -p tsconfig.json --watch",
48
+ "test": "vitest run",
49
+ "test:coverage": "vitest run --coverage",
50
+ "clean": "rimraf dist"
51
+ }
52
+ }