@root-signals/scorable-cli 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +113 -0
  2. package/dist/client.d.ts +1 -0
  3. package/dist/client.d.ts.map +1 -1
  4. package/dist/client.js +1 -0
  5. package/dist/commands/annotation/index.d.ts +3 -0
  6. package/dist/commands/annotation/index.d.ts.map +1 -0
  7. package/dist/commands/annotation/index.js +145 -0
  8. package/dist/commands/auth/index.d.ts.map +1 -1
  9. package/dist/commands/auth/index.js +68 -1
  10. package/dist/commands/calibration-run/index.d.ts +3 -0
  11. package/dist/commands/calibration-run/index.d.ts.map +1 -0
  12. package/dist/commands/calibration-run/index.js +83 -0
  13. package/dist/commands/dataset-item/index.d.ts +3 -0
  14. package/dist/commands/dataset-item/index.d.ts.map +1 -0
  15. package/dist/commands/dataset-item/index.js +160 -0
  16. package/dist/commands/evaluator/calibrate.d.ts +3 -0
  17. package/dist/commands/evaluator/calibrate.d.ts.map +1 -0
  18. package/dist/commands/evaluator/calibrate.js +29 -0
  19. package/dist/commands/evaluator/create.d.ts.map +1 -1
  20. package/dist/commands/evaluator/create.js +8 -0
  21. package/dist/commands/evaluator/duplicate.d.ts.map +1 -1
  22. package/dist/commands/evaluator/duplicate.js +5 -2
  23. package/dist/commands/evaluator/execute-by-name.d.ts +1 -0
  24. package/dist/commands/evaluator/execute-by-name.d.ts.map +1 -1
  25. package/dist/commands/evaluator/execute-by-name.js +5 -0
  26. package/dist/commands/evaluator/execute.d.ts +1 -0
  27. package/dist/commands/evaluator/execute.d.ts.map +1 -1
  28. package/dist/commands/evaluator/execute.js +5 -0
  29. package/dist/commands/evaluator/export-yaml.d.ts.map +1 -1
  30. package/dist/commands/evaluator/export-yaml.js +10 -1
  31. package/dist/commands/evaluator/import-yaml.d.ts.map +1 -1
  32. package/dist/commands/evaluator/import-yaml.js +10 -1
  33. package/dist/commands/evaluator/index.d.ts.map +1 -1
  34. package/dist/commands/evaluator/index.js +2 -0
  35. package/dist/commands/evaluator/list.d.ts.map +1 -1
  36. package/dist/commands/evaluator/list.js +5 -0
  37. package/dist/commands/evaluator/update.d.ts.map +1 -1
  38. package/dist/commands/evaluator/update.js +8 -0
  39. package/dist/commands/execution-log/list.d.ts.map +1 -1
  40. package/dist/commands/execution-log/list.js +5 -0
  41. package/dist/commands/judge/create.d.ts.map +1 -1
  42. package/dist/commands/judge/create.js +5 -0
  43. package/dist/commands/judge/duplicate.d.ts.map +1 -1
  44. package/dist/commands/judge/duplicate.js +5 -2
  45. package/dist/commands/judge/exec-openai-generic.d.ts.map +1 -1
  46. package/dist/commands/judge/exec-openai-generic.js +5 -0
  47. package/dist/commands/judge/exec-openai.d.ts.map +1 -1
  48. package/dist/commands/judge/exec-openai.js +5 -0
  49. package/dist/commands/judge/execute-by-name.d.ts +1 -0
  50. package/dist/commands/judge/execute-by-name.d.ts.map +1 -1
  51. package/dist/commands/judge/execute-by-name.js +5 -0
  52. package/dist/commands/judge/execute.d.ts +1 -0
  53. package/dist/commands/judge/execute.d.ts.map +1 -1
  54. package/dist/commands/judge/execute.js +5 -0
  55. package/dist/commands/judge/generate.d.ts.map +1 -1
  56. package/dist/commands/judge/generate.js +4 -0
  57. package/dist/commands/judge/list.d.ts.map +1 -1
  58. package/dist/commands/judge/list.js +5 -0
  59. package/dist/commands/judge/update.d.ts.map +1 -1
  60. package/dist/commands/judge/update.js +5 -0
  61. package/dist/commands/project/create.d.ts +3 -0
  62. package/dist/commands/project/create.d.ts.map +1 -0
  63. package/dist/commands/project/create.js +31 -0
  64. package/dist/commands/project/delete.d.ts +3 -0
  65. package/dist/commands/project/delete.d.ts.map +1 -0
  66. package/dist/commands/project/delete.js +37 -0
  67. package/dist/commands/project/get.d.ts +3 -0
  68. package/dist/commands/project/get.d.ts.map +1 -0
  69. package/dist/commands/project/get.js +23 -0
  70. package/dist/commands/project/index.d.ts +3 -0
  71. package/dist/commands/project/index.d.ts.map +1 -0
  72. package/dist/commands/project/index.js +17 -0
  73. package/dist/commands/project/list.d.ts +3 -0
  74. package/dist/commands/project/list.d.ts.map +1 -0
  75. package/dist/commands/project/list.js +39 -0
  76. package/dist/commands/project/set-default.d.ts +3 -0
  77. package/dist/commands/project/set-default.d.ts.map +1 -0
  78. package/dist/commands/project/set-default.js +23 -0
  79. package/dist/commands/project/update.d.ts +3 -0
  80. package/dist/commands/project/update.d.ts.map +1 -0
  81. package/dist/commands/project/update.js +37 -0
  82. package/dist/commands/prompt-test/init.d.ts.map +1 -1
  83. package/dist/commands/prompt-test/init.js +9 -2
  84. package/dist/commands/prompt-test/run.d.ts +1 -1
  85. package/dist/commands/prompt-test/run.d.ts.map +1 -1
  86. package/dist/commands/prompt-test/run.js +9 -2
  87. package/dist/index.d.ts.map +1 -1
  88. package/dist/index.js +8 -0
  89. package/dist/lib/project-id.d.ts +9 -0
  90. package/dist/lib/project-id.d.ts.map +1 -0
  91. package/dist/lib/project-id.js +23 -0
  92. package/dist/output.d.ts +2 -1
  93. package/dist/output.d.ts.map +1 -1
  94. package/dist/output.js +71 -12
  95. package/dist/types.d.ts +1 -0
  96. package/dist/types.d.ts.map +1 -1
  97. package/package.json +4 -4
package/dist/output.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import chalk from "chalk";
2
2
  import Table from "cli-table3";
3
+ import { ScorableError } from "@root-signals/scorable";
3
4
  import { CliError } from "./types.js";
4
5
  const UNICODE_CHARS = {
5
6
  top: "─",
@@ -24,12 +25,42 @@ export function printJson(data) {
24
25
  export function printError(msg) {
25
26
  console.error(chalk.red("✖") + " " + msg);
26
27
  }
28
+ function extractBackendMessage(details) {
29
+ if (!details || typeof details !== "object")
30
+ return undefined;
31
+ const d = details;
32
+ if (typeof d["detail"] === "string")
33
+ return d["detail"];
34
+ if (typeof d["title"] === "string")
35
+ return d["title"];
36
+ if (typeof d["error"] === "string")
37
+ return d["error"];
38
+ if (typeof d["message"] === "string")
39
+ return d["message"];
40
+ for (const v of Object.values(d)) {
41
+ if (Array.isArray(v) && v.length && typeof v[0] === "string") {
42
+ return v[0];
43
+ }
44
+ }
45
+ return undefined;
46
+ }
27
47
  export function handleSdkError(e) {
28
48
  if (e instanceof CliError)
29
49
  throw e;
30
- const message = e instanceof Error ? e.message || String(e) : String(e);
31
- printError(message);
32
- throw new CliError(1, message);
50
+ let message;
51
+ if (e instanceof ScorableError) {
52
+ const detail = extractBackendMessage(e.details);
53
+ message = detail ?? e.detail ?? e.title ?? e.message ?? `API Error ${e.status}: ${e.code}`;
54
+ }
55
+ else if (e instanceof Error) {
56
+ message = e.message || String(e);
57
+ }
58
+ else {
59
+ message = String(e);
60
+ }
61
+ const oneLine = message.replace(/\s*\n\s*/g, " ").trim();
62
+ printError(oneLine);
63
+ throw new CliError(1, oneLine);
33
64
  }
34
65
  export function printSuccess(msg) {
35
66
  console.log(chalk.green("✔") + " " + msg);
@@ -46,16 +77,19 @@ export function printMessage(msg) {
46
77
  function truncate(s, max) {
47
78
  return s.length > max ? s.slice(0, max - 1) + "…" : s;
48
79
  }
80
+ function projectIdCell(value) {
81
+ return value ?? "";
82
+ }
49
83
  export function printJudgeTable(judges, nextCursor) {
50
84
  const table = new Table({
51
- head: ["ID", "Name", "Intent", "Created At"].map((h) => chalk.bold.cyan(h)),
85
+ head: ["ID", "Name", "Intent", "Project ID", "Created At"].map((h) => chalk.bold.cyan(h)),
52
86
  chars: UNICODE_CHARS,
53
- colWidths: [38, 30, 52, 12],
87
+ colWidths: [38, 24, 40, 38, 12],
54
88
  wordWrap: true,
55
89
  });
56
90
  for (const j of judges) {
57
91
  const date = (j.created_at ?? "").slice(0, 10);
58
- table.push([j.id, j.name, truncate(j.intent ?? "", 50), date]);
92
+ table.push([j.id, j.name, truncate(j.intent ?? "", 38), projectIdCell(j.project_id), date]);
59
93
  }
60
94
  console.log(table.toString());
61
95
  if (nextCursor) {
@@ -65,14 +99,32 @@ export function printJudgeTable(judges, nextCursor) {
65
99
  }
66
100
  export function printEvaluatorTable(evaluators, nextCursor) {
67
101
  const table = new Table({
68
- head: ["ID", "Name", "Created At"].map((h) => chalk.bold.cyan(h)),
102
+ head: ["ID", "Name", "Project ID", "Created At"].map((h) => chalk.bold.cyan(h)),
69
103
  chars: UNICODE_CHARS,
70
- colWidths: [38, 30, 12],
104
+ colWidths: [38, 30, 38, 12],
71
105
  wordWrap: true,
72
106
  });
73
107
  for (const e of evaluators) {
74
108
  const date = (e.created_at ?? "").slice(0, 10);
75
- table.push([e.id, e.name, date]);
109
+ table.push([e.id, e.name, projectIdCell(e.project_id), date]);
110
+ }
111
+ console.log(table.toString());
112
+ if (nextCursor) {
113
+ const cursor = nextCursor.split("cursor=")[1] ?? nextCursor;
114
+ printInfo(`Next page available. Use --cursor "${cursor}"`);
115
+ }
116
+ }
117
+ export function printProjectTable(projects, nextCursor) {
118
+ const table = new Table({
119
+ head: ["ID", "Name", "Description", "Created At"].map((h) => chalk.bold.cyan(h)),
120
+ chars: UNICODE_CHARS,
121
+ colWidths: [38, 30, 40, 12],
122
+ wordWrap: true,
123
+ });
124
+ for (const p of projects) {
125
+ const date = (p.created_at ?? "").slice(0, 10);
126
+ const name = p.is_default ? `${p.name} (default)` : p.name;
127
+ table.push([p.id, name, truncate(p.description ?? "", 38), date]);
76
128
  }
77
129
  console.log(table.toString());
78
130
  if (nextCursor) {
@@ -99,15 +151,22 @@ export function printModelTable(models, nextCursor) {
99
151
  }
100
152
  export function printExecutionLogTable(logs, nextCursor) {
101
153
  const table = new Table({
102
- head: ["ID", "Item Name", "Type", "Score", "Created At"].map((h) => chalk.bold.cyan(h)),
154
+ head: ["ID", "Item Name", "Type", "Score", "Project ID", "Created At"].map((h) => chalk.bold.cyan(h)),
103
155
  chars: UNICODE_CHARS,
104
- colWidths: [38, 30, 15, 8, 12],
156
+ colWidths: [38, 24, 15, 8, 38, 12],
105
157
  wordWrap: true,
106
158
  });
107
159
  for (const l of logs) {
108
160
  const date = (l.created_at ?? "").slice(0, 10);
109
161
  const score = l.score != null ? String(l.score) : "";
110
- table.push([l.id, truncate(l.executed_item_name, 28), l.execution_type, score, date]);
162
+ table.push([
163
+ l.id,
164
+ truncate(l.executed_item_name, 22),
165
+ l.execution_type,
166
+ score,
167
+ projectIdCell(l.project_id),
168
+ date,
169
+ ]);
111
170
  }
112
171
  console.log(table.toString());
113
172
  if (nextCursor) {
package/dist/types.d.ts CHANGED
@@ -39,6 +39,7 @@ export interface PromptTestConfig {
39
39
  evaluators: EvaluatorConfig[];
40
40
  response_schema?: Record<string, unknown>;
41
41
  dataset_id?: string;
42
+ project_id?: string;
42
43
  }
43
44
  export declare class CliError extends Error {
44
45
  readonly exitCode: number;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,KAAK,CAAC;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,QAAQ,EAAE,MAAM;gBAAhB,QAAQ,EAAE,MAAM,EAChC,OAAO,EAAE,MAAM;CAKlB"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,KAAK,CAAC;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,QAAQ,EAAE,MAAM;gBAAhB,QAAQ,EAAE,MAAM,EAChC,OAAO,EAAE,MAAM;CAKlB"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@root-signals/scorable-cli",
3
- "version": "0.13.0",
3
+ "version": "0.15.0",
4
4
  "description": "CLI for Scorable",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Scorable",
@@ -32,10 +32,10 @@
32
32
  },
33
33
  "dependencies": {
34
34
  "@inquirer/prompts": "^8.3.2",
35
- "@root-signals/scorable": "^0.10.0",
35
+ "@root-signals/scorable": "^0.12.0",
36
36
  "chalk": "^5.6.2",
37
37
  "cli-table3": "^0.6.5",
38
- "commander": "^14.0.3",
38
+ "commander": "^15.0.0",
39
39
  "js-yaml": "^4.1.1",
40
40
  "ora": "^9.3.0",
41
41
  "zod": "^4.4.3"
@@ -43,7 +43,7 @@
43
43
  "devDependencies": {
44
44
  "@types/js-yaml": "^4.0.9",
45
45
  "@types/node": "^25.5.0",
46
- "oxfmt": "^0.41.0",
46
+ "oxfmt": "^0.53.0",
47
47
  "oxlint": "^1.56.0",
48
48
  "typescript": "^5.9.3",
49
49
  "vitest": "^4.1.5"