@respan/cli 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/commands/datasets/create-span.js +5 -3
  2. package/dist/commands/datasets/create.js +5 -4
  3. package/dist/commands/datasets/get-span.js +5 -1
  4. package/dist/commands/datasets/get.js +1 -1
  5. package/dist/commands/datasets/list.js +5 -1
  6. package/dist/commands/datasets/spans.js +1 -1
  7. package/dist/commands/datasets/update.js +3 -6
  8. package/dist/commands/{datasets/add-spans.d.ts → eval/index.d.ts} +2 -3
  9. package/dist/commands/eval/index.js +26 -0
  10. package/dist/commands/evaluators/create.js +10 -7
  11. package/dist/commands/evaluators/get.js +1 -1
  12. package/dist/commands/evaluators/list.js +1 -1
  13. package/dist/commands/evaluators/run.js +8 -7
  14. package/dist/commands/evaluators/update.js +6 -7
  15. package/dist/commands/experiments/create.js +9 -8
  16. package/dist/commands/experiments/get.js +1 -1
  17. package/dist/commands/experiments/list.js +1 -1
  18. package/dist/commands/logs/create.js +9 -7
  19. package/dist/commands/logs/get.js +1 -1
  20. package/dist/commands/logs/list.js +4 -3
  21. package/dist/commands/logs/summary.js +3 -2
  22. package/dist/commands/prompts/create-version.js +9 -9
  23. package/dist/commands/prompts/create.js +5 -4
  24. package/dist/commands/prompts/get.js +1 -1
  25. package/dist/commands/prompts/list.js +1 -1
  26. package/dist/commands/prompts/update.js +3 -6
  27. package/dist/commands/prompts/versions.js +1 -1
  28. package/dist/commands/setup/doctor.d.ts +0 -1
  29. package/dist/commands/setup/doctor.js +25 -40
  30. package/dist/commands/setup/gateway.d.ts +16 -0
  31. package/dist/commands/setup/gateway.js +33 -0
  32. package/dist/commands/setup/index.d.ts +4 -33
  33. package/dist/commands/setup/index.js +11 -907
  34. package/dist/commands/setup/tracing.d.ts +16 -0
  35. package/dist/commands/setup/tracing.js +33 -0
  36. package/dist/commands/traces/get.js +1 -1
  37. package/dist/commands/traces/list.js +8 -10
  38. package/dist/commands/traces/summary.js +6 -2
  39. package/dist/commands/users/get.js +1 -1
  40. package/dist/commands/users/list.js +5 -7
  41. package/dist/lib/agents.d.ts +16 -0
  42. package/dist/lib/agents.js +75 -0
  43. package/dist/lib/auth.js +24 -0
  44. package/dist/lib/base-command.d.ts +2 -0
  45. package/dist/lib/base-command.js +11 -1
  46. package/dist/lib/colors.d.ts +7 -0
  47. package/dist/lib/colors.js +10 -0
  48. package/dist/lib/eval/runner.d.ts +15 -0
  49. package/dist/lib/eval/runner.js +292 -0
  50. package/dist/lib/eval/schema.d.ts +67 -0
  51. package/dist/lib/eval/schema.js +22 -0
  52. package/dist/lib/integrate.d.ts +6 -0
  53. package/dist/lib/integrate.js +11 -0
  54. package/dist/lib/setup-base-command.d.ts +84 -0
  55. package/dist/lib/setup-base-command.js +649 -0
  56. package/dist/lib/skill-content.d.ts +1 -0
  57. package/dist/lib/skill-content.js +16 -0
  58. package/dist/lib/skill-refs.generated.d.ts +6 -0
  59. package/dist/lib/skill-refs.generated.js +1489 -0
  60. package/oclif.manifest.json +385 -230
  61. package/package.json +8 -3
  62. package/dist/commands/datasets/add-spans.js +0 -24
@@ -0,0 +1,16 @@
1
+ import { SetupBaseCommand } from '../../lib/setup-base-command.js';
2
+ export default class SetupTracing extends SetupBaseCommand {
3
+ static description: string;
4
+ static examples: string[];
5
+ static flags: {
6
+ agent: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
7
+ 'no-instrument': import("@oclif/core/interfaces").BooleanFlag<boolean>;
8
+ json: import("@oclif/core/interfaces").BooleanFlag<boolean>;
9
+ csv: import("@oclif/core/interfaces").BooleanFlag<boolean>;
10
+ 'api-key': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
11
+ 'base-url': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
12
+ profile: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
13
+ verbose: import("@oclif/core/interfaces").BooleanFlag<boolean>;
14
+ };
15
+ run(): Promise<void>;
16
+ }
@@ -0,0 +1,33 @@
1
+ import { Flags } from '@oclif/core';
2
+ import { SetupBaseCommand } from '../../lib/setup-base-command.js';
3
+ class SetupTracing extends SetupBaseCommand {
4
+ async run() {
5
+ const { flags } = await this.parse(SetupTracing);
6
+ this.globalFlags = flags;
7
+ await this.runSetup('tracing', {
8
+ agent: flags.agent,
9
+ noInstrument: flags['no-instrument'],
10
+ });
11
+ }
12
+ }
13
+ SetupTracing.description = `Set up Respan SDK tracing in this project.
14
+
15
+ Runs the tracing setup flow directly (skips the "Tracing or Gateway?"
16
+ question). Sets up your API key, installs the Respan skill, and optionally
17
+ opens your coding agent to instrument the project.`;
18
+ SetupTracing.examples = [
19
+ 'respan setup tracing',
20
+ 'respan setup tracing --agent claude-code',
21
+ 'respan setup tracing --no-instrument',
22
+ ];
23
+ SetupTracing.flags = {
24
+ ...SetupBaseCommand.baseFlags,
25
+ agent: Flags.string({
26
+ description: 'Agent to configure (claude-code, cursor, codex-cli, gemini-cli, opencode)',
27
+ }),
28
+ 'no-instrument': Flags.boolean({
29
+ description: 'Skip opening the agent after setup',
30
+ default: false,
31
+ }),
32
+ };
33
+ export default SetupTracing;
@@ -6,7 +6,7 @@ class TracesGet extends BaseCommand {
6
6
  this.globalFlags = flags;
7
7
  try {
8
8
  const client = this.getClient();
9
- const data = await this.spin('Fetching trace', () => client.traces.retrieveTrace({ trace_unique_id: args.id }));
9
+ const data = await this.spin('Fetching trace', () => client.traces.retrieveTrace({ Authorization: this.getAuthHeader(), trace_unique_id: args.id }));
10
10
  this.log(JSON.stringify(data, null, 2));
11
11
  }
12
12
  catch (error) {
@@ -8,22 +8,20 @@ class TracesList extends BaseCommand {
8
8
  this.globalFlags = flags;
9
9
  try {
10
10
  const client = this.getClient();
11
- let bodyFilters = {};
11
+ let bodyFilters;
12
12
  if (flags.filter && flags.filter.length > 0) {
13
13
  bodyFilters = parseFilters(flags.filter);
14
14
  }
15
- const queryParams = {
15
+ const data = await this.spin('Fetching traces', () => client.traces.listTraces({
16
+ Authorization: this.getAuthHeader(),
16
17
  page_size: flags.limit,
17
18
  page: flags.page,
18
19
  sort_by: flags['sort-by'],
19
- };
20
- if (flags['start-time'])
21
- queryParams.start_time = flags['start-time'];
22
- if (flags['end-time'])
23
- queryParams.end_time = flags['end-time'];
24
- if (flags.environment)
25
- queryParams.environment = flags.environment;
26
- const data = await this.spin('Fetching traces', () => client.traces.list({ filters: bodyFilters }, { queryParams }));
20
+ ...(flags['start-time'] ? { start_time: flags['start-time'] } : {}),
21
+ ...(flags['end-time'] ? { end_time: flags['end-time'] } : {}),
22
+ ...(flags.environment ? { environment: flags.environment } : {}),
23
+ ...(bodyFilters ? { filters: bodyFilters } : {}),
24
+ }));
27
25
  this.outputResult(data, [
28
26
  'trace_unique_id', 'name', 'duration', 'span_count', 'total_cost', 'error_count', 'start_time',
29
27
  ]);
@@ -6,7 +6,11 @@ class TracesSummary extends BaseCommand {
6
6
  this.globalFlags = flags;
7
7
  try {
8
8
  const client = this.getClient();
9
- const data = await this.spin('Fetching traces summary', () => client.traces.retrieveTracesSummary({ start_time: flags['start-time'], end_time: flags['end-time'] }));
9
+ const data = await this.spin('Fetching traces summary', () => client.spans.getSpansSummary({
10
+ Authorization: this.getAuthHeader(),
11
+ start_time: flags['start-time'],
12
+ end_time: flags['end-time'],
13
+ }));
10
14
  this.log(JSON.stringify(data, null, 2));
11
15
  }
12
16
  catch (error) {
@@ -14,7 +18,7 @@ class TracesSummary extends BaseCommand {
14
18
  }
15
19
  }
16
20
  }
17
- TracesSummary.description = 'Get a summary of traces for a time range';
21
+ TracesSummary.description = 'Get a span-level summary of activity for a time range (counts, latency, cost, errors aggregated across spans)';
18
22
  TracesSummary.flags = {
19
23
  ...BaseCommand.baseFlags,
20
24
  'start-time': Flags.string({ description: 'Start time (ISO 8601)', required: true }),
@@ -6,7 +6,7 @@ class UsersGet extends BaseCommand {
6
6
  this.globalFlags = flags;
7
7
  try {
8
8
  const client = this.getClient();
9
- const data = await this.spin('Fetching user', () => client.users.retrieveUser({ customer_identifier: args.id }));
9
+ const data = await this.spin('Fetching user', () => client.users.retrieveUser({ Authorization: this.getAuthHeader(), customer_identifier: args.id }));
10
10
  this.log(JSON.stringify(data, null, 2));
11
11
  }
12
12
  catch (error) {
@@ -7,15 +7,13 @@ class UsersList extends BaseCommand {
7
7
  this.globalFlags = flags;
8
8
  try {
9
9
  const client = this.getClient();
10
- const params = {
10
+ const data = await this.spin('Fetching users', () => client.users.listCustomers({
11
+ Authorization: this.getAuthHeader(),
11
12
  page_size: flags.limit,
12
13
  page: flags.page,
13
- };
14
- if (flags['sort-by'])
15
- params.sort_by = flags['sort-by'];
16
- if (flags.environment)
17
- params.environment = flags.environment;
18
- const data = await this.spin('Fetching users', () => client.users.list(params));
14
+ ...(flags['sort-by'] ? { sort_by: flags['sort-by'] } : {}),
15
+ ...(flags.environment ? { environment: flags.environment } : {}),
16
+ }));
19
17
  this.outputResult(data, [
20
18
  'customer_identifier', 'name', 'email', 'number_of_requests', 'total_cost', 'last_active_timeframe',
21
19
  ]);
@@ -0,0 +1,16 @@
1
+ export type CliTool = 'claude-code' | 'cursor' | 'codex-cli' | 'gemini-cli' | 'opencode';
2
+ export interface ToolMeta {
3
+ name: string;
4
+ binary: string;
5
+ description: string;
6
+ configDirs: string[];
7
+ }
8
+ export declare const CLI_TOOLS: Record<CliTool, ToolMeta>;
9
+ export interface DetectionSignal {
10
+ tool: CliTool;
11
+ onPath: boolean;
12
+ hasConfigDir: boolean;
13
+ reason: string;
14
+ }
15
+ export declare function isBinaryInstalled(binary: string): boolean;
16
+ export declare function detectAgents(projectRoot: string, home: string): DetectionSignal[];
@@ -0,0 +1,75 @@
1
+ // Coding-agent registry and detection primitives.
2
+ //
3
+ // This is the single source of truth for the agents Respan can configure.
4
+ // commands/setup/index.ts and commands/setup/doctor.ts previously maintained
5
+ // parallel copies of this metadata.
6
+ import * as fs from 'node:fs';
7
+ import * as path from 'node:path';
8
+ import { execSync } from 'node:child_process';
9
+ export const CLI_TOOLS = {
10
+ 'claude-code': {
11
+ name: 'Claude Code',
12
+ binary: 'claude',
13
+ description: 'Anthropic\'s coding agent',
14
+ configDirs: ['~/.claude', '.claude'],
15
+ },
16
+ 'cursor': {
17
+ name: 'Cursor',
18
+ binary: 'cursor',
19
+ description: 'AI-powered code editor',
20
+ configDirs: ['.cursor', '.cursorrc'],
21
+ },
22
+ 'codex-cli': {
23
+ name: 'Codex CLI',
24
+ binary: 'codex',
25
+ description: 'OpenAI\'s coding agent',
26
+ configDirs: ['~/.codex', '.codex'],
27
+ },
28
+ 'gemini-cli': {
29
+ name: 'Gemini CLI',
30
+ binary: 'gemini',
31
+ description: 'Google\'s coding agent',
32
+ configDirs: ['~/.gemini', '.gemini'],
33
+ },
34
+ 'opencode': {
35
+ name: 'OpenCode',
36
+ binary: 'opencode',
37
+ description: 'Open-source coding agent',
38
+ configDirs: ['.opencode'],
39
+ },
40
+ };
41
+ export function isBinaryInstalled(binary) {
42
+ try {
43
+ execSync(`command -v ${binary}`, { stdio: 'pipe' });
44
+ return true;
45
+ }
46
+ catch {
47
+ return false;
48
+ }
49
+ }
50
+ export function detectAgents(projectRoot, home) {
51
+ const signals = [];
52
+ for (const [id, meta] of Object.entries(CLI_TOOLS)) {
53
+ const onPath = isBinaryInstalled(meta.binary);
54
+ const hasConfigDir = meta.configDirs.some((dir) => {
55
+ const resolved = dir.startsWith('~')
56
+ ? path.join(home, dir.slice(1))
57
+ : dir.startsWith('.')
58
+ ? path.join(projectRoot, dir)
59
+ : dir;
60
+ return fs.existsSync(resolved);
61
+ });
62
+ const reasons = [];
63
+ if (onPath)
64
+ reasons.push('binary on PATH');
65
+ if (hasConfigDir)
66
+ reasons.push('config directory found');
67
+ signals.push({
68
+ tool: id,
69
+ onPath,
70
+ hasConfigDir,
71
+ reason: reasons.length > 0 ? reasons.join(', ') : 'not detected',
72
+ });
73
+ }
74
+ return signals;
75
+ }
package/dist/lib/auth.js CHANGED
@@ -1,4 +1,6 @@
1
+ import * as path from 'node:path';
1
2
  import { getCredential } from './config.js';
3
+ import { findProjectRoot, readTextFile, extractEnvVar } from './integrate.js';
2
4
  export const DEFAULT_BASE_URL = 'https://api.respan.ai';
3
5
  export const ENTERPRISE_BASE_URL = 'https://endpoint.respan.ai';
4
6
  function normalizeBaseUrl(baseUrl) {
@@ -9,6 +11,19 @@ function resolveConfiguredBaseUrl(credential, flagBaseUrl) {
9
11
  // the main persistent configuration path for CLI users.
10
12
  return normalizeBaseUrl(flagBaseUrl || process.env.RESPAN_API_BASE_URL || credential?.baseUrl || DEFAULT_BASE_URL);
11
13
  }
14
+ /**
15
+ * Read RESPAN_API_KEY from the project's `.env`. setup writes it there as the
16
+ * project source of truth, so in-project commands (e.g. `respan logs list`)
17
+ * work right after setup without a separate `respan auth login`.
18
+ */
19
+ function readProjectEnvKey() {
20
+ try {
21
+ return extractEnvVar(readTextFile(path.join(findProjectRoot(), '.env')), 'RESPAN_API_KEY');
22
+ }
23
+ catch {
24
+ return undefined;
25
+ }
26
+ }
12
27
  export function resolveAuth(flags) {
13
28
  const credential = getCredential(flags.profile);
14
29
  const baseUrl = resolveConfiguredBaseUrl(credential, flags['base-url']);
@@ -21,6 +36,15 @@ export function resolveAuth(flags) {
21
36
  baseUrl,
22
37
  };
23
38
  }
39
+ // The project .env key wins over a saved credential — but an explicit
40
+ // --profile selects a specific credential (and its base URL), so don't let
41
+ // an ambient .env key override that choice and break enterprise setups.
42
+ if (!flags.profile) {
43
+ const projectEnvKey = readProjectEnvKey();
44
+ if (projectEnvKey) {
45
+ return { apiKey: projectEnvKey, baseUrl };
46
+ }
47
+ }
24
48
  if (credential) {
25
49
  return credentialToAuth(credential, baseUrl);
26
50
  }
@@ -14,6 +14,8 @@ export declare abstract class BaseCommand extends Command {
14
14
  protected globalFlags: GlobalFlags;
15
15
  protected getAuth(): AuthConfig;
16
16
  protected getClient(): RespanClient;
17
+ protected getAuthHeader(): string;
18
+ protected getBaseUrl(): string;
17
19
  protected getOutputFormat(): 'json' | 'csv' | 'table';
18
20
  protected outputResult(data: unknown, columns?: string[]): void;
19
21
  protected spin<T>(label: string, fn: () => Promise<T>): Promise<T>;
@@ -17,10 +17,20 @@ export class BaseCommand extends Command {
17
17
  if (!token)
18
18
  throw new Error('No API key or access token available.');
19
19
  return new RespanClient({
20
- token,
20
+ headers: { Authorization: `Bearer ${token}` },
21
21
  ...(auth.baseUrl ? { environment: auth.baseUrl } : {}),
22
22
  });
23
23
  }
24
+ getAuthHeader() {
25
+ const auth = this.getAuth();
26
+ const token = auth.apiKey || auth.accessToken;
27
+ if (!token)
28
+ throw new Error('No API key or access token available.');
29
+ return `Bearer ${token}`;
30
+ }
31
+ getBaseUrl() {
32
+ return this.getAuth().baseUrl || 'https://api.respan.ai';
33
+ }
24
34
  getOutputFormat() {
25
35
  if (this.globalFlags.json)
26
36
  return 'json';
@@ -0,0 +1,7 @@
1
+ export declare const PC = "\u001B[38;2;100;131;240m";
2
+ export declare const RESET = "\u001B[0m";
3
+ export declare const DIM = "\u001B[2m";
4
+ export declare const GREEN = "\u001B[32m";
5
+ export declare const BOLD = "\u001B[1m";
6
+ export declare const RED = "\u001B[31m";
7
+ export declare const YELLOW = "\u001B[33m";
@@ -0,0 +1,10 @@
1
+ // Shared ANSI color/style constants for setup-related commands.
2
+ // Consolidated from the copies previously inlined in commands/setup/index.ts
3
+ // and commands/setup/doctor.ts.
4
+ export const PC = '\x1b[38;2;100;131;240m';
5
+ export const RESET = '\x1b[0m';
6
+ export const DIM = '\x1b[2m';
7
+ export const GREEN = '\x1b[32m';
8
+ export const BOLD = '\x1b[1m';
9
+ export const RED = '\x1b[31m';
10
+ export const YELLOW = '\x1b[33m';
@@ -0,0 +1,15 @@
1
+ import { RespanClient } from '@respan/respan-api';
2
+ export interface RunEvalResult {
3
+ prompt_id: string;
4
+ prompt_version_id?: string;
5
+ prompt_version_deployed?: number;
6
+ dataset_id: string;
7
+ evaluator_workflow_ids: string[];
8
+ experiment_id: string;
9
+ created_prompt: boolean;
10
+ created_dataset: boolean;
11
+ created_experiment: boolean;
12
+ evaluators_created: number;
13
+ rows_added: number;
14
+ }
15
+ export declare function runEval(filePath: string, client: RespanClient, authHeader: string, baseUrl: string, log?: (msg: string) => void): Promise<RunEvalResult>;
@@ -0,0 +1,292 @@
1
+ import { readFileSync, writeFileSync } from 'node:fs';
2
+ import { parseEvalFile } from './schema.js';
3
+ export async function runEval(filePath, client, authHeader, baseUrl, log = () => { }) {
4
+ const raw = readFileSync(filePath, 'utf8');
5
+ const evalFile = parseEvalFile(raw);
6
+ const promptResult = await ensurePrompt(evalFile, client, authHeader, log);
7
+ persist(filePath, evalFile);
8
+ const datasetResult = await ensureDataset(evalFile, client, authHeader, log);
9
+ persist(filePath, evalFile);
10
+ const evaluatorResult = await ensureEvaluatorWorkflows(evalFile, client, authHeader, baseUrl, log);
11
+ persist(filePath, evalFile);
12
+ const experimentResult = await ensureExperiment(evalFile, promptResult.id, datasetResult.id, evaluatorResult.workflow_ids, client, authHeader, log);
13
+ persist(filePath, evalFile);
14
+ return {
15
+ prompt_id: promptResult.id,
16
+ prompt_version_id: promptResult.version_id,
17
+ prompt_version_deployed: promptResult.deployed_version,
18
+ dataset_id: datasetResult.id,
19
+ evaluator_workflow_ids: evaluatorResult.workflow_ids,
20
+ experiment_id: experimentResult.id,
21
+ created_prompt: promptResult.created,
22
+ created_dataset: datasetResult.created,
23
+ created_experiment: experimentResult.created,
24
+ evaluators_created: evaluatorResult.created_count,
25
+ rows_added: datasetResult.rows_added,
26
+ };
27
+ }
28
+ async function ensurePrompt(evalFile, client, Authorization, log) {
29
+ const { prompt } = evalFile;
30
+ if (prompt.id) {
31
+ log(`prompt: reusing ${prompt.id}`);
32
+ return { id: prompt.id, version_id: prompt.version_id, created: false };
33
+ }
34
+ log(`prompt: creating "${prompt.name}"`);
35
+ const created = (await client.prompts.createPrompt({
36
+ Authorization,
37
+ name: prompt.name,
38
+ ...(prompt.description ? { description: prompt.description } : {}),
39
+ }));
40
+ const promptId = String(created.id ?? created.prompt_id ?? '');
41
+ if (!promptId)
42
+ throw new Error('createPrompt: response missing id');
43
+ prompt.id = promptId;
44
+ log(`prompt: creating version on ${promptId}`);
45
+ const version = (await client.prompts.createPromptVersion({
46
+ Authorization,
47
+ prompt_id: promptId,
48
+ messages: prompt.messages,
49
+ model: prompt.model,
50
+ ...(prompt.temperature !== undefined ? { temperature: prompt.temperature } : {}),
51
+ ...(prompt.max_tokens !== undefined ? { max_tokens: prompt.max_tokens } : {}),
52
+ ...(prompt.top_p !== undefined ? { top_p: prompt.top_p } : {}),
53
+ }));
54
+ const versionId = String(version.id ?? version.version_id ?? '');
55
+ if (versionId)
56
+ prompt.version_id = versionId;
57
+ const versionNumber = typeof version.version === 'number'
58
+ ? version.version
59
+ : Number.parseInt(String(version.version ?? '1'), 10);
60
+ log(`prompt: committing draft version ${versionNumber}`);
61
+ await client.prompts.commitPromptVersion({ Authorization, prompt_id: promptId });
62
+ log(`prompt: deploying version ${versionNumber}`);
63
+ await client.prompts.deployPromptVersion({ Authorization, prompt_id: promptId, version: versionNumber });
64
+ return {
65
+ id: promptId,
66
+ version_id: versionId || undefined,
67
+ deployed_version: versionNumber,
68
+ created: true,
69
+ };
70
+ }
71
+ async function ensureDataset(evalFile, client, Authorization, log) {
72
+ const { dataset } = evalFile;
73
+ if (dataset.id) {
74
+ log(`dataset: reusing ${dataset.id}`);
75
+ return { id: dataset.id, created: false, rows_added: 0 };
76
+ }
77
+ log(`dataset: creating "${dataset.name}"`);
78
+ const created = (await client.datasets.createDataset({
79
+ Authorization,
80
+ name: dataset.name,
81
+ is_empty: true,
82
+ ...(dataset.description ? { description: dataset.description } : {}),
83
+ }));
84
+ const datasetId = String(created.id ?? created.dataset_id ?? '');
85
+ if (!datasetId)
86
+ throw new Error('createDataset: response missing id');
87
+ dataset.id = datasetId;
88
+ const logs = dataset.rows.map((row) => {
89
+ const { expected_output, ...variables } = row;
90
+ return {
91
+ input: variables,
92
+ ...(expected_output !== undefined ? { expected_output } : {}),
93
+ };
94
+ });
95
+ const result = (await client.datasets.bulkCreateDatasetLogs({
96
+ Authorization,
97
+ dataset_id: datasetId,
98
+ logs,
99
+ }));
100
+ const successCount = typeof result.success_count === 'number' ? result.success_count : 0;
101
+ const errorCount = typeof result.error_count === 'number' ? result.error_count : 0;
102
+ log(`dataset: added ${successCount}/${logs.length} row(s)` + (errorCount ? ` (${errorCount} errors)` : ''));
103
+ return { id: datasetId, created: true, rows_added: successCount };
104
+ }
105
+ async function ensureEvaluatorWorkflows(evalFile, client, Authorization, baseUrl, log) {
106
+ const experiment = evalFile.experiment ?? (evalFile.experiment = {});
107
+ const evaluators = experiment.evaluators ?? [];
108
+ const explicitWorkflowIds = experiment.evaluator_workflow_ids ?? [];
109
+ let createdCount = 0;
110
+ const workflowIds = [];
111
+ // Resolve user-supplied workflow IDs to their deployed-version row IDs.
112
+ // The experiment API expects version IDs (the `evaluator_workflow_ids` field
113
+ // name is misleading), so we look up each workflow's deployed version.
114
+ for (const wfId of explicitWorkflowIds) {
115
+ const versionRowId = await getDeployedWorkflowVersionId(Authorization, baseUrl, wfId);
116
+ if (!versionRowId) {
117
+ throw new Error(`workflow ${wfId} (from experiment.evaluator_workflow_ids) has no deployed version.`);
118
+ }
119
+ workflowIds.push(versionRowId);
120
+ }
121
+ for (const ev of evaluators) {
122
+ if (!ev.evaluator_id) {
123
+ log(`evaluator: creating "${ev.name}" (${ev.type})`);
124
+ const created = (await client.evaluators.createEvaluator({
125
+ Authorization,
126
+ name: ev.name,
127
+ type: ev.type,
128
+ score_value_type: ev.score_value_type,
129
+ ...(ev.description ? { description: ev.description } : {}),
130
+ ...(ev.score_config ? { score_config: ev.score_config } : {}),
131
+ ...(ev.passing_conditions ? { passing_conditions: ev.passing_conditions } : {}),
132
+ ...(ev.llm_config ? { llm_config: ev.llm_config } : {}),
133
+ ...(ev.code_config ? { code_config: ev.code_config } : {}),
134
+ }));
135
+ const evaluatorId = String(created.id ?? created.evaluator_id ?? '');
136
+ if (!evaluatorId)
137
+ throw new Error('createEvaluator: response missing id');
138
+ ev.evaluator_id = evaluatorId;
139
+ createdCount += 1;
140
+ }
141
+ else {
142
+ log(`evaluator: reusing ${ev.evaluator_id}`);
143
+ }
144
+ if (!ev.workflow_id) {
145
+ const workflowName = `${ev.name} workflow`;
146
+ log(`workflow: creating "${workflowName}" wrapping evaluator ${ev.evaluator_id}`);
147
+ const nodeIdSeed = ev.evaluator_id.slice(0, 8);
148
+ const blocklyNodeId = safeNodeId(nodeIdSeed);
149
+ const taskId = `blockly_hidden_eval_task_${blocklyNodeId}`;
150
+ const taskLabel = `blockly_hidden_eval_${blocklyNodeId}`;
151
+ const outputField = outputFieldForScoreType(ev.score_value_type);
152
+ const taskConfig = {
153
+ evaluator_id: ev.evaluator_id,
154
+ score_value_type: ev.score_value_type,
155
+ score_config: ev.score_config ?? {},
156
+ is_auto_persist_enabled: true,
157
+ _blockly_hidden_eval: true,
158
+ _blockly_is_result: true,
159
+ _blockly_node_id: blocklyNodeId,
160
+ ...(outputField ? { _blockly_output_field: outputField } : {}),
161
+ _blockly_evaluator_kind: ev.type,
162
+ ...(ev.llm_config ? { llm_config: ev.llm_config } : {}),
163
+ ...(ev.code_config ? { code_config: ev.code_config } : {}),
164
+ };
165
+ const createdWf = (await client.workflows.createWorkflow({
166
+ Authorization,
167
+ type: 'evaluators',
168
+ trigger_event_type: 'eval_only',
169
+ name: workflowName,
170
+ tasks: [{
171
+ id: taskId,
172
+ type: 'eval',
173
+ generation_method: ev.type,
174
+ label: taskLabel,
175
+ config: taskConfig,
176
+ next: null,
177
+ }],
178
+ }));
179
+ const workflowId = String(createdWf.workflow_id ?? createdWf.id ?? '');
180
+ if (!workflowId)
181
+ throw new Error('createWorkflow: response missing workflow_id');
182
+ ev.workflow_id = workflowId;
183
+ log(`workflow: committing draft v1`);
184
+ await commitWorkflow(Authorization, baseUrl, workflowId);
185
+ log(`workflow: deploying v1`);
186
+ const deployed = (await client.workflows.deployWorkflow({
187
+ Authorization,
188
+ workflow_id: workflowId,
189
+ version: 1,
190
+ }));
191
+ const deployedVersion = typeof deployed.version === 'number' ? deployed.version : 1;
192
+ ev.workflow_version_deployed = deployedVersion;
193
+ const versionRowId = String(deployed.id ?? '');
194
+ if (versionRowId)
195
+ ev.workflow_version_id = versionRowId;
196
+ }
197
+ else {
198
+ log(`workflow: reusing ${ev.workflow_id}`);
199
+ }
200
+ if (!ev.workflow_version_id) {
201
+ log(`workflow: looking up deployed version row id for ${ev.workflow_id}`);
202
+ const versionRowId = await getDeployedWorkflowVersionId(Authorization, baseUrl, ev.workflow_id);
203
+ if (!versionRowId) {
204
+ throw new Error(`workflow ${ev.workflow_id} has no deployed version; rerun after committing/deploying.`);
205
+ }
206
+ ev.workflow_version_id = versionRowId;
207
+ }
208
+ workflowIds.push(ev.workflow_version_id);
209
+ }
210
+ return { workflow_ids: workflowIds, created_count: createdCount };
211
+ }
212
+ async function ensureExperiment(evalFile, promptId, datasetId, workflowIds, client, Authorization, log) {
213
+ const experiment = evalFile.experiment ?? (evalFile.experiment = {});
214
+ if (experiment.id) {
215
+ log(`experiment: reusing ${experiment.id}`);
216
+ return { id: experiment.id, created: false };
217
+ }
218
+ const name = experiment.name ?? evalFile.experiment_name ?? `${evalFile.name} run`;
219
+ const evaluatorSlugs = experiment.evaluator_slugs ?? [];
220
+ if (workflowIds.length === 0 && evaluatorSlugs.length === 0) {
221
+ throw new Error('experiment requires at least one evaluator. Add `experiment.evaluators[]` to your eval file ' +
222
+ '(or `experiment.evaluator_workflow_ids` / `experiment.evaluator_slugs` for existing ones).');
223
+ }
224
+ const evaluatorSummary = workflowIds.length > 0
225
+ ? `evaluator_workflow_ids: ${workflowIds.join(', ')}`
226
+ : `evaluator_slugs: ${evaluatorSlugs.join(', ')}`;
227
+ log(`experiment: creating "${name}" with ${evaluatorSummary}`);
228
+ const created = (await client.experiments.createExperiment({
229
+ Authorization,
230
+ name,
231
+ dataset_id: datasetId,
232
+ ...(experiment.description ? { description: experiment.description } : {}),
233
+ workflow: [{ type: 'prompt', config: { prompt_id: promptId } }],
234
+ ...(workflowIds.length > 0 ? { evaluator_workflow_ids: workflowIds } : {}),
235
+ ...(evaluatorSlugs.length > 0 ? { evaluator_slugs: evaluatorSlugs } : {}),
236
+ }));
237
+ const experimentId = String(created.id ?? created.experiment_id ?? '');
238
+ if (!experimentId)
239
+ throw new Error('createExperiment: response missing id');
240
+ experiment.id = experimentId;
241
+ if (!experiment.name)
242
+ experiment.name = name;
243
+ return { id: experimentId, created: true };
244
+ }
245
+ function safeNodeId(raw) {
246
+ const sanitized = raw.replace(/[^a-zA-Z0-9_]/g, '_').slice(0, 30);
247
+ return sanitized || 'node';
248
+ }
249
+ function outputFieldForScoreType(scoreValueType) {
250
+ switch (scoreValueType) {
251
+ case 'boolean':
252
+ return 'boolean_value';
253
+ case 'single_select':
254
+ case 'text':
255
+ case 'comment':
256
+ return 'string_value';
257
+ case 'multi_select':
258
+ case 'json':
259
+ return null;
260
+ default:
261
+ return 'primary_score';
262
+ }
263
+ }
264
+ async function getDeployedWorkflowVersionId(Authorization, baseUrl, workflowId) {
265
+ const url = `${normalizeBaseUrl(baseUrl)}/api/workflows/${encodeURIComponent(workflowId)}/versions/`;
266
+ const response = await fetch(url, { headers: { Authorization } });
267
+ if (!response.ok)
268
+ return null;
269
+ const body = (await response.json());
270
+ const results = body.results ?? [];
271
+ const deployed = results.find((v) => v.is_enabled === true && v.is_read_only === true)
272
+ ?? results.find((v) => v.is_read_only === true);
273
+ return deployed ? String(deployed.id ?? '') : null;
274
+ }
275
+ function normalizeBaseUrl(baseUrl) {
276
+ return baseUrl.replace(/\/+$/, '').replace(/\/api$/, '');
277
+ }
278
+ async function commitWorkflow(Authorization, baseUrl, workflowId) {
279
+ const url = `${normalizeBaseUrl(baseUrl)}/api/workflows/${encodeURIComponent(workflowId)}/commits/`;
280
+ const response = await fetch(url, {
281
+ method: 'POST',
282
+ headers: { 'Content-Type': 'application/json', Authorization },
283
+ body: JSON.stringify({}),
284
+ });
285
+ if (!response.ok) {
286
+ const text = await response.text().catch(() => '');
287
+ throw new Error(`commitWorkflow ${response.status}: ${text || response.statusText}`);
288
+ }
289
+ }
290
+ function persist(filePath, evalFile) {
291
+ writeFileSync(filePath, JSON.stringify(evalFile, null, 2) + '\n', 'utf8');
292
+ }