@eigenpal/cli 0.14.3 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -52713,10 +52713,11 @@ function compileTypedScript(opts) {
52713
52713
  });
52714
52714
  }
52715
52715
  const actualParamNames = actualParams;
52716
- if (actualParamNames.length !== expectedParams.length || actualParamNames.some((n, i) => n !== expectedParams[i])) {
52716
+ const parametersMismatch = opts.kind === "evaluator" ? actualParamNames.length !== expectedParams.length || actualParamNames.some((name, index) => name !== expectedParams[index]) : actualParamNames.length !== expectedParams.length || new Set(actualParamNames).size !== actualParamNames.length || actualParamNames.some((name) => !expectedParams.includes(name));
52717
+ if (parametersMismatch) {
52717
52718
  return failWith({
52718
52719
  kind: "wrong-shape",
52719
- message: `Parameter list must be \`(${expectedParams.join(", ")})\` in this order (got \`(${actualParamNames.join(", ")})\`).`,
52720
+ message: opts.kind === "evaluator" ? `Parameter list must be \`(${expectedParams.join(", ")})\` in this order (got \`(${actualParamNames.join(", ")})\`).` : `Parameter names must match the configured inputs \`(${expectedParams.join(", ")})\`; their order does not matter (got \`(${actualParamNames.join(", ")})\`).`,
52720
52721
  line: fn.loc?.start.line,
52721
52722
  column: fn.loc?.start.column
52722
52723
  });
@@ -53482,8 +53483,8 @@ var init_step_configs = __esm(() => {
53482
53483
  }).superRefine(refineJsonToXlsxLayout);
53483
53484
  TransformJsonToXlsxOutputSchema = JsonToXlsxOutputSchema;
53484
53485
  TransformScriptConfigSchema = object({
53485
- inputs: record(string2(), string2()).optional().describe("Named inputs mapped from template expressions. Keys become the function parameter list in declaration order: `inputs: { items, taxRate }` ⇒ `function script(items: …, taxRate: …): R { … }`."),
53486
- function: string2().min(1).max(SCRIPT_FN_MAX_BYTES, `script function is too long (max ${(SCRIPT_FN_MAX_BYTES / 1000).toFixed(0)}k characters)`).describe("TypeScript function declaration. Must be `function script(args): R { … }` where the parameter list equals `Object.keys(inputs)` in order and `R` is a return type annotation. The annotation IS this step's output schema."),
53486
+ inputs: record(string2(), string2()).optional().describe("Named inputs mapped from template expressions. Keys become function parameters; parameter order is independent of YAML/JSON key order: `inputs: { items, taxRate }` accepts `function script(taxRate: …, items: …): R { … }`."),
53487
+ function: string2().min(1).max(SCRIPT_FN_MAX_BYTES, `script function is too long (max ${(SCRIPT_FN_MAX_BYTES / 1000).toFixed(0)}k characters)`).describe("TypeScript function declaration. Must be `function script(args): R { … }` where parameter names exactly match the configured input keys in any order and `R` is a return type annotation. The annotation IS this step's output schema."),
53487
53488
  timeout: number2().positive().default(5000).optional().describe("Max execution time in milliseconds (default: 5000)"),
53488
53489
  memoryLimit: number2().positive().default(10 * 1024 * 1024).optional().describe("Max memory in bytes (default: 10MB)")
53489
53490
  }).superRefine((cfg, ctx) => {
@@ -53863,7 +53864,7 @@ var init_step_configs = __esm(() => {
53863
53864
  type: "transform.script",
53864
53865
  category: "transform",
53865
53866
  name: "Script",
53866
- description: "Execute a TypeScript function in a QuickJS sandbox. Input keys become the function's parameter list, in declaration order, and the required `: R` return-type annotation IS this step's output schema: `inputs: { items, taxRate }` ⇒ `function script(items: …, taxRate: …): R { … }`.",
53867
+ description: "Execute a TypeScript function in a QuickJS sandbox. Input keys become function parameters in any order, and the required `: R` return-type annotation IS this step's output schema: `inputs: { items, taxRate }` accepts `function script(taxRate: …, items: …): R { … }`.",
53867
53868
  configSchema: TransformScriptConfigSchema,
53868
53869
  outputSchema: TransformScriptOutputSchema,
53869
53870
  configInWith: true
@@ -54027,36 +54028,36 @@ var init_workflow = __esm(() => {
54027
54028
  init_retry();
54028
54029
  init_steps();
54029
54030
  WORKFLOW_NAME_PATTERN = /^[a-z0-9][a-z0-9_-]*$/;
54030
- WorkflowNameSchema = string2().min(1).max(64).regex(WORKFLOW_NAME_PATTERN, 'Workflow name must start with a lowercase letter or digit and contain only lowercase letters, digits, "_" or "-".');
54031
+ WorkflowNameSchema = string2().min(1).max(64).regex(WORKFLOW_NAME_PATTERN, 'Workflow name must start with a lowercase letter or digit and contain only lowercase letters, digits, "_" or "-".').describe("URL-safe workflow slug: 1-64 lowercase letters, digits, underscores, or hyphens; must start with a letter or digit.");
54031
54032
  WorkflowInputPropertySchema = lazy(() => object({
54032
- name: string2().min(1).regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, "Property names must start with a letter or underscore and contain only letters, numbers, and underscores"),
54033
- type: string2(),
54034
- description: string2().optional(),
54035
- required: boolean2().optional(),
54036
- values: array(string2()).optional(),
54033
+ name: string2().min(1).regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, "Property names must start with a letter or underscore and contain only letters, numbers, and underscores").describe("JSON object key, addressable in templates with dot notation."),
54034
+ type: string2().describe("One of string, enum, number, integer, boolean, array, or object."),
54035
+ description: string2().optional().describe("Human-readable field meaning."),
54036
+ required: boolean2().optional().describe("Whether this property must be present; defaults to true when omitted."),
54037
+ values: array(string2()).optional().describe("Allowed strings when type is enum."),
54037
54038
  items: object({
54038
- type: string2(),
54039
- values: array(string2()).optional(),
54040
- properties: array(WorkflowInputPropertySchema).optional()
54041
- }).optional(),
54042
- properties: array(WorkflowInputPropertySchema).optional()
54039
+ type: string2().describe("Array element type: string, enum, number, integer, boolean, or object."),
54040
+ values: array(string2()).optional().describe("Allowed strings when items.type is enum."),
54041
+ properties: array(WorkflowInputPropertySchema).optional().describe("Recursive object fields when items.type is object.")
54042
+ }).optional().describe("Element definition when type is array."),
54043
+ properties: array(WorkflowInputPropertySchema).optional().describe("Recursive fields when type is object.")
54043
54044
  }));
54044
54045
  WorkflowInputDefSchema = object({
54045
- name: string2().min(1),
54046
- type: string2(),
54047
- description: string2().optional(),
54048
- required: boolean2().default(true),
54049
- default: unknown().optional(),
54050
- values: array(string2()).optional(),
54046
+ name: string2().min(1).describe("Top-level input name used as `input.<name>` in template expressions."),
54047
+ type: string2().describe("One of string, enum, number, integer, boolean, array, object, or file."),
54048
+ description: string2().optional().describe("Human-readable input meaning shown to callers."),
54049
+ required: boolean2().default(true).describe("Whether callers must provide this input."),
54050
+ default: unknown().optional().describe("Value used when an optional input is omitted."),
54051
+ values: array(string2()).optional().describe("Closed set of allowed strings when type is enum."),
54051
54052
  items: object({
54052
- type: string2(),
54053
- values: array(string2()).optional(),
54054
- properties: array(WorkflowInputPropertySchema).optional()
54055
- }).optional(),
54056
- properties: array(WorkflowInputPropertySchema).optional(),
54057
- source: string2().min(1).optional(),
54058
- mimeType: string2().min(1).optional(),
54059
- extension: string2().min(1).optional()
54053
+ type: string2().describe("Array element type: string, enum, number, integer, boolean, object, or file."),
54054
+ values: array(string2()).optional().describe("Closed set of allowed strings when items.type is enum."),
54055
+ properties: array(WorkflowInputPropertySchema).optional().describe("Recursive fields when items.type is object.")
54056
+ }).optional().describe("Element definition when type is array."),
54057
+ properties: array(WorkflowInputPropertySchema).optional().describe("Recursive field definitions when type is object."),
54058
+ source: string2().min(1).optional().describe("Registered external file resolver for single-tenant string-id file inputs, for example gpfs; valid only with type file."),
54059
+ mimeType: string2().min(1).optional().describe("MIME hint such as application/pdf for a sourced file; mutually exclusive with extension."),
54060
+ extension: string2().min(1).optional().describe("Extension hint such as pdf for a sourced file; mutually exclusive with mimeType.")
54060
54061
  }).superRefine((def, ctx) => {
54061
54062
  if (def.properties && def.type !== "object") {
54062
54063
  ctx.addIssue({
@@ -54115,20 +54116,20 @@ var init_workflow = __esm(() => {
54115
54116
  }
54116
54117
  });
54117
54118
  ManualTriggerMethodSchema = object({
54118
- type: literal("manual"),
54119
- enabled: boolean2().optional(),
54120
- inputSchema: JsonSchemaSchema.optional()
54119
+ type: literal("manual").describe("Dashboard/manual run trigger."),
54120
+ enabled: boolean2().optional().describe("Set false to disable dashboard runs; manual triggering is enabled by default."),
54121
+ inputSchema: JsonSchemaSchema.optional().describe("Optional JSON Schema for trigger input.")
54121
54122
  });
54122
54123
  ApiTriggerMethodSchema = object({
54123
- type: literal("api"),
54124
- inputSchema: JsonSchemaSchema.optional()
54124
+ type: literal("api").describe("Public API run trigger."),
54125
+ inputSchema: JsonSchemaSchema.optional().describe("Optional JSON Schema for API input.")
54125
54126
  });
54126
54127
  EmailTriggerMethodSchema = object({
54127
- type: literal("email"),
54128
+ type: literal("email").describe("Inbound email trigger."),
54128
54129
  whitelist: object({
54129
- domains: array(string2()).optional(),
54130
- emails: array(string2()).optional()
54131
- }).optional()
54130
+ domains: array(string2()).optional().describe("Allowed sender domains."),
54131
+ emails: array(string2()).optional().describe("Allowed exact sender email addresses.")
54132
+ }).optional().describe("Optional sender allowlist by exact email address or domain.")
54132
54133
  });
54133
54134
  TriggerMethodSchema = discriminatedUnion("type", [
54134
54135
  ManualTriggerMethodSchema,
@@ -54137,23 +54138,23 @@ var init_workflow = __esm(() => {
54137
54138
  ]);
54138
54139
  TriggerMethodsSchema = array(TriggerMethodSchema).default([{ type: "manual" }]);
54139
54140
  WorkflowSettingsSchema = object({
54140
- timeout: number2().positive().optional(),
54141
- retry: WorkflowRetryPolicySchema.optional(),
54141
+ timeout: number2().positive().optional().describe("Default step timeout in milliseconds."),
54142
+ retry: WorkflowRetryPolicySchema.optional().describe("Default durable retry policy for steps."),
54142
54143
  retries: number2().int().min(0).optional(),
54143
54144
  retryDelay: number2().positive().optional()
54144
54145
  }).strict();
54145
54146
  WorkflowDefinitionSchema = object({
54146
54147
  name: WorkflowNameSchema,
54147
- kind: _enum(["workflow"]).default("workflow").optional(),
54148
- version: string2().default("1.0.0"),
54149
- description: string2().optional(),
54150
- enabled: boolean2().default(true),
54151
- triggerMethods: array(TriggerMethodSchema).default([{ type: "manual" }]),
54152
- inputs: array(WorkflowInputDefSchema).optional(),
54153
- steps: array(StepSchema),
54154
- output: union([record(string2(), string2()), string2()]).optional(),
54155
- settings: WorkflowSettingsSchema.optional(),
54156
- defaultModel: string2().optional()
54148
+ kind: _enum(["workflow"]).default("workflow").optional().describe("Deprecated compatibility discriminator; omit for new workflows."),
54149
+ version: string2().default("1.0.0").describe("Author-provided semantic version label stored with the definition."),
54150
+ description: string2().optional().describe("Human-readable purpose of the workflow."),
54151
+ enabled: boolean2().default(true).describe("Whether the workflow may be run."),
54152
+ triggerMethods: array(TriggerMethodSchema).default([{ type: "manual" }]).describe("Ways this workflow can be invoked: manual, api, or email."),
54153
+ inputs: array(WorkflowInputDefSchema).optional().describe("Top-level inputs available under `input` in template expressions."),
54154
+ steps: array(StepSchema).describe("Ordered workflow steps; names must be unique."),
54155
+ output: union([record(string2(), string2()), string2()]).optional().describe("Final output as named field-to-template mappings or one passthrough template expression."),
54156
+ settings: WorkflowSettingsSchema.optional().describe("Workflow-wide timeout and retry defaults."),
54157
+ defaultModel: string2().optional().describe("Default configured LLM provider id used by AI steps when the step does not select one.")
54157
54158
  });
54158
54159
  StoredWorkflowDefinitionSchema = object({
54159
54160
  id: string2(),
@@ -54251,6 +54252,23 @@ var init_file_source = __esm(() => {
54251
54252
  });
54252
54253
 
54253
54254
  // ../types/src/eval/compare-output.ts
54255
+ function tokenizeEvalPath(path) {
54256
+ if (!path || path === "$")
54257
+ return [];
54258
+ return path.match(/[^.[\]]+|\[\]|\[\d+\]/g) ?? [];
54259
+ }
54260
+ function evalPathTokenMatches(pattern, candidate) {
54261
+ return pattern === candidate || pattern === "[]" && /^\[\d+\]$/.test(candidate);
54262
+ }
54263
+ function evalPathIsDescendant(descendant, ancestor) {
54264
+ if (descendant === ancestor)
54265
+ return false;
54266
+ if (ancestor === "$" || ancestor === "")
54267
+ return descendant !== "$" && descendant !== "";
54268
+ const ancestorTokens = tokenizeEvalPath(ancestor);
54269
+ const descendantTokens = tokenizeEvalPath(descendant);
54270
+ return descendantTokens.length > ancestorTokens.length && ancestorTokens.every((token, index) => evalPathTokenMatches(token, descendantTokens[index] ?? ""));
54271
+ }
54254
54272
  var MAX_UNORDERED_ARRAY_ITEMS = 1000, MAX_UNORDERED_PAIR_COMPARISONS = 1e6, MAX_UNORDERED_COMPARISON_OPERATIONS = 2000000;
54255
54273
  var init_compare_output = () => {};
54256
54274
 
@@ -54259,13 +54277,13 @@ var DATASET_NAME_PATTERN, DatasetNameSchema, DatasetMetaSchema, DatasetImportMod
54259
54277
  var init_dataset_archive = __esm(() => {
54260
54278
  init_zod();
54261
54279
  DATASET_NAME_PATTERN = /^[a-z0-9][a-z0-9-_]*$/;
54262
- DatasetNameSchema = string2().min(1).max(128).regex(DATASET_NAME_PATTERN, "name must be lowercase kebab/snake-case");
54280
+ DatasetNameSchema = string2().min(1).max(128).regex(DATASET_NAME_PATTERN, "name must be lowercase kebab/snake-case").describe("Example folder name: 1-128 lowercase letters, digits, hyphens, or underscores; must start with a letter or digit.");
54263
54281
  DatasetMetaSchema = object({
54264
- rowOrder: number2().int().min(0).optional(),
54265
- annotation: string2().max(2000).optional(),
54266
- overrides: record(string2(), unknown()).optional()
54282
+ rowOrder: number2().int().min(0).optional().describe("Non-negative display order hint for this example."),
54283
+ annotation: string2().max(2000).optional().describe("Free-form example note, limited to 2000 characters."),
54284
+ overrides: record(string2(), unknown()).optional().describe('Per-step output overrides as `{ "steps": { "<stepName>": <outputObject> } }`; overridden steps are skipped or partially merged during evaluation.')
54267
54285
  });
54268
- DatasetImportModeSchema = _enum(["append", "replace"]);
54286
+ DatasetImportModeSchema = _enum(["append", "replace"]).describe("append keeps existing examples; replace removes them before import.");
54269
54287
  DatasetImportRequestSchema = object({
54270
54288
  mode: DatasetImportModeSchema
54271
54289
  });
@@ -54292,7 +54310,8 @@ var init_dataset_archive = __esm(() => {
54292
54310
  label: string2().nullable(),
54293
54311
  weight: number2(),
54294
54312
  createdAt: string2().datetime({ offset: true }),
54295
- error: string2().nullable()
54313
+ error: string2().nullable(),
54314
+ details: unknown().nullable().optional().describe("Evaluator-specific structured details. Exact-diff includes mismatches with path, expected, and actual values.")
54296
54315
  });
54297
54316
  EvalResultsExportJsonSchema = object({
54298
54317
  workflowId: string2().min(1),
@@ -54308,7 +54327,7 @@ var init_evaluator_config = __esm(() => {
54308
54327
  init_ai_client();
54309
54328
  init_typed_script();
54310
54329
  init_compare_output();
54311
- EvaluatorTypeSchema = _enum(["exact-diff", "llm-judge", "custom-script"]);
54330
+ EvaluatorTypeSchema = _enum(["exact-diff", "llm-judge", "custom-script"]).describe("Evaluator implementation selected by each evaluators.yaml entry.");
54312
54331
  ScoreInUnitInterval = number2().min(0, { message: "score must be >= 0" }).max(1, { message: "score must be <= 1" });
54313
54332
  LEGACY_EXACT_DIFF_COMPARISON_KEYS = [
54314
54333
  "paths",
@@ -54337,12 +54356,24 @@ var init_evaluator_config = __esm(() => {
54337
54356
  }
54338
54357
  });
54339
54358
  ExactDiffPathRuleSchema = object({
54359
+ ignore: literal(true).optional().describe("Exclude this path and every descendant from comparison. Cannot be combined with other settings on the same rule."),
54340
54360
  order: _enum(["ordered", "unordered"]).optional().describe(`Array matching mode at this path. Independent of \`items\`. Set on the array path (\`lineItems\`), not the item path (\`lineItems[]\`). \`unordered\` matches distinct items in any order. Arrays stay ordered when omitted. Unordered matching is limited to ${MAX_UNORDERED_ARRAY_ITEMS} items, ${MAX_UNORDERED_PAIR_COMPARISONS} candidate pairs, and ${MAX_UNORDERED_COMPARISON_OPERATIONS} recursive operations per comparison.`),
54341
54361
  items: ExactDiffArrayItemsSchema.optional().describe("How expected and actual array items relate. Set on the array path (`lineItems`), not the item path (`lineItems[]`). Independent of `order`. Ordered arrays use positional prefix matching: `at-least` (default) requires expected to match actual from index 0 and allows trailing actual extras; `at-most` requires actual to match expected from index 0 and allows trailing expected extras; `exactly` is equal-cardinality positional comparison. Unordered arrays match distinct items in any order: `at-least` requires every expected item and allows extra actual items (including extras with missing or duplicate identities); `at-most` requires every actual item and allows missing expected items (every actual item being checked needs a valid unique identity); `exactly` is a bijection (both sides need valid unique identities)."),
54342
54362
  matchBy: ExactDiffMatchBySchema.optional().describe("Identity fields for unordered arrays of objects. Requires `order: unordered` on the same rule. Set on the array path (`lineItems`), not the item path (`lineItems[]`). A relative path (`sku`) or unique paths (`[country, sku]`) inside each item. Pairs expected and actual objects by exact identity, then diffs the paired objects with inherited nested rules. Identity type and value must both match: numeric `10` differs from string `'10'`; no coercion or numeric tolerance. Omitted = structural matching. Applies only at the array path where it is set and is not inherited by descendants. Nested object fields are allowed; array indexes, commas, and wildcards are not. Empty or comma-only values are rejected; composite paths must be unique."),
54343
54363
  allowExtraFields: boolean2().optional().describe("When false, extra actual keys at this object fail the diff. When omitted, extra keys are allowed. Applies only to objects."),
54344
54364
  numericTolerance: number2().min(0).optional().describe("Maximum absolute difference for numeric leaves at or under this path. Omitted = inherit, then historical relative epsilon. Explicit `0` is exact.")
54345
54365
  }).strict().superRefine((rule, ctx) => {
54366
+ if (rule.ignore === true) {
54367
+ const otherFields = Object.keys(rule).filter((field) => field !== "ignore");
54368
+ if (otherFields.length > 0) {
54369
+ ctx.addIssue({
54370
+ code: ZodIssueCode.custom,
54371
+ path: ["ignore"],
54372
+ message: "`ignore: true` cannot be combined with other settings on the same rule"
54373
+ });
54374
+ }
54375
+ return;
54376
+ }
54346
54377
  if (rule.matchBy === undefined)
54347
54378
  return;
54348
54379
  if (rule.order !== "unordered") {
@@ -54354,7 +54385,7 @@ var init_evaluator_config = __esm(() => {
54354
54385
  }
54355
54386
  });
54356
54387
  ExactDiffConfigSchema = object({
54357
- rules: record(string2(), ExactDiffPathRuleSchema).optional().describe("Per-path comparison rules. Keys select compared paths (`$` = full output). More-specific keys inherit then override parent rules. `{}` selects a path using inherited platform defaults. Do not mix with legacy `paths`, `unorderedPaths`, `numericTolerance`, or `allowExtraFields`."),
54388
+ rules: record(string2(), ExactDiffPathRuleSchema).optional().describe("Per-path comparison rules. Non-ignored keys select compared paths (`$` = full output); an ignore-only map means full output minus those paths. More-specific keys inherit then override parent rules. `{}` selects a path using inherited platform defaults; `{ ignore: true }` excludes it. Do not mix with legacy `paths`, `unorderedPaths`, `numericTolerance`, or `allowExtraFields`."),
54358
54389
  numericTolerance: number2().min(0).optional().describe("Legacy global numeric tolerance (absolute). When omitted, comparison uses historical relative epsilon. Prefer `rules.<path>.numericTolerance`. Accepted only when `rules` is omitted."),
54359
54390
  allowExtraFields: boolean2().optional().describe("Legacy global extra-field/extra-item flag. Prefer `rules.<path>.allowExtraFields` and `rules.<path>.items`. `true` normalizes to object extras allowed and array `items: at-least`; `false` normalizes to object extras rejected and array `items: exactly`. Accepted only when `rules` is omitted."),
54360
54391
  passThreshold: ScoreInUnitInterval.optional().describe(LEGACY_EVALUATOR_THRESHOLD_DESCRIPTION),
@@ -54406,6 +54437,19 @@ var init_evaluator_config = __esm(() => {
54406
54437
  });
54407
54438
  }
54408
54439
  }
54440
+ const ignoredPaths = Object.entries(config.rules ?? {}).filter(([, rule]) => rule.ignore === true).map(([path]) => path);
54441
+ for (const [path, rule] of Object.entries(config.rules ?? {})) {
54442
+ if (rule.ignore === true)
54443
+ continue;
54444
+ const ignoredAncestor = ignoredPaths.find((ignoredPath) => ignoredPath !== path && evalPathIsDescendant(path, ignoredPath));
54445
+ if (ignoredAncestor) {
54446
+ ctx.addIssue({
54447
+ code: ZodIssueCode.custom,
54448
+ path: ["rules", path],
54449
+ message: `cannot select \`${path}\` because ignored ancestor \`${ignoredAncestor}\` removes it before comparison`
54450
+ });
54451
+ }
54452
+ }
54409
54453
  }).default({});
54410
54454
  JudgeModeSchema = _enum(["continuous", "discrete"]);
54411
54455
  LabelsRecord = record(string2().min(1), ScoreInUnitInterval);
@@ -54460,16 +54504,22 @@ var init_evaluator_config = __esm(() => {
54460
54504
  weight: number2().min(0).default(1).describe("Relative importance in the weighted mean. The overall score is Σ(weightᵢ × scoreᵢ) / Σweight. Set to 0 to score-only without affecting overall.")
54461
54505
  });
54462
54506
  EvaluatorConfigSchema = discriminatedUnion("type", [
54463
- EvaluatorBaseEntrySchema.extend({ type: literal("exact-diff"), config: ExactDiffConfigSchema }),
54464
- EvaluatorBaseEntrySchema.extend({ type: literal("llm-judge"), config: LlmJudgeConfigSchema }),
54465
54507
  EvaluatorBaseEntrySchema.extend({
54466
- type: literal("custom-script"),
54467
- config: CustomScriptConfigSchema
54508
+ type: literal("exact-diff").describe("Deterministic structured-output comparison."),
54509
+ config: ExactDiffConfigSchema.describe("Configuration for exact-diff.")
54510
+ }),
54511
+ EvaluatorBaseEntrySchema.extend({
54512
+ type: literal("llm-judge").describe("Model-graded semantic comparison."),
54513
+ config: LlmJudgeConfigSchema.describe("Configuration for llm-judge.")
54514
+ }),
54515
+ EvaluatorBaseEntrySchema.extend({
54516
+ type: literal("custom-script").describe("Typed deterministic scoring script."),
54517
+ config: CustomScriptConfigSchema.describe("Configuration for custom-script.")
54468
54518
  })
54469
54519
  ]);
54470
54520
  EvalConfigYamlSchema = object({
54471
- evaluators: array(EvaluatorConfigSchema).default([]),
54472
- passThreshold: ScoreInUnitInterval.optional()
54521
+ evaluators: array(EvaluatorConfigSchema).default([]).describe("Evaluator entries run after each experiment execution."),
54522
+ passThreshold: ScoreInUnitInterval.optional().describe("Workflow-level pass threshold for the weighted mean score. When omitted, legacy evaluator thresholds are combined for compatibility, otherwise the platform default is 0.7.")
54473
54523
  });
54474
54524
  EvaluatorResultSchema = object({
54475
54525
  score: ScoreInUnitInterval.nullable(),
@@ -184044,7 +184094,7 @@ function getAlignmentSize(text, tabWidth, startIndex = 0) {
184044
184094
  }
184045
184095
  return size;
184046
184096
  }
184047
- function isObject2(object) {
184097
+ function isObject3(object) {
184048
184098
  return object !== null && typeof object === "object";
184049
184099
  }
184050
184100
  function skip(characters) {
@@ -196834,7 +196884,7 @@ ${codeblock}`, options8);
196834
196884
  get_string_width_default2 = getStringWidth2;
196835
196885
  get_alignment_size_default = getAlignmentSize;
196836
196886
  ast_path_default = AstPath;
196837
- is_object_default = isObject2;
196887
+ is_object_default = isObject3;
196838
196888
  skipWhitespace = skip(/\s/u);
196839
196889
  skipSpaces = skip(" \t");
196840
196890
  skipToLineEnd = skip(",; \t");
@@ -205318,7 +205368,7 @@ import { fileURLToPath as fileURLToPath8 } from "url";
205318
205368
  // package.json
205319
205369
  var package_default = {
205320
205370
  name: "@eigenpal/cli",
205321
- version: "0.14.3",
205371
+ version: "0.15.1",
205322
205372
  description: "Eigenpal CLI — author and run document workflows from your terminal. Agent-ready.",
205323
205373
  keywords: [
205324
205374
  "eigenpal",
@@ -205357,15 +205407,18 @@ var package_default = {
205357
205407
  },
205358
205408
  scripts: {
205359
205409
  dev: "bun src/cli.ts",
205360
- build: "bun run generate:skill && bun build src/cli.ts --outdir dist --target node --format esm && rm -rf dist/templates dist/skill && cp -R src/templates dist/templates && cp -R src/skill dist/skill",
205361
- generate: "bun run generate:cli-docs && bun run generate:skill",
205362
- "generate:check": "bun run generate:cli-docs:check && bun run generate:skill:check",
205410
+ build: "bun run generate:skill && bun build src/cli.ts --outdir dist --target node --format esm && rm -rf dist/templates dist/skill && cp -R src/templates dist/templates && cp -R src/skill dist/skill && cp src/docs-bundle.json dist/docs-bundle.json",
205411
+ generate: "bun run generate:cli-docs && bun run generate:skill && bun run generate:docs",
205412
+ "generate:check": "bun run generate:cli-docs:check && bun run generate:skill:check && bun run generate:docs:check && bun test scripts/docs-bundle-completeness.test.ts",
205413
+ "generate:docs": "bun scripts/generate-docs-bundle.ts",
205414
+ "generate:docs:check": "bun scripts/generate-docs-bundle.ts --check",
205363
205415
  "generate:skill": "bun scripts/generate-skill-reference.ts",
205364
205416
  "generate:skill:check": "bun scripts/generate-skill-reference.ts --check",
205365
205417
  "generate:cli-docs": "bun scripts/generate-cli-docs.ts",
205366
205418
  "generate:cli-docs:check": "bun scripts/generate-cli-docs.ts --check",
205367
205419
  "generate:logo": "bun scripts/render-logo-ascii.ts --cells 20",
205368
205420
  lint: "eslint src/",
205421
+ test: "bun test src/ scripts/",
205369
205422
  typecheck: "tsc --noEmit"
205370
205423
  },
205371
205424
  dependencies: {
@@ -205873,13 +205926,20 @@ function table(rows, columns) {
205873
205926
  `);
205874
205927
  }
205875
205928
  function addJsonFlag(cmd) {
205876
- return cmd.option("--json", "Output the raw server response as JSON");
205929
+ return cmd.option("--json", "Emit machine-readable JSON on stdout");
205877
205930
  }
205878
205931
  function withBaseUrl(cmd) {
205879
205932
  return cmd.option("--base-url <url>", "Server base URL");
205880
205933
  }
205881
205934
  function intArg(value) {
205882
- return Number.parseInt(value, 10);
205935
+ if (!/^-?\d+$/.test(value)) {
205936
+ throw new InvalidArgumentError2("Expected a whole number");
205937
+ }
205938
+ const parsed = Number(value);
205939
+ if (!Number.isSafeInteger(parsed)) {
205940
+ throw new InvalidArgumentError2("Expected a safe whole number");
205941
+ }
205942
+ return parsed;
205883
205943
  }
205884
205944
  function withPagination(cmd, defaultLimit = 50) {
205885
205945
  return cmd.option("--limit <n>", "Page size", intArg, defaultLimit).option("--offset <n>", "Page offset", intArg, 0);
@@ -205911,15 +205971,15 @@ function renderListResult(raw, columns, opts = {}) {
205911
205971
  if (opts.json) {
205912
205972
  console.log(JSON.stringify(raw, null, 2));
205913
205973
  if (rows.length > 0)
205914
- writeRecordCountHint(rows.length, total, label);
205974
+ writeRecordCountHint(rows.length, total, label, false);
205915
205975
  return;
205916
205976
  }
205917
205977
  console.log(table(rows, columns));
205918
205978
  if (rows.length > 0)
205919
205979
  writeRecordCountHint(rows.length, total, label);
205920
205980
  }
205921
- function writeRecordCountHint(shown, total, label) {
205922
- process.stderr.write(`${import_picocolors.default.dim(`${shown}${total > shown ? ` of ${total}` : ""} ${label}${total === 1 ? "" : "s"} · use --json for the raw payload`)}
205981
+ function writeRecordCountHint(shown, total, label, suggestJson = true) {
205982
+ process.stderr.write(`${import_picocolors.default.dim(`${shown}${total > shown ? ` of ${total}` : ""} ${label}${total === 1 ? "" : "s"}${suggestJson ? " · use --json for machine-readable output" : ""}`)}
205923
205983
  `);
205924
205984
  }
205925
205985
 
@@ -208792,16 +208852,18 @@ async function writeBase64File(file, contentBase64) {
208792
208852
  await fs.mkdir(path6.dirname(file), { recursive: true });
208793
208853
  await fs.writeFile(file, Buffer.from(contentBase64, "base64"));
208794
208854
  }
208795
- async function pollRun(client, executionId, interval, maxWait) {
208855
+ async function pollRun(client, executionId, interval, maxWait, json = false) {
208796
208856
  const started = Date.now();
208797
208857
  const base = `/v1/runs/${encodeURIComponent(executionId)}`;
208798
208858
  for (;; ) {
208799
- const payload = await client.get(base);
208859
+ const payload = await client.get(`${base}?expand=execution`);
208800
208860
  const status = payload.execution?.status;
208801
208861
  if (payload.finished || isTerminalExecutionStatus(status)) {
208802
208862
  return await client.get(`${base}?expand=usage,execution`);
208803
208863
  }
208804
208864
  if (Date.now() - started > maxWait * 1000) {
208865
+ if (json)
208866
+ printJson(payload);
208805
208867
  process.stderr.write(`Timed out waiting for run ${executionId}
208806
208868
  `);
208807
208869
  process.exit(2);
@@ -208809,13 +208871,27 @@ async function pollRun(client, executionId, interval, maxWait) {
208809
208871
  await new Promise((resolve) => setTimeout(resolve, interval * 1000));
208810
208872
  }
208811
208873
  }
208812
- async function pollExperiment(client, automationId, batchId, interval, maxWait) {
208874
+ async function pollExperiment(client, automationId, batchId, interval, maxWait, json = false) {
208813
208875
  const started = Date.now();
208876
+ let notFoundRetries = 0;
208877
+ let lastPayload;
208814
208878
  for (;; ) {
208815
- const payload = await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(batchId)}`);
208879
+ let payload;
208880
+ try {
208881
+ payload = await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(batchId)}`);
208882
+ lastPayload = payload;
208883
+ } catch (error) {
208884
+ if (!(error instanceof ApiError) || error.status !== 404 || notFoundRetries >= 2)
208885
+ throw error;
208886
+ notFoundRetries++;
208887
+ await new Promise((resolve) => setTimeout(resolve, interval * 1000));
208888
+ continue;
208889
+ }
208816
208890
  if (payload.status === "completed")
208817
208891
  return payload;
208818
208892
  if (Date.now() - started > maxWait * 1000) {
208893
+ if (json)
208894
+ printJson(lastPayload ?? { batchId, status: "pending" });
208819
208895
  process.stderr.write(`Timed out waiting for experiment ${batchId}
208820
208896
  `);
208821
208897
  process.exit(2);
@@ -208835,7 +208911,7 @@ function renderRunPayload(payload, opts) {
208835
208911
  function setExitCodeForFailedTerminalRun(payload) {
208836
208912
  const run = payload;
208837
208913
  const status = run?.execution?.status;
208838
- if (status === "failed" || status === "cancelled" || run?.finished === true && run?.error) {
208914
+ if (status === "failed" || status === "cancelled" || status === "rejected" || run?.finished === true && run?.error) {
208839
208915
  process.exitCode = 1;
208840
208916
  }
208841
208917
  }
@@ -209489,6 +209565,89 @@ function dotenvQuote(value) {
209489
209565
  // src/commands/agents/experiments.ts
209490
209566
  import { promises as fs5 } from "node:fs";
209491
209567
 
209568
+ // src/lib/json-select.ts
209569
+ function selectJsonValue(value, path) {
209570
+ const tokens = parseJsonSelectPath(path);
209571
+ let values = [value];
209572
+ let usedWildcard = false;
209573
+ for (const token of tokens) {
209574
+ const next = [];
209575
+ for (const current of values) {
209576
+ if (token === "*") {
209577
+ usedWildcard = true;
209578
+ if (Array.isArray(current))
209579
+ next.push(...current);
209580
+ continue;
209581
+ }
209582
+ if (typeof token === "number") {
209583
+ if (Array.isArray(current) && token < current.length)
209584
+ next.push(current[token]);
209585
+ continue;
209586
+ }
209587
+ if (current && typeof current === "object" && !Array.isArray(current)) {
209588
+ const object = current;
209589
+ if (Object.hasOwn(object, token))
209590
+ next.push(object[token]);
209591
+ }
209592
+ }
209593
+ values = next;
209594
+ }
209595
+ if (values.length === 0) {
209596
+ throw new Error(`--select path "${path}" did not match the JSON payload`);
209597
+ }
209598
+ return usedWildcard ? values : values[0];
209599
+ }
209600
+ function parseJsonSelectPath(path) {
209601
+ const trimmed = path.trim();
209602
+ if (trimmed.startsWith("$") && trimmed !== "$" && !trimmed.startsWith("$.") && !trimmed.startsWith("$[")) {
209603
+ throw invalidPath(path);
209604
+ }
209605
+ const source = trimmed.replace(/^\$(?:\.)?/, "");
209606
+ if (!source)
209607
+ return [];
209608
+ const tokens = [];
209609
+ let cursor = 0;
209610
+ while (cursor < source.length) {
209611
+ if (source[cursor] === ".") {
209612
+ if (cursor === 0 || cursor === source.length - 1 || source[cursor - 1] === "." || source[cursor + 1] === "." || source[cursor + 1] === "[") {
209613
+ throw invalidPath(path);
209614
+ }
209615
+ cursor++;
209616
+ continue;
209617
+ }
209618
+ if (source[cursor] === "[") {
209619
+ if (cursor > 0 && source[cursor - 1] === ".")
209620
+ throw invalidPath(path);
209621
+ const close = source.indexOf("]", cursor + 1);
209622
+ if (close === -1)
209623
+ throw invalidPath(path);
209624
+ const inner = source.slice(cursor + 1, close);
209625
+ if (inner === "" || inner === "*")
209626
+ tokens.push("*");
209627
+ else if (/^(0|[1-9]\d*)$/.test(inner))
209628
+ tokens.push(Number(inner));
209629
+ else
209630
+ throw invalidPath(path);
209631
+ cursor = close + 1;
209632
+ if (cursor < source.length && source[cursor] !== "." && source[cursor] !== "[") {
209633
+ throw invalidPath(path);
209634
+ }
209635
+ continue;
209636
+ }
209637
+ if (cursor > 0 && source[cursor - 1] !== ".")
209638
+ throw invalidPath(path);
209639
+ const match = /^[A-Za-z_$][A-Za-z0-9_$-]*/.exec(source.slice(cursor));
209640
+ if (!match)
209641
+ throw invalidPath(path);
209642
+ tokens.push(match[0]);
209643
+ cursor += match[0].length;
209644
+ }
209645
+ return tokens;
209646
+ }
209647
+ function invalidPath(path) {
209648
+ return new Error(`Invalid --select path "${path}". Use dot keys, numeric indexes, or [] wildcards (for example output.subjects[].name).`);
209649
+ }
209650
+
209492
209651
  // src/commands/workflow/experiment-compare.ts
209493
209652
  function normalizeCompareSort(value) {
209494
209653
  switch (value) {
@@ -209571,6 +209730,90 @@ function buildBatchDiff(args) {
209571
209730
  }
209572
209731
  };
209573
209732
  }
209733
+ function buildExperimentOutputDiff(args) {
209734
+ const a = new Map(args.rowsA.map((row) => [exampleLabelOf(row), row]));
209735
+ const b = new Map(args.rowsB.map((row) => [exampleLabelOf(row), row]));
209736
+ const shared = [...a.keys()].filter((example) => b.has(example)).sort();
209737
+ const rows = shared.map((example) => {
209738
+ const rowA = a.get(example);
209739
+ const rowB = b.get(example);
209740
+ const comparable = rowA.status === "completed" && rowB.status === "completed" && rowA.output !== undefined && rowB.output !== undefined;
209741
+ if (!comparable) {
209742
+ return {
209743
+ example,
209744
+ executionA: rowA.executionId,
209745
+ executionB: rowB.executionId,
209746
+ status: "incomparable",
209747
+ differences: [
209748
+ {
209749
+ path: "$",
209750
+ type: "incomparable",
209751
+ expected: { status: rowA.status, error: rowA.error },
209752
+ actual: { status: rowB.status, error: rowB.error }
209753
+ }
209754
+ ]
209755
+ };
209756
+ }
209757
+ const differences = diffOutputJson(rowA.output, rowB.output);
209758
+ return {
209759
+ example,
209760
+ executionA: rowA.executionId,
209761
+ executionB: rowB.executionId,
209762
+ status: differences.length === 0 ? "identical" : "changed",
209763
+ differences
209764
+ };
209765
+ });
209766
+ return {
209767
+ batchA: args.batchIdA,
209768
+ batchB: args.batchIdB,
209769
+ rows,
209770
+ summary: {
209771
+ sharedExamples: shared.length,
209772
+ identical: rows.filter((row) => row.status === "identical").length,
209773
+ changed: rows.filter((row) => row.status === "changed").length,
209774
+ incomparable: rows.filter((row) => row.status === "incomparable").length,
209775
+ onlyInA: [...a.keys()].filter((example) => !b.has(example)).sort(),
209776
+ onlyInB: [...b.keys()].filter((example) => !a.has(example)).sort()
209777
+ }
209778
+ };
209779
+ }
209780
+ function diffOutputJson(expected, actual, path = "$", limit = 200) {
209781
+ if (Object.is(expected, actual))
209782
+ return [];
209783
+ if (Array.isArray(expected) && Array.isArray(actual)) {
209784
+ const differences = [];
209785
+ for (let index = 0;index < Math.max(expected.length, actual.length); index++) {
209786
+ if (differences.length >= limit)
209787
+ break;
209788
+ const child = `${path}[${index}]`;
209789
+ if (index >= expected.length) {
209790
+ differences.push({ path: child, type: "extra", actual: actual[index] });
209791
+ } else if (index >= actual.length) {
209792
+ differences.push({ path: child, type: "missing", expected: expected[index] });
209793
+ } else {
209794
+ differences.push(...diffOutputJson(expected[index], actual[index], child, limit - differences.length));
209795
+ }
209796
+ }
209797
+ return differences;
209798
+ }
209799
+ if (isObject2(expected) && isObject2(actual)) {
209800
+ const differences = [];
209801
+ for (const key of new Set([...Object.keys(expected), ...Object.keys(actual)])) {
209802
+ if (differences.length >= limit)
209803
+ break;
209804
+ const child = path === "$" ? `$.${key}` : `${path}.${key}`;
209805
+ if (!Object.hasOwn(expected, key)) {
209806
+ differences.push({ path: child, type: "extra", actual: actual[key] });
209807
+ } else if (!Object.hasOwn(actual, key)) {
209808
+ differences.push({ path: child, type: "missing", expected: expected[key] });
209809
+ } else {
209810
+ differences.push(...diffOutputJson(expected[key], actual[key], child, limit - differences.length));
209811
+ }
209812
+ }
209813
+ return differences;
209814
+ }
209815
+ return [{ path, type: "changed", expected, actual }];
209816
+ }
209574
209817
  function aggregateByEvaluator(rows) {
209575
209818
  const buckets = new Map;
209576
209819
  for (const r of rows) {
@@ -209755,6 +209998,33 @@ function renderBatchDiffHuman(diff) {
209755
209998
  process.stderr.write(`${ui.dim("mean Δ:")} ${meanDeltaText} ` + `${ui.dim("examples shared:")} ${diff.summary.sharedExamples} ` + `${ui.dim("only in A:")} ${diff.summary.onlyInA.length} ` + `${ui.dim("only in B:")} ${diff.summary.onlyInB.length}
209756
209999
  `);
209757
210000
  }
210001
+ function renderExperimentOutputDiffHuman(diff) {
210002
+ process.stderr.write(`${ui.dim("A =")} ${ui.bold(diff.batchA)} ${ui.dim("B =")} ${ui.bold(diff.batchB)}
210003
+
210004
+ `);
210005
+ const visible = diff.rows.flatMap((row) => row.differences.map((difference) => ({
210006
+ example: row.example,
210007
+ path: difference.path,
210008
+ type: difference.type,
210009
+ expected: compactValue(difference.expected),
210010
+ actual: compactValue(difference.actual)
210011
+ })));
210012
+ if (visible.length > 0) {
210013
+ console.log(table(visible, [
210014
+ { key: "example", header: "example" },
210015
+ { key: "path", header: "path" },
210016
+ { key: "type", header: "change" },
210017
+ { key: "expected", header: "A" },
210018
+ { key: "actual", header: "B" }
210019
+ ]));
210020
+ } else {
210021
+ process.stderr.write(`${ui.dim("No actual-output differences in shared examples.")}
210022
+ `);
210023
+ }
210024
+ process.stderr.write(`
210025
+ ${ui.dim("shared:")} ${diff.summary.sharedExamples} ` + `${ui.ok(`identical: ${diff.summary.identical}`)} ` + `${ui.warn(`changed: ${diff.summary.changed}`)} ` + `${ui.warn(`incomparable: ${diff.summary.incomparable}`)} ` + `${ui.dim("only in A:")} ${diff.summary.onlyInA.length} ` + `${ui.dim("only in B:")} ${diff.summary.onlyInB.length}
210026
+ `);
210027
+ }
209758
210028
  async function fetchEvalResults(client, experimentId) {
209759
210029
  const ref = await client.get(`/v1/experiments/${encodeURIComponent(experimentId)}`);
209760
210030
  const raw = await client.get(`/v1/automations/${encodeURIComponent(ref.automationId)}/experiments/${encodeURIComponent(experimentId)}/export`, { format: "json" });
@@ -209763,6 +210033,214 @@ async function fetchEvalResults(client, experimentId) {
209763
210033
  }
209764
210034
  return raw;
209765
210035
  }
210036
+ async function fetchExperimentOutputs(client, experimentId) {
210037
+ const ref = await client.get(`/v1/experiments/${encodeURIComponent(experimentId)}`);
210038
+ const detail = await client.get(`/v1/automations/${encodeURIComponent(ref.automationId)}/experiments/${encodeURIComponent(experimentId)}`);
210039
+ const runs = detail.runs ?? [];
210040
+ const rows = await mapWithConcurrency(runs, 6, async (run) => {
210041
+ const payload = await client.get(`/v1/runs/${encodeURIComponent(run.id)}`);
210042
+ return {
210043
+ executionId: run.id,
210044
+ exampleId: run.exampleId ?? null,
210045
+ exampleName: run.exampleName ?? null,
210046
+ status: payload.execution?.status ?? payload.status ?? payload.result?.status ?? null,
210047
+ error: payload.error ?? payload.execution?.error ?? payload.result?.error ?? null,
210048
+ output: payload.output !== undefined ? payload.output : payload.result?.output
210049
+ };
210050
+ });
210051
+ return { automationId: ref.automationId, rows };
210052
+ }
210053
+ function isObject2(value) {
210054
+ return value != null && typeof value === "object" && !Array.isArray(value);
210055
+ }
210056
+ function compactValue(value) {
210057
+ if (value === undefined)
210058
+ return "—";
210059
+ const rendered = JSON.stringify(value);
210060
+ if (rendered === undefined)
210061
+ return String(value);
210062
+ return rendered.length > 80 ? `${rendered.slice(0, 77)}…` : rendered;
210063
+ }
210064
+ async function mapWithConcurrency(values, concurrency, fn) {
210065
+ const results = new Array(values.length);
210066
+ let cursor = 0;
210067
+ await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, async () => {
210068
+ while (cursor < values.length) {
210069
+ const index = cursor++;
210070
+ results[index] = await fn(values[index]);
210071
+ }
210072
+ }));
210073
+ return results;
210074
+ }
210075
+
210076
+ // src/commands/workflow/experiment-results.ts
210077
+ function experimentRowsFromDetail(detail) {
210078
+ const namesByRun = new Map((detail.runs ?? []).map((run) => [run.id, run.exampleName]));
210079
+ return Object.entries(detail.resultsByRun ?? {}).flatMap(([executionId, rows]) => rows.map((row) => ({
210080
+ ...row,
210081
+ executionId,
210082
+ exampleName: namesByRun.get(executionId) ?? null,
210083
+ score: numberOrNull(row.score),
210084
+ weight: numberOrNull(row.weight) ?? 1
210085
+ })));
210086
+ }
210087
+ function analyzeExperimentResults(args) {
210088
+ let rows = args.evaluator ? args.rows.filter((row) => row.evaluatorName === args.evaluator) : [...args.rows];
210089
+ if (args.failedOnly) {
210090
+ rows = rows.filter((row) => row.error != null || row.passed === false);
210091
+ }
210092
+ const includedExecutionIds = new Set(rows.map((row) => row.executionId));
210093
+ const runs = args.evaluator || args.failedOnly ? args.runs.filter((run) => includedExecutionIds.has(run.id)) : args.runs;
210094
+ const rowsByRun = groupBy(rows, (row) => row.executionId);
210095
+ const counts = { total: runs.length, passed: 0, failed: 0, errors: 0 };
210096
+ const runScores = [];
210097
+ const usePersistedAggregate = !args.evaluator && !args.failedOnly;
210098
+ for (const run of runs) {
210099
+ const runRows = rowsByRun.get(run.id) ?? [];
210100
+ if (usePersistedAggregate && run.evalPassed != null) {
210101
+ if (["failed", "cancelled", "rejected"].includes(run.status) && run.evalPassed !== true) {
210102
+ counts.errors++;
210103
+ } else if (run.evalPassed) {
210104
+ counts.passed++;
210105
+ } else {
210106
+ counts.failed++;
210107
+ }
210108
+ if (typeof run.evalScore === "number")
210109
+ runScores.push(run.evalScore);
210110
+ continue;
210111
+ }
210112
+ if (["failed", "cancelled", "rejected"].includes(run.status) || runRows.length === 0 || runRows.some((row) => row.error != null)) {
210113
+ counts.errors++;
210114
+ continue;
210115
+ }
210116
+ if (runRows.some((row) => row.passed === false))
210117
+ counts.failed++;
210118
+ else
210119
+ counts.passed++;
210120
+ const score = weightedAverage(runRows);
210121
+ if (score != null)
210122
+ runScores.push(score);
210123
+ }
210124
+ const byEvaluator = [...groupBy(rows, (row) => row.evaluatorName)].map(([evaluator, evaluatorRows]) => ({
210125
+ evaluator,
210126
+ total: evaluatorRows.length,
210127
+ passed: evaluatorRows.filter((row) => row.error == null && row.passed === true).length,
210128
+ failed: evaluatorRows.filter((row) => row.error == null && row.passed === false).length,
210129
+ errors: evaluatorRows.filter((row) => row.error != null).length,
210130
+ averageScore: average(evaluatorRows.flatMap((row) => typeof row.score === "number" ? [row.score] : []))
210131
+ }));
210132
+ byEvaluator.sort((a, b) => a.evaluator.localeCompare(b.evaluator));
210133
+ return {
210134
+ summary: {
210135
+ ...counts,
210136
+ averageScore: average(runScores),
210137
+ byEvaluator
210138
+ },
210139
+ discrepancies: flattenDiscrepancies(rows),
210140
+ results: rows
210141
+ };
210142
+ }
210143
+ function flattenDiscrepancies(rows) {
210144
+ const discrepancies = [];
210145
+ for (const row of rows) {
210146
+ const details = objectOrNull(row.details);
210147
+ const mismatches = Array.isArray(details?.mismatches) ? details.mismatches : [];
210148
+ for (const mismatch of mismatches) {
210149
+ const item = objectOrNull(mismatch);
210150
+ if (!item || typeof item.path !== "string")
210151
+ continue;
210152
+ discrepancies.push({
210153
+ example: row.exampleName || row.executionId,
210154
+ executionId: row.executionId,
210155
+ evaluator: row.evaluatorName,
210156
+ path: item.path,
210157
+ expected: item.expected,
210158
+ actual: item.actual
210159
+ });
210160
+ }
210161
+ }
210162
+ return discrepancies;
210163
+ }
210164
+ function renderExperimentResultsSummary(analysis) {
210165
+ const summary = analysis.summary;
210166
+ process.stderr.write(`${ui.bold("Experiment results")} total=${summary.total} ${ui.ok(`passed=${summary.passed}`)} ` + `${ui.warn(`failed=${summary.failed}`)} errors=${summary.errors} ` + `average=${formatScore2(summary.averageScore)}
210167
+ `);
210168
+ if (summary.byEvaluator.length > 0) {
210169
+ console.log(table(summary.byEvaluator, [
210170
+ { key: "evaluator", header: "evaluator" },
210171
+ { key: "total", header: "total", align: "right" },
210172
+ { key: "passed", header: "pass", align: "right" },
210173
+ { key: "failed", header: "fail", align: "right" },
210174
+ { key: "errors", header: "error", align: "right" },
210175
+ {
210176
+ key: "averageScore",
210177
+ header: "average",
210178
+ align: "right",
210179
+ format: (value) => formatScore2(value)
210180
+ }
210181
+ ]));
210182
+ }
210183
+ if (analysis.discrepancies.length > 0) {
210184
+ process.stderr.write(`
210185
+ ${ui.bold("Discrepancies")}
210186
+ `);
210187
+ console.log(table(analysis.discrepancies, [
210188
+ { key: "example", header: "example" },
210189
+ { key: "evaluator", header: "evaluator" },
210190
+ { key: "path", header: "path" },
210191
+ { key: "expected", header: "expected", format: compactJson },
210192
+ { key: "actual", header: "actual", format: compactJson }
210193
+ ]));
210194
+ }
210195
+ }
210196
+ function weightedAverage(rows) {
210197
+ let total = 0;
210198
+ let weight = 0;
210199
+ for (const row of rows) {
210200
+ if (typeof row.score !== "number")
210201
+ continue;
210202
+ const rowWeight = Number.isFinite(row.weight) ? row.weight : 1;
210203
+ total += row.score * rowWeight;
210204
+ weight += rowWeight;
210205
+ }
210206
+ return weight > 0 ? total / weight : null;
210207
+ }
210208
+ function average(values) {
210209
+ return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
210210
+ }
210211
+ function numberOrNull(value) {
210212
+ if (typeof value === "number")
210213
+ return Number.isFinite(value) ? value : null;
210214
+ if (typeof value === "string") {
210215
+ const parsed = Number(value);
210216
+ return Number.isFinite(parsed) ? parsed : null;
210217
+ }
210218
+ return null;
210219
+ }
210220
+ function groupBy(values, key) {
210221
+ const groups = new Map;
210222
+ for (const value of values) {
210223
+ const name = key(value);
210224
+ const existing = groups.get(name);
210225
+ if (existing)
210226
+ existing.push(value);
210227
+ else
210228
+ groups.set(name, [value]);
210229
+ }
210230
+ return groups;
210231
+ }
210232
+ function objectOrNull(value) {
210233
+ return value && typeof value === "object" && !Array.isArray(value) ? value : null;
210234
+ }
210235
+ function formatScore2(value) {
210236
+ return value == null ? "—" : value.toFixed(3);
210237
+ }
210238
+ function compactJson(value) {
210239
+ const rendered = JSON.stringify(value);
210240
+ if (rendered === undefined)
210241
+ return "undefined";
210242
+ return rendered.length > 80 ? `${rendered.slice(0, 77)}…` : rendered;
210243
+ }
209766
210244
 
209767
210245
  // src/commands/agents/experiments.ts
209768
210246
  function registerExperimentCommands(agent) {
@@ -209770,11 +210248,11 @@ function registerExperimentCommands(agent) {
209770
210248
  process.stderr.write("`eigenpal agents experiment` requires a subcommand. Run `eigenpal agents experiment --help`.\n");
209771
210249
  process.exit(2);
209772
210250
  });
209773
- addJsonFlag(withBaseUrl(experiment.command("run <agent-id-or-slug>"))).description("Start an experiment over dataset examples.").option("--example-id <id>", "Run one dataset example").option("--wait", "Poll until the experiment reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).action(action(runExperiment));
209774
- addJsonFlag(withBaseUrl(experiment.command("status <agent-id-or-slug> <batch-id>"))).description("Get experiment status.").option("--watch", "Poll until complete").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).option("--include <parts>", "Reserved for future detailed parts").action(action(experimentStatus));
209775
- withBaseUrl(experiment.command("results <agent-id-or-slug> [batch-id]")).description("Print experiment results as JSON or CSV.").requiredOption("--format <csv|json>", "Output format", parseResultsFormat).option("--out <path>", "Write output to file").action(action(experimentResults));
210251
+ addJsonFlag(withBaseUrl(experiment.command("run <agent-id-or-slug>"))).description("Start an experiment over dataset examples.").option("--example-id <id>", "Run one dataset example").option("--wait", "Poll until the experiment reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(runExperiment));
210252
+ addJsonFlag(withBaseUrl(experiment.command("status <agent-id-or-slug> <batch-id>"))).description("Get experiment status.").option("--watch", "Poll until complete").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(experimentStatus));
210253
+ addJsonFlag(withBaseUrl(experiment.command("results <agent-id-or-slug> [batch-id]"))).description("Print experiment results as JSON or CSV.").option("--format <csv|json>", "Output format (default json)", parseResultsFormat, "json").option("--out <path>", "Write output to file").option("--summary", "Show total/pass/fail/error counts, average score, and evaluator rollups").option("--failed-only", "Keep only failed or errored evaluator results").option("--evaluator <name>", "Keep only results from this evaluator").option("--select <path>", "Print only a nested JSON value (for example summary.byEvaluator or discrepancies[].path)").action(action(experimentResults));
209776
210254
  addJsonFlag(withPagination(withBaseUrl(experiment.command("list <agent-id-or-slug>")), 50)).description("List experiments.").option("--batch-id <id>", "Filter to one batch id").action(action(listExperiments));
209777
- addJsonFlag(withBaseUrl(experiment.command("compare <batch-id-a> <batch-id-b>"))).description("Diff eval scores between two experiment batches.").option("--sort <abs-delta-desc|delta-asc|delta-desc|name>", "Row sort order (default: biggest movers first)", "abs-delta-desc").option("--regression-threshold <n>", "Δ below this is flagged as a regression (default 0.05)", Number.parseFloat, 0.05).action(action(compareExperiments));
210255
+ addJsonFlag(withBaseUrl(experiment.command("compare <batch-id-a> <batch-id-b>"))).description("Compare evaluator scores or actual outputs between two experiment batches.").option("--outputs", "Compare actual run outputs instead of evaluator scores").option("--sort <abs-delta-desc|delta-asc|delta-desc|name>", "Row sort order (default: biggest movers first)", "abs-delta-desc").option("--regression-threshold <n>", "Δ below this is flagged as a regression (default 0.05)", Number.parseFloat, 0.05).action(action(compareExperiments));
209778
210256
  addJsonFlag(withBaseUrl(experiment.command("cancel <agent-id-or-slug> <batch-id>"))).description("Cancel every active execution in an experiment.").option("--yes", "Skip confirmation (required in CI / agent terminals without a TTY)").action(action(cancelExperiment));
209779
210257
  }
209780
210258
  async function runExperiment(agentId, opts) {
@@ -209784,26 +210262,73 @@ async function runExperiment(agentId, opts) {
209784
210262
  ...opts.exampleId ? { examples: [opts.exampleId] } : {}
209785
210263
  });
209786
210264
  const experimentId = payload.id ?? payload.batchId;
210265
+ if (opts.wait && !experimentId) {
210266
+ throw new Error("Experiment start response did not include a batch id; cannot honor --wait");
210267
+ }
209787
210268
  if (opts.wait && experimentId) {
209788
- payload = await pollExperiment(client, automationId, experimentId, opts.interval, 1800);
210269
+ payload = await pollExperiment(client, automationId, experimentId, opts.interval, opts.maxWait, opts.json);
210270
+ }
210271
+ const stablePayload = {
210272
+ ...payload,
210273
+ batchId: experimentId,
210274
+ total: payload.runCount ?? payload.total
210275
+ };
210276
+ renderGeneric(stablePayload, opts, `${opts.wait ? "Completed" : "Started"} experiment ${experimentId ?? ""}`);
210277
+ const failedCount = payload.failedCount;
210278
+ if (opts.wait && typeof failedCount === "number" && failedCount > 0) {
210279
+ process.exitCode = 1;
209789
210280
  }
209790
- renderGeneric(payload, opts, `Started experiment ${experimentId ?? ""}`);
209791
210281
  }
209792
210282
  async function experimentStatus(agentId, batchId, opts) {
209793
210283
  const client = buildClient(opts);
209794
210284
  const automationId = agentAutomationId(agentId);
209795
- const payload = opts.watch ? await pollExperiment(client, automationId, batchId, opts.interval, opts.maxWait) : await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(batchId)}`);
210285
+ const payload = opts.watch ? await pollExperiment(client, automationId, batchId, opts.interval, opts.maxWait, opts.json) : await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(batchId)}`);
209796
210286
  renderGeneric(payload, opts, `Experiment ${batchId}`);
210287
+ if (opts.watch) {
210288
+ const counts = payload;
210289
+ if ([counts.failedCount, counts.cancelledCount, counts.rejectedCount].some((count) => typeof count === "number" && count > 0)) {
210290
+ process.exitCode = 1;
210291
+ }
210292
+ }
209797
210293
  }
209798
210294
  async function experimentResults(agentId, batchId, opts) {
209799
210295
  const client = buildClient(opts);
209800
210296
  const automationId = agentAutomationId(agentId);
210297
+ if (opts.json && opts.format !== "json") {
210298
+ throw new Error("--json cannot be combined with --format csv");
210299
+ }
209801
210300
  const selected = batchId ?? String((await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments`, {
209802
210301
  limit: "1",
209803
210302
  offset: "0"
209804
210303
  })).data?.[0]?.id ?? "");
209805
210304
  if (!selected)
209806
210305
  throw new Error("No experiment batch found");
210306
+ const analyze = opts.summary || opts.failedOnly || opts.evaluator !== undefined || opts.select !== undefined;
210307
+ if (analyze) {
210308
+ if (opts.format !== "json")
210309
+ throw new Error("Experiment analysis options require --format json");
210310
+ const detail = await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(selected)}`);
210311
+ const analysis = analyzeExperimentResults({
210312
+ rows: experimentRowsFromDetail(detail),
210313
+ runs: detail.runs ?? [],
210314
+ evaluator: opts.evaluator,
210315
+ failedOnly: opts.failedOnly
210316
+ });
210317
+ const output = opts.select ? selectJsonValue(analysis, opts.select) : analysis;
210318
+ if (opts.summary && !opts.json && !opts.out && !opts.select) {
210319
+ renderExperimentResultsSummary(analysis);
210320
+ return;
210321
+ }
210322
+ const content = `${JSON.stringify(output, null, 2)}
210323
+ `;
210324
+ if (opts.out) {
210325
+ await fs5.writeFile(opts.out, content);
210326
+ success(`Wrote ${opts.out}`);
210327
+ } else {
210328
+ process.stdout.write(content);
210329
+ }
210330
+ return;
210331
+ }
209807
210332
  const payload = await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(selected)}`);
209808
210333
  const content = opts.format === "json" ? JSON.stringify(payload, null, 2) : toCsv(payload.runs, ["id", "status", "exampleId"]);
209809
210334
  if (opts.out) {
@@ -209836,6 +210361,25 @@ async function compareExperiments(batchIdA, batchIdB, opts) {
209836
210361
  throw new Error("--regression-threshold must be a non-negative number");
209837
210362
  }
209838
210363
  const client = buildClient(opts);
210364
+ if (opts.outputs) {
210365
+ const [outputsA, outputsB] = await Promise.all([
210366
+ fetchExperimentOutputs(client, batchIdA),
210367
+ fetchExperimentOutputs(client, batchIdB)
210368
+ ]);
210369
+ if (outputsA.automationId !== outputsB.automationId) {
210370
+ throw new Error("Experiment batches must belong to the same automation");
210371
+ }
210372
+ const diff = buildExperimentOutputDiff({
210373
+ batchIdA,
210374
+ batchIdB,
210375
+ rowsA: outputsA.rows,
210376
+ rowsB: outputsB.rows
210377
+ });
210378
+ if (opts.json)
210379
+ return printJson(diff);
210380
+ renderExperimentOutputDiffHuman(diff);
210381
+ return;
210382
+ }
209839
210383
  const [rowsA, rowsB] = await Promise.all([
209840
210384
  fetchEvalResults(client, batchIdA),
209841
210385
  fetchEvalResults(client, batchIdB)
@@ -209884,6 +210428,9 @@ async function listAgentFiles(agentId, opts) {
209884
210428
  console.log(table((payload.files ?? []).map((name) => ({ name })), [{ key: "name", header: "PATH" }]));
209885
210429
  }
209886
210430
  async function getAgentFile(agentId, remotePath, opts) {
210431
+ if (opts.json && opts.out) {
210432
+ throw new Error("--json and --out are mutually exclusive");
210433
+ }
209887
210434
  const client = buildClient(opts);
209888
210435
  const payload = await readAgentFile(client, agentId, remotePath);
209889
210436
  if (opts.json)
@@ -210003,7 +210550,7 @@ function registerSessionCommands(agent) {
210003
210550
  addJsonFlag(withPagination(withBaseUrl(session.command("list <agent-id-or-slug>")), 50)).description("List builder sessions for an agent.").action(action(listSessions));
210004
210551
  addJsonFlag(withBaseUrl(session.command("get <session-id>"))).description("Get a builder session and messages.").action(action(getSession));
210005
210552
  addJsonFlag(withBaseUrl(session.command("start <agent-id-or-slug>"))).description("Start a builder session.").option("--title <title>", "Session title").action(action(startSession));
210006
- addJsonFlag(withBaseUrl(session.command("message <session-id>"))).description("Append a message to a builder session.").requiredOption("--text <message>", "Message text").option("--wait", "Reserved; server acknowledges after enqueueing the message").action(action(messageSession));
210553
+ addJsonFlag(withBaseUrl(session.command("message <session-id>"))).description("Append a message to a builder session.").requiredOption("--text <message>", "Message text").action(action(messageSession));
210007
210554
  addJsonFlag(withBaseUrl(session.command("stop <session-id>"))).description("Stop a builder session.").option("--yes", "Skip confirmation (required in CI / agent terminals without a TTY)").action(action(stopSession));
210008
210555
  }
210009
210556
  async function listSessions(agentId, opts) {
@@ -237879,6 +238426,7 @@ var EXT_MIME = {
237879
238426
  ".doc": "application/msword",
237880
238427
  ".xls": "application/vnd.ms-excel",
237881
238428
  ".rtf": "application/rtf",
238429
+ ".msg": "application/vnd.ms-outlook",
237882
238430
  ".json": "application/json",
237883
238431
  ".xml": "application/xml",
237884
238432
  ".html": "text/html"
@@ -237900,7 +238448,6 @@ function resolveEvalBaseDir(dir) {
237900
238448
 
237901
238449
  // src/commands/workflow/eval-example.ts
237902
238450
  var RUN_POLL_INTERVAL_MS = 2000;
237903
- var RUN_MAX_WAIT_MS = 5 * 60 * 1000;
237904
238451
  function resolveEvalGraceMs() {
237905
238452
  const raw = env.EIGENPAL_EVAL_GRACE_MS;
237906
238453
  if (raw !== undefined) {
@@ -237949,7 +238496,7 @@ async function resolveExample(client, workflowId, name) {
237949
238496
  offset += rows.length;
237950
238497
  }
237951
238498
  }
237952
- async function pollRunTerminal(client, runId) {
238499
+ async function pollRunTerminal(client, runId, intervalMs, maxWaitMs) {
237953
238500
  const start = Date.now();
237954
238501
  const base = `/v1/runs/${encodeURIComponent(runId)}`;
237955
238502
  for (;; ) {
@@ -237958,10 +238505,10 @@ async function pollRunTerminal(client, runId) {
237958
238505
  if (run.finished || status && TERMINAL.has(status)) {
237959
238506
  return await client.get(`${base}?expand=execution`);
237960
238507
  }
237961
- if (Date.now() - start > RUN_MAX_WAIT_MS) {
238508
+ if (Date.now() - start > maxWaitMs) {
237962
238509
  return { ...run, status: "timeout", execution: { status: "timeout" } };
237963
238510
  }
237964
- await new Promise((r) => setTimeout(r, RUN_POLL_INTERVAL_MS));
238511
+ await new Promise((r) => setTimeout(r, intervalMs));
237965
238512
  }
237966
238513
  }
237967
238514
  async function pollEvalRollup(client, runId) {
@@ -237991,15 +238538,18 @@ function toArtifactPayload(run, runId) {
237991
238538
  stepExecutions: []
237992
238539
  };
237993
238540
  }
237994
- async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workflowId, exampleNames) {
238541
+ async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workflowId, exampleNames, options = {}) {
237995
238542
  const hasEvaluators = await fetchHasEvaluators(client, workflowId);
237996
238543
  const mode = hasEvaluators ? "evaluators" : "diff";
237997
- info(`Running ${ui.bold(`"${workflowIdOrSlug}"`)} ${ui.dim(`(${workflowId}, ${exampleNames.length} example(s))`)}`);
237998
- dim(hasEvaluators ? "Server example run; grading with configured evaluators." : "Server example run; no evaluators configured, grading by structural diff vs expected.");
238544
+ if (!options.quiet) {
238545
+ info(`Running ${ui.bold(`"${workflowIdOrSlug}"`)} ${ui.dim(`(${workflowId}, ${exampleNames.length} example(s))`)}`);
238546
+ dim(hasEvaluators ? "Server example run; grading with configured evaluators." : "Server example run; no evaluators configured, grading by structural diff vs expected.");
238547
+ }
237999
238548
  const evalBaseDir = resolveEvalBaseDir(dir);
238000
238549
  const results = [];
238001
238550
  for (const name of exampleNames) {
238002
- process.stderr.write(`${ui.dim("→")} ${name}
238551
+ if (!options.quiet)
238552
+ process.stderr.write(`${ui.dim("→")} ${name}
238003
238553
  `);
238004
238554
  try {
238005
238555
  const example = await resolveExample(client, workflowId, name);
@@ -238010,9 +238560,10 @@ async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workfl
238010
238560
  const runId = started.id;
238011
238561
  if (typeof runId !== "string")
238012
238562
  throw new Error("Example run did not return a run id");
238013
- process.stderr.write(` ${ui.dim(`run: ${runId}`)}
238563
+ if (!options.quiet)
238564
+ process.stderr.write(` ${ui.dim(`run: ${runId}`)}
238014
238565
  `);
238015
- const run = await pollRunTerminal(client, runId);
238566
+ const run = await pollRunTerminal(client, runId, options.intervalMs ?? RUN_POLL_INTERVAL_MS, options.maxWaitMs ?? 5 * 60 * 1000);
238016
238567
  const status = run.execution?.status ?? run.status;
238017
238568
  const runOk = status === "completed";
238018
238569
  const result = {
@@ -238038,14 +238589,17 @@ async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workfl
238038
238589
  error: r.error ?? null
238039
238590
  }));
238040
238591
  } else {
238041
- warn(` ${name}: no evaluator score after ${EVAL_GRACE_MS / 1000}s; grading by diff.`);
238592
+ if (!options.quiet) {
238593
+ warn(` ${name}: no evaluator score after ${EVAL_GRACE_MS / 1000}s; grading by diff.`);
238594
+ }
238042
238595
  gradeByDiff(result, example.expected, runOutput(run));
238043
238596
  }
238044
238597
  } else if (runOk) {
238045
238598
  gradeByDiff(result, example.expected, runOutput(run));
238046
238599
  }
238047
238600
  results.push(result);
238048
- reportLine(result);
238601
+ if (!options.quiet)
238602
+ reportLine(result);
238049
238603
  if (evalBaseDir) {
238050
238604
  try {
238051
238605
  await writeExecutionArtifacts(client, `${evalBaseDir}/${name}`, toArtifactPayload(run, runId));
@@ -238059,11 +238613,13 @@ async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workfl
238059
238613
  mode: "none"
238060
238614
  };
238061
238615
  results.push(result);
238062
- reportLine(result);
238616
+ if (!options.quiet)
238617
+ reportLine(result);
238063
238618
  }
238064
238619
  }
238065
238620
  const summary = summarize(workflowIdOrSlug, mode, results);
238066
- process.stderr.write(`
238621
+ if (!options.quiet)
238622
+ process.stderr.write(`
238067
238623
  ${formatEvalSummary(summary)}
238068
238624
  `);
238069
238625
  return summary;
@@ -238157,8 +238713,8 @@ function registerRunsCommands(program) {
238157
238713
  });
238158
238714
  const listRunsCmd = addJsonFlag(withPagination(withBaseUrl(runs.command("list [source]")), 50)).alias("ls").description("List runs across workflows and agents, optionally scoped to one source.").option("--type <type>", "Filter by run type: workflow|agent").option("--status <status>", "Filter by run status").option("--source-ref <ref>", "Filter agent runs by source ref").option("--batch-id <id>", "Filter eval/experiment runs by batch id").option("--example-id <id>", "Filter eval runs by example id").option("--example-id-contains <text>", "Filter eval runs by example id substring").option("--from <date>", "Created at or after this ISO timestamp or relative date").option("--to <date>", "Created before this ISO timestamp or relative date").option("--created-after <date>", "Alias for --from").option("--created-before <date>", "Alias for --to").option("--completed-after <date>", "Completed at or after this ISO timestamp or relative date").option("--completed-before <date>", "Completed before this ISO timestamp or relative date").option("--compact", "Render compact run rows").option("--sort <field>", "Sort field; only createdAt is supported").option("--order <asc|desc>", "Sort order; only desc is supported");
238159
238715
  listRunsCmd.action(action(listRuns));
238160
- addJsonFlag(withBaseUrl(runs.command("get <run-id>"))).description("Get one run.").option("--step <name>", "For workflow runs, show only this step (or comma-separated list)").option("--expand <sections>", "Comma-separated server expand sections for GET /api/v1/runs/:id: input, usage, execution, debug. Use execution for status/schemaValid; completed runs include top-level output/files/error.").option("--include <parts>", "Comma-separated local workflow step projection fields (error, duration, inputRef, …). Legacy expand names are mapped when possible; use --expand for server sections.", "review").action(action(getRun));
238161
- addJsonFlag(withBaseUrl(runs.command("compare <reference-run-id> <run-id>"))).description("Compare one run against another run. PDF/DOCX use pdftotext/python3; XLSX compares structured workbook content (not ZIP bytes).").option("--baseline", "Compare actual outputs from both runs instead of expected artifacts").option("--step <name>", "For workflow runs, restrict comparison to one step").option("--out <dir>", "Write comparison artifacts to this directory").option("--normalize-dates", "Normalize YYYYMMDD and YYYY-MM-DD tokens in filenames/text").option("--fail-on-diff", "Exit 1 when comparison status is fail").action(action(compareRun));
238716
+ addJsonFlag(withBaseUrl(runs.command("get <run-id>"))).description("Get one run.").option("--step <name>", "For workflow runs, show only this step (or comma-separated list)").option("--select <path>", "Print only a nested JSON value; implies JSON output (for example output.subjects or output.subjects[].name)").option("--expand <sections>", "Comma-separated server expand sections for GET /api/v1/runs/:id: input, usage, execution, debug. Use execution for status/schemaValid; completed runs include top-level output/files/error.").option("--include <parts>", "Comma-separated local workflow step projection fields (error, duration, inputRef, …). Legacy expand names are mapped when possible; use --expand for server sections.", "review").action(action(getRun));
238717
+ addJsonFlag(withBaseUrl(runs.command("compare <reference-run-id> <run-id>"))).description("Compare one run against another. Repeated workflow steps are matched by stable branch/iteration path.").option("--baseline", "Compare actual outputs from both runs instead of expected artifacts").option("--step <name-or-path>", "For workflow runs, restrict by step name or stable path").option("--out <dir>", "Write comparison artifacts to this directory").option("--normalize-dates", "Normalize YYYYMMDD and YYYY-MM-DD tokens in filenames/text").option("--fail-on-diff", "Exit 1 when comparison status is fail").action(action(compareRun));
238162
238718
  addJsonFlag(withBaseUrl(runs.command("rerun <run-id>"))).description("Create a new run from a previous run's stored input snapshot.").option("--version <version>", "Version/source ref for the new run: latest (default), original, or an explicit ref (same grammar as eigenpal run)").option("--source-ref <ref>", "Alias for --version").option("--wait", "Poll until the rerun reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(rerunRun));
238163
238719
  addJsonFlag(withBaseUrl(runs.command("promote <run-id>"))).description("Promote a run into a dataset example on the run's automation.").option("--name <name>", "Name for the created example").addHelpText("after", `
238164
238720
  Examples
@@ -238174,7 +238730,7 @@ Use \`eigenpal runs reviews update\` first when you need to correct output.
238174
238730
  });
238175
238731
  addJsonFlag(withBaseUrl(artifacts.command("list <run-id>"))).description("List available run artifacts without downloading them.").action(action(listRunArtifacts));
238176
238732
  withBaseUrl(artifacts.command("fetch <run-id>")).description("Download run artifacts by canonical artifact path.").option("--out <dir>", "Output directory").option("--include <parts>", "Comma-separated parts: output,input,metadata,issues,trace,lockfile,expected,all", "all").option("--path <path>", "Fetch one exact artifact path from `artifacts list`; repeatable", collectRepeated, []).option("--json", "Output a JSON summary of written artifacts").action(action(fetchRunArtifacts));
238177
- addJsonFlag(withBaseUrl(artifacts.command("inspect [run-id] [artifact-path]"))).description("Inspect a run XLSX artifact as structured JSON (sheet names, headers, typed rows). Use --file for a local workbook.").option("--file <path>", "Inspect a local .xlsx/.xls file instead of downloading a run artifact").option("--path <path>", "Canonical artifact path from `artifacts list` (alias for positional artifact-path)").option("--sheet <names>", "Comma-separated sheet names or zero-based indices to include").option("--max-sheets <n>", "Maximum sheets to include (default 20)", intArg).option("--max-rows <n>", "Maximum data rows per sheet (default 500)", intArg).option("--max-cols <n>", "Maximum columns per sheet (default 50)", intArg).action(action(inspectRunArtifact));
238733
+ withBaseUrl(artifacts.command("inspect [run-id] [artifact-path]")).description("Inspect a run XLSX artifact as structured JSON (sheet names, headers, typed rows). Use --file for a local workbook.").option("--file <path>", "Inspect a local .xlsx/.xls file instead of downloading a run artifact").option("--path <path>", "Canonical artifact path from `artifacts list` (alias for positional artifact-path)").option("--sheet <names>", "Comma-separated sheet names or zero-based indices to include").option("--max-sheets <n>", "Maximum sheets to include (default 20)", intArg).option("--max-rows <n>", "Maximum data rows per sheet (default 500)", intArg).option("--max-cols <n>", "Maximum columns per sheet (default 50)", intArg).action(action(inspectRunArtifact));
238178
238734
  withBaseUrl(runs.command("trace <run-id>")).description("Print raw trace.jsonl for a run, or write it with --out.").option("--out <file>", "Output file path").action(action(traceRun));
238179
238735
  registerRunReviewCommands(runs);
238180
238736
  registerRunExpectedCommands(runs);
@@ -238183,6 +238739,9 @@ Use \`eigenpal runs reviews update\` first when you need to correct output.
238183
238739
  Examples:
238184
238740
  $ eigenpal runs cancel exec_abc123
238185
238741
  $ eigenpal runs cancel exec_abc123 --yes # required in scripts and agent terminals
238742
+
238743
+ Cancellation is asynchronous. Human output confirms that the request was
238744
+ accepted; use \`--json\` to inspect the server response.
238186
238745
  `).action(action(cancelRun));
238187
238746
  }
238188
238747
  function registerRunReviewCommands(runs) {
@@ -238215,17 +238774,20 @@ Examples:
238215
238774
  `).action(action(deleteRunExpected));
238216
238775
  }
238217
238776
  async function runExample(agentId, opts) {
238777
+ if (opts.failOnMismatch && !opts.wait) {
238778
+ throw new Error("--fail-on-mismatch requires --wait because the evaluator verdict is asynchronous");
238779
+ }
238218
238780
  const client = buildClient(opts);
238219
238781
  const started = await client.post(`/v1/automations/${encodeURIComponent(agentAutomationId(agentId))}/experiments`, {
238220
238782
  examples: [opts.example],
238221
238783
  ...opts.sourceRef ? { sourceRef: opts.sourceRef } : {}
238222
238784
  });
238223
238785
  const runId = started.runs?.[0]?.id;
238224
- if (opts.failOnMismatch && !opts.wait) {
238225
- warn("--fail-on-mismatch has no effect without --wait; the evaluator verdict is not awaited.");
238786
+ if (opts.wait && !runId) {
238787
+ throw new Error("Example start response did not include a run id; cannot honor --wait");
238226
238788
  }
238227
238789
  if (opts.wait && runId) {
238228
- let payload = await pollRun(client, runId, opts.interval, opts.maxWait);
238790
+ let payload = await pollRun(client, runId, opts.interval, opts.maxWait, opts.json);
238229
238791
  const status = payload.execution?.status;
238230
238792
  let wait = { kind: "no-evaluators" };
238231
238793
  if (status === "completed") {
@@ -238355,43 +238917,46 @@ async function getRun(executionId, opts) {
238355
238917
  if (isWorkflowRun(run)) {
238356
238918
  const steps = filterWorkflowSteps(run.stepExecutions, opts.step);
238357
238919
  const projectionIncludes = workflowStepProjectionIncludes(projectionIncludeSource(opts));
238358
- if (opts.step) {
238359
- return printJson({ ...run, stepExecutions: projectWorkflowSteps(steps, projectionIncludes) });
238920
+ const projectedRun = opts.step ? { ...run, stepExecutions: projectWorkflowSteps(steps, projectionIncludes) } : run;
238921
+ if (opts.select) {
238922
+ return printJson(selectJsonValue(projectedRun, opts.select));
238360
238923
  }
238361
238924
  if (opts.json) {
238362
- return printJson(run);
238925
+ return printJson(projectedRun);
238363
238926
  }
238364
- console.log(renderFrame(toExecutionSnapshot(run, executionId)));
238927
+ console.log(renderFrame(toExecutionSnapshot(projectedRun, executionId)));
238365
238928
  process.stderr.write(`${ui.dim(`use --json for the full payload, then pipe to jq for field projection`)}
238366
238929
  `);
238367
238930
  return;
238368
238931
  }
238932
+ if (opts.select)
238933
+ return printJson(selectJsonValue(run, opts.select));
238369
238934
  renderRunPayload(run, opts);
238370
238935
  }
238371
238936
  function isWorkflowRun(run) {
238372
238937
  return run.type === "workflow";
238373
238938
  }
238374
- function objectOrNull(value) {
238939
+ function objectOrNull2(value) {
238375
238940
  return value && typeof value === "object" && !Array.isArray(value) ? value : null;
238376
238941
  }
238377
238942
  function toLegacyRunView(run) {
238378
- const timing = objectOrNull(run.timing);
238379
- const source = objectOrNull(run.source);
238380
- const sourceGit = objectOrNull(source?.git);
238381
- const trigger = objectOrNull(run.trigger);
238382
- const lineage = objectOrNull(run.lineage);
238383
- const execution = objectOrNull(run.execution);
238384
- const legacyRetry = objectOrNull(lineage?.retry);
238385
- const retry = objectOrNull(execution?.retry) ?? legacyRetry;
238386
- const nextRun = objectOrNull(retry?.nextRun);
238387
- const evalInfo = objectOrNull(run.eval);
238388
- const result = objectOrNull(run.result);
238389
- const input = objectOrNull(run.input);
238390
- const usage = objectOrNull(run.usage);
238391
- const tokens = objectOrNull(usage?.tokens);
238392
- const executionFiles = objectOrNull(execution?.files);
238393
- const executionExpected = objectOrNull(execution?.expected);
238394
- const debug = objectOrNull(run.debug);
238943
+ const timing = objectOrNull2(run.timing);
238944
+ const source = objectOrNull2(run.source);
238945
+ const sourceGit = objectOrNull2(source?.git);
238946
+ const trigger = objectOrNull2(run.trigger);
238947
+ const lineage = objectOrNull2(run.lineage);
238948
+ const execution = objectOrNull2(run.execution);
238949
+ const legacyRetry = objectOrNull2(lineage?.retry);
238950
+ const retry = objectOrNull2(execution?.retry) ?? legacyRetry;
238951
+ const nextRun = objectOrNull2(retry?.nextRun);
238952
+ const evalInfo = objectOrNull2(run.eval);
238953
+ const result = objectOrNull2(run.result);
238954
+ const input = objectOrNull2(run.input);
238955
+ const usage = objectOrNull2(run.usage);
238956
+ const tokens = objectOrNull2(usage?.tokens);
238957
+ const executionFiles = objectOrNull2(execution?.files);
238958
+ const executionExpected = objectOrNull2(execution?.expected);
238959
+ const debug = objectOrNull2(run.debug);
238395
238960
  const hasGroupedShape = timing != null || source != null || result != null || run.output !== undefined;
238396
238961
  if (!hasGroupedShape)
238397
238962
  return run;
@@ -238448,8 +239013,8 @@ function toLegacyRunView(run) {
238448
239013
  durationMs: usage.durationMs
238449
239014
  } : undefined,
238450
239015
  stepExecutions: execution?.steps,
238451
- nextRetryId: nextRun?.id ?? objectOrNull(execution?.nextRetry)?.id,
238452
- nextRetryStatus: nextRun?.status ?? objectOrNull(execution?.nextRetry)?.status,
239016
+ nextRetryId: nextRun?.id ?? objectOrNull2(execution?.nextRetry)?.id,
239017
+ nextRetryStatus: nextRun?.status ?? objectOrNull2(execution?.nextRetry)?.status,
238453
239018
  review: execution?.review,
238454
239019
  expected: executionExpected?.output,
238455
239020
  expectedFiles: executionExpected?.files,
@@ -238508,7 +239073,7 @@ function projectWorkflowStep(step, includes) {
238508
239073
  return out;
238509
239074
  }
238510
239075
  function toExecutionSnapshot(run, fallbackId) {
238511
- const execution = objectOrNull(run.execution);
239076
+ const execution = objectOrNull2(run.execution);
238512
239077
  return {
238513
239078
  id: String(run.id ?? run.executionId ?? fallbackId),
238514
239079
  status: String(run.status ?? execution?.status ?? "unknown"),
@@ -238556,8 +239121,11 @@ async function rerunRun(executionId, opts) {
238556
239121
  const path = `/v1/runs/${encodeURIComponent(executionId)}/rerun${query ? `?${query}` : ""}`;
238557
239122
  let payload = await client.post(path);
238558
239123
  const rerunId = String(payload.id ?? "");
239124
+ if (opts.wait && !rerunId) {
239125
+ throw new Error("Rerun response did not include an id; cannot honor --wait");
239126
+ }
238559
239127
  if (opts.wait && rerunId) {
238560
- payload = await pollRun(client, rerunId, opts.interval, opts.maxWait);
239128
+ payload = await pollRun(client, rerunId, opts.interval, opts.maxWait, opts.json);
238561
239129
  renderRunPayload(payload, opts);
238562
239130
  setExitCodeForFailedTerminalRun(payload);
238563
239131
  return;
@@ -238654,8 +239222,6 @@ async function inspectRunArtifact(runId, artifactPathArg, opts) {
238654
239222
  ...source.kind === "local" ? { file: source.path } : { runId: source.runId, artifactPath: source.artifactPath },
238655
239223
  ...inspect
238656
239224
  };
238657
- if (opts.json)
238658
- return printJson(payload);
238659
239225
  printJson(payload);
238660
239226
  }
238661
239227
  async function listRunArtifacts(executionId, opts) {
@@ -238849,21 +239415,21 @@ async function compareRun(referenceId, executionId, opts) {
238849
239415
  function compareWorkflowRuns(referenceId, reference, targetId, target, stepFilter) {
238850
239416
  const referenceSteps = workflowSteps(reference.stepExecutions);
238851
239417
  const targetSteps = workflowSteps(target.stepExecutions);
238852
- const stepNames = new Set;
238853
- for (const step of referenceSteps)
238854
- if (step.stepName)
238855
- stepNames.add(step.stepName);
238856
- for (const step of targetSteps)
238857
- if (step.stepName)
238858
- stepNames.add(step.stepName);
239418
+ const referenceByPath = new Map(referenceSteps.map((step) => [workflowStepComparisonPath(step), step]));
239419
+ const targetByPath = new Map(targetSteps.map((step) => [workflowStepComparisonPath(step), step]));
239420
+ const stepPaths = new Set([...referenceByPath.keys(), ...targetByPath.keys()]);
238859
239421
  const wanted = stepFilter ? new Set(stepFilter.split(",").map((step) => step.trim()).filter(Boolean)) : null;
238860
- const steps = [...stepNames].sort().filter((stepName) => !wanted || wanted.has(stepName)).map((stepName) => {
238861
- const referenceStep = referenceSteps.find((step) => step.stepName === stepName);
238862
- const targetStep = targetSteps.find((step) => step.stepName === stepName);
239422
+ const steps = [...stepPaths].sort().map((stepPath) => {
239423
+ const referenceStep = referenceByPath.get(stepPath);
239424
+ const targetStep = targetByPath.get(stepPath);
239425
+ const stepName = String(referenceStep?.stepName ?? targetStep?.stepName ?? stepPath);
239426
+ return { stepPath, stepName, referenceStep, targetStep };
239427
+ }).filter(({ stepName, stepPath }) => !wanted || wanted.has(stepName) || wanted.has(stepPath)).map(({ stepPath, stepName, referenceStep, targetStep }) => {
238863
239428
  const referenceDuration = typeof referenceStep?.durationMs === "number" ? referenceStep.durationMs : null;
238864
239429
  const targetDuration = typeof targetStep?.durationMs === "number" ? targetStep.durationMs : null;
238865
239430
  return {
238866
239431
  stepName,
239432
+ stepPath,
238867
239433
  referenceStatus: String(referenceStep?.status ?? "-"),
238868
239434
  targetStatus: String(targetStep?.status ?? "-"),
238869
239435
  durationDeltaMs: referenceDuration != null && targetDuration != null ? targetDuration - referenceDuration : null,
@@ -238880,6 +239446,31 @@ function compareWorkflowRuns(referenceId, reference, targetId, target, stepFilte
238880
239446
  finalOutputState
238881
239447
  };
238882
239448
  }
239449
+ function workflowStepComparisonPath(step) {
239450
+ const frames = Array.isArray(step.scopeStack) ? step.scopeStack : [];
239451
+ const parts = [];
239452
+ for (const frame of frames) {
239453
+ const type = String(frame.type ?? "");
239454
+ const stepName = String(frame.stepName ?? "");
239455
+ if (!stepName || type === "root")
239456
+ continue;
239457
+ if (type === "parallel-map-iteration" || type === "foreach-iteration") {
239458
+ const index = typeof frame.iterationIndex === "number" ? frame.iterationIndex : "?";
239459
+ parts.push(`${stepName}[${index}]`);
239460
+ } else if (type === "parallel-branch") {
239461
+ parts.push(`${stepName}.${String(frame.branchName ?? "?")}`);
239462
+ } else if (type === "if-then") {
239463
+ parts.push(`${stepName}.then`);
239464
+ } else if (type === "if-else") {
239465
+ parts.push(`${stepName}.else`);
239466
+ } else {
239467
+ parts.push(stepName);
239468
+ }
239469
+ }
239470
+ if (step.stepName)
239471
+ parts.push(step.stepName);
239472
+ return parts.join(".") || step.stepPath || step.stepName || step.scopeHash || "unknown-step";
239473
+ }
238883
239474
  function renderWorkflowComparisonReport(report) {
238884
239475
  console.log(`${ui.dim("A =")} ${ui.bold(report.comparedWithRunId)} ${ui.dim("B =")} ${ui.bold(report.runId)}`);
238885
239476
  console.log("");
@@ -238887,7 +239478,7 @@ function renderWorkflowComparisonReport(report) {
238887
239478
  console.log(ui.dim(`------------------------ --------- --------- ---------- ------`));
238888
239479
  for (const step of report.steps) {
238889
239480
  const delta = step.durationDeltaMs == null ? "-" : `${step.durationDeltaMs >= 0 ? "+" : ""}${step.durationDeltaMs}ms`;
238890
- console.log(`${step.stepName.padEnd(28)} ${padColored(colorStatus(step.referenceStatus), step.referenceStatus, 11)} ${padColored(colorStatus(step.targetStatus), step.targetStatus, 11)} ${delta.padEnd(12)} ${colorOutputState(step.outputState)}`);
239481
+ console.log(`${step.stepPath.padEnd(28)} ${padColored(colorStatus(step.referenceStatus), step.referenceStatus, 11)} ${padColored(colorStatus(step.targetStatus), step.targetStatus, 11)} ${delta.padEnd(12)} ${colorOutputState(step.outputState)}`);
238891
239482
  }
238892
239483
  console.log("");
238893
239484
  console.log(`${ui.dim("final output:")} ${colorOutputState(report.finalOutputState)}`);
@@ -238940,9 +239531,9 @@ async function listRuns(source, opts) {
238940
239531
  };
238941
239532
  const payload = await client.get("/v1/runs", params);
238942
239533
  const viewRuns = payload.runs.map(toLegacyRunView);
238943
- const rows = opts.compact ? viewRuns.map(compactRunRow) : viewRuns;
238944
239534
  if (opts.json)
238945
- return printJson({ ...payload, runs: rows });
239535
+ return printJson({ ...payload, runs: viewRuns });
239536
+ const rows = opts.compact ? viewRuns.map(compactRunRow) : viewRuns;
238946
239537
  if (payload.scanLimited)
238947
239538
  warn("Filtered run list scanned only the first matching window.");
238948
239539
  if (payload.noResolvedAnchor)
@@ -239124,7 +239715,7 @@ async function deleteRunExpected(executionId, name, opts) {
239124
239715
  async function watchRunCommand(executionId, opts) {
239125
239716
  const client = buildClient(opts);
239126
239717
  if (opts.json) {
239127
- const payload = await pollRun(client, executionId, opts.interval, opts.maxWait);
239718
+ const payload = await pollRun(client, executionId, opts.interval, opts.maxWait, opts.json);
239128
239719
  renderRunPayload(payload, opts);
239129
239720
  setExitCodeForFailedTerminalRun(payload);
239130
239721
  return;
@@ -239146,7 +239737,7 @@ Detached after ${opts.maxWait}s without reaching terminal status. Re-run \`eigen
239146
239737
  `);
239147
239738
  process.exit(2);
239148
239739
  }
239149
- if (result.final.status === "failed" || result.final.status === "cancelled") {
239740
+ if (result.final.status === "failed" || result.final.status === "cancelled" || result.final.status === "rejected") {
239150
239741
  process.exitCode = 1;
239151
239742
  }
239152
239743
  }
@@ -239154,7 +239745,11 @@ async function cancelRun(executionId, opts) {
239154
239745
  requireYesInNonInteractive(opts.yes, "Cancel run");
239155
239746
  const client = buildClient(opts);
239156
239747
  const payload = await client.post(`/v1/runs/${encodeURIComponent(executionId)}/cancel`, {});
239157
- renderRunPayload(payload, opts);
239748
+ if (opts.json) {
239749
+ printJson(payload);
239750
+ return;
239751
+ }
239752
+ success(`Cancellation requested for run ${executionId}.`);
239158
239753
  }
239159
239754
  async function downloadTraceText(client, executionId) {
239160
239755
  const payload = await client.get(`/v1/runs/${encodeURIComponent(executionId)}/trace`);
@@ -239915,14 +240510,147 @@ async function completion(shell, program) {
239915
240510
  `);
239916
240511
  }
239917
240512
 
240513
+ // src/commands/docs.ts
240514
+ import { existsSync as existsSync13, readFileSync as readFileSync13 } from "node:fs";
240515
+ var docsCache = null;
240516
+ function bundledDocs() {
240517
+ if (docsCache)
240518
+ return docsCache;
240519
+ const candidates = [
240520
+ new URL("./docs-bundle.json", import.meta.url),
240521
+ new URL("../docs-bundle.json", import.meta.url)
240522
+ ];
240523
+ const bundleUrl = candidates.find((candidate) => existsSync13(candidate));
240524
+ if (!bundleUrl) {
240525
+ throw new Error("Bundled documentation asset is missing. Reinstall the Eigenpal CLI.");
240526
+ }
240527
+ const parsed = JSON.parse(readFileSync13(bundleUrl, "utf8"));
240528
+ if (!Array.isArray(parsed.entries)) {
240529
+ throw new Error("Bundled documentation asset is invalid. Reinstall the Eigenpal CLI.");
240530
+ }
240531
+ docsCache = parsed.entries;
240532
+ return docsCache;
240533
+ }
240534
+ function listDocs(prefix, source) {
240535
+ const docs = bundledDocs();
240536
+ const normalizedPrefix = normalizeTopic(prefix ?? "");
240537
+ return docs.filter((doc) => (!normalizedPrefix || doc.topic.startsWith(normalizedPrefix)) && (!source || doc.source === source));
240538
+ }
240539
+ function resolveDoc(topic) {
240540
+ const docs = bundledDocs();
240541
+ const normalized = normalizeTopic(topic);
240542
+ const exact = docs.find((doc) => doc.topic === normalized);
240543
+ if (exact)
240544
+ return exact;
240545
+ const suffixMatches = docs.filter((doc) => doc.topic.endsWith(`/${normalized}`) || doc.topic === normalized);
240546
+ if (suffixMatches.length === 1)
240547
+ return suffixMatches[0];
240548
+ if (suffixMatches.length > 1) {
240549
+ throw new Error(`Documentation topic "${topic}" is ambiguous. Use one of: ${suffixMatches.map((doc) => doc.topic).join(", ")}`);
240550
+ }
240551
+ throw new Error(`Unknown documentation topic "${topic}". Run \`eigenpal docs search ${JSON.stringify(topic)}\` or \`eigenpal docs list\`.`);
240552
+ }
240553
+ function searchDocs(query, limit = 20) {
240554
+ const docs = bundledDocs();
240555
+ const terms = query.toLocaleLowerCase().split(/\s+/).filter(Boolean);
240556
+ if (terms.length === 0)
240557
+ throw new Error("Search query must not be empty.");
240558
+ return docs.map((doc) => {
240559
+ const title = `${doc.topic}
240560
+ ${doc.title}
240561
+ ${doc.description}`.toLocaleLowerCase();
240562
+ const content = doc.content.toLocaleLowerCase();
240563
+ if (!terms.every((term) => title.includes(term) || content.includes(term)))
240564
+ return null;
240565
+ const score = terms.reduce((sum, term) => sum + (title.includes(term) ? 10 : 0) + occurrences(content, term), 0);
240566
+ return { ...doc, snippet: matchingSnippet(doc.content, terms), score };
240567
+ }).filter((doc) => doc !== null).sort((a, b) => b.score - a.score || a.topic.localeCompare(b.topic)).slice(0, Math.max(1, limit)).map(({ score: _score, ...doc }) => doc);
240568
+ }
240569
+ function registerDocsCommands(program) {
240570
+ const command = program.command("docs").description("List, search, and read the complete release-matched Eigenpal documentation offline.").action(() => {
240571
+ process.stderr.write("`eigenpal docs` requires a subcommand: `list`, `search`, or `read`.\nRun `eigenpal docs --help` to see examples.\n");
240572
+ process.exit(2);
240573
+ });
240574
+ addJsonFlag(command.command("list [prefix]")).description("List bundled documentation topics, optionally under a path prefix.").option("--source <source>", "Filter by public, agent-reference, or api").action(action(async (prefix, opts) => {
240575
+ const results = listDocs(prefix, opts.source);
240576
+ if (opts.json) {
240577
+ printJson(results.map(({ content: _content, ...doc }) => doc));
240578
+ return;
240579
+ }
240580
+ console.log(table(results, [
240581
+ { key: "topic", header: "TOPIC" },
240582
+ { key: "source", header: "SOURCE" },
240583
+ { key: "title", header: "TITLE" }
240584
+ ]));
240585
+ }));
240586
+ addJsonFlag(command.command("search <query>")).description("Search every public page and detailed agent reference.").option("--limit <n>", "Maximum matches", intArg, 20).action(action(async (query, opts) => {
240587
+ const results = searchDocs(query, opts.limit);
240588
+ if (opts.json) {
240589
+ printJson(results.map(({ content: _content, ...doc }) => doc));
240590
+ return;
240591
+ }
240592
+ console.log(table(results.map((doc) => ({ ...doc, match: doc.snippet || doc.description })), [
240593
+ { key: "topic", header: "TOPIC" },
240594
+ { key: "source", header: "SOURCE" },
240595
+ { key: "match", header: "MATCH" }
240596
+ ]));
240597
+ }));
240598
+ addJsonFlag(command.command("read <topic>")).description("Print one bundled documentation topic to stdout.").action(action(async (topic, opts) => {
240599
+ const doc = resolveDoc(topic);
240600
+ if (opts.json) {
240601
+ printJson(doc);
240602
+ return;
240603
+ }
240604
+ process.stdout.write(doc.content.endsWith(`
240605
+ `) ? doc.content : `${doc.content}
240606
+ `);
240607
+ }));
240608
+ command.addHelpText("after", `
240609
+ Examples:
240610
+ $ eigenpal docs search "evaluator path syntax"
240611
+ $ eigenpal docs read reference/evaluators
240612
+ $ eigenpal docs read steps/transform/script
240613
+ $ eigenpal docs list api-reference --json
240614
+
240615
+ The bundle contains every public docs page, the OpenAPI specification, and the
240616
+ detailed agent references generated from Eigenpal's real schemas and CLI help.
240617
+ It ships with the CLI, needs no authentication, and matches that CLI release.
240618
+ `);
240619
+ }
240620
+ function normalizeTopic(value) {
240621
+ let topic = value.trim();
240622
+ try {
240623
+ const url = new URL(topic);
240624
+ topic = url.pathname;
240625
+ } catch {}
240626
+ return topic.replace(/^\/+/, "").replace(/^docs\//, "").replace(/\.(md|mdx|json)$/, "").replace(/\/+$/, "");
240627
+ }
240628
+ function occurrences(content, term) {
240629
+ let count = 0;
240630
+ let index = 0;
240631
+ while ((index = content.indexOf(term, index)) >= 0) {
240632
+ count += 1;
240633
+ index += term.length;
240634
+ }
240635
+ return Math.min(count, 20);
240636
+ }
240637
+ function matchingSnippet(content, terms) {
240638
+ const lines = content.split(/\r?\n/);
240639
+ const line = lines.find((candidate) => {
240640
+ const lower = candidate.toLocaleLowerCase();
240641
+ return terms.some((term) => lower.includes(term));
240642
+ }) ?? "";
240643
+ return line.replace(/^[#>*\s-]+/, "").replace(/[`[\]{}]/g, "").trim().slice(0, 140);
240644
+ }
240645
+
239918
240646
  // src/commands/init/index.ts
239919
240647
  init_dist4();
239920
240648
  import {
239921
240649
  copyFileSync,
239922
- existsSync as existsSync13,
240650
+ existsSync as existsSync14,
239923
240651
  mkdirSync as mkdirSync5,
239924
240652
  readdirSync as readdirSync4,
239925
- readFileSync as readFileSync13,
240653
+ readFileSync as readFileSync14,
239926
240654
  statSync as statSync4,
239927
240655
  writeFileSync as writeFileSync7
239928
240656
  } from "node:fs";
@@ -239940,10 +240668,10 @@ function isValidProjectName(name) {
239940
240668
  function resolveTemplatesRoot() {
239941
240669
  const here = dirname4(fileURLToPath6(import.meta.url));
239942
240670
  const srcCandidate = resolve4(here, "..", "..", "templates");
239943
- if (existsSync13(srcCandidate))
240671
+ if (existsSync14(srcCandidate))
239944
240672
  return srcCandidate;
239945
240673
  const distCandidate = resolve4(here, "templates");
239946
- if (existsSync13(distCandidate))
240674
+ if (existsSync14(distCandidate))
239947
240675
  return distCandidate;
239948
240676
  throw new Error(`Cannot locate bundled templates. Looked in:
239949
240677
  ${srcCandidate}
@@ -239955,7 +240683,7 @@ async function init(name, opts) {
239955
240683
  }
239956
240684
  const target = resolve4(process.cwd(), opts.dir ?? name);
239957
240685
  assertSafeTargetDir(target, opts.dir);
239958
- if (existsSync13(target) && readdirSync4(target).length > 0) {
240686
+ if (existsSync14(target) && readdirSync4(target).length > 0) {
239959
240687
  throw new Error(`Target directory ${relative2(process.cwd(), target)} exists and is not empty.`);
239960
240688
  }
239961
240689
  let template;
@@ -239970,7 +240698,7 @@ async function init(name, opts) {
239970
240698
  template = await promptForTemplate();
239971
240699
  }
239972
240700
  const sourceRoot = join7(resolveTemplatesRoot(), template);
239973
- if (!existsSync13(sourceRoot)) {
240701
+ if (!existsSync14(sourceRoot)) {
239974
240702
  throw new Error(`Template directory missing on disk: ${sourceRoot}`);
239975
240703
  }
239976
240704
  const written = copyTemplate(sourceRoot, target, name);
@@ -240023,7 +240751,7 @@ function copyTemplate(src, dst, projectName) {
240023
240751
  }
240024
240752
  mkdirSync5(dirname4(targetPath), { recursive: true });
240025
240753
  if (isTextFile(sourcePath)) {
240026
- const text = readFileSync13(sourcePath, "utf-8").replaceAll(PLACEHOLDER, projectName);
240754
+ const text = readFileSync14(sourcePath, "utf-8").replaceAll(PLACEHOLDER, projectName);
240027
240755
  writeFileSync7(targetPath, text);
240028
240756
  } else {
240029
240757
  copyFileSync(sourcePath, targetPath);
@@ -240168,7 +240896,7 @@ function newIdempotencyKey() {
240168
240896
  return randomUUID();
240169
240897
  }
240170
240898
  async function uploadReusableFile(client, input) {
240171
- const contentType = input.mimeType ?? guessMimeType(input.filename) ?? "application/octet-stream";
240899
+ const contentType = input.mimeType || guessMimeType(input.filename) || "application/octet-stream";
240172
240900
  const idempotencyKey = input.idempotencyKey ?? newIdempotencyKey();
240173
240901
  const negotiation = await client.post(apiPath("/files/uploads"), {
240174
240902
  filename: input.filename,
@@ -240223,7 +240951,7 @@ async function buildPreparedRunRequest(client, input) {
240223
240951
  fieldName: spec.fieldName,
240224
240952
  content: data,
240225
240953
  filename: path17.basename(spec.filePath),
240226
- mimeType: guessMimeType(path17.basename(spec.filePath)) ?? "application/octet-stream"
240954
+ mimeType: guessMimeType(path17.basename(spec.filePath)) || "application/octet-stream"
240227
240955
  });
240228
240956
  }
240229
240957
  for (const file of input.files ?? []) {
@@ -240232,7 +240960,7 @@ async function buildPreparedRunRequest(client, input) {
240232
240960
  fieldName: file.fieldName,
240233
240961
  content,
240234
240962
  filename: file.filename,
240235
- mimeType: file.mimeType ?? guessMimeType(file.filename) ?? "application/octet-stream"
240963
+ mimeType: file.mimeType || guessMimeType(file.filename) || "application/octet-stream"
240236
240964
  });
240237
240965
  }
240238
240966
  const preUploadIndices = indicesRequiringPreUpload(localFiles.map((file) => ({ size: file.content.byteLength })));
@@ -240291,7 +241019,7 @@ function upgradeMixedFieldPreUpload(localFiles, preUploadIndices) {
240291
241019
  }
240292
241020
  function appendFilePart(form, fieldName, content, filename, mimeType) {
240293
241021
  assertValidFileFieldName(fieldName);
240294
- const type = mimeType ?? guessMimeType(filename) ?? "application/octet-stream";
241022
+ const type = mimeType || guessMimeType(filename) || "application/octet-stream";
240295
241023
  form.append(`files.${fieldName}`, new Blob([content], { type }), filename);
240296
241024
  }
240297
241025
  function assertValidFileFieldName(fieldName) {
@@ -240318,7 +241046,7 @@ function parseInputFileSpecs(inputFile) {
240318
241046
 
240319
241047
  // src/lib/format-eigenpal.ts
240320
241048
  import { spawnSync as spawnSync5 } from "child_process";
240321
- import { existsSync as existsSync14, readFileSync as readFileSync14 } from "fs";
241049
+ import { existsSync as existsSync15, readFileSync as readFileSync15 } from "fs";
240322
241050
  import { join as join8, resolve as resolve5 } from "path";
240323
241051
  var PRETTRC_NAMES = [
240324
241052
  ".prettierrc",
@@ -240332,14 +241060,14 @@ var PRETTRC_NAMES = [
240332
241060
  ];
240333
241061
  function hasPrettierConfig(cwd) {
240334
241062
  for (const name of PRETTRC_NAMES) {
240335
- if (existsSync14(join8(cwd, name)))
241063
+ if (existsSync15(join8(cwd, name)))
240336
241064
  return true;
240337
241065
  }
240338
241066
  try {
240339
241067
  const pkgPath = join8(cwd, "package.json");
240340
- if (!existsSync14(pkgPath))
241068
+ if (!existsSync15(pkgPath))
240341
241069
  return false;
240342
- const raw = readFileSync14(pkgPath, "utf-8");
241070
+ const raw = readFileSync15(pkgPath, "utf-8");
240343
241071
  const pkg = JSON.parse(raw);
240344
241072
  return pkg != null && typeof pkg === "object" && "prettier" in pkg;
240345
241073
  } catch {
@@ -240349,7 +241077,7 @@ function hasPrettierConfig(cwd) {
240349
241077
  function formatEigenpalDirIfAvailable(dir) {
240350
241078
  const cwd = process.cwd();
240351
241079
  const absDir = resolve5(cwd, dir);
240352
- if (!existsSync14(absDir))
241080
+ if (!existsSync15(absDir))
240353
241081
  return;
240354
241082
  if (!hasPrettierConfig(cwd))
240355
241083
  return;
@@ -240369,9 +241097,13 @@ async function runSavedWorkflowExamples(workflow, examples, opts) {
240369
241097
  const client = new ApiClient(config);
240370
241098
  const workflowId = await resolveWorkflowId(client, workflow);
240371
241099
  if (opts.version && !["latest", "current", "undefined"].includes(opts.version)) {
240372
- warn(`--example runs the current published version; ignoring @${opts.version}.`);
241100
+ throw new Error(`--example cannot run workflow version @${opts.version}; publish or activate that version first, then use @latest`);
240373
241101
  }
240374
- const summary = await runWorkflowExamplesWithEval(client, config.dir, workflow, workflowId, examples);
241102
+ const summary = await runWorkflowExamplesWithEval(client, config.dir, workflow, workflowId, examples, {
241103
+ quiet: opts.json,
241104
+ intervalMs: Math.max(0, opts.interval ?? 2) * 1000,
241105
+ maxWaitMs: Math.max(1, opts.maxWait ?? 1800) * 1000
241106
+ });
240375
241107
  if (opts.json)
240376
241108
  console.log(JSON.stringify(summary, null, 2));
240377
241109
  if (summary.errored > 0)
@@ -240453,8 +241185,11 @@ async function runTarget(target, opts) {
240453
241185
  }
240454
241186
  const runId = String(payload.id ?? "");
240455
241187
  let waitedForTerminalRun = false;
241188
+ if (opts.wait && !runId) {
241189
+ throw new Error("Run start response did not include an id; cannot honor --wait");
241190
+ }
240456
241191
  if (opts.wait && runId) {
240457
- payload = await pollRun(client, runId, opts.interval, opts.maxWait);
241192
+ payload = await pollRun(client, runId, opts.interval, opts.maxWait, opts.json);
240458
241193
  waitedForTerminalRun = true;
240459
241194
  }
240460
241195
  renderRunPayload(payload, opts);
@@ -240469,10 +241204,10 @@ async function rerunTarget(runId, opts) {
240469
241204
  init_dist4();
240470
241205
  import { createHash as createHash5 } from "node:crypto";
240471
241206
  import {
240472
- existsSync as existsSync15,
241207
+ existsSync as existsSync16,
240473
241208
  lstatSync,
240474
241209
  mkdirSync as mkdirSync6,
240475
- readFileSync as readFileSync15,
241210
+ readFileSync as readFileSync16,
240476
241211
  readdirSync as readdirSync5,
240477
241212
  rmSync as rmSync2,
240478
241213
  rmdirSync,
@@ -240555,10 +241290,10 @@ var MANIFEST_NAME = ".eigenpal-manifest.json";
240555
241290
  function resolveSkillSource() {
240556
241291
  const here = dirname5(fileURLToPath7(import.meta.url));
240557
241292
  const srcCandidate = resolve6(here, "..", "..", "skill");
240558
- if (existsSync15(srcCandidate))
241293
+ if (existsSync16(srcCandidate))
240559
241294
  return srcCandidate;
240560
241295
  const distCandidate = resolve6(here, "skill");
240561
- if (existsSync15(distCandidate))
241296
+ if (existsSync16(distCandidate))
240562
241297
  return distCandidate;
240563
241298
  throw new Error(`Cannot locate bundled skill content. Looked in:
240564
241299
  ${srcCandidate}
@@ -240595,15 +241330,15 @@ async function installSkill(opts) {
240595
241330
  for (const relativePath of sourceFiles) {
240596
241331
  const sourcePath = join10(source, relativePath);
240597
241332
  const targetPath = join10(target, relativePath);
240598
- const sourceBytes = templateSourceFile(relativePath, readFileSync15(sourcePath), cliVersion);
241333
+ const sourceBytes = templateSourceFile(relativePath, readFileSync16(sourcePath), cliVersion);
240599
241334
  const sourceHash = sha256(sourceBytes);
240600
- if (!existsSync15(targetPath)) {
241335
+ if (!existsSync16(targetPath)) {
240601
241336
  mkdirSync6(dirname5(targetPath), { recursive: true });
240602
241337
  writeFileSync8(targetPath, sourceBytes);
240603
241338
  written += 1;
240604
241339
  continue;
240605
241340
  }
240606
- const targetBytes = readFileSync15(targetPath);
241341
+ const targetBytes = readFileSync16(targetPath);
240607
241342
  const targetHash = sha256(targetBytes);
240608
241343
  if (targetHash === sourceHash) {
240609
241344
  skipped += 1;
@@ -240635,9 +241370,9 @@ async function installSkill(opts) {
240635
241370
  const manifestEntries = [];
240636
241371
  for (const relativePath of sourceFiles) {
240637
241372
  const targetPath = join10(target, relativePath);
240638
- if (!existsSync15(targetPath))
241373
+ if (!existsSync16(targetPath))
240639
241374
  continue;
240640
- manifestEntries.push({ path: relativePath, sha256: sha256(readFileSync15(targetPath)) });
241375
+ manifestEntries.push({ path: relativePath, sha256: sha256(readFileSync16(targetPath)) });
240641
241376
  }
240642
241377
  const manifest = {
240643
241378
  cliVersion,
@@ -240678,9 +241413,9 @@ async function uninstallSkill(opts) {
240678
241413
  console.error(`Refusing to touch ${path}: manifest path escapes ${targetReal}`);
240679
241414
  continue;
240680
241415
  }
240681
- if (!existsSync15(path))
241416
+ if (!existsSync16(path))
240682
241417
  continue;
240683
- const currentHash = sha256(readFileSync15(path));
241418
+ const currentHash = sha256(readFileSync16(path));
240684
241419
  if (currentHash !== entry.sha256 && !opts.yes) {
240685
241420
  kept += 1;
240686
241421
  continue;
@@ -240777,6 +241512,9 @@ async function uninstallSkillTools(opts) {
240777
241512
  await uninstallSkill({ target: opts.target, yes: opts.yes });
240778
241513
  return;
240779
241514
  }
241515
+ if (!opts.all && !hasToolIds && (opts.yes || !isInteractiveStdout())) {
241516
+ throw new Error("Non-interactive: pass tool ids (e.g. `skill uninstall claude`) or `--all`.");
241517
+ }
240780
241518
  const cwd = process.cwd();
240781
241519
  const installedTools = TOOLS.filter((t) => isInstalled(cwd, t));
240782
241520
  if (installedTools.length === 0) {
@@ -240795,8 +241533,6 @@ async function uninstallSkillTools(opts) {
240795
241533
  info(`Not installed (skipping): ${notInstalled.join(", ")}`);
240796
241534
  targets = targets.filter((t) => installedIds.has(t.id));
240797
241535
  }
240798
- } else if (opts.yes || !isInteractiveStdout()) {
240799
- throw new Error("Non-interactive: pass tool ids (e.g. `skill uninstall claude`) or `--all`.");
240800
241536
  } else {
240801
241537
  const picked = await ve({
240802
241538
  message: "Select tools to uninstall (only installed tools shown)",
@@ -240880,10 +241616,10 @@ function hintFor(tool, installed, detected) {
240880
241616
  return tool.relativePath;
240881
241617
  }
240882
241618
  function isInstalled(cwd, tool) {
240883
- return existsSync15(join10(cwd, tool.relativePath, MANIFEST_NAME));
241619
+ return existsSync16(join10(cwd, tool.relativePath, MANIFEST_NAME));
240884
241620
  }
240885
241621
  function isProjectUsing(cwd, tool) {
240886
- return tool.detectPaths.some((p) => existsSync15(join10(cwd, p)));
241622
+ return tool.detectPaths.some((p) => existsSync16(join10(cwd, p)));
240887
241623
  }
240888
241624
  function listFiles(root) {
240889
241625
  const out = [];
@@ -240907,10 +241643,10 @@ function listFiles(root) {
240907
241643
  }
240908
241644
  function readManifest(target) {
240909
241645
  const path = join10(target, MANIFEST_NAME);
240910
- if (!existsSync15(path))
241646
+ if (!existsSync16(path))
240911
241647
  return null;
240912
241648
  try {
240913
- return JSON.parse(readFileSync15(path, "utf-8"));
241649
+ return JSON.parse(readFileSync16(path, "utf-8"));
240914
241650
  } catch {
240915
241651
  return null;
240916
241652
  }
@@ -240924,8 +241660,8 @@ function readCliVersion() {
240924
241660
  resolve6(here, "..", "package.json")
240925
241661
  ];
240926
241662
  for (const candidate of candidates) {
240927
- if (existsSync15(candidate)) {
240928
- const pkg = JSON.parse(readFileSync15(candidate, "utf-8"));
241663
+ if (existsSync16(candidate)) {
241664
+ const pkg = JSON.parse(readFileSync16(candidate, "utf-8"));
240929
241665
  if (pkg.name && pkg.name !== "@eigenpal/cli")
240930
241666
  continue;
240931
241667
  if (pkg.version)
@@ -240936,7 +241672,7 @@ function readCliVersion() {
240936
241672
  return "unknown";
240937
241673
  }
240938
241674
  function pruneEmptyDirs(root) {
240939
- if (!existsSync15(root))
241675
+ if (!existsSync16(root))
240940
241676
  return;
240941
241677
  const stack = [];
240942
241678
  const collect = (dir) => {
@@ -241726,12 +242462,12 @@ function looksLikeFreeform(prop) {
241726
242462
  return prop.pattern !== undefined || prop.format !== undefined;
241727
242463
  }
241728
242464
  // src/commands/workflow/index.ts
241729
- import { existsSync as existsSync21, promises as fs20 } from "node:fs";
242465
+ import { existsSync as existsSync22, promises as fs20 } from "node:fs";
241730
242466
  import path18 from "node:path";
241731
242467
 
241732
242468
  // src/lib/local-templates.ts
241733
242469
  init_src();
241734
- import { existsSync as existsSync16, promises as fs17, realpathSync as realpathSync3, statSync as statSync5 } from "node:fs";
242470
+ import { existsSync as existsSync17, promises as fs17, realpathSync as realpathSync3, statSync as statSync5 } from "node:fs";
241735
242471
  import { basename, dirname as dirname6, isAbsolute, relative as relative4, resolve as resolve7, sep } from "node:path";
241736
242472
 
241737
242473
  // src/lib/office-template.ts
@@ -241974,7 +242710,7 @@ function walkTemplateSteps(steps, visit) {
241974
242710
  }
241975
242711
  function resolveWorkflowProjectRoot(workflowFile) {
241976
242712
  const abs = resolve7(workflowFile);
241977
- if (!existsSync16(abs)) {
242713
+ if (!existsSync17(abs)) {
241978
242714
  throw new Error(`Workflow file not found: ${abs}`);
241979
242715
  }
241980
242716
  return dirname6(realpathSync3(abs));
@@ -241997,7 +242733,7 @@ function resolveWorkflowTemplatePath(workflowFile, templatePath, opts = {}) {
241997
242733
  if (!opts.allowExternal) {
241998
242734
  assertInsideProject(projectRoot, templatePath, candidate);
241999
242735
  }
242000
- if (!existsSync16(candidate)) {
242736
+ if (!existsSync17(candidate)) {
242001
242737
  throw new Error(`Local template not found: ${candidate}`);
242002
242738
  }
242003
242739
  const real = realpathSync3(candidate);
@@ -242179,11 +242915,11 @@ function rewriteLocalTemplateYaml(yamlText, resolved) {
242179
242915
  }
242180
242916
 
242181
242917
  // src/commands/workflow/clear.ts
242182
- import { existsSync as existsSync17, readdirSync as readdirSync6, rmSync as rmSync3, statSync as statSync6 } from "fs";
242918
+ import { existsSync as existsSync18, readdirSync as readdirSync6, rmSync as rmSync3, statSync as statSync6 } from "fs";
242183
242919
  import { basename as basename2, join as join11 } from "path";
242184
242920
  var EMPTY_REPORT = { removed: 0, kept: 0, touched: 0 };
242185
242921
  function getDirectoryEntries(path) {
242186
- if (!existsSync17(path))
242922
+ if (!existsSync18(path))
242187
242923
  return [];
242188
242924
  return readdirSync6(path).filter((entry) => statSync6(join11(path, entry)).isDirectory()).sort();
242189
242925
  }
@@ -242204,7 +242940,7 @@ function removeDirEntriesWithRetention(path, removeAll) {
242204
242940
  return { removed, kept: keepSet.size, touched: entries.length };
242205
242941
  }
242206
242942
  function removeJudgeSummaryFilesWithRetention(exampleDir, removeAll) {
242207
- if (!existsSync17(exampleDir))
242943
+ if (!existsSync18(exampleDir))
242208
242944
  return EMPTY_REPORT;
242209
242945
  const stampByFile = new Map;
242210
242946
  for (const fileName of readdirSync6(exampleDir).sort()) {
@@ -242276,7 +243012,7 @@ async function clearEvalOutputs(dir, exampleNames = [], options = {}) {
242276
243012
  runSection({
242277
243013
  exampleDirs,
242278
243014
  clean: (exampleDir) => removeDirEntriesWithRetention(join11(exampleDir, "executions"), removeAll),
242279
- shouldCount: (exampleDir) => existsSync17(join11(exampleDir, "executions")),
243015
+ shouldCount: (exampleDir) => existsSync18(join11(exampleDir, "executions")),
242280
243016
  successLabel: "Execution cleanup complete",
242281
243017
  scannedLabel: "examples scanned:",
242282
243018
  seenLabel: "runs seen: ",
@@ -242288,7 +243024,7 @@ async function clearEvalOutputs(dir, exampleNames = [], options = {}) {
242288
243024
  runSection({
242289
243025
  exampleDirs,
242290
243026
  clean: (exampleDir) => removeJudgeSummaryFilesWithRetention(exampleDir, removeAll),
242291
- shouldCount: (exampleDir) => existsSync17(exampleDir),
243027
+ shouldCount: (exampleDir) => existsSync18(exampleDir),
242292
243028
  successLabel: "Judge summary cleanup complete",
242293
243029
  scannedLabel: "examples scanned: ",
242294
243030
  seenLabel: "summary files seen: ",
@@ -242401,7 +243137,7 @@ function registerStepExecCommands(workflow) {
242401
243137
 
242402
243138
  // src/commands/workflow/templates.ts
242403
243139
  init_src();
242404
- import { existsSync as existsSync18, promises as fs19 } from "node:fs";
243140
+ import { existsSync as existsSync19, promises as fs19 } from "node:fs";
242405
243141
  import { basename as basename3, resolve as resolve8 } from "node:path";
242406
243142
  function buildClient2(opts) {
242407
243143
  const config = resolveConfig(opts);
@@ -242582,7 +243318,7 @@ function registerTemplateCommands(parent) {
242582
243318
  filename = downloaded.filename;
242583
243319
  } else {
242584
243320
  const filePath = resolve8(templateArg);
242585
- if (!existsSync18(filePath)) {
243321
+ if (!existsSync19(filePath)) {
242586
243322
  throw new Error(`Template file not found: ${filePath}. Pass a local .docx/.xlsx path or a tmpl_… id.`);
242587
243323
  }
242588
243324
  bytes = await fs19.readFile(filePath);
@@ -242624,12 +243360,12 @@ function registerTemplateCommands(parent) {
242624
243360
 
242625
243361
  // src/commands/workflow/validation.ts
242626
243362
  init_src();
242627
- import { existsSync as existsSync20, readdirSync as readdirSync7, readFileSync as readFileSync17, statSync as statSync7 } from "fs";
243363
+ import { existsSync as existsSync21, readdirSync as readdirSync7, readFileSync as readFileSync18, statSync as statSync7 } from "fs";
242628
243364
  import { join as join12, resolve as resolve9 } from "path";
242629
243365
 
242630
243366
  // src/lib/template-diagnostics.ts
242631
243367
  init_src();
242632
- import { existsSync as existsSync19, readFileSync as readFileSync16 } from "node:fs";
243368
+ import { existsSync as existsSync20, readFileSync as readFileSync17 } from "node:fs";
242633
243369
  function walkSteps(steps, pathPrefix, visit) {
242634
243370
  if (!Array.isArray(steps))
242635
243371
  return;
@@ -242754,7 +243490,7 @@ async function diagnoseTemplateSteps(args) {
242754
243490
  });
242755
243491
  continue;
242756
243492
  }
242757
- if (!existsSync19(absolute)) {
243493
+ if (!existsSync20(absolute)) {
242758
243494
  diagnostics.push({
242759
243495
  field: `${info.fieldPath}.template`,
242760
243496
  message: `File not found: ${absolute}`,
@@ -242763,7 +243499,7 @@ async function diagnoseTemplateSteps(args) {
242763
243499
  continue;
242764
243500
  }
242765
243501
  try {
242766
- const bytes = readFileSync16(absolute);
243502
+ const bytes = readFileSync17(absolute);
242767
243503
  const inspection = inspectOfficeTemplateBytes(bytes, absolute);
242768
243504
  diagnostics.push(...formatDiagnostics(info, inspection));
242769
243505
  } catch (err) {
@@ -242877,8 +243613,17 @@ function countInvokeWorkflowSteps(steps) {
242877
243613
  if (step.type === "action.invoke-workflow")
242878
243614
  count++;
242879
243615
  count += countInvokeWorkflowSteps(step.steps);
243616
+ count += countInvokeWorkflowSteps(step.body);
242880
243617
  count += countInvokeWorkflowSteps(step.then);
242881
243618
  count += countInvokeWorkflowSteps(step.else);
243619
+ count += countInvokeWorkflowSteps(step.default);
243620
+ if (Array.isArray(step.cases)) {
243621
+ for (const switchCase of step.cases) {
243622
+ if (switchCase && typeof switchCase === "object") {
243623
+ count += countInvokeWorkflowSteps(switchCase.steps);
243624
+ }
243625
+ }
243626
+ }
242882
243627
  if (Array.isArray(step.branches)) {
242883
243628
  for (const branch of step.branches) {
242884
243629
  if (branch && typeof branch === "object") {
@@ -242890,7 +243635,7 @@ function countInvokeWorkflowSteps(steps) {
242890
243635
  return count;
242891
243636
  }
242892
243637
  function validateWorkflowYaml(path) {
242893
- if (!existsSync20(path)) {
243638
+ if (!existsSync21(path)) {
242894
243639
  return {
242895
243640
  issues: [{ field: path, message: "File not found." }],
242896
243641
  warnings: [],
@@ -242899,7 +243644,7 @@ function validateWorkflowYaml(path) {
242899
243644
  steps: []
242900
243645
  };
242901
243646
  }
242902
- const text = readFileSync17(path, "utf-8");
243647
+ const text = readFileSync18(path, "utf-8");
242903
243648
  try {
242904
243649
  const definition = parseWorkflow(text);
242905
243650
  let warnings = [];
@@ -243000,10 +243745,10 @@ function printSchemaQualityWarnings(path, warnings) {
243000
243745
  }
243001
243746
  }
243002
243747
  function validateEvaluatorsYaml(path) {
243003
- if (!existsSync20(path)) {
243748
+ if (!existsSync21(path)) {
243004
243749
  return [{ field: path, message: "File not found." }];
243005
243750
  }
243006
- const text = readFileSync17(path, "utf-8");
243751
+ const text = readFileSync18(path, "utf-8");
243007
243752
  let parsed;
243008
243753
  try {
243009
243754
  parsed = $parse(text);
@@ -243020,17 +243765,17 @@ function validateEvaluatorsYaml(path) {
243020
243765
  }
243021
243766
  function validateDatasetFolder(root) {
243022
243767
  const issues = [];
243023
- if (!existsSync20(root)) {
243768
+ if (!existsSync21(root)) {
243024
243769
  return [{ field: root, message: "Folder not found." }];
243025
243770
  }
243026
- if (existsSync20(join12(root, "manifest.json"))) {
243771
+ if (existsSync21(join12(root, "manifest.json"))) {
243027
243772
  issues.push({
243028
243773
  field: "manifest.json",
243029
243774
  message: "Top-level manifest.json is the legacy format. Re-export the dataset using the folder convention."
243030
243775
  });
243031
243776
  }
243032
243777
  const examplesDir = join12(root, "examples");
243033
- if (!existsSync20(examplesDir)) {
243778
+ if (!existsSync21(examplesDir)) {
243034
243779
  issues.push({ field: "examples/", message: "No examples/ directory found at dataset root." });
243035
243780
  return issues;
243036
243781
  }
@@ -243046,7 +243791,7 @@ function validateDatasetFolder(root) {
243046
243791
  continue;
243047
243792
  }
243048
243793
  const inputJsonPath = join12(exampleFolder, "input.json");
243049
- const hasInputJson = existsSync20(inputJsonPath);
243794
+ const hasInputJson = existsSync21(inputJsonPath);
243050
243795
  if (!hasInputJson) {
243051
243796
  issues.push({
243052
243797
  field: `examples/${exampleName}/input.json`,
@@ -243056,7 +243801,7 @@ function validateDatasetFolder(root) {
243056
243801
  }
243057
243802
  let args;
243058
243803
  try {
243059
- const parsed = JSON.parse(readFileSync17(inputJsonPath, "utf-8"));
243804
+ const parsed = JSON.parse(readFileSync18(inputJsonPath, "utf-8"));
243060
243805
  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
243061
243806
  issues.push({
243062
243807
  field: `examples/${exampleName}/input.json`,
@@ -243073,7 +243818,7 @@ function validateDatasetFolder(root) {
243073
243818
  continue;
243074
243819
  }
243075
243820
  const inputDir = join12(exampleFolder, "input");
243076
- if (existsSync20(join12(inputDir, "arguments.json"))) {
243821
+ if (existsSync21(join12(inputDir, "arguments.json"))) {
243077
243822
  issues.push({
243078
243823
  field: `examples/${exampleName}/input/arguments.json`,
243079
243824
  message: "Legacy input/arguments.json is no longer accepted. Use input.json instead."
@@ -243090,7 +243835,7 @@ function validateDatasetFolder(root) {
243090
243835
  }
243091
243836
  const expectedDir = join12(exampleFolder, "expected");
243092
243837
  const expectedJsonPath = join12(exampleFolder, "expected.json");
243093
- if (existsSync20(expectedDir)) {
243838
+ if (existsSync21(expectedDir)) {
243094
243839
  for (const entry of readdirSync7(expectedDir)) {
243095
243840
  if (entry === "output.json" || entry === "error.json") {
243096
243841
  issues.push({
@@ -243126,10 +243871,10 @@ function validateDatasetFolder(root) {
243126
243871
  }
243127
243872
  }
243128
243873
  }
243129
- if (existsSync20(expectedJsonPath)) {
243874
+ if (existsSync21(expectedJsonPath)) {
243130
243875
  let parsed;
243131
243876
  try {
243132
- parsed = JSON.parse(readFileSync17(expectedJsonPath, "utf-8"));
243877
+ parsed = JSON.parse(readFileSync18(expectedJsonPath, "utf-8"));
243133
243878
  } catch {
243134
243879
  issues.push({
243135
243880
  field: `examples/${exampleName}/expected.json`,
@@ -243208,7 +243953,7 @@ function validateFileRefs2(args) {
243208
243953
  });
243209
243954
  continue;
243210
243955
  }
243211
- if (!existsSync20(join12(args.filesRoot, relative))) {
243956
+ if (!existsSync21(join12(args.filesRoot, relative))) {
243212
243957
  issues.push({
243213
243958
  field: `${args.fieldPrefix}:${ref.jsonPath}`,
243214
243959
  message: `Referenced file does not exist: ${ref.path}.`
@@ -243238,16 +243983,55 @@ async function onlineInvokeIssues(yamlText, opts) {
243238
243983
  const res = await client.post("/api/workflows/validate", { yaml: yamlText });
243239
243984
  return res.issues ?? [];
243240
243985
  }
243986
+ function discoverWorkflowProjectRoots(rootPath) {
243987
+ const root = resolve9(rootPath);
243988
+ if (existsSync21(join12(root, "workflow.yaml")))
243989
+ return [root];
243990
+ const candidates = [
243991
+ join12(root, "eigenpal", "workflows"),
243992
+ join12(root, "workflows"),
243993
+ root.endsWith(join12("eigenpal", "workflows")) ? root : ""
243994
+ ].filter(Boolean);
243995
+ for (const parent of candidates) {
243996
+ if (!existsSync21(parent) || !statSync7(parent).isDirectory())
243997
+ continue;
243998
+ const projects = readdirSync7(parent, { withFileTypes: true }).filter((entry) => entry.isDirectory() && !entry.isSymbolicLink() && existsSync21(join12(parent, entry.name, "workflow.yaml"))).map((entry) => join12(parent, entry.name)).sort();
243999
+ if (projects.length > 0)
244000
+ return projects;
244001
+ }
244002
+ return [];
244003
+ }
244004
+ async function validateProjectRoot(root, opts) {
244005
+ const wfFile = join12(root, "workflow.yaml");
244006
+ const wfResult = validateWorkflowYaml(wfFile);
244007
+ const wfPath = relPath(root, wfFile);
244008
+ const wfOk = printIssues("workflow.yaml", wfPath, wfResult.issues);
244009
+ if (wfOk) {
244010
+ printSchemaQualityWarnings(wfPath, wfResult.warnings);
244011
+ printInvokeOnlineHint(wfResult.invokeStepCount, wfResult.templateStepCount, opts.online);
244012
+ }
244013
+ const evOk = printIssues("evaluators.yaml", relPath(root, join12(root, "evaluators.yaml")), validateEvaluatorsYaml(join12(root, "evaluators.yaml")));
244014
+ const dsOk = printIssues("dataset/", relPath(root, join12(root, "dataset")), validateDatasetFolder(join12(root, "dataset")));
244015
+ const templateOk = wfOk && existsSync21(wfFile) ? await runTemplateDiagnostics(wfFile, wfResult.steps, opts) : true;
244016
+ let onlineOk = true;
244017
+ if (wfOk && opts.online && existsSync21(wfFile)) {
244018
+ onlineOk = printIssues("invoke refs (server)", wfPath, await onlineInvokeIssues(readFileSync18(wfFile, "utf-8"), opts));
244019
+ }
244020
+ return wfOk && evOk && dsOk && templateOk && onlineOk;
244021
+ }
243241
244022
  function registerAllValidateCommand(parent) {
243242
- const cmd = parent.command("validate [path]").description("Local-only validation. Without [path]: runs the templated three-way check (./workflow.yaml + ./evaluators.yaml + ./dataset/). With [path] pointing at a YAML file: validates that workflow.yaml. For per-noun targeting use `evaluators validate` or `dataset validate`.").option("--dir <path>", "Project root (defaults to cwd; resolves the three default paths from here)").option("--online", "Authenticate and also validate action.invoke-workflow targets, transform.template tmpl_… references, and explicitly selected OCR/vision/text models against this tenant environment (existence, format, revision pairing, tokens vs data, XLSX {{ }} mistakes, configured catalog). Local template: paths are inspected on disk with or without this flag. Model checks use the configured catalog only — they do not probe live provider health.").option("--allow-external-templates", "Allow local template: paths whose real path is outside the workflow project directory (the folder that contains the YAML file). Off by default; ../ and symlink escapes are rejected.").addHelpText("after", `
244023
+ const cmd = parent.command("validate [path]").description("Local-only validation. Without [path]: runs the templated three-way check (./workflow.yaml + ./evaluators.yaml + ./dataset/) in the project root. When the root has no workflow.yaml, discovers nested projects under eigenpal/workflows/<slug>/ or workflows/<slug>/ and validates each. With [path] pointing at a YAML file: validates that workflow.yaml only. For per-noun targeting use `evaluators validate` or `dataset validate`.").option("--dir <path>", "Project root (defaults to cwd; resolves the three default paths from here)").option("--online", "Authenticate and also validate action.invoke-workflow targets, transform.template tmpl_… references, and explicitly selected OCR/vision/text models against this tenant environment (existence, format, revision pairing, tokens vs data, XLSX {{ }} mistakes, configured catalog). Local template: paths are inspected on disk with or without this flag. Model checks use the configured catalog only — they do not probe live provider health.").option("--allow-external-templates", "Allow local template: paths whose real path is outside the workflow project directory (the folder that contains the YAML file). Off by default; ../ and symlink escapes are rejected.").addHelpText("after", `
243243
244024
  Note
244025
+ ` + ` When [path] or --dir points at a repository root (or eigenpal/workflows),
244026
+ ` + ` every discovered workflow project is validated and failures are grouped
244027
+ ` + ` by project path.
243244
244028
  ` + ` Local validation cannot resolve action.invoke-workflow targets, tmpl_…
243245
244029
  ` + ` template ids, or whether selected OCR/vision/text models exist in this
243246
244030
  ` + " environment (they live on the server). Run `validate --online` to check\n" + " them before pushing. Source-controlled `template: ./file.xlsx` paths are\n" + ` inspected on disk without --online and must stay inside the workflow
243247
244031
  ` + ` project unless you pass --allow-external-templates.
243248
244032
  `);
243249
244033
  withBaseUrl(cmd).action(action(async (path, opts) => {
243250
- if (path) {
244034
+ if (path && (!existsSync21(resolve9(path)) || !statSync7(resolve9(path)).isDirectory())) {
243251
244035
  const target = resolve9(path);
243252
244036
  const wfResult = validateWorkflowYaml(target);
243253
244037
  const wfOk = printIssues("workflow.yaml", target, wfResult.issues);
@@ -243258,29 +244042,29 @@ Note
243258
244042
  const templateOk = wfOk ? await runTemplateDiagnostics(target, wfResult.steps, opts) : true;
243259
244043
  let onlineOk = true;
243260
244044
  if (wfOk && opts.online) {
243261
- onlineOk = printIssues("invoke refs (server)", target, await onlineInvokeIssues(readFileSync17(target, "utf-8"), opts));
244045
+ onlineOk = printIssues("invoke refs (server)", target, await onlineInvokeIssues(readFileSync18(target, "utf-8"), opts));
243262
244046
  }
243263
244047
  if (!wfOk || !templateOk || !onlineOk)
243264
244048
  process.exit(1);
243265
244049
  return;
243266
244050
  }
243267
- const root = rootDir(opts);
243268
- const wfFile = join12(root, "workflow.yaml");
243269
- const wfResult = validateWorkflowYaml(wfFile);
243270
- const wfPath = relPath(root, wfFile);
243271
- const wfOk = printIssues("workflow.yaml", wfPath, wfResult.issues);
243272
- if (wfOk) {
243273
- printSchemaQualityWarnings(wfPath, wfResult.warnings);
243274
- printInvokeOnlineHint(wfResult.invokeStepCount, wfResult.templateStepCount, opts.online);
243275
- }
243276
- const evOk = printIssues("evaluators.yaml", relPath(root, join12(root, "evaluators.yaml")), validateEvaluatorsYaml(join12(root, "evaluators.yaml")));
243277
- const dsOk = printIssues("dataset/", relPath(root, join12(root, "dataset")), validateDatasetFolder(join12(root, "dataset")));
243278
- let onlineOk = true;
243279
- const templateOk = wfOk && existsSync20(wfFile) ? await runTemplateDiagnostics(wfFile, wfResult.steps, opts) : true;
243280
- if (wfOk && opts.online && existsSync20(wfFile)) {
243281
- onlineOk = printIssues("invoke refs (server)", wfPath, await onlineInvokeIssues(readFileSync17(wfFile, "utf-8"), opts));
243282
- }
243283
- if (!wfOk || !evOk || !dsOk || !templateOk || !onlineOk)
244051
+ const requestedRoot = path ? resolve9(path) : rootDir(opts);
244052
+ const projects = discoverWorkflowProjectRoots(requestedRoot);
244053
+ if (projects.length === 0) {
244054
+ if (!await validateProjectRoot(requestedRoot, opts))
244055
+ process.exit(1);
244056
+ return;
244057
+ }
244058
+ let valid = true;
244059
+ for (const project of projects) {
244060
+ if (projects.length > 1) {
244061
+ process.stderr.write(`${ui.bold(relPath(requestedRoot, project))}
244062
+ `);
244063
+ }
244064
+ if (!await validateProjectRoot(project, opts))
244065
+ valid = false;
244066
+ }
244067
+ if (!valid)
243284
244068
  process.exit(1);
243285
244069
  }));
243286
244070
  }
@@ -243509,7 +244293,7 @@ new uploads are deleted. The file on disk is not rewritten.
243509
244293
  defaultValue: "workflow.yaml",
243510
244294
  validate: (value) => {
243511
244295
  const p = value || "workflow.yaml";
243512
- if (!existsSync21(p))
244296
+ if (!existsSync22(p))
243513
244297
  return `Not found: ${p}`;
243514
244298
  return;
243515
244299
  }
@@ -244148,7 +244932,7 @@ function registerExperimentCommands2(parent) {
244148
244932
  }
244149
244933
  ], { ...opts, entityLabel: "experiment" });
244150
244934
  }));
244151
- const runCmd = experiment.command("run <workflow-id>").description(descriptionFor("workflow_run_experiment")).option("--example-id <id>", "Run only this example (repeatable)", (val2, prev) => [...prev, val2], []).option("--wait", "Poll until terminal; non-zero exit on passRate < 1.0", false).option("--interval <n>", "Polling interval in seconds (default 10)", intArg, 10).addHelpText("after", `
244935
+ const runCmd = experiment.command("run <workflow-id>").description(descriptionFor("workflow_run_experiment")).option("--example-id <id>", "Run only this example (repeatable)", (val2, prev) => [...prev, val2], []).option("--wait", "Poll until terminal; non-zero exit on passRate < 1.0", false).option("--interval <n>", "Polling interval in seconds (default 10)", intArg, 10).option("--max-wait <seconds>", "Maximum wait before exit code 2 (default 1800)", intArg, 1800).addHelpText("after", `
244152
244936
  Examples:
244153
244937
  $ eigenpal workflow experiment run wf_abc123
244154
244938
  $ eigenpal workflow experiment run my-extraction --example-id evx_111
@@ -244181,15 +244965,40 @@ returns immediately with \`{ batchId, total }\`; poll
244181
244965
  return;
244182
244966
  }
244183
244967
  const intervalMs = Math.max(1, opts.interval) * 1000;
244968
+ const deadline = Date.now() + Math.max(1, opts.maxWait) * 1000;
244184
244969
  while (true) {
244185
244970
  const status = await client.get(`/v1/automations/${workflowId}/experiments/${experimentId}`);
244186
244971
  if (status.status === "completed") {
244187
- printJson3(status);
244972
+ if (opts.json) {
244973
+ printJson3({
244974
+ ...status,
244975
+ batchId: experimentId,
244976
+ total: status.runCount ?? status.total
244977
+ });
244978
+ } else {
244979
+ const completed = status.completedCount ?? status.completed;
244980
+ const runTotal = status.runCount ?? status.total;
244981
+ const score = typeof status.evalScore === "number" ? ` average=${status.evalScore.toFixed(3)}` : "";
244982
+ success(`Experiment ${ui.bold(experimentId)} completed (${completed}/${runTotal}${score})`);
244983
+ }
244188
244984
  if (typeof status.passRate === "number" && status.passRate < 1 || typeof status.failedCount === "number" && status.failedCount > 0)
244189
244985
  process.exit(1);
244190
244986
  return;
244191
244987
  }
244192
- console.error(`${ui.dim(`[${experimentId}]`)} ${status.completedCount ?? status.completed}/${status.runCount ?? status.total} ${ui.dim(`(status=${status.status})`)}`);
244988
+ if (!opts.json) {
244989
+ console.error(`${ui.dim(`[${experimentId}]`)} ${status.completedCount ?? status.completed}/${status.runCount ?? status.total} ${ui.dim(`(status=${status.status})`)}`);
244990
+ }
244991
+ if (Date.now() >= deadline) {
244992
+ if (opts.json) {
244993
+ printJson3({
244994
+ ...status,
244995
+ batchId: experimentId,
244996
+ total: status.runCount ?? status.total
244997
+ });
244998
+ }
244999
+ error2(`Timed out waiting for experiment ${experimentId} after ${opts.maxWait}s`);
245000
+ process.exit(2);
245001
+ }
244193
245002
  await new Promise((r) => setTimeout(r, intervalMs));
244194
245003
  }
244195
245004
  }));
@@ -244199,6 +245008,10 @@ returns immediately with \`{ batchId, total }\`; poll
244199
245008
  error2("--short and --watch are mutually exclusive (short is for spot-check polling).");
244200
245009
  process.exit(2);
244201
245010
  }
245011
+ if (opts.short && opts.json) {
245012
+ error2("--short and --json are mutually exclusive (short is human spot-check output).");
245013
+ process.exit(2);
245014
+ }
244202
245015
  const includes = new Set((opts.include ?? "").split(",").map((s) => s.trim()).filter(Boolean));
244203
245016
  const includePayload = includes.has("payload");
244204
245017
  const { client, workflowId } = await buildClientForWorkflow(workflow, opts);
@@ -244307,14 +245120,41 @@ Behavior:
244307
245120
  }
244308
245121
  return value;
244309
245122
  };
244310
- const resultsCmd = experiment.command("results <workflow-id> [batchId]").description(descriptionFor("workflow_get_experiment_results")).requiredOption("--format <csv|json>", "Output format", parseExportFormat).option("--out <path>", "Output file. When omitted, the binary streams to stdout.");
244311
- withBaseUrl(resultsCmd).action(action(async (workflow, batchId, opts) => {
245123
+ const resultsCmd = experiment.command("results <workflow-id> [batchId]").description(descriptionFor("workflow_get_experiment_results")).option("--format <csv|json>", "Output format (default json)", parseExportFormat, "json").option("--out <path>", "Output file. When omitted, the binary streams to stdout.").option("--summary", "Show total/pass/fail/error counts, average score, and evaluator rollups").option("--failed-only", "Keep only failed or errored evaluator results").option("--evaluator <name>", "Keep only results from this evaluator").option("--select <path>", "Print only a nested JSON value (for example summary.byEvaluator or discrepancies[].path)");
245124
+ addJsonFlag(withBaseUrl(resultsCmd)).action(action(async (workflow, batchId, opts) => {
244312
245125
  const { client, workflowId } = await buildClientForWorkflow(workflow, opts);
245126
+ if (opts.json && opts.format !== "json") {
245127
+ throw new Error("--json cannot be combined with --format csv");
245128
+ }
245129
+ const analyze = opts.summary || opts.failedOnly || opts.evaluator !== undefined || opts.select !== undefined;
245130
+ if (analyze) {
245131
+ if (!batchId) {
245132
+ throw new Error("--summary, --failed-only, --evaluator, and --select require a single experiment batch id");
245133
+ }
245134
+ if (opts.format !== "json") {
245135
+ throw new Error("Experiment analysis options require --format json");
245136
+ }
245137
+ const detailPayload = await client.get(`/v1/automations/${workflowId}/experiments/${encodeURIComponent(batchId)}`);
245138
+ const analysis = analyzeExperimentResults({
245139
+ rows: experimentRowsFromDetail(detailPayload),
245140
+ runs: detailPayload.runs ?? [],
245141
+ evaluator: opts.evaluator,
245142
+ failedOnly: opts.failedOnly
245143
+ });
245144
+ const selected = opts.select ? selectJsonValue(analysis, opts.select) : analysis;
245145
+ if (opts.summary && !opts.json && !opts.out && !opts.select) {
245146
+ renderExperimentResultsSummary(analysis);
245147
+ return;
245148
+ }
245149
+ await writeOrPrint(opts.out, `${JSON.stringify(selected, null, 2)}
245150
+ `);
245151
+ return;
245152
+ }
244313
245153
  const params = new URLSearchParams({ format: opts.format });
244314
245154
  const exportPath = batchId ? `/v1/automations/${workflowId}/experiments/${batchId}/export?${params.toString()}` : `/v1/automations/${workflowId}/experiments/export?${params.toString()}`;
244315
245155
  await downloadStreamToFile2(client, exportPath, opts.out);
244316
245156
  }));
244317
- const compareCmd = experiment.command("compare <batchIdA> <batchIdB>").description("Diff eval scores between two experiment batches.").option("--sort <abs-delta-desc|delta-asc|delta-desc|name>", "Row sort order (default: biggest movers first)", "abs-delta-desc").option("--regression-threshold <n>", "Δ below this is flagged as a regression (default 0.05)", Number.parseFloat, 0.05).addHelpText("after", `
245157
+ const compareCmd = experiment.command("compare <batchIdA> <batchIdB>").description("Compare evaluator scores or actual outputs between two experiment batches.").option("--outputs", "Compare actual run outputs instead of evaluator scores").option("--sort <abs-delta-desc|delta-asc|delta-desc|name>", "Row sort order (default: biggest movers first)", "abs-delta-desc").option("--regression-threshold <n>", "Δ below this is flagged as a regression (default 0.05)", Number.parseFloat, 0.05).addHelpText("after", `
244318
245158
  Examples:
244319
245159
  $ eigenpal workflow experiment compare evb_old evb_new
244320
245160
  $ eigenpal workflow experiment compare evb_old evb_new --regression-threshold 0.10
@@ -244334,6 +245174,26 @@ batches must belong to the same workflow within the caller's tenant.
244334
245174
  throw new Error("--regression-threshold must be a non-negative number");
244335
245175
  }
244336
245176
  const client = buildClient3(opts);
245177
+ if (opts.outputs) {
245178
+ const [outputsA, outputsB] = await Promise.all([
245179
+ fetchExperimentOutputs(client, batchIdA),
245180
+ fetchExperimentOutputs(client, batchIdB)
245181
+ ]);
245182
+ if (outputsA.automationId !== outputsB.automationId) {
245183
+ throw new Error("Experiment batches must belong to the same automation");
245184
+ }
245185
+ const outputDiff = buildExperimentOutputDiff({
245186
+ batchIdA,
245187
+ batchIdB,
245188
+ rowsA: outputsA.rows,
245189
+ rowsB: outputsB.rows
245190
+ });
245191
+ if (opts.json)
245192
+ printJson3(outputDiff);
245193
+ else
245194
+ renderExperimentOutputDiffHuman(outputDiff);
245195
+ return;
245196
+ }
244337
245197
  const [bodyA, bodyB] = await Promise.all([
244338
245198
  fetchEvalResults(client, batchIdA),
244339
245199
  fetchEvalResults(client, batchIdB)
@@ -244385,7 +245245,7 @@ Behavior:
244385
245245
  const dest = opts.pull ? opts.pullOnComplete ?? `./results-${batchId}.${opts.format}` : null;
244386
245246
  if (dest) {
244387
245247
  const dir = path18.dirname(dest);
244388
- if (!existsSync21(dir)) {
245248
+ if (!existsSync22(dir)) {
244389
245249
  error2(`Destination directory does not exist: ${dir}. Create it first or pass a path under an existing directory.`);
244390
245250
  process.exit(2);
244391
245251
  }
@@ -245158,6 +246018,7 @@ registerModelsCommands(program2);
245158
246018
  registerRunCommands(program2);
245159
246019
  registerRunsCommands(program2);
245160
246020
  registerGitCommands(program2);
246021
+ registerDocsCommands(program2);
245161
246022
  program2.command("completion <shell>").description("Print a shell completion script to stdout (supported shells: bash, zsh, fish). Pipe into the appropriate completion file for your shell — see the script header for install instructions.").action(async (shell) => {
245162
246023
  try {
245163
246024
  await completion(shell, program2);