@eigenpal/cli 0.14.3 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +11 -9
- package/dist/cli.js +1110 -249
- package/dist/docs-bundle.json +1356 -0
- package/dist/skill/SKILL.md +33 -0
- package/dist/skill/reference/cli/agents.md +111 -106
- package/dist/skill/reference/cli/docs.md +84 -0
- package/dist/skill/reference/cli/models.md +1 -1
- package/dist/skill/reference/cli/rerun.md +1 -1
- package/dist/skill/reference/cli/run.md +1 -1
- package/dist/skill/reference/cli/runs.md +67 -67
- package/dist/skill/reference/cli/surface.md +4 -0
- package/dist/skill/reference/cli/workflow.md +114 -107
- package/dist/skill/reference/dataset-format.md +151 -0
- package/dist/skill/reference/debugging.md +9 -0
- package/dist/skill/reference/evaluators.md +361 -4
- package/dist/skill/reference/step-types.md +4519 -15
- package/dist/skill/reference/workflow-yaml.md +1458 -23
- package/package.json +7 -4
package/dist/cli.js
CHANGED
|
@@ -52713,10 +52713,11 @@ function compileTypedScript(opts) {
|
|
|
52713
52713
|
});
|
|
52714
52714
|
}
|
|
52715
52715
|
const actualParamNames = actualParams;
|
|
52716
|
-
|
|
52716
|
+
const parametersMismatch = opts.kind === "evaluator" ? actualParamNames.length !== expectedParams.length || actualParamNames.some((name, index) => name !== expectedParams[index]) : actualParamNames.length !== expectedParams.length || new Set(actualParamNames).size !== actualParamNames.length || actualParamNames.some((name) => !expectedParams.includes(name));
|
|
52717
|
+
if (parametersMismatch) {
|
|
52717
52718
|
return failWith({
|
|
52718
52719
|
kind: "wrong-shape",
|
|
52719
|
-
message: `Parameter list must be \`(${expectedParams.join(", ")})\` in this order (got \`(${actualParamNames.join(", ")})\`).`,
|
|
52720
|
+
message: opts.kind === "evaluator" ? `Parameter list must be \`(${expectedParams.join(", ")})\` in this order (got \`(${actualParamNames.join(", ")})\`).` : `Parameter names must match the configured inputs \`(${expectedParams.join(", ")})\`; their order does not matter (got \`(${actualParamNames.join(", ")})\`).`,
|
|
52720
52721
|
line: fn.loc?.start.line,
|
|
52721
52722
|
column: fn.loc?.start.column
|
|
52722
52723
|
});
|
|
@@ -53482,8 +53483,8 @@ var init_step_configs = __esm(() => {
|
|
|
53482
53483
|
}).superRefine(refineJsonToXlsxLayout);
|
|
53483
53484
|
TransformJsonToXlsxOutputSchema = JsonToXlsxOutputSchema;
|
|
53484
53485
|
TransformScriptConfigSchema = object({
|
|
53485
|
-
inputs: record(string2(), string2()).optional().describe("Named inputs mapped from template expressions. Keys become
|
|
53486
|
-
function: string2().min(1).max(SCRIPT_FN_MAX_BYTES, `script function is too long (max ${(SCRIPT_FN_MAX_BYTES / 1000).toFixed(0)}k characters)`).describe("TypeScript function declaration. Must be `function script(args): R { … }` where the
|
|
53486
|
+
inputs: record(string2(), string2()).optional().describe("Named inputs mapped from template expressions. Keys become function parameters; parameter order is independent of YAML/JSON key order: `inputs: { items, taxRate }` accepts `function script(taxRate: …, items: …): R { … }`."),
|
|
53487
|
+
function: string2().min(1).max(SCRIPT_FN_MAX_BYTES, `script function is too long (max ${(SCRIPT_FN_MAX_BYTES / 1000).toFixed(0)}k characters)`).describe("TypeScript function declaration. Must be `function script(args): R { … }` where parameter names exactly match the configured input keys in any order and `R` is a return type annotation. The annotation IS this step's output schema."),
|
|
53487
53488
|
timeout: number2().positive().default(5000).optional().describe("Max execution time in milliseconds (default: 5000)"),
|
|
53488
53489
|
memoryLimit: number2().positive().default(10 * 1024 * 1024).optional().describe("Max memory in bytes (default: 10MB)")
|
|
53489
53490
|
}).superRefine((cfg, ctx) => {
|
|
@@ -53863,7 +53864,7 @@ var init_step_configs = __esm(() => {
|
|
|
53863
53864
|
type: "transform.script",
|
|
53864
53865
|
category: "transform",
|
|
53865
53866
|
name: "Script",
|
|
53866
|
-
description: "Execute a TypeScript function in a QuickJS sandbox. Input keys become
|
|
53867
|
+
description: "Execute a TypeScript function in a QuickJS sandbox. Input keys become function parameters in any order, and the required `: R` return-type annotation IS this step's output schema: `inputs: { items, taxRate }` accepts `function script(taxRate: …, items: …): R { … }`.",
|
|
53867
53868
|
configSchema: TransformScriptConfigSchema,
|
|
53868
53869
|
outputSchema: TransformScriptOutputSchema,
|
|
53869
53870
|
configInWith: true
|
|
@@ -54027,36 +54028,36 @@ var init_workflow = __esm(() => {
|
|
|
54027
54028
|
init_retry();
|
|
54028
54029
|
init_steps();
|
|
54029
54030
|
WORKFLOW_NAME_PATTERN = /^[a-z0-9][a-z0-9_-]*$/;
|
|
54030
|
-
WorkflowNameSchema = string2().min(1).max(64).regex(WORKFLOW_NAME_PATTERN, 'Workflow name must start with a lowercase letter or digit and contain only lowercase letters, digits, "_" or "-".');
|
|
54031
|
+
WorkflowNameSchema = string2().min(1).max(64).regex(WORKFLOW_NAME_PATTERN, 'Workflow name must start with a lowercase letter or digit and contain only lowercase letters, digits, "_" or "-".').describe("URL-safe workflow slug: 1-64 lowercase letters, digits, underscores, or hyphens; must start with a letter or digit.");
|
|
54031
54032
|
WorkflowInputPropertySchema = lazy(() => object({
|
|
54032
|
-
name: string2().min(1).regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, "Property names must start with a letter or underscore and contain only letters, numbers, and underscores"),
|
|
54033
|
-
type: string2(),
|
|
54034
|
-
description: string2().optional(),
|
|
54035
|
-
required: boolean2().optional(),
|
|
54036
|
-
values: array(string2()).optional(),
|
|
54033
|
+
name: string2().min(1).regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, "Property names must start with a letter or underscore and contain only letters, numbers, and underscores").describe("JSON object key, addressable in templates with dot notation."),
|
|
54034
|
+
type: string2().describe("One of string, enum, number, integer, boolean, array, or object."),
|
|
54035
|
+
description: string2().optional().describe("Human-readable field meaning."),
|
|
54036
|
+
required: boolean2().optional().describe("Whether this property must be present; defaults to true when omitted."),
|
|
54037
|
+
values: array(string2()).optional().describe("Allowed strings when type is enum."),
|
|
54037
54038
|
items: object({
|
|
54038
|
-
type: string2(),
|
|
54039
|
-
values: array(string2()).optional(),
|
|
54040
|
-
properties: array(WorkflowInputPropertySchema).optional()
|
|
54041
|
-
}).optional(),
|
|
54042
|
-
properties: array(WorkflowInputPropertySchema).optional()
|
|
54039
|
+
type: string2().describe("Array element type: string, enum, number, integer, boolean, or object."),
|
|
54040
|
+
values: array(string2()).optional().describe("Allowed strings when items.type is enum."),
|
|
54041
|
+
properties: array(WorkflowInputPropertySchema).optional().describe("Recursive object fields when items.type is object.")
|
|
54042
|
+
}).optional().describe("Element definition when type is array."),
|
|
54043
|
+
properties: array(WorkflowInputPropertySchema).optional().describe("Recursive fields when type is object.")
|
|
54043
54044
|
}));
|
|
54044
54045
|
WorkflowInputDefSchema = object({
|
|
54045
|
-
name: string2().min(1),
|
|
54046
|
-
type: string2(),
|
|
54047
|
-
description: string2().optional(),
|
|
54048
|
-
required: boolean2().default(true),
|
|
54049
|
-
default: unknown().optional(),
|
|
54050
|
-
values: array(string2()).optional(),
|
|
54046
|
+
name: string2().min(1).describe("Top-level input name used as `input.<name>` in template expressions."),
|
|
54047
|
+
type: string2().describe("One of string, enum, number, integer, boolean, array, object, or file."),
|
|
54048
|
+
description: string2().optional().describe("Human-readable input meaning shown to callers."),
|
|
54049
|
+
required: boolean2().default(true).describe("Whether callers must provide this input."),
|
|
54050
|
+
default: unknown().optional().describe("Value used when an optional input is omitted."),
|
|
54051
|
+
values: array(string2()).optional().describe("Closed set of allowed strings when type is enum."),
|
|
54051
54052
|
items: object({
|
|
54052
|
-
type: string2(),
|
|
54053
|
-
values: array(string2()).optional(),
|
|
54054
|
-
properties: array(WorkflowInputPropertySchema).optional()
|
|
54055
|
-
}).optional(),
|
|
54056
|
-
properties: array(WorkflowInputPropertySchema).optional(),
|
|
54057
|
-
source: string2().min(1).optional(),
|
|
54058
|
-
mimeType: string2().min(1).optional(),
|
|
54059
|
-
extension: string2().min(1).optional()
|
|
54053
|
+
type: string2().describe("Array element type: string, enum, number, integer, boolean, object, or file."),
|
|
54054
|
+
values: array(string2()).optional().describe("Closed set of allowed strings when items.type is enum."),
|
|
54055
|
+
properties: array(WorkflowInputPropertySchema).optional().describe("Recursive fields when items.type is object.")
|
|
54056
|
+
}).optional().describe("Element definition when type is array."),
|
|
54057
|
+
properties: array(WorkflowInputPropertySchema).optional().describe("Recursive field definitions when type is object."),
|
|
54058
|
+
source: string2().min(1).optional().describe("Registered external file resolver for single-tenant string-id file inputs, for example gpfs; valid only with type file."),
|
|
54059
|
+
mimeType: string2().min(1).optional().describe("MIME hint such as application/pdf for a sourced file; mutually exclusive with extension."),
|
|
54060
|
+
extension: string2().min(1).optional().describe("Extension hint such as pdf for a sourced file; mutually exclusive with mimeType.")
|
|
54060
54061
|
}).superRefine((def, ctx) => {
|
|
54061
54062
|
if (def.properties && def.type !== "object") {
|
|
54062
54063
|
ctx.addIssue({
|
|
@@ -54115,20 +54116,20 @@ var init_workflow = __esm(() => {
|
|
|
54115
54116
|
}
|
|
54116
54117
|
});
|
|
54117
54118
|
ManualTriggerMethodSchema = object({
|
|
54118
|
-
type: literal("manual"),
|
|
54119
|
-
enabled: boolean2().optional(),
|
|
54120
|
-
inputSchema: JsonSchemaSchema.optional()
|
|
54119
|
+
type: literal("manual").describe("Dashboard/manual run trigger."),
|
|
54120
|
+
enabled: boolean2().optional().describe("Set false to disable dashboard runs; manual triggering is enabled by default."),
|
|
54121
|
+
inputSchema: JsonSchemaSchema.optional().describe("Optional JSON Schema for trigger input.")
|
|
54121
54122
|
});
|
|
54122
54123
|
ApiTriggerMethodSchema = object({
|
|
54123
|
-
type: literal("api"),
|
|
54124
|
-
inputSchema: JsonSchemaSchema.optional()
|
|
54124
|
+
type: literal("api").describe("Public API run trigger."),
|
|
54125
|
+
inputSchema: JsonSchemaSchema.optional().describe("Optional JSON Schema for API input.")
|
|
54125
54126
|
});
|
|
54126
54127
|
EmailTriggerMethodSchema = object({
|
|
54127
|
-
type: literal("email"),
|
|
54128
|
+
type: literal("email").describe("Inbound email trigger."),
|
|
54128
54129
|
whitelist: object({
|
|
54129
|
-
domains: array(string2()).optional(),
|
|
54130
|
-
emails: array(string2()).optional()
|
|
54131
|
-
}).optional()
|
|
54130
|
+
domains: array(string2()).optional().describe("Allowed sender domains."),
|
|
54131
|
+
emails: array(string2()).optional().describe("Allowed exact sender email addresses.")
|
|
54132
|
+
}).optional().describe("Optional sender allowlist by exact email address or domain.")
|
|
54132
54133
|
});
|
|
54133
54134
|
TriggerMethodSchema = discriminatedUnion("type", [
|
|
54134
54135
|
ManualTriggerMethodSchema,
|
|
@@ -54137,23 +54138,23 @@ var init_workflow = __esm(() => {
|
|
|
54137
54138
|
]);
|
|
54138
54139
|
TriggerMethodsSchema = array(TriggerMethodSchema).default([{ type: "manual" }]);
|
|
54139
54140
|
WorkflowSettingsSchema = object({
|
|
54140
|
-
timeout: number2().positive().optional(),
|
|
54141
|
-
retry: WorkflowRetryPolicySchema.optional(),
|
|
54141
|
+
timeout: number2().positive().optional().describe("Default step timeout in milliseconds."),
|
|
54142
|
+
retry: WorkflowRetryPolicySchema.optional().describe("Default durable retry policy for steps."),
|
|
54142
54143
|
retries: number2().int().min(0).optional(),
|
|
54143
54144
|
retryDelay: number2().positive().optional()
|
|
54144
54145
|
}).strict();
|
|
54145
54146
|
WorkflowDefinitionSchema = object({
|
|
54146
54147
|
name: WorkflowNameSchema,
|
|
54147
|
-
kind: _enum(["workflow"]).default("workflow").optional(),
|
|
54148
|
-
version: string2().default("1.0.0"),
|
|
54149
|
-
description: string2().optional(),
|
|
54150
|
-
enabled: boolean2().default(true),
|
|
54151
|
-
triggerMethods: array(TriggerMethodSchema).default([{ type: "manual" }]),
|
|
54152
|
-
inputs: array(WorkflowInputDefSchema).optional(),
|
|
54153
|
-
steps: array(StepSchema),
|
|
54154
|
-
output: union([record(string2(), string2()), string2()]).optional(),
|
|
54155
|
-
settings: WorkflowSettingsSchema.optional(),
|
|
54156
|
-
defaultModel: string2().optional()
|
|
54148
|
+
kind: _enum(["workflow"]).default("workflow").optional().describe("Deprecated compatibility discriminator; omit for new workflows."),
|
|
54149
|
+
version: string2().default("1.0.0").describe("Author-provided semantic version label stored with the definition."),
|
|
54150
|
+
description: string2().optional().describe("Human-readable purpose of the workflow."),
|
|
54151
|
+
enabled: boolean2().default(true).describe("Whether the workflow may be run."),
|
|
54152
|
+
triggerMethods: array(TriggerMethodSchema).default([{ type: "manual" }]).describe("Ways this workflow can be invoked: manual, api, or email."),
|
|
54153
|
+
inputs: array(WorkflowInputDefSchema).optional().describe("Top-level inputs available under `input` in template expressions."),
|
|
54154
|
+
steps: array(StepSchema).describe("Ordered workflow steps; names must be unique."),
|
|
54155
|
+
output: union([record(string2(), string2()), string2()]).optional().describe("Final output as named field-to-template mappings or one passthrough template expression."),
|
|
54156
|
+
settings: WorkflowSettingsSchema.optional().describe("Workflow-wide timeout and retry defaults."),
|
|
54157
|
+
defaultModel: string2().optional().describe("Default configured LLM provider id used by AI steps when the step does not select one.")
|
|
54157
54158
|
});
|
|
54158
54159
|
StoredWorkflowDefinitionSchema = object({
|
|
54159
54160
|
id: string2(),
|
|
@@ -54251,6 +54252,23 @@ var init_file_source = __esm(() => {
|
|
|
54251
54252
|
});
|
|
54252
54253
|
|
|
54253
54254
|
// ../types/src/eval/compare-output.ts
|
|
54255
|
+
function tokenizeEvalPath(path) {
|
|
54256
|
+
if (!path || path === "$")
|
|
54257
|
+
return [];
|
|
54258
|
+
return path.match(/[^.[\]]+|\[\]|\[\d+\]/g) ?? [];
|
|
54259
|
+
}
|
|
54260
|
+
function evalPathTokenMatches(pattern, candidate) {
|
|
54261
|
+
return pattern === candidate || pattern === "[]" && /^\[\d+\]$/.test(candidate);
|
|
54262
|
+
}
|
|
54263
|
+
function evalPathIsDescendant(descendant, ancestor) {
|
|
54264
|
+
if (descendant === ancestor)
|
|
54265
|
+
return false;
|
|
54266
|
+
if (ancestor === "$" || ancestor === "")
|
|
54267
|
+
return descendant !== "$" && descendant !== "";
|
|
54268
|
+
const ancestorTokens = tokenizeEvalPath(ancestor);
|
|
54269
|
+
const descendantTokens = tokenizeEvalPath(descendant);
|
|
54270
|
+
return descendantTokens.length > ancestorTokens.length && ancestorTokens.every((token, index) => evalPathTokenMatches(token, descendantTokens[index] ?? ""));
|
|
54271
|
+
}
|
|
54254
54272
|
var MAX_UNORDERED_ARRAY_ITEMS = 1000, MAX_UNORDERED_PAIR_COMPARISONS = 1e6, MAX_UNORDERED_COMPARISON_OPERATIONS = 2000000;
|
|
54255
54273
|
var init_compare_output = () => {};
|
|
54256
54274
|
|
|
@@ -54259,13 +54277,13 @@ var DATASET_NAME_PATTERN, DatasetNameSchema, DatasetMetaSchema, DatasetImportMod
|
|
|
54259
54277
|
var init_dataset_archive = __esm(() => {
|
|
54260
54278
|
init_zod();
|
|
54261
54279
|
DATASET_NAME_PATTERN = /^[a-z0-9][a-z0-9-_]*$/;
|
|
54262
|
-
DatasetNameSchema = string2().min(1).max(128).regex(DATASET_NAME_PATTERN, "name must be lowercase kebab/snake-case");
|
|
54280
|
+
DatasetNameSchema = string2().min(1).max(128).regex(DATASET_NAME_PATTERN, "name must be lowercase kebab/snake-case").describe("Example folder name: 1-128 lowercase letters, digits, hyphens, or underscores; must start with a letter or digit.");
|
|
54263
54281
|
DatasetMetaSchema = object({
|
|
54264
|
-
rowOrder: number2().int().min(0).optional(),
|
|
54265
|
-
annotation: string2().max(2000).optional(),
|
|
54266
|
-
overrides: record(string2(), unknown()).optional()
|
|
54282
|
+
rowOrder: number2().int().min(0).optional().describe("Non-negative display order hint for this example."),
|
|
54283
|
+
annotation: string2().max(2000).optional().describe("Free-form example note, limited to 2000 characters."),
|
|
54284
|
+
overrides: record(string2(), unknown()).optional().describe('Per-step output overrides as `{ "steps": { "<stepName>": <outputObject> } }`; overridden steps are skipped or partially merged during evaluation.')
|
|
54267
54285
|
});
|
|
54268
|
-
DatasetImportModeSchema = _enum(["append", "replace"]);
|
|
54286
|
+
DatasetImportModeSchema = _enum(["append", "replace"]).describe("append keeps existing examples; replace removes them before import.");
|
|
54269
54287
|
DatasetImportRequestSchema = object({
|
|
54270
54288
|
mode: DatasetImportModeSchema
|
|
54271
54289
|
});
|
|
@@ -54292,7 +54310,8 @@ var init_dataset_archive = __esm(() => {
|
|
|
54292
54310
|
label: string2().nullable(),
|
|
54293
54311
|
weight: number2(),
|
|
54294
54312
|
createdAt: string2().datetime({ offset: true }),
|
|
54295
|
-
error: string2().nullable()
|
|
54313
|
+
error: string2().nullable(),
|
|
54314
|
+
details: unknown().nullable().optional().describe("Evaluator-specific structured details. Exact-diff includes mismatches with path, expected, and actual values.")
|
|
54296
54315
|
});
|
|
54297
54316
|
EvalResultsExportJsonSchema = object({
|
|
54298
54317
|
workflowId: string2().min(1),
|
|
@@ -54308,7 +54327,7 @@ var init_evaluator_config = __esm(() => {
|
|
|
54308
54327
|
init_ai_client();
|
|
54309
54328
|
init_typed_script();
|
|
54310
54329
|
init_compare_output();
|
|
54311
|
-
EvaluatorTypeSchema = _enum(["exact-diff", "llm-judge", "custom-script"]);
|
|
54330
|
+
EvaluatorTypeSchema = _enum(["exact-diff", "llm-judge", "custom-script"]).describe("Evaluator implementation selected by each evaluators.yaml entry.");
|
|
54312
54331
|
ScoreInUnitInterval = number2().min(0, { message: "score must be >= 0" }).max(1, { message: "score must be <= 1" });
|
|
54313
54332
|
LEGACY_EXACT_DIFF_COMPARISON_KEYS = [
|
|
54314
54333
|
"paths",
|
|
@@ -54337,12 +54356,24 @@ var init_evaluator_config = __esm(() => {
|
|
|
54337
54356
|
}
|
|
54338
54357
|
});
|
|
54339
54358
|
ExactDiffPathRuleSchema = object({
|
|
54359
|
+
ignore: literal(true).optional().describe("Exclude this path and every descendant from comparison. Cannot be combined with other settings on the same rule."),
|
|
54340
54360
|
order: _enum(["ordered", "unordered"]).optional().describe(`Array matching mode at this path. Independent of \`items\`. Set on the array path (\`lineItems\`), not the item path (\`lineItems[]\`). \`unordered\` matches distinct items in any order. Arrays stay ordered when omitted. Unordered matching is limited to ${MAX_UNORDERED_ARRAY_ITEMS} items, ${MAX_UNORDERED_PAIR_COMPARISONS} candidate pairs, and ${MAX_UNORDERED_COMPARISON_OPERATIONS} recursive operations per comparison.`),
|
|
54341
54361
|
items: ExactDiffArrayItemsSchema.optional().describe("How expected and actual array items relate. Set on the array path (`lineItems`), not the item path (`lineItems[]`). Independent of `order`. Ordered arrays use positional prefix matching: `at-least` (default) requires expected to match actual from index 0 and allows trailing actual extras; `at-most` requires actual to match expected from index 0 and allows trailing expected extras; `exactly` is equal-cardinality positional comparison. Unordered arrays match distinct items in any order: `at-least` requires every expected item and allows extra actual items (including extras with missing or duplicate identities); `at-most` requires every actual item and allows missing expected items (every actual item being checked needs a valid unique identity); `exactly` is a bijection (both sides need valid unique identities)."),
|
|
54342
54362
|
matchBy: ExactDiffMatchBySchema.optional().describe("Identity fields for unordered arrays of objects. Requires `order: unordered` on the same rule. Set on the array path (`lineItems`), not the item path (`lineItems[]`). A relative path (`sku`) or unique paths (`[country, sku]`) inside each item. Pairs expected and actual objects by exact identity, then diffs the paired objects with inherited nested rules. Identity type and value must both match: numeric `10` differs from string `'10'`; no coercion or numeric tolerance. Omitted = structural matching. Applies only at the array path where it is set and is not inherited by descendants. Nested object fields are allowed; array indexes, commas, and wildcards are not. Empty or comma-only values are rejected; composite paths must be unique."),
|
|
54343
54363
|
allowExtraFields: boolean2().optional().describe("When false, extra actual keys at this object fail the diff. When omitted, extra keys are allowed. Applies only to objects."),
|
|
54344
54364
|
numericTolerance: number2().min(0).optional().describe("Maximum absolute difference for numeric leaves at or under this path. Omitted = inherit, then historical relative epsilon. Explicit `0` is exact.")
|
|
54345
54365
|
}).strict().superRefine((rule, ctx) => {
|
|
54366
|
+
if (rule.ignore === true) {
|
|
54367
|
+
const otherFields = Object.keys(rule).filter((field) => field !== "ignore");
|
|
54368
|
+
if (otherFields.length > 0) {
|
|
54369
|
+
ctx.addIssue({
|
|
54370
|
+
code: ZodIssueCode.custom,
|
|
54371
|
+
path: ["ignore"],
|
|
54372
|
+
message: "`ignore: true` cannot be combined with other settings on the same rule"
|
|
54373
|
+
});
|
|
54374
|
+
}
|
|
54375
|
+
return;
|
|
54376
|
+
}
|
|
54346
54377
|
if (rule.matchBy === undefined)
|
|
54347
54378
|
return;
|
|
54348
54379
|
if (rule.order !== "unordered") {
|
|
@@ -54354,7 +54385,7 @@ var init_evaluator_config = __esm(() => {
|
|
|
54354
54385
|
}
|
|
54355
54386
|
});
|
|
54356
54387
|
ExactDiffConfigSchema = object({
|
|
54357
|
-
rules: record(string2(), ExactDiffPathRuleSchema).optional().describe("Per-path comparison rules.
|
|
54388
|
+
rules: record(string2(), ExactDiffPathRuleSchema).optional().describe("Per-path comparison rules. Non-ignored keys select compared paths (`$` = full output); an ignore-only map means full output minus those paths. More-specific keys inherit then override parent rules. `{}` selects a path using inherited platform defaults; `{ ignore: true }` excludes it. Do not mix with legacy `paths`, `unorderedPaths`, `numericTolerance`, or `allowExtraFields`."),
|
|
54358
54389
|
numericTolerance: number2().min(0).optional().describe("Legacy global numeric tolerance (absolute). When omitted, comparison uses historical relative epsilon. Prefer `rules.<path>.numericTolerance`. Accepted only when `rules` is omitted."),
|
|
54359
54390
|
allowExtraFields: boolean2().optional().describe("Legacy global extra-field/extra-item flag. Prefer `rules.<path>.allowExtraFields` and `rules.<path>.items`. `true` normalizes to object extras allowed and array `items: at-least`; `false` normalizes to object extras rejected and array `items: exactly`. Accepted only when `rules` is omitted."),
|
|
54360
54391
|
passThreshold: ScoreInUnitInterval.optional().describe(LEGACY_EVALUATOR_THRESHOLD_DESCRIPTION),
|
|
@@ -54406,6 +54437,19 @@ var init_evaluator_config = __esm(() => {
|
|
|
54406
54437
|
});
|
|
54407
54438
|
}
|
|
54408
54439
|
}
|
|
54440
|
+
const ignoredPaths = Object.entries(config.rules ?? {}).filter(([, rule]) => rule.ignore === true).map(([path]) => path);
|
|
54441
|
+
for (const [path, rule] of Object.entries(config.rules ?? {})) {
|
|
54442
|
+
if (rule.ignore === true)
|
|
54443
|
+
continue;
|
|
54444
|
+
const ignoredAncestor = ignoredPaths.find((ignoredPath) => ignoredPath !== path && evalPathIsDescendant(path, ignoredPath));
|
|
54445
|
+
if (ignoredAncestor) {
|
|
54446
|
+
ctx.addIssue({
|
|
54447
|
+
code: ZodIssueCode.custom,
|
|
54448
|
+
path: ["rules", path],
|
|
54449
|
+
message: `cannot select \`${path}\` because ignored ancestor \`${ignoredAncestor}\` removes it before comparison`
|
|
54450
|
+
});
|
|
54451
|
+
}
|
|
54452
|
+
}
|
|
54409
54453
|
}).default({});
|
|
54410
54454
|
JudgeModeSchema = _enum(["continuous", "discrete"]);
|
|
54411
54455
|
LabelsRecord = record(string2().min(1), ScoreInUnitInterval);
|
|
@@ -54460,16 +54504,22 @@ var init_evaluator_config = __esm(() => {
|
|
|
54460
54504
|
weight: number2().min(0).default(1).describe("Relative importance in the weighted mean. The overall score is Σ(weightᵢ × scoreᵢ) / Σweight. Set to 0 to score-only without affecting overall.")
|
|
54461
54505
|
});
|
|
54462
54506
|
EvaluatorConfigSchema = discriminatedUnion("type", [
|
|
54463
|
-
EvaluatorBaseEntrySchema.extend({ type: literal("exact-diff"), config: ExactDiffConfigSchema }),
|
|
54464
|
-
EvaluatorBaseEntrySchema.extend({ type: literal("llm-judge"), config: LlmJudgeConfigSchema }),
|
|
54465
54507
|
EvaluatorBaseEntrySchema.extend({
|
|
54466
|
-
type: literal("
|
|
54467
|
-
config:
|
|
54508
|
+
type: literal("exact-diff").describe("Deterministic structured-output comparison."),
|
|
54509
|
+
config: ExactDiffConfigSchema.describe("Configuration for exact-diff.")
|
|
54510
|
+
}),
|
|
54511
|
+
EvaluatorBaseEntrySchema.extend({
|
|
54512
|
+
type: literal("llm-judge").describe("Model-graded semantic comparison."),
|
|
54513
|
+
config: LlmJudgeConfigSchema.describe("Configuration for llm-judge.")
|
|
54514
|
+
}),
|
|
54515
|
+
EvaluatorBaseEntrySchema.extend({
|
|
54516
|
+
type: literal("custom-script").describe("Typed deterministic scoring script."),
|
|
54517
|
+
config: CustomScriptConfigSchema.describe("Configuration for custom-script.")
|
|
54468
54518
|
})
|
|
54469
54519
|
]);
|
|
54470
54520
|
EvalConfigYamlSchema = object({
|
|
54471
|
-
evaluators: array(EvaluatorConfigSchema).default([]),
|
|
54472
|
-
passThreshold: ScoreInUnitInterval.optional()
|
|
54521
|
+
evaluators: array(EvaluatorConfigSchema).default([]).describe("Evaluator entries run after each experiment execution."),
|
|
54522
|
+
passThreshold: ScoreInUnitInterval.optional().describe("Workflow-level pass threshold for the weighted mean score. When omitted, legacy evaluator thresholds are combined for compatibility, otherwise the platform default is 0.7.")
|
|
54473
54523
|
});
|
|
54474
54524
|
EvaluatorResultSchema = object({
|
|
54475
54525
|
score: ScoreInUnitInterval.nullable(),
|
|
@@ -184044,7 +184094,7 @@ function getAlignmentSize(text, tabWidth, startIndex = 0) {
|
|
|
184044
184094
|
}
|
|
184045
184095
|
return size;
|
|
184046
184096
|
}
|
|
184047
|
-
function
|
|
184097
|
+
function isObject3(object) {
|
|
184048
184098
|
return object !== null && typeof object === "object";
|
|
184049
184099
|
}
|
|
184050
184100
|
function skip(characters) {
|
|
@@ -196834,7 +196884,7 @@ ${codeblock}`, options8);
|
|
|
196834
196884
|
get_string_width_default2 = getStringWidth2;
|
|
196835
196885
|
get_alignment_size_default = getAlignmentSize;
|
|
196836
196886
|
ast_path_default = AstPath;
|
|
196837
|
-
is_object_default =
|
|
196887
|
+
is_object_default = isObject3;
|
|
196838
196888
|
skipWhitespace = skip(/\s/u);
|
|
196839
196889
|
skipSpaces = skip(" \t");
|
|
196840
196890
|
skipToLineEnd = skip(",; \t");
|
|
@@ -205318,7 +205368,7 @@ import { fileURLToPath as fileURLToPath8 } from "url";
|
|
|
205318
205368
|
// package.json
|
|
205319
205369
|
var package_default = {
|
|
205320
205370
|
name: "@eigenpal/cli",
|
|
205321
|
-
version: "0.
|
|
205371
|
+
version: "0.15.1",
|
|
205322
205372
|
description: "Eigenpal CLI — author and run document workflows from your terminal. Agent-ready.",
|
|
205323
205373
|
keywords: [
|
|
205324
205374
|
"eigenpal",
|
|
@@ -205357,15 +205407,18 @@ var package_default = {
|
|
|
205357
205407
|
},
|
|
205358
205408
|
scripts: {
|
|
205359
205409
|
dev: "bun src/cli.ts",
|
|
205360
|
-
build: "bun run generate:skill && bun build src/cli.ts --outdir dist --target node --format esm && rm -rf dist/templates dist/skill && cp -R src/templates dist/templates && cp -R src/skill dist/skill",
|
|
205361
|
-
generate: "bun run generate:cli-docs && bun run generate:skill",
|
|
205362
|
-
"generate:check": "bun run generate:cli-docs:check && bun run generate:skill:check",
|
|
205410
|
+
build: "bun run generate:skill && bun build src/cli.ts --outdir dist --target node --format esm && rm -rf dist/templates dist/skill && cp -R src/templates dist/templates && cp -R src/skill dist/skill && cp src/docs-bundle.json dist/docs-bundle.json",
|
|
205411
|
+
generate: "bun run generate:cli-docs && bun run generate:skill && bun run generate:docs",
|
|
205412
|
+
"generate:check": "bun run generate:cli-docs:check && bun run generate:skill:check && bun run generate:docs:check && bun test scripts/docs-bundle-completeness.test.ts",
|
|
205413
|
+
"generate:docs": "bun scripts/generate-docs-bundle.ts",
|
|
205414
|
+
"generate:docs:check": "bun scripts/generate-docs-bundle.ts --check",
|
|
205363
205415
|
"generate:skill": "bun scripts/generate-skill-reference.ts",
|
|
205364
205416
|
"generate:skill:check": "bun scripts/generate-skill-reference.ts --check",
|
|
205365
205417
|
"generate:cli-docs": "bun scripts/generate-cli-docs.ts",
|
|
205366
205418
|
"generate:cli-docs:check": "bun scripts/generate-cli-docs.ts --check",
|
|
205367
205419
|
"generate:logo": "bun scripts/render-logo-ascii.ts --cells 20",
|
|
205368
205420
|
lint: "eslint src/",
|
|
205421
|
+
test: "bun test src/ scripts/",
|
|
205369
205422
|
typecheck: "tsc --noEmit"
|
|
205370
205423
|
},
|
|
205371
205424
|
dependencies: {
|
|
@@ -205873,13 +205926,20 @@ function table(rows, columns) {
|
|
|
205873
205926
|
`);
|
|
205874
205927
|
}
|
|
205875
205928
|
function addJsonFlag(cmd) {
|
|
205876
|
-
return cmd.option("--json", "
|
|
205929
|
+
return cmd.option("--json", "Emit machine-readable JSON on stdout");
|
|
205877
205930
|
}
|
|
205878
205931
|
function withBaseUrl(cmd) {
|
|
205879
205932
|
return cmd.option("--base-url <url>", "Server base URL");
|
|
205880
205933
|
}
|
|
205881
205934
|
function intArg(value) {
|
|
205882
|
-
|
|
205935
|
+
if (!/^-?\d+$/.test(value)) {
|
|
205936
|
+
throw new InvalidArgumentError2("Expected a whole number");
|
|
205937
|
+
}
|
|
205938
|
+
const parsed = Number(value);
|
|
205939
|
+
if (!Number.isSafeInteger(parsed)) {
|
|
205940
|
+
throw new InvalidArgumentError2("Expected a safe whole number");
|
|
205941
|
+
}
|
|
205942
|
+
return parsed;
|
|
205883
205943
|
}
|
|
205884
205944
|
function withPagination(cmd, defaultLimit = 50) {
|
|
205885
205945
|
return cmd.option("--limit <n>", "Page size", intArg, defaultLimit).option("--offset <n>", "Page offset", intArg, 0);
|
|
@@ -205911,15 +205971,15 @@ function renderListResult(raw, columns, opts = {}) {
|
|
|
205911
205971
|
if (opts.json) {
|
|
205912
205972
|
console.log(JSON.stringify(raw, null, 2));
|
|
205913
205973
|
if (rows.length > 0)
|
|
205914
|
-
writeRecordCountHint(rows.length, total, label);
|
|
205974
|
+
writeRecordCountHint(rows.length, total, label, false);
|
|
205915
205975
|
return;
|
|
205916
205976
|
}
|
|
205917
205977
|
console.log(table(rows, columns));
|
|
205918
205978
|
if (rows.length > 0)
|
|
205919
205979
|
writeRecordCountHint(rows.length, total, label);
|
|
205920
205980
|
}
|
|
205921
|
-
function writeRecordCountHint(shown, total, label) {
|
|
205922
|
-
process.stderr.write(`${import_picocolors.default.dim(`${shown}${total > shown ? ` of ${total}` : ""} ${label}${total === 1 ? "" : "s"} · use --json for
|
|
205981
|
+
function writeRecordCountHint(shown, total, label, suggestJson = true) {
|
|
205982
|
+
process.stderr.write(`${import_picocolors.default.dim(`${shown}${total > shown ? ` of ${total}` : ""} ${label}${total === 1 ? "" : "s"}${suggestJson ? " · use --json for machine-readable output" : ""}`)}
|
|
205923
205983
|
`);
|
|
205924
205984
|
}
|
|
205925
205985
|
|
|
@@ -208792,16 +208852,18 @@ async function writeBase64File(file, contentBase64) {
|
|
|
208792
208852
|
await fs.mkdir(path6.dirname(file), { recursive: true });
|
|
208793
208853
|
await fs.writeFile(file, Buffer.from(contentBase64, "base64"));
|
|
208794
208854
|
}
|
|
208795
|
-
async function pollRun(client, executionId, interval, maxWait) {
|
|
208855
|
+
async function pollRun(client, executionId, interval, maxWait, json = false) {
|
|
208796
208856
|
const started = Date.now();
|
|
208797
208857
|
const base = `/v1/runs/${encodeURIComponent(executionId)}`;
|
|
208798
208858
|
for (;; ) {
|
|
208799
|
-
const payload = await client.get(base);
|
|
208859
|
+
const payload = await client.get(`${base}?expand=execution`);
|
|
208800
208860
|
const status = payload.execution?.status;
|
|
208801
208861
|
if (payload.finished || isTerminalExecutionStatus(status)) {
|
|
208802
208862
|
return await client.get(`${base}?expand=usage,execution`);
|
|
208803
208863
|
}
|
|
208804
208864
|
if (Date.now() - started > maxWait * 1000) {
|
|
208865
|
+
if (json)
|
|
208866
|
+
printJson(payload);
|
|
208805
208867
|
process.stderr.write(`Timed out waiting for run ${executionId}
|
|
208806
208868
|
`);
|
|
208807
208869
|
process.exit(2);
|
|
@@ -208809,13 +208871,27 @@ async function pollRun(client, executionId, interval, maxWait) {
|
|
|
208809
208871
|
await new Promise((resolve) => setTimeout(resolve, interval * 1000));
|
|
208810
208872
|
}
|
|
208811
208873
|
}
|
|
208812
|
-
async function pollExperiment(client, automationId, batchId, interval, maxWait) {
|
|
208874
|
+
async function pollExperiment(client, automationId, batchId, interval, maxWait, json = false) {
|
|
208813
208875
|
const started = Date.now();
|
|
208876
|
+
let notFoundRetries = 0;
|
|
208877
|
+
let lastPayload;
|
|
208814
208878
|
for (;; ) {
|
|
208815
|
-
|
|
208879
|
+
let payload;
|
|
208880
|
+
try {
|
|
208881
|
+
payload = await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(batchId)}`);
|
|
208882
|
+
lastPayload = payload;
|
|
208883
|
+
} catch (error) {
|
|
208884
|
+
if (!(error instanceof ApiError) || error.status !== 404 || notFoundRetries >= 2)
|
|
208885
|
+
throw error;
|
|
208886
|
+
notFoundRetries++;
|
|
208887
|
+
await new Promise((resolve) => setTimeout(resolve, interval * 1000));
|
|
208888
|
+
continue;
|
|
208889
|
+
}
|
|
208816
208890
|
if (payload.status === "completed")
|
|
208817
208891
|
return payload;
|
|
208818
208892
|
if (Date.now() - started > maxWait * 1000) {
|
|
208893
|
+
if (json)
|
|
208894
|
+
printJson(lastPayload ?? { batchId, status: "pending" });
|
|
208819
208895
|
process.stderr.write(`Timed out waiting for experiment ${batchId}
|
|
208820
208896
|
`);
|
|
208821
208897
|
process.exit(2);
|
|
@@ -208835,7 +208911,7 @@ function renderRunPayload(payload, opts) {
|
|
|
208835
208911
|
function setExitCodeForFailedTerminalRun(payload) {
|
|
208836
208912
|
const run = payload;
|
|
208837
208913
|
const status = run?.execution?.status;
|
|
208838
|
-
if (status === "failed" || status === "cancelled" || run?.finished === true && run?.error) {
|
|
208914
|
+
if (status === "failed" || status === "cancelled" || status === "rejected" || run?.finished === true && run?.error) {
|
|
208839
208915
|
process.exitCode = 1;
|
|
208840
208916
|
}
|
|
208841
208917
|
}
|
|
@@ -209489,6 +209565,89 @@ function dotenvQuote(value) {
|
|
|
209489
209565
|
// src/commands/agents/experiments.ts
|
|
209490
209566
|
import { promises as fs5 } from "node:fs";
|
|
209491
209567
|
|
|
209568
|
+
// src/lib/json-select.ts
|
|
209569
|
+
function selectJsonValue(value, path) {
|
|
209570
|
+
const tokens = parseJsonSelectPath(path);
|
|
209571
|
+
let values = [value];
|
|
209572
|
+
let usedWildcard = false;
|
|
209573
|
+
for (const token of tokens) {
|
|
209574
|
+
const next = [];
|
|
209575
|
+
for (const current of values) {
|
|
209576
|
+
if (token === "*") {
|
|
209577
|
+
usedWildcard = true;
|
|
209578
|
+
if (Array.isArray(current))
|
|
209579
|
+
next.push(...current);
|
|
209580
|
+
continue;
|
|
209581
|
+
}
|
|
209582
|
+
if (typeof token === "number") {
|
|
209583
|
+
if (Array.isArray(current) && token < current.length)
|
|
209584
|
+
next.push(current[token]);
|
|
209585
|
+
continue;
|
|
209586
|
+
}
|
|
209587
|
+
if (current && typeof current === "object" && !Array.isArray(current)) {
|
|
209588
|
+
const object = current;
|
|
209589
|
+
if (Object.hasOwn(object, token))
|
|
209590
|
+
next.push(object[token]);
|
|
209591
|
+
}
|
|
209592
|
+
}
|
|
209593
|
+
values = next;
|
|
209594
|
+
}
|
|
209595
|
+
if (values.length === 0) {
|
|
209596
|
+
throw new Error(`--select path "${path}" did not match the JSON payload`);
|
|
209597
|
+
}
|
|
209598
|
+
return usedWildcard ? values : values[0];
|
|
209599
|
+
}
|
|
209600
|
+
function parseJsonSelectPath(path) {
|
|
209601
|
+
const trimmed = path.trim();
|
|
209602
|
+
if (trimmed.startsWith("$") && trimmed !== "$" && !trimmed.startsWith("$.") && !trimmed.startsWith("$[")) {
|
|
209603
|
+
throw invalidPath(path);
|
|
209604
|
+
}
|
|
209605
|
+
const source = trimmed.replace(/^\$(?:\.)?/, "");
|
|
209606
|
+
if (!source)
|
|
209607
|
+
return [];
|
|
209608
|
+
const tokens = [];
|
|
209609
|
+
let cursor = 0;
|
|
209610
|
+
while (cursor < source.length) {
|
|
209611
|
+
if (source[cursor] === ".") {
|
|
209612
|
+
if (cursor === 0 || cursor === source.length - 1 || source[cursor - 1] === "." || source[cursor + 1] === "." || source[cursor + 1] === "[") {
|
|
209613
|
+
throw invalidPath(path);
|
|
209614
|
+
}
|
|
209615
|
+
cursor++;
|
|
209616
|
+
continue;
|
|
209617
|
+
}
|
|
209618
|
+
if (source[cursor] === "[") {
|
|
209619
|
+
if (cursor > 0 && source[cursor - 1] === ".")
|
|
209620
|
+
throw invalidPath(path);
|
|
209621
|
+
const close = source.indexOf("]", cursor + 1);
|
|
209622
|
+
if (close === -1)
|
|
209623
|
+
throw invalidPath(path);
|
|
209624
|
+
const inner = source.slice(cursor + 1, close);
|
|
209625
|
+
if (inner === "" || inner === "*")
|
|
209626
|
+
tokens.push("*");
|
|
209627
|
+
else if (/^(0|[1-9]\d*)$/.test(inner))
|
|
209628
|
+
tokens.push(Number(inner));
|
|
209629
|
+
else
|
|
209630
|
+
throw invalidPath(path);
|
|
209631
|
+
cursor = close + 1;
|
|
209632
|
+
if (cursor < source.length && source[cursor] !== "." && source[cursor] !== "[") {
|
|
209633
|
+
throw invalidPath(path);
|
|
209634
|
+
}
|
|
209635
|
+
continue;
|
|
209636
|
+
}
|
|
209637
|
+
if (cursor > 0 && source[cursor - 1] !== ".")
|
|
209638
|
+
throw invalidPath(path);
|
|
209639
|
+
const match = /^[A-Za-z_$][A-Za-z0-9_$-]*/.exec(source.slice(cursor));
|
|
209640
|
+
if (!match)
|
|
209641
|
+
throw invalidPath(path);
|
|
209642
|
+
tokens.push(match[0]);
|
|
209643
|
+
cursor += match[0].length;
|
|
209644
|
+
}
|
|
209645
|
+
return tokens;
|
|
209646
|
+
}
|
|
209647
|
+
function invalidPath(path) {
|
|
209648
|
+
return new Error(`Invalid --select path "${path}". Use dot keys, numeric indexes, or [] wildcards (for example output.subjects[].name).`);
|
|
209649
|
+
}
|
|
209650
|
+
|
|
209492
209651
|
// src/commands/workflow/experiment-compare.ts
|
|
209493
209652
|
function normalizeCompareSort(value) {
|
|
209494
209653
|
switch (value) {
|
|
@@ -209571,6 +209730,90 @@ function buildBatchDiff(args) {
|
|
|
209571
209730
|
}
|
|
209572
209731
|
};
|
|
209573
209732
|
}
|
|
209733
|
+
function buildExperimentOutputDiff(args) {
|
|
209734
|
+
const a = new Map(args.rowsA.map((row) => [exampleLabelOf(row), row]));
|
|
209735
|
+
const b = new Map(args.rowsB.map((row) => [exampleLabelOf(row), row]));
|
|
209736
|
+
const shared = [...a.keys()].filter((example) => b.has(example)).sort();
|
|
209737
|
+
const rows = shared.map((example) => {
|
|
209738
|
+
const rowA = a.get(example);
|
|
209739
|
+
const rowB = b.get(example);
|
|
209740
|
+
const comparable = rowA.status === "completed" && rowB.status === "completed" && rowA.output !== undefined && rowB.output !== undefined;
|
|
209741
|
+
if (!comparable) {
|
|
209742
|
+
return {
|
|
209743
|
+
example,
|
|
209744
|
+
executionA: rowA.executionId,
|
|
209745
|
+
executionB: rowB.executionId,
|
|
209746
|
+
status: "incomparable",
|
|
209747
|
+
differences: [
|
|
209748
|
+
{
|
|
209749
|
+
path: "$",
|
|
209750
|
+
type: "incomparable",
|
|
209751
|
+
expected: { status: rowA.status, error: rowA.error },
|
|
209752
|
+
actual: { status: rowB.status, error: rowB.error }
|
|
209753
|
+
}
|
|
209754
|
+
]
|
|
209755
|
+
};
|
|
209756
|
+
}
|
|
209757
|
+
const differences = diffOutputJson(rowA.output, rowB.output);
|
|
209758
|
+
return {
|
|
209759
|
+
example,
|
|
209760
|
+
executionA: rowA.executionId,
|
|
209761
|
+
executionB: rowB.executionId,
|
|
209762
|
+
status: differences.length === 0 ? "identical" : "changed",
|
|
209763
|
+
differences
|
|
209764
|
+
};
|
|
209765
|
+
});
|
|
209766
|
+
return {
|
|
209767
|
+
batchA: args.batchIdA,
|
|
209768
|
+
batchB: args.batchIdB,
|
|
209769
|
+
rows,
|
|
209770
|
+
summary: {
|
|
209771
|
+
sharedExamples: shared.length,
|
|
209772
|
+
identical: rows.filter((row) => row.status === "identical").length,
|
|
209773
|
+
changed: rows.filter((row) => row.status === "changed").length,
|
|
209774
|
+
incomparable: rows.filter((row) => row.status === "incomparable").length,
|
|
209775
|
+
onlyInA: [...a.keys()].filter((example) => !b.has(example)).sort(),
|
|
209776
|
+
onlyInB: [...b.keys()].filter((example) => !a.has(example)).sort()
|
|
209777
|
+
}
|
|
209778
|
+
};
|
|
209779
|
+
}
|
|
209780
|
+
function diffOutputJson(expected, actual, path = "$", limit = 200) {
|
|
209781
|
+
if (Object.is(expected, actual))
|
|
209782
|
+
return [];
|
|
209783
|
+
if (Array.isArray(expected) && Array.isArray(actual)) {
|
|
209784
|
+
const differences = [];
|
|
209785
|
+
for (let index = 0;index < Math.max(expected.length, actual.length); index++) {
|
|
209786
|
+
if (differences.length >= limit)
|
|
209787
|
+
break;
|
|
209788
|
+
const child = `${path}[${index}]`;
|
|
209789
|
+
if (index >= expected.length) {
|
|
209790
|
+
differences.push({ path: child, type: "extra", actual: actual[index] });
|
|
209791
|
+
} else if (index >= actual.length) {
|
|
209792
|
+
differences.push({ path: child, type: "missing", expected: expected[index] });
|
|
209793
|
+
} else {
|
|
209794
|
+
differences.push(...diffOutputJson(expected[index], actual[index], child, limit - differences.length));
|
|
209795
|
+
}
|
|
209796
|
+
}
|
|
209797
|
+
return differences;
|
|
209798
|
+
}
|
|
209799
|
+
if (isObject2(expected) && isObject2(actual)) {
|
|
209800
|
+
const differences = [];
|
|
209801
|
+
for (const key of new Set([...Object.keys(expected), ...Object.keys(actual)])) {
|
|
209802
|
+
if (differences.length >= limit)
|
|
209803
|
+
break;
|
|
209804
|
+
const child = path === "$" ? `$.${key}` : `${path}.${key}`;
|
|
209805
|
+
if (!Object.hasOwn(expected, key)) {
|
|
209806
|
+
differences.push({ path: child, type: "extra", actual: actual[key] });
|
|
209807
|
+
} else if (!Object.hasOwn(actual, key)) {
|
|
209808
|
+
differences.push({ path: child, type: "missing", expected: expected[key] });
|
|
209809
|
+
} else {
|
|
209810
|
+
differences.push(...diffOutputJson(expected[key], actual[key], child, limit - differences.length));
|
|
209811
|
+
}
|
|
209812
|
+
}
|
|
209813
|
+
return differences;
|
|
209814
|
+
}
|
|
209815
|
+
return [{ path, type: "changed", expected, actual }];
|
|
209816
|
+
}
|
|
209574
209817
|
function aggregateByEvaluator(rows) {
|
|
209575
209818
|
const buckets = new Map;
|
|
209576
209819
|
for (const r of rows) {
|
|
@@ -209755,6 +209998,33 @@ function renderBatchDiffHuman(diff) {
|
|
|
209755
209998
|
process.stderr.write(`${ui.dim("mean Δ:")} ${meanDeltaText} ` + `${ui.dim("examples shared:")} ${diff.summary.sharedExamples} ` + `${ui.dim("only in A:")} ${diff.summary.onlyInA.length} ` + `${ui.dim("only in B:")} ${diff.summary.onlyInB.length}
|
|
209756
209999
|
`);
|
|
209757
210000
|
}
|
|
210001
|
+
function renderExperimentOutputDiffHuman(diff) {
|
|
210002
|
+
process.stderr.write(`${ui.dim("A =")} ${ui.bold(diff.batchA)} ${ui.dim("B =")} ${ui.bold(diff.batchB)}
|
|
210003
|
+
|
|
210004
|
+
`);
|
|
210005
|
+
const visible = diff.rows.flatMap((row) => row.differences.map((difference) => ({
|
|
210006
|
+
example: row.example,
|
|
210007
|
+
path: difference.path,
|
|
210008
|
+
type: difference.type,
|
|
210009
|
+
expected: compactValue(difference.expected),
|
|
210010
|
+
actual: compactValue(difference.actual)
|
|
210011
|
+
})));
|
|
210012
|
+
if (visible.length > 0) {
|
|
210013
|
+
console.log(table(visible, [
|
|
210014
|
+
{ key: "example", header: "example" },
|
|
210015
|
+
{ key: "path", header: "path" },
|
|
210016
|
+
{ key: "type", header: "change" },
|
|
210017
|
+
{ key: "expected", header: "A" },
|
|
210018
|
+
{ key: "actual", header: "B" }
|
|
210019
|
+
]));
|
|
210020
|
+
} else {
|
|
210021
|
+
process.stderr.write(`${ui.dim("No actual-output differences in shared examples.")}
|
|
210022
|
+
`);
|
|
210023
|
+
}
|
|
210024
|
+
process.stderr.write(`
|
|
210025
|
+
${ui.dim("shared:")} ${diff.summary.sharedExamples} ` + `${ui.ok(`identical: ${diff.summary.identical}`)} ` + `${ui.warn(`changed: ${diff.summary.changed}`)} ` + `${ui.warn(`incomparable: ${diff.summary.incomparable}`)} ` + `${ui.dim("only in A:")} ${diff.summary.onlyInA.length} ` + `${ui.dim("only in B:")} ${diff.summary.onlyInB.length}
|
|
210026
|
+
`);
|
|
210027
|
+
}
|
|
209758
210028
|
async function fetchEvalResults(client, experimentId) {
|
|
209759
210029
|
const ref = await client.get(`/v1/experiments/${encodeURIComponent(experimentId)}`);
|
|
209760
210030
|
const raw = await client.get(`/v1/automations/${encodeURIComponent(ref.automationId)}/experiments/${encodeURIComponent(experimentId)}/export`, { format: "json" });
|
|
@@ -209763,6 +210033,214 @@ async function fetchEvalResults(client, experimentId) {
|
|
|
209763
210033
|
}
|
|
209764
210034
|
return raw;
|
|
209765
210035
|
}
|
|
210036
|
+
async function fetchExperimentOutputs(client, experimentId) {
|
|
210037
|
+
const ref = await client.get(`/v1/experiments/${encodeURIComponent(experimentId)}`);
|
|
210038
|
+
const detail = await client.get(`/v1/automations/${encodeURIComponent(ref.automationId)}/experiments/${encodeURIComponent(experimentId)}`);
|
|
210039
|
+
const runs = detail.runs ?? [];
|
|
210040
|
+
const rows = await mapWithConcurrency(runs, 6, async (run) => {
|
|
210041
|
+
const payload = await client.get(`/v1/runs/${encodeURIComponent(run.id)}`);
|
|
210042
|
+
return {
|
|
210043
|
+
executionId: run.id,
|
|
210044
|
+
exampleId: run.exampleId ?? null,
|
|
210045
|
+
exampleName: run.exampleName ?? null,
|
|
210046
|
+
status: payload.execution?.status ?? payload.status ?? payload.result?.status ?? null,
|
|
210047
|
+
error: payload.error ?? payload.execution?.error ?? payload.result?.error ?? null,
|
|
210048
|
+
output: payload.output !== undefined ? payload.output : payload.result?.output
|
|
210049
|
+
};
|
|
210050
|
+
});
|
|
210051
|
+
return { automationId: ref.automationId, rows };
|
|
210052
|
+
}
|
|
210053
|
+
function isObject2(value) {
|
|
210054
|
+
return value != null && typeof value === "object" && !Array.isArray(value);
|
|
210055
|
+
}
|
|
210056
|
+
function compactValue(value) {
|
|
210057
|
+
if (value === undefined)
|
|
210058
|
+
return "—";
|
|
210059
|
+
const rendered = JSON.stringify(value);
|
|
210060
|
+
if (rendered === undefined)
|
|
210061
|
+
return String(value);
|
|
210062
|
+
return rendered.length > 80 ? `${rendered.slice(0, 77)}…` : rendered;
|
|
210063
|
+
}
|
|
210064
|
+
async function mapWithConcurrency(values, concurrency, fn) {
|
|
210065
|
+
const results = new Array(values.length);
|
|
210066
|
+
let cursor = 0;
|
|
210067
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, async () => {
|
|
210068
|
+
while (cursor < values.length) {
|
|
210069
|
+
const index = cursor++;
|
|
210070
|
+
results[index] = await fn(values[index]);
|
|
210071
|
+
}
|
|
210072
|
+
}));
|
|
210073
|
+
return results;
|
|
210074
|
+
}
|
|
210075
|
+
|
|
210076
|
+
// src/commands/workflow/experiment-results.ts
|
|
210077
|
+
function experimentRowsFromDetail(detail) {
|
|
210078
|
+
const namesByRun = new Map((detail.runs ?? []).map((run) => [run.id, run.exampleName]));
|
|
210079
|
+
return Object.entries(detail.resultsByRun ?? {}).flatMap(([executionId, rows]) => rows.map((row) => ({
|
|
210080
|
+
...row,
|
|
210081
|
+
executionId,
|
|
210082
|
+
exampleName: namesByRun.get(executionId) ?? null,
|
|
210083
|
+
score: numberOrNull(row.score),
|
|
210084
|
+
weight: numberOrNull(row.weight) ?? 1
|
|
210085
|
+
})));
|
|
210086
|
+
}
|
|
210087
|
+
function analyzeExperimentResults(args) {
|
|
210088
|
+
let rows = args.evaluator ? args.rows.filter((row) => row.evaluatorName === args.evaluator) : [...args.rows];
|
|
210089
|
+
if (args.failedOnly) {
|
|
210090
|
+
rows = rows.filter((row) => row.error != null || row.passed === false);
|
|
210091
|
+
}
|
|
210092
|
+
const includedExecutionIds = new Set(rows.map((row) => row.executionId));
|
|
210093
|
+
const runs = args.evaluator || args.failedOnly ? args.runs.filter((run) => includedExecutionIds.has(run.id)) : args.runs;
|
|
210094
|
+
const rowsByRun = groupBy(rows, (row) => row.executionId);
|
|
210095
|
+
const counts = { total: runs.length, passed: 0, failed: 0, errors: 0 };
|
|
210096
|
+
const runScores = [];
|
|
210097
|
+
const usePersistedAggregate = !args.evaluator && !args.failedOnly;
|
|
210098
|
+
for (const run of runs) {
|
|
210099
|
+
const runRows = rowsByRun.get(run.id) ?? [];
|
|
210100
|
+
if (usePersistedAggregate && run.evalPassed != null) {
|
|
210101
|
+
if (["failed", "cancelled", "rejected"].includes(run.status) && run.evalPassed !== true) {
|
|
210102
|
+
counts.errors++;
|
|
210103
|
+
} else if (run.evalPassed) {
|
|
210104
|
+
counts.passed++;
|
|
210105
|
+
} else {
|
|
210106
|
+
counts.failed++;
|
|
210107
|
+
}
|
|
210108
|
+
if (typeof run.evalScore === "number")
|
|
210109
|
+
runScores.push(run.evalScore);
|
|
210110
|
+
continue;
|
|
210111
|
+
}
|
|
210112
|
+
if (["failed", "cancelled", "rejected"].includes(run.status) || runRows.length === 0 || runRows.some((row) => row.error != null)) {
|
|
210113
|
+
counts.errors++;
|
|
210114
|
+
continue;
|
|
210115
|
+
}
|
|
210116
|
+
if (runRows.some((row) => row.passed === false))
|
|
210117
|
+
counts.failed++;
|
|
210118
|
+
else
|
|
210119
|
+
counts.passed++;
|
|
210120
|
+
const score = weightedAverage(runRows);
|
|
210121
|
+
if (score != null)
|
|
210122
|
+
runScores.push(score);
|
|
210123
|
+
}
|
|
210124
|
+
const byEvaluator = [...groupBy(rows, (row) => row.evaluatorName)].map(([evaluator, evaluatorRows]) => ({
|
|
210125
|
+
evaluator,
|
|
210126
|
+
total: evaluatorRows.length,
|
|
210127
|
+
passed: evaluatorRows.filter((row) => row.error == null && row.passed === true).length,
|
|
210128
|
+
failed: evaluatorRows.filter((row) => row.error == null && row.passed === false).length,
|
|
210129
|
+
errors: evaluatorRows.filter((row) => row.error != null).length,
|
|
210130
|
+
averageScore: average(evaluatorRows.flatMap((row) => typeof row.score === "number" ? [row.score] : []))
|
|
210131
|
+
}));
|
|
210132
|
+
byEvaluator.sort((a, b) => a.evaluator.localeCompare(b.evaluator));
|
|
210133
|
+
return {
|
|
210134
|
+
summary: {
|
|
210135
|
+
...counts,
|
|
210136
|
+
averageScore: average(runScores),
|
|
210137
|
+
byEvaluator
|
|
210138
|
+
},
|
|
210139
|
+
discrepancies: flattenDiscrepancies(rows),
|
|
210140
|
+
results: rows
|
|
210141
|
+
};
|
|
210142
|
+
}
|
|
210143
|
+
function flattenDiscrepancies(rows) {
|
|
210144
|
+
const discrepancies = [];
|
|
210145
|
+
for (const row of rows) {
|
|
210146
|
+
const details = objectOrNull(row.details);
|
|
210147
|
+
const mismatches = Array.isArray(details?.mismatches) ? details.mismatches : [];
|
|
210148
|
+
for (const mismatch of mismatches) {
|
|
210149
|
+
const item = objectOrNull(mismatch);
|
|
210150
|
+
if (!item || typeof item.path !== "string")
|
|
210151
|
+
continue;
|
|
210152
|
+
discrepancies.push({
|
|
210153
|
+
example: row.exampleName || row.executionId,
|
|
210154
|
+
executionId: row.executionId,
|
|
210155
|
+
evaluator: row.evaluatorName,
|
|
210156
|
+
path: item.path,
|
|
210157
|
+
expected: item.expected,
|
|
210158
|
+
actual: item.actual
|
|
210159
|
+
});
|
|
210160
|
+
}
|
|
210161
|
+
}
|
|
210162
|
+
return discrepancies;
|
|
210163
|
+
}
|
|
210164
|
+
function renderExperimentResultsSummary(analysis) {
|
|
210165
|
+
const summary = analysis.summary;
|
|
210166
|
+
process.stderr.write(`${ui.bold("Experiment results")} total=${summary.total} ${ui.ok(`passed=${summary.passed}`)} ` + `${ui.warn(`failed=${summary.failed}`)} errors=${summary.errors} ` + `average=${formatScore2(summary.averageScore)}
|
|
210167
|
+
`);
|
|
210168
|
+
if (summary.byEvaluator.length > 0) {
|
|
210169
|
+
console.log(table(summary.byEvaluator, [
|
|
210170
|
+
{ key: "evaluator", header: "evaluator" },
|
|
210171
|
+
{ key: "total", header: "total", align: "right" },
|
|
210172
|
+
{ key: "passed", header: "pass", align: "right" },
|
|
210173
|
+
{ key: "failed", header: "fail", align: "right" },
|
|
210174
|
+
{ key: "errors", header: "error", align: "right" },
|
|
210175
|
+
{
|
|
210176
|
+
key: "averageScore",
|
|
210177
|
+
header: "average",
|
|
210178
|
+
align: "right",
|
|
210179
|
+
format: (value) => formatScore2(value)
|
|
210180
|
+
}
|
|
210181
|
+
]));
|
|
210182
|
+
}
|
|
210183
|
+
if (analysis.discrepancies.length > 0) {
|
|
210184
|
+
process.stderr.write(`
|
|
210185
|
+
${ui.bold("Discrepancies")}
|
|
210186
|
+
`);
|
|
210187
|
+
console.log(table(analysis.discrepancies, [
|
|
210188
|
+
{ key: "example", header: "example" },
|
|
210189
|
+
{ key: "evaluator", header: "evaluator" },
|
|
210190
|
+
{ key: "path", header: "path" },
|
|
210191
|
+
{ key: "expected", header: "expected", format: compactJson },
|
|
210192
|
+
{ key: "actual", header: "actual", format: compactJson }
|
|
210193
|
+
]));
|
|
210194
|
+
}
|
|
210195
|
+
}
|
|
210196
|
+
function weightedAverage(rows) {
|
|
210197
|
+
let total = 0;
|
|
210198
|
+
let weight = 0;
|
|
210199
|
+
for (const row of rows) {
|
|
210200
|
+
if (typeof row.score !== "number")
|
|
210201
|
+
continue;
|
|
210202
|
+
const rowWeight = Number.isFinite(row.weight) ? row.weight : 1;
|
|
210203
|
+
total += row.score * rowWeight;
|
|
210204
|
+
weight += rowWeight;
|
|
210205
|
+
}
|
|
210206
|
+
return weight > 0 ? total / weight : null;
|
|
210207
|
+
}
|
|
210208
|
+
function average(values) {
|
|
210209
|
+
return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
|
|
210210
|
+
}
|
|
210211
|
+
function numberOrNull(value) {
|
|
210212
|
+
if (typeof value === "number")
|
|
210213
|
+
return Number.isFinite(value) ? value : null;
|
|
210214
|
+
if (typeof value === "string") {
|
|
210215
|
+
const parsed = Number(value);
|
|
210216
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
210217
|
+
}
|
|
210218
|
+
return null;
|
|
210219
|
+
}
|
|
210220
|
+
function groupBy(values, key) {
|
|
210221
|
+
const groups = new Map;
|
|
210222
|
+
for (const value of values) {
|
|
210223
|
+
const name = key(value);
|
|
210224
|
+
const existing = groups.get(name);
|
|
210225
|
+
if (existing)
|
|
210226
|
+
existing.push(value);
|
|
210227
|
+
else
|
|
210228
|
+
groups.set(name, [value]);
|
|
210229
|
+
}
|
|
210230
|
+
return groups;
|
|
210231
|
+
}
|
|
210232
|
+
function objectOrNull(value) {
|
|
210233
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
210234
|
+
}
|
|
210235
|
+
function formatScore2(value) {
|
|
210236
|
+
return value == null ? "—" : value.toFixed(3);
|
|
210237
|
+
}
|
|
210238
|
+
function compactJson(value) {
|
|
210239
|
+
const rendered = JSON.stringify(value);
|
|
210240
|
+
if (rendered === undefined)
|
|
210241
|
+
return "undefined";
|
|
210242
|
+
return rendered.length > 80 ? `${rendered.slice(0, 77)}…` : rendered;
|
|
210243
|
+
}
|
|
209766
210244
|
|
|
209767
210245
|
// src/commands/agents/experiments.ts
|
|
209768
210246
|
function registerExperimentCommands(agent) {
|
|
@@ -209770,11 +210248,11 @@ function registerExperimentCommands(agent) {
|
|
|
209770
210248
|
process.stderr.write("`eigenpal agents experiment` requires a subcommand. Run `eigenpal agents experiment --help`.\n");
|
|
209771
210249
|
process.exit(2);
|
|
209772
210250
|
});
|
|
209773
|
-
addJsonFlag(withBaseUrl(experiment.command("run <agent-id-or-slug>"))).description("Start an experiment over dataset examples.").option("--example-id <id>", "Run one dataset example").option("--wait", "Poll until the experiment reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).action(action(runExperiment));
|
|
209774
|
-
addJsonFlag(withBaseUrl(experiment.command("status <agent-id-or-slug> <batch-id>"))).description("Get experiment status.").option("--watch", "Poll until complete").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).
|
|
209775
|
-
withBaseUrl(experiment.command("results <agent-id-or-slug> [batch-id]")).description("Print experiment results as JSON or CSV.").
|
|
210251
|
+
addJsonFlag(withBaseUrl(experiment.command("run <agent-id-or-slug>"))).description("Start an experiment over dataset examples.").option("--example-id <id>", "Run one dataset example").option("--wait", "Poll until the experiment reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(runExperiment));
|
|
210252
|
+
addJsonFlag(withBaseUrl(experiment.command("status <agent-id-or-slug> <batch-id>"))).description("Get experiment status.").option("--watch", "Poll until complete").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(experimentStatus));
|
|
210253
|
+
addJsonFlag(withBaseUrl(experiment.command("results <agent-id-or-slug> [batch-id]"))).description("Print experiment results as JSON or CSV.").option("--format <csv|json>", "Output format (default json)", parseResultsFormat, "json").option("--out <path>", "Write output to file").option("--summary", "Show total/pass/fail/error counts, average score, and evaluator rollups").option("--failed-only", "Keep only failed or errored evaluator results").option("--evaluator <name>", "Keep only results from this evaluator").option("--select <path>", "Print only a nested JSON value (for example summary.byEvaluator or discrepancies[].path)").action(action(experimentResults));
|
|
209776
210254
|
addJsonFlag(withPagination(withBaseUrl(experiment.command("list <agent-id-or-slug>")), 50)).description("List experiments.").option("--batch-id <id>", "Filter to one batch id").action(action(listExperiments));
|
|
209777
|
-
addJsonFlag(withBaseUrl(experiment.command("compare <batch-id-a> <batch-id-b>"))).description("
|
|
210255
|
+
addJsonFlag(withBaseUrl(experiment.command("compare <batch-id-a> <batch-id-b>"))).description("Compare evaluator scores or actual outputs between two experiment batches.").option("--outputs", "Compare actual run outputs instead of evaluator scores").option("--sort <abs-delta-desc|delta-asc|delta-desc|name>", "Row sort order (default: biggest movers first)", "abs-delta-desc").option("--regression-threshold <n>", "Δ below this is flagged as a regression (default 0.05)", Number.parseFloat, 0.05).action(action(compareExperiments));
|
|
209778
210256
|
addJsonFlag(withBaseUrl(experiment.command("cancel <agent-id-or-slug> <batch-id>"))).description("Cancel every active execution in an experiment.").option("--yes", "Skip confirmation (required in CI / agent terminals without a TTY)").action(action(cancelExperiment));
|
|
209779
210257
|
}
|
|
209780
210258
|
async function runExperiment(agentId, opts) {
|
|
@@ -209784,26 +210262,73 @@ async function runExperiment(agentId, opts) {
|
|
|
209784
210262
|
...opts.exampleId ? { examples: [opts.exampleId] } : {}
|
|
209785
210263
|
});
|
|
209786
210264
|
const experimentId = payload.id ?? payload.batchId;
|
|
210265
|
+
if (opts.wait && !experimentId) {
|
|
210266
|
+
throw new Error("Experiment start response did not include a batch id; cannot honor --wait");
|
|
210267
|
+
}
|
|
209787
210268
|
if (opts.wait && experimentId) {
|
|
209788
|
-
payload = await pollExperiment(client, automationId, experimentId, opts.interval,
|
|
210269
|
+
payload = await pollExperiment(client, automationId, experimentId, opts.interval, opts.maxWait, opts.json);
|
|
210270
|
+
}
|
|
210271
|
+
const stablePayload = {
|
|
210272
|
+
...payload,
|
|
210273
|
+
batchId: experimentId,
|
|
210274
|
+
total: payload.runCount ?? payload.total
|
|
210275
|
+
};
|
|
210276
|
+
renderGeneric(stablePayload, opts, `${opts.wait ? "Completed" : "Started"} experiment ${experimentId ?? ""}`);
|
|
210277
|
+
const failedCount = payload.failedCount;
|
|
210278
|
+
if (opts.wait && typeof failedCount === "number" && failedCount > 0) {
|
|
210279
|
+
process.exitCode = 1;
|
|
209789
210280
|
}
|
|
209790
|
-
renderGeneric(payload, opts, `Started experiment ${experimentId ?? ""}`);
|
|
209791
210281
|
}
|
|
209792
210282
|
async function experimentStatus(agentId, batchId, opts) {
|
|
209793
210283
|
const client = buildClient(opts);
|
|
209794
210284
|
const automationId = agentAutomationId(agentId);
|
|
209795
|
-
const payload = opts.watch ? await pollExperiment(client, automationId, batchId, opts.interval, opts.maxWait) : await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(batchId)}`);
|
|
210285
|
+
const payload = opts.watch ? await pollExperiment(client, automationId, batchId, opts.interval, opts.maxWait, opts.json) : await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(batchId)}`);
|
|
209796
210286
|
renderGeneric(payload, opts, `Experiment ${batchId}`);
|
|
210287
|
+
if (opts.watch) {
|
|
210288
|
+
const counts = payload;
|
|
210289
|
+
if ([counts.failedCount, counts.cancelledCount, counts.rejectedCount].some((count) => typeof count === "number" && count > 0)) {
|
|
210290
|
+
process.exitCode = 1;
|
|
210291
|
+
}
|
|
210292
|
+
}
|
|
209797
210293
|
}
|
|
209798
210294
|
async function experimentResults(agentId, batchId, opts) {
|
|
209799
210295
|
const client = buildClient(opts);
|
|
209800
210296
|
const automationId = agentAutomationId(agentId);
|
|
210297
|
+
if (opts.json && opts.format !== "json") {
|
|
210298
|
+
throw new Error("--json cannot be combined with --format csv");
|
|
210299
|
+
}
|
|
209801
210300
|
const selected = batchId ?? String((await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments`, {
|
|
209802
210301
|
limit: "1",
|
|
209803
210302
|
offset: "0"
|
|
209804
210303
|
})).data?.[0]?.id ?? "");
|
|
209805
210304
|
if (!selected)
|
|
209806
210305
|
throw new Error("No experiment batch found");
|
|
210306
|
+
const analyze = opts.summary || opts.failedOnly || opts.evaluator !== undefined || opts.select !== undefined;
|
|
210307
|
+
if (analyze) {
|
|
210308
|
+
if (opts.format !== "json")
|
|
210309
|
+
throw new Error("Experiment analysis options require --format json");
|
|
210310
|
+
const detail = await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(selected)}`);
|
|
210311
|
+
const analysis = analyzeExperimentResults({
|
|
210312
|
+
rows: experimentRowsFromDetail(detail),
|
|
210313
|
+
runs: detail.runs ?? [],
|
|
210314
|
+
evaluator: opts.evaluator,
|
|
210315
|
+
failedOnly: opts.failedOnly
|
|
210316
|
+
});
|
|
210317
|
+
const output = opts.select ? selectJsonValue(analysis, opts.select) : analysis;
|
|
210318
|
+
if (opts.summary && !opts.json && !opts.out && !opts.select) {
|
|
210319
|
+
renderExperimentResultsSummary(analysis);
|
|
210320
|
+
return;
|
|
210321
|
+
}
|
|
210322
|
+
const content = `${JSON.stringify(output, null, 2)}
|
|
210323
|
+
`;
|
|
210324
|
+
if (opts.out) {
|
|
210325
|
+
await fs5.writeFile(opts.out, content);
|
|
210326
|
+
success(`Wrote ${opts.out}`);
|
|
210327
|
+
} else {
|
|
210328
|
+
process.stdout.write(content);
|
|
210329
|
+
}
|
|
210330
|
+
return;
|
|
210331
|
+
}
|
|
209807
210332
|
const payload = await client.get(`/v1/automations/${encodeURIComponent(automationId)}/experiments/${encodeURIComponent(selected)}`);
|
|
209808
210333
|
const content = opts.format === "json" ? JSON.stringify(payload, null, 2) : toCsv(payload.runs, ["id", "status", "exampleId"]);
|
|
209809
210334
|
if (opts.out) {
|
|
@@ -209836,6 +210361,25 @@ async function compareExperiments(batchIdA, batchIdB, opts) {
|
|
|
209836
210361
|
throw new Error("--regression-threshold must be a non-negative number");
|
|
209837
210362
|
}
|
|
209838
210363
|
const client = buildClient(opts);
|
|
210364
|
+
if (opts.outputs) {
|
|
210365
|
+
const [outputsA, outputsB] = await Promise.all([
|
|
210366
|
+
fetchExperimentOutputs(client, batchIdA),
|
|
210367
|
+
fetchExperimentOutputs(client, batchIdB)
|
|
210368
|
+
]);
|
|
210369
|
+
if (outputsA.automationId !== outputsB.automationId) {
|
|
210370
|
+
throw new Error("Experiment batches must belong to the same automation");
|
|
210371
|
+
}
|
|
210372
|
+
const diff = buildExperimentOutputDiff({
|
|
210373
|
+
batchIdA,
|
|
210374
|
+
batchIdB,
|
|
210375
|
+
rowsA: outputsA.rows,
|
|
210376
|
+
rowsB: outputsB.rows
|
|
210377
|
+
});
|
|
210378
|
+
if (opts.json)
|
|
210379
|
+
return printJson(diff);
|
|
210380
|
+
renderExperimentOutputDiffHuman(diff);
|
|
210381
|
+
return;
|
|
210382
|
+
}
|
|
209839
210383
|
const [rowsA, rowsB] = await Promise.all([
|
|
209840
210384
|
fetchEvalResults(client, batchIdA),
|
|
209841
210385
|
fetchEvalResults(client, batchIdB)
|
|
@@ -209884,6 +210428,9 @@ async function listAgentFiles(agentId, opts) {
|
|
|
209884
210428
|
console.log(table((payload.files ?? []).map((name) => ({ name })), [{ key: "name", header: "PATH" }]));
|
|
209885
210429
|
}
|
|
209886
210430
|
async function getAgentFile(agentId, remotePath, opts) {
|
|
210431
|
+
if (opts.json && opts.out) {
|
|
210432
|
+
throw new Error("--json and --out are mutually exclusive");
|
|
210433
|
+
}
|
|
209887
210434
|
const client = buildClient(opts);
|
|
209888
210435
|
const payload = await readAgentFile(client, agentId, remotePath);
|
|
209889
210436
|
if (opts.json)
|
|
@@ -210003,7 +210550,7 @@ function registerSessionCommands(agent) {
|
|
|
210003
210550
|
addJsonFlag(withPagination(withBaseUrl(session.command("list <agent-id-or-slug>")), 50)).description("List builder sessions for an agent.").action(action(listSessions));
|
|
210004
210551
|
addJsonFlag(withBaseUrl(session.command("get <session-id>"))).description("Get a builder session and messages.").action(action(getSession));
|
|
210005
210552
|
addJsonFlag(withBaseUrl(session.command("start <agent-id-or-slug>"))).description("Start a builder session.").option("--title <title>", "Session title").action(action(startSession));
|
|
210006
|
-
addJsonFlag(withBaseUrl(session.command("message <session-id>"))).description("Append a message to a builder session.").requiredOption("--text <message>", "Message text").
|
|
210553
|
+
addJsonFlag(withBaseUrl(session.command("message <session-id>"))).description("Append a message to a builder session.").requiredOption("--text <message>", "Message text").action(action(messageSession));
|
|
210007
210554
|
addJsonFlag(withBaseUrl(session.command("stop <session-id>"))).description("Stop a builder session.").option("--yes", "Skip confirmation (required in CI / agent terminals without a TTY)").action(action(stopSession));
|
|
210008
210555
|
}
|
|
210009
210556
|
async function listSessions(agentId, opts) {
|
|
@@ -237879,6 +238426,7 @@ var EXT_MIME = {
|
|
|
237879
238426
|
".doc": "application/msword",
|
|
237880
238427
|
".xls": "application/vnd.ms-excel",
|
|
237881
238428
|
".rtf": "application/rtf",
|
|
238429
|
+
".msg": "application/vnd.ms-outlook",
|
|
237882
238430
|
".json": "application/json",
|
|
237883
238431
|
".xml": "application/xml",
|
|
237884
238432
|
".html": "text/html"
|
|
@@ -237900,7 +238448,6 @@ function resolveEvalBaseDir(dir) {
|
|
|
237900
238448
|
|
|
237901
238449
|
// src/commands/workflow/eval-example.ts
|
|
237902
238450
|
var RUN_POLL_INTERVAL_MS = 2000;
|
|
237903
|
-
var RUN_MAX_WAIT_MS = 5 * 60 * 1000;
|
|
237904
238451
|
function resolveEvalGraceMs() {
|
|
237905
238452
|
const raw = env.EIGENPAL_EVAL_GRACE_MS;
|
|
237906
238453
|
if (raw !== undefined) {
|
|
@@ -237949,7 +238496,7 @@ async function resolveExample(client, workflowId, name) {
|
|
|
237949
238496
|
offset += rows.length;
|
|
237950
238497
|
}
|
|
237951
238498
|
}
|
|
237952
|
-
async function pollRunTerminal(client, runId) {
|
|
238499
|
+
async function pollRunTerminal(client, runId, intervalMs, maxWaitMs) {
|
|
237953
238500
|
const start = Date.now();
|
|
237954
238501
|
const base = `/v1/runs/${encodeURIComponent(runId)}`;
|
|
237955
238502
|
for (;; ) {
|
|
@@ -237958,10 +238505,10 @@ async function pollRunTerminal(client, runId) {
|
|
|
237958
238505
|
if (run.finished || status && TERMINAL.has(status)) {
|
|
237959
238506
|
return await client.get(`${base}?expand=execution`);
|
|
237960
238507
|
}
|
|
237961
|
-
if (Date.now() - start >
|
|
238508
|
+
if (Date.now() - start > maxWaitMs) {
|
|
237962
238509
|
return { ...run, status: "timeout", execution: { status: "timeout" } };
|
|
237963
238510
|
}
|
|
237964
|
-
await new Promise((r) => setTimeout(r,
|
|
238511
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
237965
238512
|
}
|
|
237966
238513
|
}
|
|
237967
238514
|
async function pollEvalRollup(client, runId) {
|
|
@@ -237991,15 +238538,18 @@ function toArtifactPayload(run, runId) {
|
|
|
237991
238538
|
stepExecutions: []
|
|
237992
238539
|
};
|
|
237993
238540
|
}
|
|
237994
|
-
async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workflowId, exampleNames) {
|
|
238541
|
+
async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workflowId, exampleNames, options = {}) {
|
|
237995
238542
|
const hasEvaluators = await fetchHasEvaluators(client, workflowId);
|
|
237996
238543
|
const mode = hasEvaluators ? "evaluators" : "diff";
|
|
237997
|
-
|
|
237998
|
-
|
|
238544
|
+
if (!options.quiet) {
|
|
238545
|
+
info(`Running ${ui.bold(`"${workflowIdOrSlug}"`)} ${ui.dim(`(${workflowId}, ${exampleNames.length} example(s))`)}`);
|
|
238546
|
+
dim(hasEvaluators ? "Server example run; grading with configured evaluators." : "Server example run; no evaluators configured, grading by structural diff vs expected.");
|
|
238547
|
+
}
|
|
237999
238548
|
const evalBaseDir = resolveEvalBaseDir(dir);
|
|
238000
238549
|
const results = [];
|
|
238001
238550
|
for (const name of exampleNames) {
|
|
238002
|
-
|
|
238551
|
+
if (!options.quiet)
|
|
238552
|
+
process.stderr.write(`${ui.dim("→")} ${name}
|
|
238003
238553
|
`);
|
|
238004
238554
|
try {
|
|
238005
238555
|
const example = await resolveExample(client, workflowId, name);
|
|
@@ -238010,9 +238560,10 @@ async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workfl
|
|
|
238010
238560
|
const runId = started.id;
|
|
238011
238561
|
if (typeof runId !== "string")
|
|
238012
238562
|
throw new Error("Example run did not return a run id");
|
|
238013
|
-
|
|
238563
|
+
if (!options.quiet)
|
|
238564
|
+
process.stderr.write(` ${ui.dim(`run: ${runId}`)}
|
|
238014
238565
|
`);
|
|
238015
|
-
const run = await pollRunTerminal(client, runId);
|
|
238566
|
+
const run = await pollRunTerminal(client, runId, options.intervalMs ?? RUN_POLL_INTERVAL_MS, options.maxWaitMs ?? 5 * 60 * 1000);
|
|
238016
238567
|
const status = run.execution?.status ?? run.status;
|
|
238017
238568
|
const runOk = status === "completed";
|
|
238018
238569
|
const result = {
|
|
@@ -238038,14 +238589,17 @@ async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workfl
|
|
|
238038
238589
|
error: r.error ?? null
|
|
238039
238590
|
}));
|
|
238040
238591
|
} else {
|
|
238041
|
-
|
|
238592
|
+
if (!options.quiet) {
|
|
238593
|
+
warn(` ${name}: no evaluator score after ${EVAL_GRACE_MS / 1000}s; grading by diff.`);
|
|
238594
|
+
}
|
|
238042
238595
|
gradeByDiff(result, example.expected, runOutput(run));
|
|
238043
238596
|
}
|
|
238044
238597
|
} else if (runOk) {
|
|
238045
238598
|
gradeByDiff(result, example.expected, runOutput(run));
|
|
238046
238599
|
}
|
|
238047
238600
|
results.push(result);
|
|
238048
|
-
|
|
238601
|
+
if (!options.quiet)
|
|
238602
|
+
reportLine(result);
|
|
238049
238603
|
if (evalBaseDir) {
|
|
238050
238604
|
try {
|
|
238051
238605
|
await writeExecutionArtifacts(client, `${evalBaseDir}/${name}`, toArtifactPayload(run, runId));
|
|
@@ -238059,11 +238613,13 @@ async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workfl
|
|
|
238059
238613
|
mode: "none"
|
|
238060
238614
|
};
|
|
238061
238615
|
results.push(result);
|
|
238062
|
-
|
|
238616
|
+
if (!options.quiet)
|
|
238617
|
+
reportLine(result);
|
|
238063
238618
|
}
|
|
238064
238619
|
}
|
|
238065
238620
|
const summary = summarize(workflowIdOrSlug, mode, results);
|
|
238066
|
-
|
|
238621
|
+
if (!options.quiet)
|
|
238622
|
+
process.stderr.write(`
|
|
238067
238623
|
${formatEvalSummary(summary)}
|
|
238068
238624
|
`);
|
|
238069
238625
|
return summary;
|
|
@@ -238157,8 +238713,8 @@ function registerRunsCommands(program) {
|
|
|
238157
238713
|
});
|
|
238158
238714
|
const listRunsCmd = addJsonFlag(withPagination(withBaseUrl(runs.command("list [source]")), 50)).alias("ls").description("List runs across workflows and agents, optionally scoped to one source.").option("--type <type>", "Filter by run type: workflow|agent").option("--status <status>", "Filter by run status").option("--source-ref <ref>", "Filter agent runs by source ref").option("--batch-id <id>", "Filter eval/experiment runs by batch id").option("--example-id <id>", "Filter eval runs by example id").option("--example-id-contains <text>", "Filter eval runs by example id substring").option("--from <date>", "Created at or after this ISO timestamp or relative date").option("--to <date>", "Created before this ISO timestamp or relative date").option("--created-after <date>", "Alias for --from").option("--created-before <date>", "Alias for --to").option("--completed-after <date>", "Completed at or after this ISO timestamp or relative date").option("--completed-before <date>", "Completed before this ISO timestamp or relative date").option("--compact", "Render compact run rows").option("--sort <field>", "Sort field; only createdAt is supported").option("--order <asc|desc>", "Sort order; only desc is supported");
|
|
238159
238715
|
listRunsCmd.action(action(listRuns));
|
|
238160
|
-
addJsonFlag(withBaseUrl(runs.command("get <run-id>"))).description("Get one run.").option("--step <name>", "For workflow runs, show only this step (or comma-separated list)").option("--expand <sections>", "Comma-separated server expand sections for GET /api/v1/runs/:id: input, usage, execution, debug. Use execution for status/schemaValid; completed runs include top-level output/files/error.").option("--include <parts>", "Comma-separated local workflow step projection fields (error, duration, inputRef, …). Legacy expand names are mapped when possible; use --expand for server sections.", "review").action(action(getRun));
|
|
238161
|
-
addJsonFlag(withBaseUrl(runs.command("compare <reference-run-id> <run-id>"))).description("Compare one run against another
|
|
238716
|
+
addJsonFlag(withBaseUrl(runs.command("get <run-id>"))).description("Get one run.").option("--step <name>", "For workflow runs, show only this step (or comma-separated list)").option("--select <path>", "Print only a nested JSON value; implies JSON output (for example output.subjects or output.subjects[].name)").option("--expand <sections>", "Comma-separated server expand sections for GET /api/v1/runs/:id: input, usage, execution, debug. Use execution for status/schemaValid; completed runs include top-level output/files/error.").option("--include <parts>", "Comma-separated local workflow step projection fields (error, duration, inputRef, …). Legacy expand names are mapped when possible; use --expand for server sections.", "review").action(action(getRun));
|
|
238717
|
+
addJsonFlag(withBaseUrl(runs.command("compare <reference-run-id> <run-id>"))).description("Compare one run against another. Repeated workflow steps are matched by stable branch/iteration path.").option("--baseline", "Compare actual outputs from both runs instead of expected artifacts").option("--step <name-or-path>", "For workflow runs, restrict by step name or stable path").option("--out <dir>", "Write comparison artifacts to this directory").option("--normalize-dates", "Normalize YYYYMMDD and YYYY-MM-DD tokens in filenames/text").option("--fail-on-diff", "Exit 1 when comparison status is fail").action(action(compareRun));
|
|
238162
238718
|
addJsonFlag(withBaseUrl(runs.command("rerun <run-id>"))).description("Create a new run from a previous run's stored input snapshot.").option("--version <version>", "Version/source ref for the new run: latest (default), original, or an explicit ref (same grammar as eigenpal run)").option("--source-ref <ref>", "Alias for --version").option("--wait", "Poll until the rerun reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(rerunRun));
|
|
238163
238719
|
addJsonFlag(withBaseUrl(runs.command("promote <run-id>"))).description("Promote a run into a dataset example on the run's automation.").option("--name <name>", "Name for the created example").addHelpText("after", `
|
|
238164
238720
|
Examples
|
|
@@ -238174,7 +238730,7 @@ Use \`eigenpal runs reviews update\` first when you need to correct output.
|
|
|
238174
238730
|
});
|
|
238175
238731
|
addJsonFlag(withBaseUrl(artifacts.command("list <run-id>"))).description("List available run artifacts without downloading them.").action(action(listRunArtifacts));
|
|
238176
238732
|
withBaseUrl(artifacts.command("fetch <run-id>")).description("Download run artifacts by canonical artifact path.").option("--out <dir>", "Output directory").option("--include <parts>", "Comma-separated parts: output,input,metadata,issues,trace,lockfile,expected,all", "all").option("--path <path>", "Fetch one exact artifact path from `artifacts list`; repeatable", collectRepeated, []).option("--json", "Output a JSON summary of written artifacts").action(action(fetchRunArtifacts));
|
|
238177
|
-
|
|
238733
|
+
withBaseUrl(artifacts.command("inspect [run-id] [artifact-path]")).description("Inspect a run XLSX artifact as structured JSON (sheet names, headers, typed rows). Use --file for a local workbook.").option("--file <path>", "Inspect a local .xlsx/.xls file instead of downloading a run artifact").option("--path <path>", "Canonical artifact path from `artifacts list` (alias for positional artifact-path)").option("--sheet <names>", "Comma-separated sheet names or zero-based indices to include").option("--max-sheets <n>", "Maximum sheets to include (default 20)", intArg).option("--max-rows <n>", "Maximum data rows per sheet (default 500)", intArg).option("--max-cols <n>", "Maximum columns per sheet (default 50)", intArg).action(action(inspectRunArtifact));
|
|
238178
238734
|
withBaseUrl(runs.command("trace <run-id>")).description("Print raw trace.jsonl for a run, or write it with --out.").option("--out <file>", "Output file path").action(action(traceRun));
|
|
238179
238735
|
registerRunReviewCommands(runs);
|
|
238180
238736
|
registerRunExpectedCommands(runs);
|
|
@@ -238183,6 +238739,9 @@ Use \`eigenpal runs reviews update\` first when you need to correct output.
|
|
|
238183
238739
|
Examples:
|
|
238184
238740
|
$ eigenpal runs cancel exec_abc123
|
|
238185
238741
|
$ eigenpal runs cancel exec_abc123 --yes # required in scripts and agent terminals
|
|
238742
|
+
|
|
238743
|
+
Cancellation is asynchronous. Human output confirms that the request was
|
|
238744
|
+
accepted; use \`--json\` to inspect the server response.
|
|
238186
238745
|
`).action(action(cancelRun));
|
|
238187
238746
|
}
|
|
238188
238747
|
function registerRunReviewCommands(runs) {
|
|
@@ -238215,17 +238774,20 @@ Examples:
|
|
|
238215
238774
|
`).action(action(deleteRunExpected));
|
|
238216
238775
|
}
|
|
238217
238776
|
async function runExample(agentId, opts) {
|
|
238777
|
+
if (opts.failOnMismatch && !opts.wait) {
|
|
238778
|
+
throw new Error("--fail-on-mismatch requires --wait because the evaluator verdict is asynchronous");
|
|
238779
|
+
}
|
|
238218
238780
|
const client = buildClient(opts);
|
|
238219
238781
|
const started = await client.post(`/v1/automations/${encodeURIComponent(agentAutomationId(agentId))}/experiments`, {
|
|
238220
238782
|
examples: [opts.example],
|
|
238221
238783
|
...opts.sourceRef ? { sourceRef: opts.sourceRef } : {}
|
|
238222
238784
|
});
|
|
238223
238785
|
const runId = started.runs?.[0]?.id;
|
|
238224
|
-
if (opts.
|
|
238225
|
-
|
|
238786
|
+
if (opts.wait && !runId) {
|
|
238787
|
+
throw new Error("Example start response did not include a run id; cannot honor --wait");
|
|
238226
238788
|
}
|
|
238227
238789
|
if (opts.wait && runId) {
|
|
238228
|
-
let payload = await pollRun(client, runId, opts.interval, opts.maxWait);
|
|
238790
|
+
let payload = await pollRun(client, runId, opts.interval, opts.maxWait, opts.json);
|
|
238229
238791
|
const status = payload.execution?.status;
|
|
238230
238792
|
let wait = { kind: "no-evaluators" };
|
|
238231
238793
|
if (status === "completed") {
|
|
@@ -238355,43 +238917,46 @@ async function getRun(executionId, opts) {
|
|
|
238355
238917
|
if (isWorkflowRun(run)) {
|
|
238356
238918
|
const steps = filterWorkflowSteps(run.stepExecutions, opts.step);
|
|
238357
238919
|
const projectionIncludes = workflowStepProjectionIncludes(projectionIncludeSource(opts));
|
|
238358
|
-
|
|
238359
|
-
|
|
238920
|
+
const projectedRun = opts.step ? { ...run, stepExecutions: projectWorkflowSteps(steps, projectionIncludes) } : run;
|
|
238921
|
+
if (opts.select) {
|
|
238922
|
+
return printJson(selectJsonValue(projectedRun, opts.select));
|
|
238360
238923
|
}
|
|
238361
238924
|
if (opts.json) {
|
|
238362
|
-
return printJson(
|
|
238925
|
+
return printJson(projectedRun);
|
|
238363
238926
|
}
|
|
238364
|
-
console.log(renderFrame(toExecutionSnapshot(
|
|
238927
|
+
console.log(renderFrame(toExecutionSnapshot(projectedRun, executionId)));
|
|
238365
238928
|
process.stderr.write(`${ui.dim(`use --json for the full payload, then pipe to jq for field projection`)}
|
|
238366
238929
|
`);
|
|
238367
238930
|
return;
|
|
238368
238931
|
}
|
|
238932
|
+
if (opts.select)
|
|
238933
|
+
return printJson(selectJsonValue(run, opts.select));
|
|
238369
238934
|
renderRunPayload(run, opts);
|
|
238370
238935
|
}
|
|
238371
238936
|
function isWorkflowRun(run) {
|
|
238372
238937
|
return run.type === "workflow";
|
|
238373
238938
|
}
|
|
238374
|
-
function
|
|
238939
|
+
function objectOrNull2(value) {
|
|
238375
238940
|
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
238376
238941
|
}
|
|
238377
238942
|
function toLegacyRunView(run) {
|
|
238378
|
-
const timing =
|
|
238379
|
-
const source =
|
|
238380
|
-
const sourceGit =
|
|
238381
|
-
const trigger =
|
|
238382
|
-
const lineage =
|
|
238383
|
-
const execution =
|
|
238384
|
-
const legacyRetry =
|
|
238385
|
-
const retry =
|
|
238386
|
-
const nextRun =
|
|
238387
|
-
const evalInfo =
|
|
238388
|
-
const result =
|
|
238389
|
-
const input =
|
|
238390
|
-
const usage =
|
|
238391
|
-
const tokens =
|
|
238392
|
-
const executionFiles =
|
|
238393
|
-
const executionExpected =
|
|
238394
|
-
const debug =
|
|
238943
|
+
const timing = objectOrNull2(run.timing);
|
|
238944
|
+
const source = objectOrNull2(run.source);
|
|
238945
|
+
const sourceGit = objectOrNull2(source?.git);
|
|
238946
|
+
const trigger = objectOrNull2(run.trigger);
|
|
238947
|
+
const lineage = objectOrNull2(run.lineage);
|
|
238948
|
+
const execution = objectOrNull2(run.execution);
|
|
238949
|
+
const legacyRetry = objectOrNull2(lineage?.retry);
|
|
238950
|
+
const retry = objectOrNull2(execution?.retry) ?? legacyRetry;
|
|
238951
|
+
const nextRun = objectOrNull2(retry?.nextRun);
|
|
238952
|
+
const evalInfo = objectOrNull2(run.eval);
|
|
238953
|
+
const result = objectOrNull2(run.result);
|
|
238954
|
+
const input = objectOrNull2(run.input);
|
|
238955
|
+
const usage = objectOrNull2(run.usage);
|
|
238956
|
+
const tokens = objectOrNull2(usage?.tokens);
|
|
238957
|
+
const executionFiles = objectOrNull2(execution?.files);
|
|
238958
|
+
const executionExpected = objectOrNull2(execution?.expected);
|
|
238959
|
+
const debug = objectOrNull2(run.debug);
|
|
238395
238960
|
const hasGroupedShape = timing != null || source != null || result != null || run.output !== undefined;
|
|
238396
238961
|
if (!hasGroupedShape)
|
|
238397
238962
|
return run;
|
|
@@ -238448,8 +239013,8 @@ function toLegacyRunView(run) {
|
|
|
238448
239013
|
durationMs: usage.durationMs
|
|
238449
239014
|
} : undefined,
|
|
238450
239015
|
stepExecutions: execution?.steps,
|
|
238451
|
-
nextRetryId: nextRun?.id ??
|
|
238452
|
-
nextRetryStatus: nextRun?.status ??
|
|
239016
|
+
nextRetryId: nextRun?.id ?? objectOrNull2(execution?.nextRetry)?.id,
|
|
239017
|
+
nextRetryStatus: nextRun?.status ?? objectOrNull2(execution?.nextRetry)?.status,
|
|
238453
239018
|
review: execution?.review,
|
|
238454
239019
|
expected: executionExpected?.output,
|
|
238455
239020
|
expectedFiles: executionExpected?.files,
|
|
@@ -238508,7 +239073,7 @@ function projectWorkflowStep(step, includes) {
|
|
|
238508
239073
|
return out;
|
|
238509
239074
|
}
|
|
238510
239075
|
function toExecutionSnapshot(run, fallbackId) {
|
|
238511
|
-
const execution =
|
|
239076
|
+
const execution = objectOrNull2(run.execution);
|
|
238512
239077
|
return {
|
|
238513
239078
|
id: String(run.id ?? run.executionId ?? fallbackId),
|
|
238514
239079
|
status: String(run.status ?? execution?.status ?? "unknown"),
|
|
@@ -238556,8 +239121,11 @@ async function rerunRun(executionId, opts) {
|
|
|
238556
239121
|
const path = `/v1/runs/${encodeURIComponent(executionId)}/rerun${query ? `?${query}` : ""}`;
|
|
238557
239122
|
let payload = await client.post(path);
|
|
238558
239123
|
const rerunId = String(payload.id ?? "");
|
|
239124
|
+
if (opts.wait && !rerunId) {
|
|
239125
|
+
throw new Error("Rerun response did not include an id; cannot honor --wait");
|
|
239126
|
+
}
|
|
238559
239127
|
if (opts.wait && rerunId) {
|
|
238560
|
-
payload = await pollRun(client, rerunId, opts.interval, opts.maxWait);
|
|
239128
|
+
payload = await pollRun(client, rerunId, opts.interval, opts.maxWait, opts.json);
|
|
238561
239129
|
renderRunPayload(payload, opts);
|
|
238562
239130
|
setExitCodeForFailedTerminalRun(payload);
|
|
238563
239131
|
return;
|
|
@@ -238654,8 +239222,6 @@ async function inspectRunArtifact(runId, artifactPathArg, opts) {
|
|
|
238654
239222
|
...source.kind === "local" ? { file: source.path } : { runId: source.runId, artifactPath: source.artifactPath },
|
|
238655
239223
|
...inspect
|
|
238656
239224
|
};
|
|
238657
|
-
if (opts.json)
|
|
238658
|
-
return printJson(payload);
|
|
238659
239225
|
printJson(payload);
|
|
238660
239226
|
}
|
|
238661
239227
|
async function listRunArtifacts(executionId, opts) {
|
|
@@ -238849,21 +239415,21 @@ async function compareRun(referenceId, executionId, opts) {
|
|
|
238849
239415
|
function compareWorkflowRuns(referenceId, reference, targetId, target, stepFilter) {
|
|
238850
239416
|
const referenceSteps = workflowSteps(reference.stepExecutions);
|
|
238851
239417
|
const targetSteps = workflowSteps(target.stepExecutions);
|
|
238852
|
-
const
|
|
238853
|
-
|
|
238854
|
-
|
|
238855
|
-
stepNames.add(step.stepName);
|
|
238856
|
-
for (const step of targetSteps)
|
|
238857
|
-
if (step.stepName)
|
|
238858
|
-
stepNames.add(step.stepName);
|
|
239418
|
+
const referenceByPath = new Map(referenceSteps.map((step) => [workflowStepComparisonPath(step), step]));
|
|
239419
|
+
const targetByPath = new Map(targetSteps.map((step) => [workflowStepComparisonPath(step), step]));
|
|
239420
|
+
const stepPaths = new Set([...referenceByPath.keys(), ...targetByPath.keys()]);
|
|
238859
239421
|
const wanted = stepFilter ? new Set(stepFilter.split(",").map((step) => step.trim()).filter(Boolean)) : null;
|
|
238860
|
-
const steps = [...
|
|
238861
|
-
const referenceStep =
|
|
238862
|
-
const targetStep =
|
|
239422
|
+
const steps = [...stepPaths].sort().map((stepPath) => {
|
|
239423
|
+
const referenceStep = referenceByPath.get(stepPath);
|
|
239424
|
+
const targetStep = targetByPath.get(stepPath);
|
|
239425
|
+
const stepName = String(referenceStep?.stepName ?? targetStep?.stepName ?? stepPath);
|
|
239426
|
+
return { stepPath, stepName, referenceStep, targetStep };
|
|
239427
|
+
}).filter(({ stepName, stepPath }) => !wanted || wanted.has(stepName) || wanted.has(stepPath)).map(({ stepPath, stepName, referenceStep, targetStep }) => {
|
|
238863
239428
|
const referenceDuration = typeof referenceStep?.durationMs === "number" ? referenceStep.durationMs : null;
|
|
238864
239429
|
const targetDuration = typeof targetStep?.durationMs === "number" ? targetStep.durationMs : null;
|
|
238865
239430
|
return {
|
|
238866
239431
|
stepName,
|
|
239432
|
+
stepPath,
|
|
238867
239433
|
referenceStatus: String(referenceStep?.status ?? "-"),
|
|
238868
239434
|
targetStatus: String(targetStep?.status ?? "-"),
|
|
238869
239435
|
durationDeltaMs: referenceDuration != null && targetDuration != null ? targetDuration - referenceDuration : null,
|
|
@@ -238880,6 +239446,31 @@ function compareWorkflowRuns(referenceId, reference, targetId, target, stepFilte
|
|
|
238880
239446
|
finalOutputState
|
|
238881
239447
|
};
|
|
238882
239448
|
}
|
|
239449
|
+
function workflowStepComparisonPath(step) {
|
|
239450
|
+
const frames = Array.isArray(step.scopeStack) ? step.scopeStack : [];
|
|
239451
|
+
const parts = [];
|
|
239452
|
+
for (const frame of frames) {
|
|
239453
|
+
const type = String(frame.type ?? "");
|
|
239454
|
+
const stepName = String(frame.stepName ?? "");
|
|
239455
|
+
if (!stepName || type === "root")
|
|
239456
|
+
continue;
|
|
239457
|
+
if (type === "parallel-map-iteration" || type === "foreach-iteration") {
|
|
239458
|
+
const index = typeof frame.iterationIndex === "number" ? frame.iterationIndex : "?";
|
|
239459
|
+
parts.push(`${stepName}[${index}]`);
|
|
239460
|
+
} else if (type === "parallel-branch") {
|
|
239461
|
+
parts.push(`${stepName}.${String(frame.branchName ?? "?")}`);
|
|
239462
|
+
} else if (type === "if-then") {
|
|
239463
|
+
parts.push(`${stepName}.then`);
|
|
239464
|
+
} else if (type === "if-else") {
|
|
239465
|
+
parts.push(`${stepName}.else`);
|
|
239466
|
+
} else {
|
|
239467
|
+
parts.push(stepName);
|
|
239468
|
+
}
|
|
239469
|
+
}
|
|
239470
|
+
if (step.stepName)
|
|
239471
|
+
parts.push(step.stepName);
|
|
239472
|
+
return parts.join(".") || step.stepPath || step.stepName || step.scopeHash || "unknown-step";
|
|
239473
|
+
}
|
|
238883
239474
|
function renderWorkflowComparisonReport(report) {
|
|
238884
239475
|
console.log(`${ui.dim("A =")} ${ui.bold(report.comparedWithRunId)} ${ui.dim("B =")} ${ui.bold(report.runId)}`);
|
|
238885
239476
|
console.log("");
|
|
@@ -238887,7 +239478,7 @@ function renderWorkflowComparisonReport(report) {
|
|
|
238887
239478
|
console.log(ui.dim(`------------------------ --------- --------- ---------- ------`));
|
|
238888
239479
|
for (const step of report.steps) {
|
|
238889
239480
|
const delta = step.durationDeltaMs == null ? "-" : `${step.durationDeltaMs >= 0 ? "+" : ""}${step.durationDeltaMs}ms`;
|
|
238890
|
-
console.log(`${step.
|
|
239481
|
+
console.log(`${step.stepPath.padEnd(28)} ${padColored(colorStatus(step.referenceStatus), step.referenceStatus, 11)} ${padColored(colorStatus(step.targetStatus), step.targetStatus, 11)} ${delta.padEnd(12)} ${colorOutputState(step.outputState)}`);
|
|
238891
239482
|
}
|
|
238892
239483
|
console.log("");
|
|
238893
239484
|
console.log(`${ui.dim("final output:")} ${colorOutputState(report.finalOutputState)}`);
|
|
@@ -238940,9 +239531,9 @@ async function listRuns(source, opts) {
|
|
|
238940
239531
|
};
|
|
238941
239532
|
const payload = await client.get("/v1/runs", params);
|
|
238942
239533
|
const viewRuns = payload.runs.map(toLegacyRunView);
|
|
238943
|
-
const rows = opts.compact ? viewRuns.map(compactRunRow) : viewRuns;
|
|
238944
239534
|
if (opts.json)
|
|
238945
|
-
return printJson({ ...payload, runs:
|
|
239535
|
+
return printJson({ ...payload, runs: viewRuns });
|
|
239536
|
+
const rows = opts.compact ? viewRuns.map(compactRunRow) : viewRuns;
|
|
238946
239537
|
if (payload.scanLimited)
|
|
238947
239538
|
warn("Filtered run list scanned only the first matching window.");
|
|
238948
239539
|
if (payload.noResolvedAnchor)
|
|
@@ -239124,7 +239715,7 @@ async function deleteRunExpected(executionId, name, opts) {
|
|
|
239124
239715
|
async function watchRunCommand(executionId, opts) {
|
|
239125
239716
|
const client = buildClient(opts);
|
|
239126
239717
|
if (opts.json) {
|
|
239127
|
-
const payload = await pollRun(client, executionId, opts.interval, opts.maxWait);
|
|
239718
|
+
const payload = await pollRun(client, executionId, opts.interval, opts.maxWait, opts.json);
|
|
239128
239719
|
renderRunPayload(payload, opts);
|
|
239129
239720
|
setExitCodeForFailedTerminalRun(payload);
|
|
239130
239721
|
return;
|
|
@@ -239146,7 +239737,7 @@ Detached after ${opts.maxWait}s without reaching terminal status. Re-run \`eigen
|
|
|
239146
239737
|
`);
|
|
239147
239738
|
process.exit(2);
|
|
239148
239739
|
}
|
|
239149
|
-
if (result.final.status === "failed" || result.final.status === "cancelled") {
|
|
239740
|
+
if (result.final.status === "failed" || result.final.status === "cancelled" || result.final.status === "rejected") {
|
|
239150
239741
|
process.exitCode = 1;
|
|
239151
239742
|
}
|
|
239152
239743
|
}
|
|
@@ -239154,7 +239745,11 @@ async function cancelRun(executionId, opts) {
|
|
|
239154
239745
|
requireYesInNonInteractive(opts.yes, "Cancel run");
|
|
239155
239746
|
const client = buildClient(opts);
|
|
239156
239747
|
const payload = await client.post(`/v1/runs/${encodeURIComponent(executionId)}/cancel`, {});
|
|
239157
|
-
|
|
239748
|
+
if (opts.json) {
|
|
239749
|
+
printJson(payload);
|
|
239750
|
+
return;
|
|
239751
|
+
}
|
|
239752
|
+
success(`Cancellation requested for run ${executionId}.`);
|
|
239158
239753
|
}
|
|
239159
239754
|
async function downloadTraceText(client, executionId) {
|
|
239160
239755
|
const payload = await client.get(`/v1/runs/${encodeURIComponent(executionId)}/trace`);
|
|
@@ -239915,14 +240510,147 @@ async function completion(shell, program) {
|
|
|
239915
240510
|
`);
|
|
239916
240511
|
}
|
|
239917
240512
|
|
|
240513
|
+
// src/commands/docs.ts
|
|
240514
|
+
import { existsSync as existsSync13, readFileSync as readFileSync13 } from "node:fs";
|
|
240515
|
+
var docsCache = null;
|
|
240516
|
+
function bundledDocs() {
|
|
240517
|
+
if (docsCache)
|
|
240518
|
+
return docsCache;
|
|
240519
|
+
const candidates = [
|
|
240520
|
+
new URL("./docs-bundle.json", import.meta.url),
|
|
240521
|
+
new URL("../docs-bundle.json", import.meta.url)
|
|
240522
|
+
];
|
|
240523
|
+
const bundleUrl = candidates.find((candidate) => existsSync13(candidate));
|
|
240524
|
+
if (!bundleUrl) {
|
|
240525
|
+
throw new Error("Bundled documentation asset is missing. Reinstall the Eigenpal CLI.");
|
|
240526
|
+
}
|
|
240527
|
+
const parsed = JSON.parse(readFileSync13(bundleUrl, "utf8"));
|
|
240528
|
+
if (!Array.isArray(parsed.entries)) {
|
|
240529
|
+
throw new Error("Bundled documentation asset is invalid. Reinstall the Eigenpal CLI.");
|
|
240530
|
+
}
|
|
240531
|
+
docsCache = parsed.entries;
|
|
240532
|
+
return docsCache;
|
|
240533
|
+
}
|
|
240534
|
+
function listDocs(prefix, source) {
|
|
240535
|
+
const docs = bundledDocs();
|
|
240536
|
+
const normalizedPrefix = normalizeTopic(prefix ?? "");
|
|
240537
|
+
return docs.filter((doc) => (!normalizedPrefix || doc.topic.startsWith(normalizedPrefix)) && (!source || doc.source === source));
|
|
240538
|
+
}
|
|
240539
|
+
function resolveDoc(topic) {
|
|
240540
|
+
const docs = bundledDocs();
|
|
240541
|
+
const normalized = normalizeTopic(topic);
|
|
240542
|
+
const exact = docs.find((doc) => doc.topic === normalized);
|
|
240543
|
+
if (exact)
|
|
240544
|
+
return exact;
|
|
240545
|
+
const suffixMatches = docs.filter((doc) => doc.topic.endsWith(`/${normalized}`) || doc.topic === normalized);
|
|
240546
|
+
if (suffixMatches.length === 1)
|
|
240547
|
+
return suffixMatches[0];
|
|
240548
|
+
if (suffixMatches.length > 1) {
|
|
240549
|
+
throw new Error(`Documentation topic "${topic}" is ambiguous. Use one of: ${suffixMatches.map((doc) => doc.topic).join(", ")}`);
|
|
240550
|
+
}
|
|
240551
|
+
throw new Error(`Unknown documentation topic "${topic}". Run \`eigenpal docs search ${JSON.stringify(topic)}\` or \`eigenpal docs list\`.`);
|
|
240552
|
+
}
|
|
240553
|
+
function searchDocs(query, limit = 20) {
|
|
240554
|
+
const docs = bundledDocs();
|
|
240555
|
+
const terms = query.toLocaleLowerCase().split(/\s+/).filter(Boolean);
|
|
240556
|
+
if (terms.length === 0)
|
|
240557
|
+
throw new Error("Search query must not be empty.");
|
|
240558
|
+
return docs.map((doc) => {
|
|
240559
|
+
const title = `${doc.topic}
|
|
240560
|
+
${doc.title}
|
|
240561
|
+
${doc.description}`.toLocaleLowerCase();
|
|
240562
|
+
const content = doc.content.toLocaleLowerCase();
|
|
240563
|
+
if (!terms.every((term) => title.includes(term) || content.includes(term)))
|
|
240564
|
+
return null;
|
|
240565
|
+
const score = terms.reduce((sum, term) => sum + (title.includes(term) ? 10 : 0) + occurrences(content, term), 0);
|
|
240566
|
+
return { ...doc, snippet: matchingSnippet(doc.content, terms), score };
|
|
240567
|
+
}).filter((doc) => doc !== null).sort((a, b) => b.score - a.score || a.topic.localeCompare(b.topic)).slice(0, Math.max(1, limit)).map(({ score: _score, ...doc }) => doc);
|
|
240568
|
+
}
|
|
240569
|
+
function registerDocsCommands(program) {
|
|
240570
|
+
const command = program.command("docs").description("List, search, and read the complete release-matched Eigenpal documentation offline.").action(() => {
|
|
240571
|
+
process.stderr.write("`eigenpal docs` requires a subcommand: `list`, `search`, or `read`.\nRun `eigenpal docs --help` to see examples.\n");
|
|
240572
|
+
process.exit(2);
|
|
240573
|
+
});
|
|
240574
|
+
addJsonFlag(command.command("list [prefix]")).description("List bundled documentation topics, optionally under a path prefix.").option("--source <source>", "Filter by public, agent-reference, or api").action(action(async (prefix, opts) => {
|
|
240575
|
+
const results = listDocs(prefix, opts.source);
|
|
240576
|
+
if (opts.json) {
|
|
240577
|
+
printJson(results.map(({ content: _content, ...doc }) => doc));
|
|
240578
|
+
return;
|
|
240579
|
+
}
|
|
240580
|
+
console.log(table(results, [
|
|
240581
|
+
{ key: "topic", header: "TOPIC" },
|
|
240582
|
+
{ key: "source", header: "SOURCE" },
|
|
240583
|
+
{ key: "title", header: "TITLE" }
|
|
240584
|
+
]));
|
|
240585
|
+
}));
|
|
240586
|
+
addJsonFlag(command.command("search <query>")).description("Search every public page and detailed agent reference.").option("--limit <n>", "Maximum matches", intArg, 20).action(action(async (query, opts) => {
|
|
240587
|
+
const results = searchDocs(query, opts.limit);
|
|
240588
|
+
if (opts.json) {
|
|
240589
|
+
printJson(results.map(({ content: _content, ...doc }) => doc));
|
|
240590
|
+
return;
|
|
240591
|
+
}
|
|
240592
|
+
console.log(table(results.map((doc) => ({ ...doc, match: doc.snippet || doc.description })), [
|
|
240593
|
+
{ key: "topic", header: "TOPIC" },
|
|
240594
|
+
{ key: "source", header: "SOURCE" },
|
|
240595
|
+
{ key: "match", header: "MATCH" }
|
|
240596
|
+
]));
|
|
240597
|
+
}));
|
|
240598
|
+
addJsonFlag(command.command("read <topic>")).description("Print one bundled documentation topic to stdout.").action(action(async (topic, opts) => {
|
|
240599
|
+
const doc = resolveDoc(topic);
|
|
240600
|
+
if (opts.json) {
|
|
240601
|
+
printJson(doc);
|
|
240602
|
+
return;
|
|
240603
|
+
}
|
|
240604
|
+
process.stdout.write(doc.content.endsWith(`
|
|
240605
|
+
`) ? doc.content : `${doc.content}
|
|
240606
|
+
`);
|
|
240607
|
+
}));
|
|
240608
|
+
command.addHelpText("after", `
|
|
240609
|
+
Examples:
|
|
240610
|
+
$ eigenpal docs search "evaluator path syntax"
|
|
240611
|
+
$ eigenpal docs read reference/evaluators
|
|
240612
|
+
$ eigenpal docs read steps/transform/script
|
|
240613
|
+
$ eigenpal docs list api-reference --json
|
|
240614
|
+
|
|
240615
|
+
The bundle contains every public docs page, the OpenAPI specification, and the
|
|
240616
|
+
detailed agent references generated from Eigenpal's real schemas and CLI help.
|
|
240617
|
+
It ships with the CLI, needs no authentication, and matches that CLI release.
|
|
240618
|
+
`);
|
|
240619
|
+
}
|
|
240620
|
+
function normalizeTopic(value) {
|
|
240621
|
+
let topic = value.trim();
|
|
240622
|
+
try {
|
|
240623
|
+
const url = new URL(topic);
|
|
240624
|
+
topic = url.pathname;
|
|
240625
|
+
} catch {}
|
|
240626
|
+
return topic.replace(/^\/+/, "").replace(/^docs\//, "").replace(/\.(md|mdx|json)$/, "").replace(/\/+$/, "");
|
|
240627
|
+
}
|
|
240628
|
+
function occurrences(content, term) {
|
|
240629
|
+
let count = 0;
|
|
240630
|
+
let index = 0;
|
|
240631
|
+
while ((index = content.indexOf(term, index)) >= 0) {
|
|
240632
|
+
count += 1;
|
|
240633
|
+
index += term.length;
|
|
240634
|
+
}
|
|
240635
|
+
return Math.min(count, 20);
|
|
240636
|
+
}
|
|
240637
|
+
function matchingSnippet(content, terms) {
|
|
240638
|
+
const lines = content.split(/\r?\n/);
|
|
240639
|
+
const line = lines.find((candidate) => {
|
|
240640
|
+
const lower = candidate.toLocaleLowerCase();
|
|
240641
|
+
return terms.some((term) => lower.includes(term));
|
|
240642
|
+
}) ?? "";
|
|
240643
|
+
return line.replace(/^[#>*\s-]+/, "").replace(/[`[\]{}]/g, "").trim().slice(0, 140);
|
|
240644
|
+
}
|
|
240645
|
+
|
|
239918
240646
|
// src/commands/init/index.ts
|
|
239919
240647
|
init_dist4();
|
|
239920
240648
|
import {
|
|
239921
240649
|
copyFileSync,
|
|
239922
|
-
existsSync as
|
|
240650
|
+
existsSync as existsSync14,
|
|
239923
240651
|
mkdirSync as mkdirSync5,
|
|
239924
240652
|
readdirSync as readdirSync4,
|
|
239925
|
-
readFileSync as
|
|
240653
|
+
readFileSync as readFileSync14,
|
|
239926
240654
|
statSync as statSync4,
|
|
239927
240655
|
writeFileSync as writeFileSync7
|
|
239928
240656
|
} from "node:fs";
|
|
@@ -239940,10 +240668,10 @@ function isValidProjectName(name) {
|
|
|
239940
240668
|
function resolveTemplatesRoot() {
|
|
239941
240669
|
const here = dirname4(fileURLToPath6(import.meta.url));
|
|
239942
240670
|
const srcCandidate = resolve4(here, "..", "..", "templates");
|
|
239943
|
-
if (
|
|
240671
|
+
if (existsSync14(srcCandidate))
|
|
239944
240672
|
return srcCandidate;
|
|
239945
240673
|
const distCandidate = resolve4(here, "templates");
|
|
239946
|
-
if (
|
|
240674
|
+
if (existsSync14(distCandidate))
|
|
239947
240675
|
return distCandidate;
|
|
239948
240676
|
throw new Error(`Cannot locate bundled templates. Looked in:
|
|
239949
240677
|
${srcCandidate}
|
|
@@ -239955,7 +240683,7 @@ async function init(name, opts) {
|
|
|
239955
240683
|
}
|
|
239956
240684
|
const target = resolve4(process.cwd(), opts.dir ?? name);
|
|
239957
240685
|
assertSafeTargetDir(target, opts.dir);
|
|
239958
|
-
if (
|
|
240686
|
+
if (existsSync14(target) && readdirSync4(target).length > 0) {
|
|
239959
240687
|
throw new Error(`Target directory ${relative2(process.cwd(), target)} exists and is not empty.`);
|
|
239960
240688
|
}
|
|
239961
240689
|
let template;
|
|
@@ -239970,7 +240698,7 @@ async function init(name, opts) {
|
|
|
239970
240698
|
template = await promptForTemplate();
|
|
239971
240699
|
}
|
|
239972
240700
|
const sourceRoot = join7(resolveTemplatesRoot(), template);
|
|
239973
|
-
if (!
|
|
240701
|
+
if (!existsSync14(sourceRoot)) {
|
|
239974
240702
|
throw new Error(`Template directory missing on disk: ${sourceRoot}`);
|
|
239975
240703
|
}
|
|
239976
240704
|
const written = copyTemplate(sourceRoot, target, name);
|
|
@@ -240023,7 +240751,7 @@ function copyTemplate(src, dst, projectName) {
|
|
|
240023
240751
|
}
|
|
240024
240752
|
mkdirSync5(dirname4(targetPath), { recursive: true });
|
|
240025
240753
|
if (isTextFile(sourcePath)) {
|
|
240026
|
-
const text =
|
|
240754
|
+
const text = readFileSync14(sourcePath, "utf-8").replaceAll(PLACEHOLDER, projectName);
|
|
240027
240755
|
writeFileSync7(targetPath, text);
|
|
240028
240756
|
} else {
|
|
240029
240757
|
copyFileSync(sourcePath, targetPath);
|
|
@@ -240168,7 +240896,7 @@ function newIdempotencyKey() {
|
|
|
240168
240896
|
return randomUUID();
|
|
240169
240897
|
}
|
|
240170
240898
|
async function uploadReusableFile(client, input) {
|
|
240171
|
-
const contentType = input.mimeType
|
|
240899
|
+
const contentType = input.mimeType || guessMimeType(input.filename) || "application/octet-stream";
|
|
240172
240900
|
const idempotencyKey = input.idempotencyKey ?? newIdempotencyKey();
|
|
240173
240901
|
const negotiation = await client.post(apiPath("/files/uploads"), {
|
|
240174
240902
|
filename: input.filename,
|
|
@@ -240223,7 +240951,7 @@ async function buildPreparedRunRequest(client, input) {
|
|
|
240223
240951
|
fieldName: spec.fieldName,
|
|
240224
240952
|
content: data,
|
|
240225
240953
|
filename: path17.basename(spec.filePath),
|
|
240226
|
-
mimeType: guessMimeType(path17.basename(spec.filePath))
|
|
240954
|
+
mimeType: guessMimeType(path17.basename(spec.filePath)) || "application/octet-stream"
|
|
240227
240955
|
});
|
|
240228
240956
|
}
|
|
240229
240957
|
for (const file of input.files ?? []) {
|
|
@@ -240232,7 +240960,7 @@ async function buildPreparedRunRequest(client, input) {
|
|
|
240232
240960
|
fieldName: file.fieldName,
|
|
240233
240961
|
content,
|
|
240234
240962
|
filename: file.filename,
|
|
240235
|
-
mimeType: file.mimeType
|
|
240963
|
+
mimeType: file.mimeType || guessMimeType(file.filename) || "application/octet-stream"
|
|
240236
240964
|
});
|
|
240237
240965
|
}
|
|
240238
240966
|
const preUploadIndices = indicesRequiringPreUpload(localFiles.map((file) => ({ size: file.content.byteLength })));
|
|
@@ -240291,7 +241019,7 @@ function upgradeMixedFieldPreUpload(localFiles, preUploadIndices) {
|
|
|
240291
241019
|
}
|
|
240292
241020
|
function appendFilePart(form, fieldName, content, filename, mimeType) {
|
|
240293
241021
|
assertValidFileFieldName(fieldName);
|
|
240294
|
-
const type = mimeType
|
|
241022
|
+
const type = mimeType || guessMimeType(filename) || "application/octet-stream";
|
|
240295
241023
|
form.append(`files.${fieldName}`, new Blob([content], { type }), filename);
|
|
240296
241024
|
}
|
|
240297
241025
|
function assertValidFileFieldName(fieldName) {
|
|
@@ -240318,7 +241046,7 @@ function parseInputFileSpecs(inputFile) {
|
|
|
240318
241046
|
|
|
240319
241047
|
// src/lib/format-eigenpal.ts
|
|
240320
241048
|
import { spawnSync as spawnSync5 } from "child_process";
|
|
240321
|
-
import { existsSync as
|
|
241049
|
+
import { existsSync as existsSync15, readFileSync as readFileSync15 } from "fs";
|
|
240322
241050
|
import { join as join8, resolve as resolve5 } from "path";
|
|
240323
241051
|
var PRETTRC_NAMES = [
|
|
240324
241052
|
".prettierrc",
|
|
@@ -240332,14 +241060,14 @@ var PRETTRC_NAMES = [
|
|
|
240332
241060
|
];
|
|
240333
241061
|
function hasPrettierConfig(cwd) {
|
|
240334
241062
|
for (const name of PRETTRC_NAMES) {
|
|
240335
|
-
if (
|
|
241063
|
+
if (existsSync15(join8(cwd, name)))
|
|
240336
241064
|
return true;
|
|
240337
241065
|
}
|
|
240338
241066
|
try {
|
|
240339
241067
|
const pkgPath = join8(cwd, "package.json");
|
|
240340
|
-
if (!
|
|
241068
|
+
if (!existsSync15(pkgPath))
|
|
240341
241069
|
return false;
|
|
240342
|
-
const raw =
|
|
241070
|
+
const raw = readFileSync15(pkgPath, "utf-8");
|
|
240343
241071
|
const pkg = JSON.parse(raw);
|
|
240344
241072
|
return pkg != null && typeof pkg === "object" && "prettier" in pkg;
|
|
240345
241073
|
} catch {
|
|
@@ -240349,7 +241077,7 @@ function hasPrettierConfig(cwd) {
|
|
|
240349
241077
|
function formatEigenpalDirIfAvailable(dir) {
|
|
240350
241078
|
const cwd = process.cwd();
|
|
240351
241079
|
const absDir = resolve5(cwd, dir);
|
|
240352
|
-
if (!
|
|
241080
|
+
if (!existsSync15(absDir))
|
|
240353
241081
|
return;
|
|
240354
241082
|
if (!hasPrettierConfig(cwd))
|
|
240355
241083
|
return;
|
|
@@ -240369,9 +241097,13 @@ async function runSavedWorkflowExamples(workflow, examples, opts) {
|
|
|
240369
241097
|
const client = new ApiClient(config);
|
|
240370
241098
|
const workflowId = await resolveWorkflowId(client, workflow);
|
|
240371
241099
|
if (opts.version && !["latest", "current", "undefined"].includes(opts.version)) {
|
|
240372
|
-
|
|
241100
|
+
throw new Error(`--example cannot run workflow version @${opts.version}; publish or activate that version first, then use @latest`);
|
|
240373
241101
|
}
|
|
240374
|
-
const summary = await runWorkflowExamplesWithEval(client, config.dir, workflow, workflowId, examples
|
|
241102
|
+
const summary = await runWorkflowExamplesWithEval(client, config.dir, workflow, workflowId, examples, {
|
|
241103
|
+
quiet: opts.json,
|
|
241104
|
+
intervalMs: Math.max(0, opts.interval ?? 2) * 1000,
|
|
241105
|
+
maxWaitMs: Math.max(1, opts.maxWait ?? 1800) * 1000
|
|
241106
|
+
});
|
|
240375
241107
|
if (opts.json)
|
|
240376
241108
|
console.log(JSON.stringify(summary, null, 2));
|
|
240377
241109
|
if (summary.errored > 0)
|
|
@@ -240453,8 +241185,11 @@ async function runTarget(target, opts) {
|
|
|
240453
241185
|
}
|
|
240454
241186
|
const runId = String(payload.id ?? "");
|
|
240455
241187
|
let waitedForTerminalRun = false;
|
|
241188
|
+
if (opts.wait && !runId) {
|
|
241189
|
+
throw new Error("Run start response did not include an id; cannot honor --wait");
|
|
241190
|
+
}
|
|
240456
241191
|
if (opts.wait && runId) {
|
|
240457
|
-
payload = await pollRun(client, runId, opts.interval, opts.maxWait);
|
|
241192
|
+
payload = await pollRun(client, runId, opts.interval, opts.maxWait, opts.json);
|
|
240458
241193
|
waitedForTerminalRun = true;
|
|
240459
241194
|
}
|
|
240460
241195
|
renderRunPayload(payload, opts);
|
|
@@ -240469,10 +241204,10 @@ async function rerunTarget(runId, opts) {
|
|
|
240469
241204
|
init_dist4();
|
|
240470
241205
|
import { createHash as createHash5 } from "node:crypto";
|
|
240471
241206
|
import {
|
|
240472
|
-
existsSync as
|
|
241207
|
+
existsSync as existsSync16,
|
|
240473
241208
|
lstatSync,
|
|
240474
241209
|
mkdirSync as mkdirSync6,
|
|
240475
|
-
readFileSync as
|
|
241210
|
+
readFileSync as readFileSync16,
|
|
240476
241211
|
readdirSync as readdirSync5,
|
|
240477
241212
|
rmSync as rmSync2,
|
|
240478
241213
|
rmdirSync,
|
|
@@ -240555,10 +241290,10 @@ var MANIFEST_NAME = ".eigenpal-manifest.json";
|
|
|
240555
241290
|
function resolveSkillSource() {
|
|
240556
241291
|
const here = dirname5(fileURLToPath7(import.meta.url));
|
|
240557
241292
|
const srcCandidate = resolve6(here, "..", "..", "skill");
|
|
240558
|
-
if (
|
|
241293
|
+
if (existsSync16(srcCandidate))
|
|
240559
241294
|
return srcCandidate;
|
|
240560
241295
|
const distCandidate = resolve6(here, "skill");
|
|
240561
|
-
if (
|
|
241296
|
+
if (existsSync16(distCandidate))
|
|
240562
241297
|
return distCandidate;
|
|
240563
241298
|
throw new Error(`Cannot locate bundled skill content. Looked in:
|
|
240564
241299
|
${srcCandidate}
|
|
@@ -240595,15 +241330,15 @@ async function installSkill(opts) {
|
|
|
240595
241330
|
for (const relativePath of sourceFiles) {
|
|
240596
241331
|
const sourcePath = join10(source, relativePath);
|
|
240597
241332
|
const targetPath = join10(target, relativePath);
|
|
240598
|
-
const sourceBytes = templateSourceFile(relativePath,
|
|
241333
|
+
const sourceBytes = templateSourceFile(relativePath, readFileSync16(sourcePath), cliVersion);
|
|
240599
241334
|
const sourceHash = sha256(sourceBytes);
|
|
240600
|
-
if (!
|
|
241335
|
+
if (!existsSync16(targetPath)) {
|
|
240601
241336
|
mkdirSync6(dirname5(targetPath), { recursive: true });
|
|
240602
241337
|
writeFileSync8(targetPath, sourceBytes);
|
|
240603
241338
|
written += 1;
|
|
240604
241339
|
continue;
|
|
240605
241340
|
}
|
|
240606
|
-
const targetBytes =
|
|
241341
|
+
const targetBytes = readFileSync16(targetPath);
|
|
240607
241342
|
const targetHash = sha256(targetBytes);
|
|
240608
241343
|
if (targetHash === sourceHash) {
|
|
240609
241344
|
skipped += 1;
|
|
@@ -240635,9 +241370,9 @@ async function installSkill(opts) {
|
|
|
240635
241370
|
const manifestEntries = [];
|
|
240636
241371
|
for (const relativePath of sourceFiles) {
|
|
240637
241372
|
const targetPath = join10(target, relativePath);
|
|
240638
|
-
if (!
|
|
241373
|
+
if (!existsSync16(targetPath))
|
|
240639
241374
|
continue;
|
|
240640
|
-
manifestEntries.push({ path: relativePath, sha256: sha256(
|
|
241375
|
+
manifestEntries.push({ path: relativePath, sha256: sha256(readFileSync16(targetPath)) });
|
|
240641
241376
|
}
|
|
240642
241377
|
const manifest = {
|
|
240643
241378
|
cliVersion,
|
|
@@ -240678,9 +241413,9 @@ async function uninstallSkill(opts) {
|
|
|
240678
241413
|
console.error(`Refusing to touch ${path}: manifest path escapes ${targetReal}`);
|
|
240679
241414
|
continue;
|
|
240680
241415
|
}
|
|
240681
|
-
if (!
|
|
241416
|
+
if (!existsSync16(path))
|
|
240682
241417
|
continue;
|
|
240683
|
-
const currentHash = sha256(
|
|
241418
|
+
const currentHash = sha256(readFileSync16(path));
|
|
240684
241419
|
if (currentHash !== entry.sha256 && !opts.yes) {
|
|
240685
241420
|
kept += 1;
|
|
240686
241421
|
continue;
|
|
@@ -240777,6 +241512,9 @@ async function uninstallSkillTools(opts) {
|
|
|
240777
241512
|
await uninstallSkill({ target: opts.target, yes: opts.yes });
|
|
240778
241513
|
return;
|
|
240779
241514
|
}
|
|
241515
|
+
if (!opts.all && !hasToolIds && (opts.yes || !isInteractiveStdout())) {
|
|
241516
|
+
throw new Error("Non-interactive: pass tool ids (e.g. `skill uninstall claude`) or `--all`.");
|
|
241517
|
+
}
|
|
240780
241518
|
const cwd = process.cwd();
|
|
240781
241519
|
const installedTools = TOOLS.filter((t) => isInstalled(cwd, t));
|
|
240782
241520
|
if (installedTools.length === 0) {
|
|
@@ -240795,8 +241533,6 @@ async function uninstallSkillTools(opts) {
|
|
|
240795
241533
|
info(`Not installed (skipping): ${notInstalled.join(", ")}`);
|
|
240796
241534
|
targets = targets.filter((t) => installedIds.has(t.id));
|
|
240797
241535
|
}
|
|
240798
|
-
} else if (opts.yes || !isInteractiveStdout()) {
|
|
240799
|
-
throw new Error("Non-interactive: pass tool ids (e.g. `skill uninstall claude`) or `--all`.");
|
|
240800
241536
|
} else {
|
|
240801
241537
|
const picked = await ve({
|
|
240802
241538
|
message: "Select tools to uninstall (only installed tools shown)",
|
|
@@ -240880,10 +241616,10 @@ function hintFor(tool, installed, detected) {
|
|
|
240880
241616
|
return tool.relativePath;
|
|
240881
241617
|
}
|
|
240882
241618
|
function isInstalled(cwd, tool) {
|
|
240883
|
-
return
|
|
241619
|
+
return existsSync16(join10(cwd, tool.relativePath, MANIFEST_NAME));
|
|
240884
241620
|
}
|
|
240885
241621
|
function isProjectUsing(cwd, tool) {
|
|
240886
|
-
return tool.detectPaths.some((p) =>
|
|
241622
|
+
return tool.detectPaths.some((p) => existsSync16(join10(cwd, p)));
|
|
240887
241623
|
}
|
|
240888
241624
|
function listFiles(root) {
|
|
240889
241625
|
const out = [];
|
|
@@ -240907,10 +241643,10 @@ function listFiles(root) {
|
|
|
240907
241643
|
}
|
|
240908
241644
|
function readManifest(target) {
|
|
240909
241645
|
const path = join10(target, MANIFEST_NAME);
|
|
240910
|
-
if (!
|
|
241646
|
+
if (!existsSync16(path))
|
|
240911
241647
|
return null;
|
|
240912
241648
|
try {
|
|
240913
|
-
return JSON.parse(
|
|
241649
|
+
return JSON.parse(readFileSync16(path, "utf-8"));
|
|
240914
241650
|
} catch {
|
|
240915
241651
|
return null;
|
|
240916
241652
|
}
|
|
@@ -240924,8 +241660,8 @@ function readCliVersion() {
|
|
|
240924
241660
|
resolve6(here, "..", "package.json")
|
|
240925
241661
|
];
|
|
240926
241662
|
for (const candidate of candidates) {
|
|
240927
|
-
if (
|
|
240928
|
-
const pkg = JSON.parse(
|
|
241663
|
+
if (existsSync16(candidate)) {
|
|
241664
|
+
const pkg = JSON.parse(readFileSync16(candidate, "utf-8"));
|
|
240929
241665
|
if (pkg.name && pkg.name !== "@eigenpal/cli")
|
|
240930
241666
|
continue;
|
|
240931
241667
|
if (pkg.version)
|
|
@@ -240936,7 +241672,7 @@ function readCliVersion() {
|
|
|
240936
241672
|
return "unknown";
|
|
240937
241673
|
}
|
|
240938
241674
|
function pruneEmptyDirs(root) {
|
|
240939
|
-
if (!
|
|
241675
|
+
if (!existsSync16(root))
|
|
240940
241676
|
return;
|
|
240941
241677
|
const stack = [];
|
|
240942
241678
|
const collect = (dir) => {
|
|
@@ -241726,12 +242462,12 @@ function looksLikeFreeform(prop) {
|
|
|
241726
242462
|
return prop.pattern !== undefined || prop.format !== undefined;
|
|
241727
242463
|
}
|
|
241728
242464
|
// src/commands/workflow/index.ts
|
|
241729
|
-
import { existsSync as
|
|
242465
|
+
import { existsSync as existsSync22, promises as fs20 } from "node:fs";
|
|
241730
242466
|
import path18 from "node:path";
|
|
241731
242467
|
|
|
241732
242468
|
// src/lib/local-templates.ts
|
|
241733
242469
|
init_src();
|
|
241734
|
-
import { existsSync as
|
|
242470
|
+
import { existsSync as existsSync17, promises as fs17, realpathSync as realpathSync3, statSync as statSync5 } from "node:fs";
|
|
241735
242471
|
import { basename, dirname as dirname6, isAbsolute, relative as relative4, resolve as resolve7, sep } from "node:path";
|
|
241736
242472
|
|
|
241737
242473
|
// src/lib/office-template.ts
|
|
@@ -241974,7 +242710,7 @@ function walkTemplateSteps(steps, visit) {
|
|
|
241974
242710
|
}
|
|
241975
242711
|
function resolveWorkflowProjectRoot(workflowFile) {
|
|
241976
242712
|
const abs = resolve7(workflowFile);
|
|
241977
|
-
if (!
|
|
242713
|
+
if (!existsSync17(abs)) {
|
|
241978
242714
|
throw new Error(`Workflow file not found: ${abs}`);
|
|
241979
242715
|
}
|
|
241980
242716
|
return dirname6(realpathSync3(abs));
|
|
@@ -241997,7 +242733,7 @@ function resolveWorkflowTemplatePath(workflowFile, templatePath, opts = {}) {
|
|
|
241997
242733
|
if (!opts.allowExternal) {
|
|
241998
242734
|
assertInsideProject(projectRoot, templatePath, candidate);
|
|
241999
242735
|
}
|
|
242000
|
-
if (!
|
|
242736
|
+
if (!existsSync17(candidate)) {
|
|
242001
242737
|
throw new Error(`Local template not found: ${candidate}`);
|
|
242002
242738
|
}
|
|
242003
242739
|
const real = realpathSync3(candidate);
|
|
@@ -242179,11 +242915,11 @@ function rewriteLocalTemplateYaml(yamlText, resolved) {
|
|
|
242179
242915
|
}
|
|
242180
242916
|
|
|
242181
242917
|
// src/commands/workflow/clear.ts
|
|
242182
|
-
import { existsSync as
|
|
242918
|
+
import { existsSync as existsSync18, readdirSync as readdirSync6, rmSync as rmSync3, statSync as statSync6 } from "fs";
|
|
242183
242919
|
import { basename as basename2, join as join11 } from "path";
|
|
242184
242920
|
var EMPTY_REPORT = { removed: 0, kept: 0, touched: 0 };
|
|
242185
242921
|
function getDirectoryEntries(path) {
|
|
242186
|
-
if (!
|
|
242922
|
+
if (!existsSync18(path))
|
|
242187
242923
|
return [];
|
|
242188
242924
|
return readdirSync6(path).filter((entry) => statSync6(join11(path, entry)).isDirectory()).sort();
|
|
242189
242925
|
}
|
|
@@ -242204,7 +242940,7 @@ function removeDirEntriesWithRetention(path, removeAll) {
|
|
|
242204
242940
|
return { removed, kept: keepSet.size, touched: entries.length };
|
|
242205
242941
|
}
|
|
242206
242942
|
function removeJudgeSummaryFilesWithRetention(exampleDir, removeAll) {
|
|
242207
|
-
if (!
|
|
242943
|
+
if (!existsSync18(exampleDir))
|
|
242208
242944
|
return EMPTY_REPORT;
|
|
242209
242945
|
const stampByFile = new Map;
|
|
242210
242946
|
for (const fileName of readdirSync6(exampleDir).sort()) {
|
|
@@ -242276,7 +243012,7 @@ async function clearEvalOutputs(dir, exampleNames = [], options = {}) {
|
|
|
242276
243012
|
runSection({
|
|
242277
243013
|
exampleDirs,
|
|
242278
243014
|
clean: (exampleDir) => removeDirEntriesWithRetention(join11(exampleDir, "executions"), removeAll),
|
|
242279
|
-
shouldCount: (exampleDir) =>
|
|
243015
|
+
shouldCount: (exampleDir) => existsSync18(join11(exampleDir, "executions")),
|
|
242280
243016
|
successLabel: "Execution cleanup complete",
|
|
242281
243017
|
scannedLabel: "examples scanned:",
|
|
242282
243018
|
seenLabel: "runs seen: ",
|
|
@@ -242288,7 +243024,7 @@ async function clearEvalOutputs(dir, exampleNames = [], options = {}) {
|
|
|
242288
243024
|
runSection({
|
|
242289
243025
|
exampleDirs,
|
|
242290
243026
|
clean: (exampleDir) => removeJudgeSummaryFilesWithRetention(exampleDir, removeAll),
|
|
242291
|
-
shouldCount: (exampleDir) =>
|
|
243027
|
+
shouldCount: (exampleDir) => existsSync18(exampleDir),
|
|
242292
243028
|
successLabel: "Judge summary cleanup complete",
|
|
242293
243029
|
scannedLabel: "examples scanned: ",
|
|
242294
243030
|
seenLabel: "summary files seen: ",
|
|
@@ -242401,7 +243137,7 @@ function registerStepExecCommands(workflow) {
|
|
|
242401
243137
|
|
|
242402
243138
|
// src/commands/workflow/templates.ts
|
|
242403
243139
|
init_src();
|
|
242404
|
-
import { existsSync as
|
|
243140
|
+
import { existsSync as existsSync19, promises as fs19 } from "node:fs";
|
|
242405
243141
|
import { basename as basename3, resolve as resolve8 } from "node:path";
|
|
242406
243142
|
function buildClient2(opts) {
|
|
242407
243143
|
const config = resolveConfig(opts);
|
|
@@ -242582,7 +243318,7 @@ function registerTemplateCommands(parent) {
|
|
|
242582
243318
|
filename = downloaded.filename;
|
|
242583
243319
|
} else {
|
|
242584
243320
|
const filePath = resolve8(templateArg);
|
|
242585
|
-
if (!
|
|
243321
|
+
if (!existsSync19(filePath)) {
|
|
242586
243322
|
throw new Error(`Template file not found: ${filePath}. Pass a local .docx/.xlsx path or a tmpl_… id.`);
|
|
242587
243323
|
}
|
|
242588
243324
|
bytes = await fs19.readFile(filePath);
|
|
@@ -242624,12 +243360,12 @@ function registerTemplateCommands(parent) {
|
|
|
242624
243360
|
|
|
242625
243361
|
// src/commands/workflow/validation.ts
|
|
242626
243362
|
init_src();
|
|
242627
|
-
import { existsSync as
|
|
243363
|
+
import { existsSync as existsSync21, readdirSync as readdirSync7, readFileSync as readFileSync18, statSync as statSync7 } from "fs";
|
|
242628
243364
|
import { join as join12, resolve as resolve9 } from "path";
|
|
242629
243365
|
|
|
242630
243366
|
// src/lib/template-diagnostics.ts
|
|
242631
243367
|
init_src();
|
|
242632
|
-
import { existsSync as
|
|
243368
|
+
import { existsSync as existsSync20, readFileSync as readFileSync17 } from "node:fs";
|
|
242633
243369
|
function walkSteps(steps, pathPrefix, visit) {
|
|
242634
243370
|
if (!Array.isArray(steps))
|
|
242635
243371
|
return;
|
|
@@ -242754,7 +243490,7 @@ async function diagnoseTemplateSteps(args) {
|
|
|
242754
243490
|
});
|
|
242755
243491
|
continue;
|
|
242756
243492
|
}
|
|
242757
|
-
if (!
|
|
243493
|
+
if (!existsSync20(absolute)) {
|
|
242758
243494
|
diagnostics.push({
|
|
242759
243495
|
field: `${info.fieldPath}.template`,
|
|
242760
243496
|
message: `File not found: ${absolute}`,
|
|
@@ -242763,7 +243499,7 @@ async function diagnoseTemplateSteps(args) {
|
|
|
242763
243499
|
continue;
|
|
242764
243500
|
}
|
|
242765
243501
|
try {
|
|
242766
|
-
const bytes =
|
|
243502
|
+
const bytes = readFileSync17(absolute);
|
|
242767
243503
|
const inspection = inspectOfficeTemplateBytes(bytes, absolute);
|
|
242768
243504
|
diagnostics.push(...formatDiagnostics(info, inspection));
|
|
242769
243505
|
} catch (err) {
|
|
@@ -242877,8 +243613,17 @@ function countInvokeWorkflowSteps(steps) {
|
|
|
242877
243613
|
if (step.type === "action.invoke-workflow")
|
|
242878
243614
|
count++;
|
|
242879
243615
|
count += countInvokeWorkflowSteps(step.steps);
|
|
243616
|
+
count += countInvokeWorkflowSteps(step.body);
|
|
242880
243617
|
count += countInvokeWorkflowSteps(step.then);
|
|
242881
243618
|
count += countInvokeWorkflowSteps(step.else);
|
|
243619
|
+
count += countInvokeWorkflowSteps(step.default);
|
|
243620
|
+
if (Array.isArray(step.cases)) {
|
|
243621
|
+
for (const switchCase of step.cases) {
|
|
243622
|
+
if (switchCase && typeof switchCase === "object") {
|
|
243623
|
+
count += countInvokeWorkflowSteps(switchCase.steps);
|
|
243624
|
+
}
|
|
243625
|
+
}
|
|
243626
|
+
}
|
|
242882
243627
|
if (Array.isArray(step.branches)) {
|
|
242883
243628
|
for (const branch of step.branches) {
|
|
242884
243629
|
if (branch && typeof branch === "object") {
|
|
@@ -242890,7 +243635,7 @@ function countInvokeWorkflowSteps(steps) {
|
|
|
242890
243635
|
return count;
|
|
242891
243636
|
}
|
|
242892
243637
|
function validateWorkflowYaml(path) {
|
|
242893
|
-
if (!
|
|
243638
|
+
if (!existsSync21(path)) {
|
|
242894
243639
|
return {
|
|
242895
243640
|
issues: [{ field: path, message: "File not found." }],
|
|
242896
243641
|
warnings: [],
|
|
@@ -242899,7 +243644,7 @@ function validateWorkflowYaml(path) {
|
|
|
242899
243644
|
steps: []
|
|
242900
243645
|
};
|
|
242901
243646
|
}
|
|
242902
|
-
const text =
|
|
243647
|
+
const text = readFileSync18(path, "utf-8");
|
|
242903
243648
|
try {
|
|
242904
243649
|
const definition = parseWorkflow(text);
|
|
242905
243650
|
let warnings = [];
|
|
@@ -243000,10 +243745,10 @@ function printSchemaQualityWarnings(path, warnings) {
|
|
|
243000
243745
|
}
|
|
243001
243746
|
}
|
|
243002
243747
|
function validateEvaluatorsYaml(path) {
|
|
243003
|
-
if (!
|
|
243748
|
+
if (!existsSync21(path)) {
|
|
243004
243749
|
return [{ field: path, message: "File not found." }];
|
|
243005
243750
|
}
|
|
243006
|
-
const text =
|
|
243751
|
+
const text = readFileSync18(path, "utf-8");
|
|
243007
243752
|
let parsed;
|
|
243008
243753
|
try {
|
|
243009
243754
|
parsed = $parse(text);
|
|
@@ -243020,17 +243765,17 @@ function validateEvaluatorsYaml(path) {
|
|
|
243020
243765
|
}
|
|
243021
243766
|
function validateDatasetFolder(root) {
|
|
243022
243767
|
const issues = [];
|
|
243023
|
-
if (!
|
|
243768
|
+
if (!existsSync21(root)) {
|
|
243024
243769
|
return [{ field: root, message: "Folder not found." }];
|
|
243025
243770
|
}
|
|
243026
|
-
if (
|
|
243771
|
+
if (existsSync21(join12(root, "manifest.json"))) {
|
|
243027
243772
|
issues.push({
|
|
243028
243773
|
field: "manifest.json",
|
|
243029
243774
|
message: "Top-level manifest.json is the legacy format. Re-export the dataset using the folder convention."
|
|
243030
243775
|
});
|
|
243031
243776
|
}
|
|
243032
243777
|
const examplesDir = join12(root, "examples");
|
|
243033
|
-
if (!
|
|
243778
|
+
if (!existsSync21(examplesDir)) {
|
|
243034
243779
|
issues.push({ field: "examples/", message: "No examples/ directory found at dataset root." });
|
|
243035
243780
|
return issues;
|
|
243036
243781
|
}
|
|
@@ -243046,7 +243791,7 @@ function validateDatasetFolder(root) {
|
|
|
243046
243791
|
continue;
|
|
243047
243792
|
}
|
|
243048
243793
|
const inputJsonPath = join12(exampleFolder, "input.json");
|
|
243049
|
-
const hasInputJson =
|
|
243794
|
+
const hasInputJson = existsSync21(inputJsonPath);
|
|
243050
243795
|
if (!hasInputJson) {
|
|
243051
243796
|
issues.push({
|
|
243052
243797
|
field: `examples/${exampleName}/input.json`,
|
|
@@ -243056,7 +243801,7 @@ function validateDatasetFolder(root) {
|
|
|
243056
243801
|
}
|
|
243057
243802
|
let args;
|
|
243058
243803
|
try {
|
|
243059
|
-
const parsed = JSON.parse(
|
|
243804
|
+
const parsed = JSON.parse(readFileSync18(inputJsonPath, "utf-8"));
|
|
243060
243805
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
243061
243806
|
issues.push({
|
|
243062
243807
|
field: `examples/${exampleName}/input.json`,
|
|
@@ -243073,7 +243818,7 @@ function validateDatasetFolder(root) {
|
|
|
243073
243818
|
continue;
|
|
243074
243819
|
}
|
|
243075
243820
|
const inputDir = join12(exampleFolder, "input");
|
|
243076
|
-
if (
|
|
243821
|
+
if (existsSync21(join12(inputDir, "arguments.json"))) {
|
|
243077
243822
|
issues.push({
|
|
243078
243823
|
field: `examples/${exampleName}/input/arguments.json`,
|
|
243079
243824
|
message: "Legacy input/arguments.json is no longer accepted. Use input.json instead."
|
|
@@ -243090,7 +243835,7 @@ function validateDatasetFolder(root) {
|
|
|
243090
243835
|
}
|
|
243091
243836
|
const expectedDir = join12(exampleFolder, "expected");
|
|
243092
243837
|
const expectedJsonPath = join12(exampleFolder, "expected.json");
|
|
243093
|
-
if (
|
|
243838
|
+
if (existsSync21(expectedDir)) {
|
|
243094
243839
|
for (const entry of readdirSync7(expectedDir)) {
|
|
243095
243840
|
if (entry === "output.json" || entry === "error.json") {
|
|
243096
243841
|
issues.push({
|
|
@@ -243126,10 +243871,10 @@ function validateDatasetFolder(root) {
|
|
|
243126
243871
|
}
|
|
243127
243872
|
}
|
|
243128
243873
|
}
|
|
243129
|
-
if (
|
|
243874
|
+
if (existsSync21(expectedJsonPath)) {
|
|
243130
243875
|
let parsed;
|
|
243131
243876
|
try {
|
|
243132
|
-
parsed = JSON.parse(
|
|
243877
|
+
parsed = JSON.parse(readFileSync18(expectedJsonPath, "utf-8"));
|
|
243133
243878
|
} catch {
|
|
243134
243879
|
issues.push({
|
|
243135
243880
|
field: `examples/${exampleName}/expected.json`,
|
|
@@ -243208,7 +243953,7 @@ function validateFileRefs2(args) {
|
|
|
243208
243953
|
});
|
|
243209
243954
|
continue;
|
|
243210
243955
|
}
|
|
243211
|
-
if (!
|
|
243956
|
+
if (!existsSync21(join12(args.filesRoot, relative))) {
|
|
243212
243957
|
issues.push({
|
|
243213
243958
|
field: `${args.fieldPrefix}:${ref.jsonPath}`,
|
|
243214
243959
|
message: `Referenced file does not exist: ${ref.path}.`
|
|
@@ -243238,16 +243983,55 @@ async function onlineInvokeIssues(yamlText, opts) {
|
|
|
243238
243983
|
const res = await client.post("/api/workflows/validate", { yaml: yamlText });
|
|
243239
243984
|
return res.issues ?? [];
|
|
243240
243985
|
}
|
|
243986
|
+
function discoverWorkflowProjectRoots(rootPath) {
|
|
243987
|
+
const root = resolve9(rootPath);
|
|
243988
|
+
if (existsSync21(join12(root, "workflow.yaml")))
|
|
243989
|
+
return [root];
|
|
243990
|
+
const candidates = [
|
|
243991
|
+
join12(root, "eigenpal", "workflows"),
|
|
243992
|
+
join12(root, "workflows"),
|
|
243993
|
+
root.endsWith(join12("eigenpal", "workflows")) ? root : ""
|
|
243994
|
+
].filter(Boolean);
|
|
243995
|
+
for (const parent of candidates) {
|
|
243996
|
+
if (!existsSync21(parent) || !statSync7(parent).isDirectory())
|
|
243997
|
+
continue;
|
|
243998
|
+
const projects = readdirSync7(parent, { withFileTypes: true }).filter((entry) => entry.isDirectory() && !entry.isSymbolicLink() && existsSync21(join12(parent, entry.name, "workflow.yaml"))).map((entry) => join12(parent, entry.name)).sort();
|
|
243999
|
+
if (projects.length > 0)
|
|
244000
|
+
return projects;
|
|
244001
|
+
}
|
|
244002
|
+
return [];
|
|
244003
|
+
}
|
|
244004
|
+
async function validateProjectRoot(root, opts) {
|
|
244005
|
+
const wfFile = join12(root, "workflow.yaml");
|
|
244006
|
+
const wfResult = validateWorkflowYaml(wfFile);
|
|
244007
|
+
const wfPath = relPath(root, wfFile);
|
|
244008
|
+
const wfOk = printIssues("workflow.yaml", wfPath, wfResult.issues);
|
|
244009
|
+
if (wfOk) {
|
|
244010
|
+
printSchemaQualityWarnings(wfPath, wfResult.warnings);
|
|
244011
|
+
printInvokeOnlineHint(wfResult.invokeStepCount, wfResult.templateStepCount, opts.online);
|
|
244012
|
+
}
|
|
244013
|
+
const evOk = printIssues("evaluators.yaml", relPath(root, join12(root, "evaluators.yaml")), validateEvaluatorsYaml(join12(root, "evaluators.yaml")));
|
|
244014
|
+
const dsOk = printIssues("dataset/", relPath(root, join12(root, "dataset")), validateDatasetFolder(join12(root, "dataset")));
|
|
244015
|
+
const templateOk = wfOk && existsSync21(wfFile) ? await runTemplateDiagnostics(wfFile, wfResult.steps, opts) : true;
|
|
244016
|
+
let onlineOk = true;
|
|
244017
|
+
if (wfOk && opts.online && existsSync21(wfFile)) {
|
|
244018
|
+
onlineOk = printIssues("invoke refs (server)", wfPath, await onlineInvokeIssues(readFileSync18(wfFile, "utf-8"), opts));
|
|
244019
|
+
}
|
|
244020
|
+
return wfOk && evOk && dsOk && templateOk && onlineOk;
|
|
244021
|
+
}
|
|
243241
244022
|
function registerAllValidateCommand(parent) {
|
|
243242
|
-
const cmd = parent.command("validate [path]").description("Local-only validation. Without [path]: runs the templated three-way check (./workflow.yaml + ./evaluators.yaml + ./dataset/). With [path] pointing at a YAML file: validates that workflow.yaml. For per-noun targeting use `evaluators validate` or `dataset validate`.").option("--dir <path>", "Project root (defaults to cwd; resolves the three default paths from here)").option("--online", "Authenticate and also validate action.invoke-workflow targets, transform.template tmpl_… references, and explicitly selected OCR/vision/text models against this tenant environment (existence, format, revision pairing, tokens vs data, XLSX {{ }} mistakes, configured catalog). Local template: paths are inspected on disk with or without this flag. Model checks use the configured catalog only — they do not probe live provider health.").option("--allow-external-templates", "Allow local template: paths whose real path is outside the workflow project directory (the folder that contains the YAML file). Off by default; ../ and symlink escapes are rejected.").addHelpText("after", `
|
|
244023
|
+
const cmd = parent.command("validate [path]").description("Local-only validation. Without [path]: runs the templated three-way check (./workflow.yaml + ./evaluators.yaml + ./dataset/) in the project root. When the root has no workflow.yaml, discovers nested projects under eigenpal/workflows/<slug>/ or workflows/<slug>/ and validates each. With [path] pointing at a YAML file: validates that workflow.yaml only. For per-noun targeting use `evaluators validate` or `dataset validate`.").option("--dir <path>", "Project root (defaults to cwd; resolves the three default paths from here)").option("--online", "Authenticate and also validate action.invoke-workflow targets, transform.template tmpl_… references, and explicitly selected OCR/vision/text models against this tenant environment (existence, format, revision pairing, tokens vs data, XLSX {{ }} mistakes, configured catalog). Local template: paths are inspected on disk with or without this flag. Model checks use the configured catalog only — they do not probe live provider health.").option("--allow-external-templates", "Allow local template: paths whose real path is outside the workflow project directory (the folder that contains the YAML file). Off by default; ../ and symlink escapes are rejected.").addHelpText("after", `
|
|
243243
244024
|
Note
|
|
244025
|
+
` + ` When [path] or --dir points at a repository root (or eigenpal/workflows),
|
|
244026
|
+
` + ` every discovered workflow project is validated and failures are grouped
|
|
244027
|
+
` + ` by project path.
|
|
243244
244028
|
` + ` Local validation cannot resolve action.invoke-workflow targets, tmpl_…
|
|
243245
244029
|
` + ` template ids, or whether selected OCR/vision/text models exist in this
|
|
243246
244030
|
` + " environment (they live on the server). Run `validate --online` to check\n" + " them before pushing. Source-controlled `template: ./file.xlsx` paths are\n" + ` inspected on disk without --online and must stay inside the workflow
|
|
243247
244031
|
` + ` project unless you pass --allow-external-templates.
|
|
243248
244032
|
`);
|
|
243249
244033
|
withBaseUrl(cmd).action(action(async (path, opts) => {
|
|
243250
|
-
if (path) {
|
|
244034
|
+
if (path && (!existsSync21(resolve9(path)) || !statSync7(resolve9(path)).isDirectory())) {
|
|
243251
244035
|
const target = resolve9(path);
|
|
243252
244036
|
const wfResult = validateWorkflowYaml(target);
|
|
243253
244037
|
const wfOk = printIssues("workflow.yaml", target, wfResult.issues);
|
|
@@ -243258,29 +244042,29 @@ Note
|
|
|
243258
244042
|
const templateOk = wfOk ? await runTemplateDiagnostics(target, wfResult.steps, opts) : true;
|
|
243259
244043
|
let onlineOk = true;
|
|
243260
244044
|
if (wfOk && opts.online) {
|
|
243261
|
-
onlineOk = printIssues("invoke refs (server)", target, await onlineInvokeIssues(
|
|
244045
|
+
onlineOk = printIssues("invoke refs (server)", target, await onlineInvokeIssues(readFileSync18(target, "utf-8"), opts));
|
|
243262
244046
|
}
|
|
243263
244047
|
if (!wfOk || !templateOk || !onlineOk)
|
|
243264
244048
|
process.exit(1);
|
|
243265
244049
|
return;
|
|
243266
244050
|
}
|
|
243267
|
-
const
|
|
243268
|
-
const
|
|
243269
|
-
|
|
243270
|
-
|
|
243271
|
-
|
|
243272
|
-
|
|
243273
|
-
|
|
243274
|
-
|
|
243275
|
-
|
|
243276
|
-
|
|
243277
|
-
|
|
243278
|
-
|
|
243279
|
-
|
|
243280
|
-
|
|
243281
|
-
|
|
243282
|
-
}
|
|
243283
|
-
if (!
|
|
244051
|
+
const requestedRoot = path ? resolve9(path) : rootDir(opts);
|
|
244052
|
+
const projects = discoverWorkflowProjectRoots(requestedRoot);
|
|
244053
|
+
if (projects.length === 0) {
|
|
244054
|
+
if (!await validateProjectRoot(requestedRoot, opts))
|
|
244055
|
+
process.exit(1);
|
|
244056
|
+
return;
|
|
244057
|
+
}
|
|
244058
|
+
let valid = true;
|
|
244059
|
+
for (const project of projects) {
|
|
244060
|
+
if (projects.length > 1) {
|
|
244061
|
+
process.stderr.write(`${ui.bold(relPath(requestedRoot, project))}
|
|
244062
|
+
`);
|
|
244063
|
+
}
|
|
244064
|
+
if (!await validateProjectRoot(project, opts))
|
|
244065
|
+
valid = false;
|
|
244066
|
+
}
|
|
244067
|
+
if (!valid)
|
|
243284
244068
|
process.exit(1);
|
|
243285
244069
|
}));
|
|
243286
244070
|
}
|
|
@@ -243509,7 +244293,7 @@ new uploads are deleted. The file on disk is not rewritten.
|
|
|
243509
244293
|
defaultValue: "workflow.yaml",
|
|
243510
244294
|
validate: (value) => {
|
|
243511
244295
|
const p = value || "workflow.yaml";
|
|
243512
|
-
if (!
|
|
244296
|
+
if (!existsSync22(p))
|
|
243513
244297
|
return `Not found: ${p}`;
|
|
243514
244298
|
return;
|
|
243515
244299
|
}
|
|
@@ -244148,7 +244932,7 @@ function registerExperimentCommands2(parent) {
|
|
|
244148
244932
|
}
|
|
244149
244933
|
], { ...opts, entityLabel: "experiment" });
|
|
244150
244934
|
}));
|
|
244151
|
-
const runCmd = experiment.command("run <workflow-id>").description(descriptionFor("workflow_run_experiment")).option("--example-id <id>", "Run only this example (repeatable)", (val2, prev) => [...prev, val2], []).option("--wait", "Poll until terminal; non-zero exit on passRate < 1.0", false).option("--interval <n>", "Polling interval in seconds (default 10)", intArg, 10).addHelpText("after", `
|
|
244935
|
+
const runCmd = experiment.command("run <workflow-id>").description(descriptionFor("workflow_run_experiment")).option("--example-id <id>", "Run only this example (repeatable)", (val2, prev) => [...prev, val2], []).option("--wait", "Poll until terminal; non-zero exit on passRate < 1.0", false).option("--interval <n>", "Polling interval in seconds (default 10)", intArg, 10).option("--max-wait <seconds>", "Maximum wait before exit code 2 (default 1800)", intArg, 1800).addHelpText("after", `
|
|
244152
244936
|
Examples:
|
|
244153
244937
|
$ eigenpal workflow experiment run wf_abc123
|
|
244154
244938
|
$ eigenpal workflow experiment run my-extraction --example-id evx_111
|
|
@@ -244181,15 +244965,40 @@ returns immediately with \`{ batchId, total }\`; poll
|
|
|
244181
244965
|
return;
|
|
244182
244966
|
}
|
|
244183
244967
|
const intervalMs = Math.max(1, opts.interval) * 1000;
|
|
244968
|
+
const deadline = Date.now() + Math.max(1, opts.maxWait) * 1000;
|
|
244184
244969
|
while (true) {
|
|
244185
244970
|
const status = await client.get(`/v1/automations/${workflowId}/experiments/${experimentId}`);
|
|
244186
244971
|
if (status.status === "completed") {
|
|
244187
|
-
|
|
244972
|
+
if (opts.json) {
|
|
244973
|
+
printJson3({
|
|
244974
|
+
...status,
|
|
244975
|
+
batchId: experimentId,
|
|
244976
|
+
total: status.runCount ?? status.total
|
|
244977
|
+
});
|
|
244978
|
+
} else {
|
|
244979
|
+
const completed = status.completedCount ?? status.completed;
|
|
244980
|
+
const runTotal = status.runCount ?? status.total;
|
|
244981
|
+
const score = typeof status.evalScore === "number" ? ` average=${status.evalScore.toFixed(3)}` : "";
|
|
244982
|
+
success(`Experiment ${ui.bold(experimentId)} completed (${completed}/${runTotal}${score})`);
|
|
244983
|
+
}
|
|
244188
244984
|
if (typeof status.passRate === "number" && status.passRate < 1 || typeof status.failedCount === "number" && status.failedCount > 0)
|
|
244189
244985
|
process.exit(1);
|
|
244190
244986
|
return;
|
|
244191
244987
|
}
|
|
244192
|
-
|
|
244988
|
+
if (!opts.json) {
|
|
244989
|
+
console.error(`${ui.dim(`[${experimentId}]`)} ${status.completedCount ?? status.completed}/${status.runCount ?? status.total} ${ui.dim(`(status=${status.status})`)}`);
|
|
244990
|
+
}
|
|
244991
|
+
if (Date.now() >= deadline) {
|
|
244992
|
+
if (opts.json) {
|
|
244993
|
+
printJson3({
|
|
244994
|
+
...status,
|
|
244995
|
+
batchId: experimentId,
|
|
244996
|
+
total: status.runCount ?? status.total
|
|
244997
|
+
});
|
|
244998
|
+
}
|
|
244999
|
+
error2(`Timed out waiting for experiment ${experimentId} after ${opts.maxWait}s`);
|
|
245000
|
+
process.exit(2);
|
|
245001
|
+
}
|
|
244193
245002
|
await new Promise((r) => setTimeout(r, intervalMs));
|
|
244194
245003
|
}
|
|
244195
245004
|
}));
|
|
@@ -244199,6 +245008,10 @@ returns immediately with \`{ batchId, total }\`; poll
|
|
|
244199
245008
|
error2("--short and --watch are mutually exclusive (short is for spot-check polling).");
|
|
244200
245009
|
process.exit(2);
|
|
244201
245010
|
}
|
|
245011
|
+
if (opts.short && opts.json) {
|
|
245012
|
+
error2("--short and --json are mutually exclusive (short is human spot-check output).");
|
|
245013
|
+
process.exit(2);
|
|
245014
|
+
}
|
|
244202
245015
|
const includes = new Set((opts.include ?? "").split(",").map((s) => s.trim()).filter(Boolean));
|
|
244203
245016
|
const includePayload = includes.has("payload");
|
|
244204
245017
|
const { client, workflowId } = await buildClientForWorkflow(workflow, opts);
|
|
@@ -244307,14 +245120,41 @@ Behavior:
|
|
|
244307
245120
|
}
|
|
244308
245121
|
return value;
|
|
244309
245122
|
};
|
|
244310
|
-
const resultsCmd = experiment.command("results <workflow-id> [batchId]").description(descriptionFor("workflow_get_experiment_results")).
|
|
244311
|
-
withBaseUrl(resultsCmd).action(action(async (workflow, batchId, opts) => {
|
|
245123
|
+
const resultsCmd = experiment.command("results <workflow-id> [batchId]").description(descriptionFor("workflow_get_experiment_results")).option("--format <csv|json>", "Output format (default json)", parseExportFormat, "json").option("--out <path>", "Output file. When omitted, the binary streams to stdout.").option("--summary", "Show total/pass/fail/error counts, average score, and evaluator rollups").option("--failed-only", "Keep only failed or errored evaluator results").option("--evaluator <name>", "Keep only results from this evaluator").option("--select <path>", "Print only a nested JSON value (for example summary.byEvaluator or discrepancies[].path)");
|
|
245124
|
+
addJsonFlag(withBaseUrl(resultsCmd)).action(action(async (workflow, batchId, opts) => {
|
|
244312
245125
|
const { client, workflowId } = await buildClientForWorkflow(workflow, opts);
|
|
245126
|
+
if (opts.json && opts.format !== "json") {
|
|
245127
|
+
throw new Error("--json cannot be combined with --format csv");
|
|
245128
|
+
}
|
|
245129
|
+
const analyze = opts.summary || opts.failedOnly || opts.evaluator !== undefined || opts.select !== undefined;
|
|
245130
|
+
if (analyze) {
|
|
245131
|
+
if (!batchId) {
|
|
245132
|
+
throw new Error("--summary, --failed-only, --evaluator, and --select require a single experiment batch id");
|
|
245133
|
+
}
|
|
245134
|
+
if (opts.format !== "json") {
|
|
245135
|
+
throw new Error("Experiment analysis options require --format json");
|
|
245136
|
+
}
|
|
245137
|
+
const detailPayload = await client.get(`/v1/automations/${workflowId}/experiments/${encodeURIComponent(batchId)}`);
|
|
245138
|
+
const analysis = analyzeExperimentResults({
|
|
245139
|
+
rows: experimentRowsFromDetail(detailPayload),
|
|
245140
|
+
runs: detailPayload.runs ?? [],
|
|
245141
|
+
evaluator: opts.evaluator,
|
|
245142
|
+
failedOnly: opts.failedOnly
|
|
245143
|
+
});
|
|
245144
|
+
const selected = opts.select ? selectJsonValue(analysis, opts.select) : analysis;
|
|
245145
|
+
if (opts.summary && !opts.json && !opts.out && !opts.select) {
|
|
245146
|
+
renderExperimentResultsSummary(analysis);
|
|
245147
|
+
return;
|
|
245148
|
+
}
|
|
245149
|
+
await writeOrPrint(opts.out, `${JSON.stringify(selected, null, 2)}
|
|
245150
|
+
`);
|
|
245151
|
+
return;
|
|
245152
|
+
}
|
|
244313
245153
|
const params = new URLSearchParams({ format: opts.format });
|
|
244314
245154
|
const exportPath = batchId ? `/v1/automations/${workflowId}/experiments/${batchId}/export?${params.toString()}` : `/v1/automations/${workflowId}/experiments/export?${params.toString()}`;
|
|
244315
245155
|
await downloadStreamToFile2(client, exportPath, opts.out);
|
|
244316
245156
|
}));
|
|
244317
|
-
const compareCmd = experiment.command("compare <batchIdA> <batchIdB>").description("
|
|
245157
|
+
const compareCmd = experiment.command("compare <batchIdA> <batchIdB>").description("Compare evaluator scores or actual outputs between two experiment batches.").option("--outputs", "Compare actual run outputs instead of evaluator scores").option("--sort <abs-delta-desc|delta-asc|delta-desc|name>", "Row sort order (default: biggest movers first)", "abs-delta-desc").option("--regression-threshold <n>", "Δ below this is flagged as a regression (default 0.05)", Number.parseFloat, 0.05).addHelpText("after", `
|
|
244318
245158
|
Examples:
|
|
244319
245159
|
$ eigenpal workflow experiment compare evb_old evb_new
|
|
244320
245160
|
$ eigenpal workflow experiment compare evb_old evb_new --regression-threshold 0.10
|
|
@@ -244334,6 +245174,26 @@ batches must belong to the same workflow within the caller's tenant.
|
|
|
244334
245174
|
throw new Error("--regression-threshold must be a non-negative number");
|
|
244335
245175
|
}
|
|
244336
245176
|
const client = buildClient3(opts);
|
|
245177
|
+
if (opts.outputs) {
|
|
245178
|
+
const [outputsA, outputsB] = await Promise.all([
|
|
245179
|
+
fetchExperimentOutputs(client, batchIdA),
|
|
245180
|
+
fetchExperimentOutputs(client, batchIdB)
|
|
245181
|
+
]);
|
|
245182
|
+
if (outputsA.automationId !== outputsB.automationId) {
|
|
245183
|
+
throw new Error("Experiment batches must belong to the same automation");
|
|
245184
|
+
}
|
|
245185
|
+
const outputDiff = buildExperimentOutputDiff({
|
|
245186
|
+
batchIdA,
|
|
245187
|
+
batchIdB,
|
|
245188
|
+
rowsA: outputsA.rows,
|
|
245189
|
+
rowsB: outputsB.rows
|
|
245190
|
+
});
|
|
245191
|
+
if (opts.json)
|
|
245192
|
+
printJson3(outputDiff);
|
|
245193
|
+
else
|
|
245194
|
+
renderExperimentOutputDiffHuman(outputDiff);
|
|
245195
|
+
return;
|
|
245196
|
+
}
|
|
244337
245197
|
const [bodyA, bodyB] = await Promise.all([
|
|
244338
245198
|
fetchEvalResults(client, batchIdA),
|
|
244339
245199
|
fetchEvalResults(client, batchIdB)
|
|
@@ -244385,7 +245245,7 @@ Behavior:
|
|
|
244385
245245
|
const dest = opts.pull ? opts.pullOnComplete ?? `./results-${batchId}.${opts.format}` : null;
|
|
244386
245246
|
if (dest) {
|
|
244387
245247
|
const dir = path18.dirname(dest);
|
|
244388
|
-
if (!
|
|
245248
|
+
if (!existsSync22(dir)) {
|
|
244389
245249
|
error2(`Destination directory does not exist: ${dir}. Create it first or pass a path under an existing directory.`);
|
|
244390
245250
|
process.exit(2);
|
|
244391
245251
|
}
|
|
@@ -245158,6 +246018,7 @@ registerModelsCommands(program2);
|
|
|
245158
246018
|
registerRunCommands(program2);
|
|
245159
246019
|
registerRunsCommands(program2);
|
|
245160
246020
|
registerGitCommands(program2);
|
|
246021
|
+
registerDocsCommands(program2);
|
|
245161
246022
|
program2.command("completion <shell>").description("Print a shell completion script to stdout (supported shells: bash, zsh, fish). Pipe into the appropriate completion file for your shell — see the script header for install instructions.").action(async (shell) => {
|
|
245162
246023
|
try {
|
|
245163
246024
|
await completion(shell, program2);
|