agent-skill-evals 0.1.0 → 0.1.1

package/dist/index-D9lGL0Qg.d.mts

@@ -0,0 +1,381 @@
+import { z } from "zod";
+
+//#region src/evidence-schema.d.ts
+declare const CommandEventSchema: z.ZodObject<{
+  command: z.ZodString;
+  args: z.ZodDefault<z.ZodArray<z.ZodString>>;
+  exitCode: z.ZodNumber;
+  signal: z.ZodOptional<z.ZodString>;
+  stdout: z.ZodOptional<z.ZodString>;
+  stderr: z.ZodOptional<z.ZodString>;
+  startedAt: z.ZodNumber;
+  durationMs: z.ZodNumber;
+  turn: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+declare const FileEventSchema: z.ZodObject<{
+  path: z.ZodString;
+  op: z.ZodEnum<{
+    create: "create";
+    modify: "modify";
+    delete: "delete";
+  }>;
+}, z.core.$strip>;
+declare const ToolCallEventSchema: z.ZodObject<{
+  tool: z.ZodString;
+  provider: z.ZodOptional<z.ZodString>;
+  server: z.ZodOptional<z.ZodString>;
+  args: z.ZodOptional<z.ZodUnknown>;
+  result: z.ZodOptional<z.ZodUnknown>;
+  startedAt: z.ZodNumber;
+  durationMs: z.ZodNumber;
+  turn: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+declare const SkillLoadEventSchema: z.ZodObject<{
+  skill: z.ZodString;
+  delivery: z.ZodEnum<{
+    mcp: "mcp";
+    native: "native";
+    explicit: "explicit";
+  }>;
+  provider: z.ZodOptional<z.ZodString>;
+  server: z.ZodOptional<z.ZodString>;
+  source: z.ZodOptional<z.ZodString>;
+  startedAt: z.ZodNumber;
+}, z.core.$strip>;
+declare const SkillAvailableEventSchema: z.ZodObject<{
+  skill: z.ZodString;
+  path: z.ZodString;
+  role: z.ZodEnum<{
+    "under-test": "under-test";
+    supporting: "supporting";
+    distractor: "distractor";
+  }>;
+}, z.core.$strip>;
+declare const UsageSchema: z.ZodObject<{
+  inputTokens: z.ZodOptional<z.ZodNumber>;
+  outputTokens: z.ZodOptional<z.ZodNumber>;
+  totalTokens: z.ZodOptional<z.ZodNumber>;
+  cacheReadTokens: z.ZodOptional<z.ZodNumber>;
+  cacheWriteTokens: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+declare const RuntimeIdentitySchema: z.ZodObject<{
+  adapter: z.ZodOptional<z.ZodString>;
+  preset: z.ZodOptional<z.ZodString>;
+  command: z.ZodOptional<z.ZodString>;
+  cliVersion: z.ZodOptional<z.ZodString>;
+  model: z.ZodOptional<z.ZodString>;
+  continuation: z.ZodOptional<z.ZodEnum<{
+    "transcript-replay": "transcript-replay";
+    "native-session": "native-session";
+  }>>;
+}, z.core.$strip>;
+declare const TurnRecordSchema: z.ZodObject<{
+  turn: z.ZodNumber;
+  role: z.ZodEnum<{
+    user: "user";
+    agent: "agent";
+  }>;
+  text: z.ZodString;
+  startedAt: z.ZodNumber;
+  durationMs: z.ZodNumber;
+  usage: z.ZodOptional<z.ZodObject<{
+    inputTokens: z.ZodOptional<z.ZodNumber>;
+    outputTokens: z.ZodOptional<z.ZodNumber>;
+    totalTokens: z.ZodOptional<z.ZodNumber>;
+    cacheReadTokens: z.ZodOptional<z.ZodNumber>;
+    cacheWriteTokens: z.ZodOptional<z.ZodNumber>;
+  }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const EvidenceSnapshotSchema: z.ZodObject<{
+  schemaVersion: z.ZodLiteral<"agent-skill-evals.evidence.v2">;
+  output: z.ZodDefault<z.ZodString>;
+  run: z.ZodObject<{
+    runDir: z.ZodString;
+    worldPath: z.ZodString;
+    fixture: z.ZodOptional<z.ZodString>;
+    durationMs: z.ZodOptional<z.ZodNumber>;
+  }, z.core.$strip>;
+  commands: z.ZodDefault<z.ZodArray<z.ZodObject<{
+    command: z.ZodString;
+    args: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    exitCode: z.ZodNumber;
+    signal: z.ZodOptional<z.ZodString>;
+    stdout: z.ZodOptional<z.ZodString>;
+    stderr: z.ZodOptional<z.ZodString>;
+    startedAt: z.ZodNumber;
+    durationMs: z.ZodNumber;
+    turn: z.ZodOptional<z.ZodNumber>;
+  }, z.core.$strip>>>;
+  filesWritten: z.ZodDefault<z.ZodArray<z.ZodObject<{
+    path: z.ZodString;
+    op: z.ZodEnum<{
+      create: "create";
+      modify: "modify";
+      delete: "delete";
+    }>;
+  }, z.core.$strip>>>;
+  toolCalls: z.ZodDefault<z.ZodArray<z.ZodObject<{
+    tool: z.ZodString;
+    provider: z.ZodOptional<z.ZodString>;
+    server: z.ZodOptional<z.ZodString>;
+    args: z.ZodOptional<z.ZodUnknown>;
+    result: z.ZodOptional<z.ZodUnknown>;
+    startedAt: z.ZodNumber;
+    durationMs: z.ZodNumber;
+    turn: z.ZodOptional<z.ZodNumber>;
+  }, z.core.$strip>>>;
+  skillsLoaded: z.ZodDefault<z.ZodArray<z.ZodObject<{
+    skill: z.ZodString;
+    delivery: z.ZodEnum<{
+      mcp: "mcp";
+      native: "native";
+      explicit: "explicit";
+    }>;
+    provider: z.ZodOptional<z.ZodString>;
+    server: z.ZodOptional<z.ZodString>;
+    source: z.ZodOptional<z.ZodString>;
+    startedAt: z.ZodNumber;
+  }, z.core.$strip>>>;
+  skillsAvailable: z.ZodDefault<z.ZodArray<z.ZodObject<{
+    skill: z.ZodString;
+    path: z.ZodString;
+    role: z.ZodEnum<{
+      "under-test": "under-test";
+      supporting: "supporting";
+      distractor: "distractor";
+    }>;
+  }, z.core.$strip>>>;
+  usage: z.ZodDefault<z.ZodObject<{
+    inputTokens: z.ZodOptional<z.ZodNumber>;
+    outputTokens: z.ZodOptional<z.ZodNumber>;
+    totalTokens: z.ZodOptional<z.ZodNumber>;
+    cacheReadTokens: z.ZodOptional<z.ZodNumber>;
+    cacheWriteTokens: z.ZodOptional<z.ZodNumber>;
+  }, z.core.$strip>>;
+  turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+    turn: z.ZodNumber;
+    role: z.ZodEnum<{
+      user: "user";
+      agent: "agent";
+    }>;
+    text: z.ZodString;
+    startedAt: z.ZodNumber;
+    durationMs: z.ZodNumber;
+    usage: z.ZodOptional<z.ZodObject<{
+      inputTokens: z.ZodOptional<z.ZodNumber>;
+      outputTokens: z.ZodOptional<z.ZodNumber>;
+      totalTokens: z.ZodOptional<z.ZodNumber>;
+      cacheReadTokens: z.ZodOptional<z.ZodNumber>;
+      cacheWriteTokens: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+  }, z.core.$strip>>>;
+  runtime: z.ZodOptional<z.ZodObject<{
+    adapter: z.ZodOptional<z.ZodString>;
+    preset: z.ZodOptional<z.ZodString>;
+    command: z.ZodOptional<z.ZodString>;
+    cliVersion: z.ZodOptional<z.ZodString>;
+    model: z.ZodOptional<z.ZodString>;
+    continuation: z.ZodOptional<z.ZodEnum<{
+      "transcript-replay": "transcript-replay";
+      "native-session": "native-session";
+    }>>;
+  }, z.core.$strip>>;
+  warnings: z.ZodOptional<z.ZodArray<z.ZodString>>;
+  extensions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
+type CommandEvent = z.infer<typeof CommandEventSchema>;
+type FileEvent = z.infer<typeof FileEventSchema>;
+type ToolCallEvent = z.infer<typeof ToolCallEventSchema>;
+type SkillLoadEvent = z.infer<typeof SkillLoadEventSchema>;
+type SkillAvailableEvent = z.infer<typeof SkillAvailableEventSchema>;
+type Usage = z.infer<typeof UsageSchema>;
+type RuntimeIdentity = z.infer<typeof RuntimeIdentitySchema>;
+type TurnRecord = z.infer<typeof TurnRecordSchema>;
+type EvidenceSnapshot = z.infer<typeof EvidenceSnapshotSchema>;
+//#endregion
+//#region src/internal-types.d.ts
+interface AgentSkillEvalsAssertionResult {
+  pass: boolean;
+  score: number;
+  reason: string;
+  componentResults?: AgentSkillEvalsAssertionResult[];
+  evidence?: unknown;
+}
+interface WorldHandle {
+  readonly path: string;
+  listFiles(glob: string): Promise<string[]>;
+  readFile(relativePath: string): Promise<string | null>;
+  exec(command: string, args?: readonly string[], opts?: {
+    timeoutMs?: number;
+    env?: Record<string, string>;
+  }): Promise<{
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+  }>;
+}
+interface EvidenceHandle {
+  output(): string;
+  commands(): readonly CommandEvent[];
+  filesWritten(): readonly FileEvent[];
+  toolCalls(): readonly ToolCallEvent[];
+  skillsLoaded(): readonly SkillLoadEvent[];
+  skillsAvailable(): readonly SkillAvailableEvent[];
+  usage(): Usage;
+  turns?(): readonly TurnRecord[];
+  runtime?(): RuntimeIdentity;
+  warnings?(): readonly string[];
+}
+//#endregion
+//#region src/agent/provider-config.d.ts
+declare const SkillEvidenceConfigSchema: z.ZodObject<{
+  mcpResource: z.ZodOptional<z.ZodObject<{
+    uriArgPaths: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    uriPatterns: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+  }, z.core.$strip>>;
+  mcpTool: z.ZodOptional<z.ZodObject<{
+    toolPatterns: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+  }, z.core.$strip>>;
+  nativeArgs: z.ZodOptional<z.ZodObject<{
+    whenArgs: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    whenAnyArgs: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    skillPathFlags: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    provider: z.ZodOptional<z.ZodString>;
+    source: z.ZodOptional<z.ZodString>;
+  }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const ProviderConfigSchema: z.ZodObject<{
+  preset: z.ZodOptional<z.ZodString>;
+  adapter: z.ZodOptional<z.ZodString>;
+  command: z.ZodOptional<z.ZodString>;
+  args: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+  extraArgs: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+  model: z.ZodOptional<z.ZodString>;
+  timeoutMs: z.ZodOptional<z.ZodNumber>;
+  baseDir: z.ZodOptional<z.ZodString>;
+  skillEvidence: z.ZodOptional<z.ZodObject<{
+    mcpResource: z.ZodOptional<z.ZodObject<{
+      uriArgPaths: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+      uriPatterns: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    }, z.core.$strip>>;
+    mcpTool: z.ZodOptional<z.ZodObject<{
+      toolPatterns: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    }, z.core.$strip>>;
+    nativeArgs: z.ZodOptional<z.ZodObject<{
+      whenArgs: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+      whenAnyArgs: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+      skillPathFlags: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+      provider: z.ZodOptional<z.ZodString>;
+      source: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+  }, z.core.$strip>>;
+  simulatedUser: z.ZodOptional<z.ZodObject<{
+    preset: z.ZodOptional<z.ZodString>;
+    adapter: z.ZodOptional<z.ZodString>;
+    command: z.ZodOptional<z.ZodString>;
+    args: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    extraArgs: z.ZodOptional<z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<readonly string[], string[]>>>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+  }, z.core.$strip>>;
+}, z.core.$strip>;
+type SkillEvidenceConfig = z.infer<typeof SkillEvidenceConfigSchema>;
+type ProviderConfig = z.infer<typeof ProviderConfigSchema>;
+//#endregion
+//#region src/agent/evidence.d.ts
+declare class EvidenceCollector {
+  private readonly skillEvidenceConfig;
+  private currentTurn?;
+  /** Usage from completed turns; the live snapshot.usage covers only the current turn. */
+  private completedTurnsUsage;
+  private snapshot;
+  constructor(skillEvidenceConfig?: SkillEvidenceConfig);
+  addCommand(e: CommandEvent): void;
+  addFileWrite(e: FileEvent): void;
+  addToolCall(e: ToolCallEvent): void;
+  private withTurn;
+  /**
+   * Start an agent turn in a multi-turn run. Adapters overwrite usage per
+   * CLI invocation (setUsage), so each turn's usage is scoped: the previous
+   * turn's counts move into the completed-turns total first.
+   */
+  beginAgentTurn(turn: number): void;
+  endAgentTurn(record: {
+    text: string;
+    startedAt: number;
+    durationMs: number;
+  }): void;
+  addUserTurn(turn: number, text: string, startedAt: number): void;
+  private addTurnRecord;
+  addSkillLoad(e: SkillLoadEvent): void;
+  addSkillAvailable(e: SkillAvailableEvent): void;
+  setUsage(u: Usage): void;
+  /** Merge non-empty runtime identity fields; later observations win. */
+  mergeRuntime(identity: RuntimeIdentity): void;
+  addWarning(warning: string): void;
+  addUsage(u: Usage): void;
+  setOutput(output: string): void;
+  setRun(run: EvidenceSnapshot["run"]): void;
+  toSnapshot(): EvidenceSnapshot;
+  static fromSnapshot(snapshot: EvidenceSnapshot): EvidenceCollector;
+}
+declare function evidenceFromSnapshot(s: EvidenceSnapshot): EvidenceHandle;
+//#endregion
+//#region src/agent/world.d.ts
+declare function makeWorldHandle(worldPath: string, recordCommand?: (event: CommandEvent) => void): WorldHandle;
+//#endregion
+//#region src/agent/metadata.d.ts
+interface AgentSkillEvalsProviderMetadata {
+  runDir: string;
+  worldPath: string;
+  evidencePath: string;
+  fixture?: string;
+  skill?: string;
+  kind?: string;
+  preconditionResults: AgentSkillEvalsAssertionResult[];
+  preconditionsPassed: boolean;
+  durationMs: number;
+  runtime?: RuntimeIdentity;
+  warnings?: string[];
+  mockEnv?: Record<string, string>;
+}
+//#endregion
+//#region src/agent/index.d.ts
+interface PromptfooContext {
+  vars?: Record<string, unknown>;
+  test?: {
+    vars?: Record<string, unknown>;
+    metadata?: Record<string, unknown>;
+  };
+}
+interface ProviderResponse {
+  output: string;
+  metadata?: Record<string, unknown>;
+  cost?: number;
+  tokenUsage?: {
+    total?: number;
+    prompt?: number;
+    completion?: number;
+    cached?: number;
+  };
+  error?: string;
+}
+interface PreparedRun {
+  runDir: string;
+  worldPath: string;
+  world: ReturnType<typeof makeWorldHandle>;
+  evidenceCollector: EvidenceCollector;
+}
+declare class AgentSkillEvalsProvider {
+  config: ProviderConfig;
+  private readonly configError?;
+  private authHome?;
+  id: () => string;
+  constructor(options?: {
+    config?: ProviderConfig;
+    id?: string;
+  });
+  callApi(prompt: string, context?: PromptfooContext): Promise<ProviderResponse>;
+}
+//#endregion
+export { evidenceFromSnapshot as a, EvidenceCollector as i, PreparedRun as n, EvidenceSnapshot as o, AgentSkillEvalsProviderMetadata as r, AgentSkillEvalsProvider as t };
+//# sourceMappingURL=index-D9lGL0Qg.d.mts.map

package/dist/index-D9lGL0Qg.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-D9lGL0Qg.d.mts","names":[],"sources":["../src/evidence-schema.ts","../src/internal-types.ts","../src/agent/provider-config.ts","../src/agent/evidence.ts","../src/agent/world.ts","../src/agent/metadata.ts","../src/agent/index.ts"],"mappings":";;;cAQM,kBAAA,EAAkB,CAAA,CAAA,SAAA;;;;;;;;;;;cAYlB,eAAA,EAAe,CAAA,CAAA,SAAA;;;;;;;;cAKf,mBAAA,EAAmB,CAAA,CAAA,SAAA;;;;;;;;;;cAWnB,oBAAA,EAAoB,CAAA,CAAA,SAAA;;;;;;;;;;;;cASpB,yBAAA,EAAyB,CAAA,CAAA,SAAA;;;;;;;;;cAMzB,WAAA,EAAW,CAAA,CAAA,SAAA;;;;;;;cAeX,qBAAA,EAAqB,CAAA,CAAA,SAAA;;;;;;;;;;;cASrB,gBAAA,EAAgB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;cAShB,sBAAA,EAAsB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAgBhB,YAAA,GAAe,CAAA,CAAE,KAAA,QAAa,kBAAA;AAAA,KAC9B,SAAA,GAAY,CAAA,CAAE,KAAA,QAAa,eAAA;AAAA,KAC3B,aAAA,GAAgB,CAAA,CAAE,KAAA,QAAa,mBAAA;AAAA,KAC/B,cAAA,GAAiB,CAAA,CAAE,KAAA,QAAa,oBAAA;AAAA,KAChC,mBAAA,GAAsB,CAAA,CAAE,KAAA,QAAa,yBAAA;AAAA,KACrC,KAAA,GAAQ,CAAA,CAAE,KAAA,QAAa,WAAA;AAAA,KAEvB,eAAA,GAAkB,CAAA,CAAE,KAAA,QAAa,qBAAA;AAAA,KACjC,UAAA,GAAa,CAAA,CAAE,KAAA,QAAa,gBAAA;AAAA,KAC5B,gBAAA,GAAmB,CAAA,CAAE,KAAA,QAAa,sBAAA;;;UChG7B,8BAAA;EACf,IAAA;EACA,KAAA;EACA,MAAA;EACA,gBAAA,GAAmB,8BAAA;EACnB,QAAA;AAAA;AAAA,UAGe,WAAA;EAAA,SACN,IAAA;EACT,SAAA,CAAU,IAAA,WAAe,OAAA;EACzB,QAAA,CAAS,YAAA,WAAuB,OAAA;EAChC,IAAA,CACE,OAAA,UACA,IAAA,sBACA,IAAA;IAAS,SAAA;IAAoB,GAAA,GAAM,MAAA;EAAA,IAClC,OAAA;IAAU,QAAA;IAAkB,MAAA;IAAgB,MAAA;EAAA;AAAA;AAAA,UAGhC,cAAA;EACf,MAAA;EACA,QAAA,aAAqB,YAAA;EACrB,YAAA,aAAyB,SAAA;EACzB,SAAA,aAAsB,aAAA;EACtB,YAAA,aAAyB,cAAA;EACzB,eAAA,aAA4B,mBAAA;EAC5B,KAAA,IAAS,KAAA;EACT,KAAA,cAAmB,UAAA;EACnB,OAAA,KAAY,eAAA;EACZ,QAAA;AAAA;;;cClCW,yBAAA,EAAyB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;cA0BzB,oBAAA,EAAoB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAarB,mBAAA,GAAsB,CAAA,CAAE,KAAA,QAAa,yBAAA;AAAA,KAErC,cAAA,GAAiB,CAAA,CAAE,KAAA,QAAa,oBAAA;;;cChB/B,iBAAA;EAAA,iBACM,mBAAA;EAAA,QACT,WAAA;;UAEA,mBAAA;EAAA,QACA,QAAA;cAgBI,mBAAA,GAAqB,mBAAA;EAIjC,UAAA,CAAW,CAAA,EAAG,YAAA;EAId,YAAA,CAAa,CAAA,EAAG,SAAA;EAIhB,WAAA,CAAY,CAAA,EAAG,aAAA;EAAA,QAOP,QAAA;;;;;;EASR,cAAA,CAAe,IAAA;EAMf,YAAA,CAAa,MAAA;IAAU,IAAA;IAAc,SAAA;IAAmB,UAAA;EAAA;EAYxD,WAAA,CAAY,IAAA,UAAc,IAAA,UAAc,SAAA;EAAA,QAIhC,aAAA;EAMR,YAAA,CAAa,CAAA,EAAG,cAAA;EAWhB,iBAAA,CAAkB,CAAA,EAAG,mBAAA;EAMrB,QAAA,CAAS,CAAA,EAAG,KAAA;;EAKZ,YAAA,CAAa,QAAA,EAAU,eAAA;EAUvB,UAAA,CAAW,OAAA;EAOX,QAAA,CAAS,CAAA,EAAG,KAAA;EAIZ,SAAA,CAAU,MAAA;EAIV,MAAA,CAAO,GAAA,EAAK,gBAAA;EAIZ,UAAA,CAAA,GAAc,gBAAA;EAAA,OAmBP,YAAA,CAAa,QAAA,EAAU,gBAAA,GAAmB,iBAAA;AAAA;AAAA,iBA4GnC,oBAAA,CAAqB,CAAA,EAAG,gBAAA,GAAmB,cAAA;;;iBC7P3C,eAAA,CACd,SAAA,UACA,aAAA,IAAiB,KAAA,EAAO,YAAA,YACvB,WAAA;;;UC/Bc,+BAAA;EACf,MAAA;EACA,SAAA;EACA,YAAA;EACA,OAAA;EACA,KAAA;EACA,IAAA;EACA,mBAAA,EAAqB,8BAAA;EACrB,mBAAA;EACA,UAAA;EACA,OAAA,GAAU,eAAA;EACV,QAAA;EACA,OAAA,GAAU,MAAA;AAAA;;;UCOF,gBAAA;EACR,IAAA,GAAO,MAAA;EACP,IAAA;IAAS,IAAA,GAAO,MAAA;IAAyB,QAAA,GAAW,MAAA;EAAA;AAAA;AAAA,UAG5C,gBAAA;EACR,MAAA;EACA,QAAA,GAAW,MAAA;EACX,IAAA;EACA,UAAA;IAAe,KAAA;IAAgB,MAAA;IAAiB,UAAA;IAAqB,MAAA;EAAA;EACrE,KAAA;AAAA;AAAA,UAGe,WAAA;EACf,MAAA;EACA,SAAA;EACA,KAAA,EAAO,UAAA,QAAkB,eAAA;EACzB,iBAAA,EAAmB,iBAAA;AAAA;AAAA,cAuQf,uBAAA;EACJ,MAAA,EAAQ,cAAA;EAAA,iBACS,WAAA;EAAA,QACT,QAAA;EACR,EAAA;cAEY,OAAA;IAAW,MAAA,GAAS,cAAA;IAAgB,EAAA;EAAA;EAY1C,OAAA,CACJ,MAAA,UACA,OAAA,GAAS,gBAAA,GACR,OAAA,CAAQ,gBAAA;AAAA"}

package/dist/presets-DwfDWckA.mjs

@@ -0,0 +1,80 @@
+//#region src/agent/presets.ts
+const AGENT_PRESETS = {
+	codex: {
+		adapter: "codex-json",
+		command: "codex",
+		args: [
+			"--ask-for-approval",
+			"never",
+			"exec",
+			"--json",
+			"--sandbox",
+			"workspace-write",
+			"--skip-git-repo-check",
+			"--ephemeral",
+			"--ignore-user-config",
+			"--ignore-rules",
+			"-"
+		],
+		testedAgainst: "Codex CLI 0.13x (exec --json)"
+	},
+	"claude-code": {
+		adapter: "claude-code-json",
+		command: "claude",
+		args: [
+			"-p",
+			"--permission-mode",
+			"bypassPermissions",
+			"--output-format",
+			"stream-json",
+			"--verbose",
+			"--no-session-persistence"
+		],
+		testedAgainst: "Claude Code CLI 2.x (-p --output-format stream-json)"
+	},
+	pi: {
+		adapter: "pi-json",
+		command: "pi",
+		args: ["--mode", "json"],
+		testedAgainst: "Pi CLI (--mode json)"
+	}
+};
+const PRESET_IDS = Object.keys(AGENT_PRESETS);
+/**
+* Merge a preset with explicit config. Explicit `adapter`/`command`/`args`
+* always win over the preset. `extraArgs` appends to whichever args list was
+* chosen; when that list ends with a lone `-` (prompt-on-stdin marker, e.g.
+* Codex), the extra args are inserted before it so the marker stays last.
+*/
+function resolveInvocation(input) {
+	const preset = input.preset ? AGENT_PRESETS[input.preset] : void 0;
+	if (input.preset && !preset) return { error: `unknown preset "${input.preset}". Supported presets: ${PRESET_IDS.join(", ")}. Use adapter/command/args directly for a custom agent.` };
+	const args = input.args ?? preset?.args;
+	const merged = {
+		...input.adapter ?? preset?.adapter ? { adapter: input.adapter ?? preset?.adapter } : {},
+		...input.command ?? preset?.command ? { command: input.command ?? preset?.command } : {},
+		...args ? { args } : {},
+		...input.preset ? { preset: input.preset } : {}
+	};
+	if (input.extraArgs && input.extraArgs.length > 0) {
+		const base = [...merged.args ?? []];
+		if (base.length > 0 && base[base.length - 1] === "-") merged.args = [
+			...base.slice(0, -1),
+			...input.extraArgs,
+			"-"
+		];
+		else merged.args = [...base, ...input.extraArgs];
+	}
+	return merged;
+}
+/** The note shown when a spawn fails and the config came from a preset. */
+function presetStalenessHint(presetId) {
+	if (!presetId) return "";
+	const preset = AGENT_PRESETS[presetId];
+	if (!preset) return "";
+	return ` The "${presetId}" preset was tested against ${preset.testedAgainst}; if the CLI updated recently its flags may have changed.`;
+}
+//#endregion
+export { resolveInvocation as i, PRESET_IDS as n, presetStalenessHint as r, AGENT_PRESETS as t };
+
+//# sourceMappingURL=presets-DwfDWckA.mjs.map

package/dist/presets-DwfDWckA.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"presets-DwfDWckA.mjs","names":[],"sources":["../src/agent/presets.ts"],"sourcesContent":["/**\n * Agent presets: known-good CLI invocations per supported agent.\n *\n * A preset owns the harness wiring (adapter, command, flags) so test authors\n * write `preset: codex` instead of hand-typing flag lists that drift as the\n * CLIs change. Every field stays overridable in provider config, and\n * `extraArgs` appends run-specific flags without re-declaring the base list.\n */\n\nexport interface AgentPreset {\n  /** Adapter id registered in the adapter catalog. */\n  adapter: string;\n  /** Executable name or path. */\n  command: string;\n  /** Known-good argument list for noninteractive eval runs. */\n  args: readonly string[];\n  /**\n   * Human-readable note about the CLI version range this preset was written\n   * against. Surfaced in stale-flag error messages, never enforced.\n   */\n  testedAgainst: string;\n}\n\nexport const AGENT_PRESETS: Readonly<Record<string, AgentPreset>> = {\n  codex: {\n    adapter: \"codex-json\",\n    command: \"codex\",\n    args: [\n      \"--ask-for-approval\",\n      \"never\",\n      \"exec\",\n      \"--json\",\n      \"--sandbox\",\n      \"workspace-write\",\n      \"--skip-git-repo-check\",\n      \"--ephemeral\",\n      \"--ignore-user-config\",\n      \"--ignore-rules\",\n      \"-\",\n    ],\n    testedAgainst: \"Codex CLI 0.13x (exec --json)\",\n  },\n  \"claude-code\": {\n    adapter: \"claude-code-json\",\n    command: \"claude\",\n    args: [\n      \"-p\",\n      \"--permission-mode\",\n      \"bypassPermissions\",\n      \"--output-format\",\n      \"stream-json\",\n      \"--verbose\",\n      \"--no-session-persistence\",\n    ],\n    testedAgainst: \"Claude Code CLI 2.x (-p --output-format stream-json)\",\n  },\n  pi: {\n    adapter: \"pi-json\",\n    command: \"pi\",\n    args: [\"--mode\", \"json\"],\n    testedAgainst: \"Pi CLI (--mode json)\",\n  },\n};\n\nexport const PRESET_IDS = Object.keys(AGENT_PRESETS);\n\nexport interface ResolvedInvocation {\n  adapter?: string;\n  command?: string;\n  args?: readonly string[];\n  preset?: string;\n}\n\nexport interface PresetInput {\n  preset?: string;\n  adapter?: string;\n  command?: string;\n  args?: readonly string[];\n  extraArgs?: readonly string[];\n}\n\n/**\n * Merge a preset with explicit config. Explicit `adapter`/`command`/`args`\n * always win over the preset. `extraArgs` appends to whichever args list was\n * chosen; when that list ends with a lone `-` (prompt-on-stdin marker, e.g.\n * Codex), the extra args are inserted before it so the marker stays last.\n */\nexport function resolveInvocation(\n  input: PresetInput,\n): ResolvedInvocation | { error: string } {\n  const preset = input.preset ? AGENT_PRESETS[input.preset] : undefined;\n  if (input.preset && !preset) {\n    return {\n      error:\n        `unknown preset \"${input.preset}\". Supported presets: ${PRESET_IDS.join(\", \")}. ` +\n        \"Use adapter/command/args directly for a custom agent.\",\n    };\n  }\n\n  const args = input.args ?? preset?.args;\n  const merged: ResolvedInvocation = {\n    ...(input.adapter ?? preset?.adapter\n      ? { adapter: input.adapter ?? preset?.adapter }\n      : {}),\n    ...(input.command ?? preset?.command\n      ? { command: input.command ?? preset?.command }\n      : {}),\n    ...(args ? { args } : {}),\n    ...(input.preset ? { preset: input.preset } : {}),\n  };\n\n  if (input.extraArgs && input.extraArgs.length > 0) {\n    const base = [...(merged.args ?? [])];\n    if (base.length > 0 && base[base.length - 1] === \"-\") {\n      merged.args = [...base.slice(0, -1), ...input.extraArgs, \"-\"];\n    } else {\n      merged.args = [...base, ...input.extraArgs];\n    }\n  }\n\n  return merged;\n}\n\n/** The note shown when a spawn fails and the config came from a preset. */\nexport function presetStalenessHint(presetId: string | undefined): string {\n  if (!presetId) return \"\";\n  const preset = AGENT_PRESETS[presetId];\n  if (!preset) return \"\";\n  return ` The \"${presetId}\" preset was tested against ${preset.testedAgainst}; if the CLI updated recently its flags may have changed.`;\n}\n"],"mappings":";AAuBA,MAAa,gBAAuD;CAClE,OAAO;EACL,SAAS;EACT,SAAS;EACT,MAAM;GACJ;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACD;EACD,eAAe;EAChB;CACD,eAAe;EACb,SAAS;EACT,SAAS;EACT,MAAM;GACJ;GACA;GACA;GACA;GACA;GACA;GACA;GACD;EACD,eAAe;EAChB;CACD,IAAI;EACF,SAAS;EACT,SAAS;EACT,MAAM,CAAC,UAAU,OAAO;EACxB,eAAe;EAChB;CACF;AAED,MAAa,aAAa,OAAO,KAAK,cAAc;;;;;;;AAuBpD,SAAgB,kBACd,OACwC;CACxC,MAAM,SAAS,MAAM,SAAS,cAAc,MAAM,UAAU,KAAA;AAC5D,KAAI,MAAM,UAAU,CAAC,OACnB,QAAO,EACL,OACE,mBAAmB,MAAM,OAAO,wBAAwB,WAAW,KAAK,KAAK,CAAC,0DAEjF;CAGH,MAAM,OAAO,MAAM,QAAQ,QAAQ;CACnC,MAAM,SAA6B;EACjC,GAAI,MAAM,WAAW,QAAQ,UACzB,EAAE,SAAS,MAAM,WAAW,QAAQ,SAAS,GAC7C,EAAE;EACN,GAAI,MAAM,WAAW,QAAQ,UACzB,EAAE,SAAS,MAAM,WAAW,QAAQ,SAAS,GAC7C,EAAE;EACN,GAAI,OAAO,EAAE,MAAM,GAAG,EAAE;EACxB,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,QAAQ,GAAG,EAAE;EACjD;AAED,KAAI,MAAM,aAAa,MAAM,UAAU,SAAS,GAAG;EACjD,MAAM,OAAO,CAAC,GAAI,OAAO,QAAQ,EAAE,CAAE;AACrC,MAAI,KAAK,SAAS,KAAK,KAAK,KAAK,SAAS,OAAO,IAC/C,QAAO,OAAO;GAAC,GAAG,KAAK,MAAM,GAAG,GAAG;GAAE,GAAG,MAAM;GAAW;GAAI;MAE7D,QAAO,OAAO,CAAC,GAAG,MAAM,GAAG,MAAM,UAAU;;AAI/C,QAAO;;;AAIT,SAAgB,oBAAoB,UAAsC;AACxE,KAAI,CAAC,SAAU,QAAO;CACtB,MAAM,SAAS,cAAc;AAC7B,KAAI,CAAC,OAAQ,QAAO;AACpB,QAAO,SAAS,SAAS,8BAA8B,OAAO,cAAc"}