@coder/aegis 0.1.0

package/lib/types/builder.d.ts

@@ -0,0 +1,144 @@
+import type { ContrastDoc, GeneratedSuite, GraderSpecDoc, OverlayActionDoc, SkillApplicable } from "./schema.js";
+export interface AuthoringGraderRef {
+    readonly kind: "aegis.grader-ref";
+    readonly id: string;
+    readonly spec: GraderSpecDoc;
+    readonly with?: Record<string, string>;
+    readonly inline?: boolean;
+}
+export interface ExperimentConfig {
+    backend: string;
+    backendOptions?: BackendOptionsInput;
+    hypothesis: string;
+    subject: ExperimentSubjectInput;
+    repeats: number;
+    budget: BudgetInput;
+    primaryMetric: string;
+    efficacyMetrics: string[];
+    guardrailMetrics: string[];
+    scheduling: SchedulingInput;
+}
+export interface BackendOptionsInput {
+    docker?: DockerBackendOptionsInput;
+    coder?: CoderBackendOptionsInput;
+}
+export interface DockerBackendOptionsInput {
+    image: string;
+}
+export interface CoderBackendOptionsInput {
+    template: string;
+}
+export interface ExperimentSubjectInput {
+    harness: string;
+    harnessVersion: string;
+    controlTransport: string;
+    model: string;
+    allowedTools: string[];
+    env?: Record<string, string>;
+}
+export interface BudgetInput {
+    maxUsd: number;
+    confirmRequired: boolean;
+    pilotMaxUsd: number;
+}
+export interface SchedulingInput {
+    order: string;
+    retry: string;
+}
+export interface SuiteInput {
+    setup?: string[];
+    defaultGraders?: AuthoringGraderRef[];
+}
+export interface BaseRepoInput {
+    git: {
+        repo: string;
+        ref: string;
+    };
+    tools?: string[];
+    env?: Record<string, string>;
+}
+export interface TestInput {
+    suite?: string;
+    baseRepo: string;
+    skillApplicable: SkillApplicable;
+    goldenOpCount?: number;
+    tags?: string[];
+    patches?: string[];
+}
+export interface AnalysisInput {
+    pairing: string;
+    perArmCI: string;
+    significance: string;
+    contrasts: Array<ContrastDoc | readonly [string, string]>;
+}
+export declare function defineExperiment(id: string, define: (evals: ExperimentBuilder) => void): GeneratedSuite;
+export declare function graderRef(id: string, spec: GraderSpecDoc, withParams?: Record<string, string>): AuthoringGraderRef;
+export declare function inlineGrader(id: string, spec: GraderSpecDoc): AuthoringGraderRef;
+export declare class ExperimentBuilder {
+    private readonly id;
+    private config?;
+    private readonly suites;
+    private readonly baseRepos;
+    private readonly variants;
+    private readonly tasks;
+    private readonly registry;
+    private activeSuite?;
+    private analysisDoc?;
+    constructor(id: string);
+    configure(config: ExperimentConfig): void;
+    suite(id: string, input?: SuiteInput): void;
+    baseRepo(id: string, input: BaseRepoInput): void;
+    variant(id: string, define: (arm: VariantBuilder) => void): void;
+    test(id: string, input: TestInput, define: (test: TestBuilder) => void): void;
+    analysis(input: AnalysisInput): void;
+    customGrader(id: string, spec: GraderSpecDoc): CustomGraderHandle;
+    build(): GeneratedSuite;
+    private registerGrader;
+    private taskDoc;
+}
+export declare class VariantBuilder {
+    private readonly context;
+    private readonly overlay;
+    constructor(context: string);
+    remove(path: string): void;
+    file(path: string): OverlayFileSwapBuilder;
+    skill(skill: string): OverlaySkillSwapBuilder;
+    promptContext(value: string): void;
+    promptContextFile(file: string): void;
+    toolAllowlist(tools: string[]): void;
+    removeToolAllowlist(): void;
+    modelID(model: string): void;
+    harnessFlag(key: string, value: string): void;
+    removeHarnessFlag(key: string): void;
+    actions(): OverlayActionDoc[];
+}
+export declare class OverlayFileSwapBuilder {
+    private readonly overlay;
+    private readonly path;
+    constructor(overlay: OverlayActionDoc[], path: string);
+    from(file: string): void;
+}
+export declare class OverlaySkillSwapBuilder {
+    private readonly overlay;
+    private readonly skill;
+    constructor(overlay: OverlayActionDoc[], skill: string);
+    from(file: string): void;
+    remove(): void;
+}
+export declare class TestBuilder {
+    private readonly context;
+    private promptValue?;
+    private readonly expectations;
+    constructor(context: string);
+    promptText(): string | undefined;
+    prompt(strings: TemplateStringsArray | string, ...values: unknown[]): void;
+    expect(grader: AuthoringGraderRef): void;
+    graders(): AuthoringGraderRef[];
+}
+export declare class CustomGraderHandle {
+    private readonly id;
+    private readonly spec;
+    constructor(id: string, spec: GraderSpecDoc);
+    use(withParams?: Record<string, string>): AuthoringGraderRef;
+}
+export declare function assertCanonicalValue(context: string, value: unknown): void;

package/lib/types/bundle.d.ts

@@ -0,0 +1,33 @@
+import { type ResolvedEvalSource } from "./layout.js";
+import { type GeneratedSuite, type GraderRegistryDoc, type TaskDoc } from "./schema.js";
+export declare const EVAL_BUNDLE_SCHEMA_VERSION = "aegis.eval_bundle/1.0";
+export interface EvalBundleDoc {
+    schema_version: typeof EVAL_BUNDLE_SCHEMA_VERSION;
+    experiment: GeneratedSuite["experiment"];
+    tasks: TaskDoc[];
+    graders: GraderRegistryDoc;
+    assets: BundleAssetDoc[];
+}
+export interface BundleAssetDoc {
+    id: string;
+    kind: "inline_tree" | "git";
+    inline?: InlineTreeAssetDoc;
+    git?: GitAssetDoc;
+}
+export interface InlineTreeAssetDoc {
+    files: InlineFileDoc[];
+}
+export interface InlineFileDoc {
+    path: string;
+    encoding?: "base64";
+    content: string;
+    executable?: boolean;
+}
+export interface GitAssetDoc {
+    repo: string;
+    ref: string;
+    digest?: string;
+}
+export declare function cloneGeneratedSuite(suite: GeneratedSuite): GeneratedSuite;
+export declare function buildEvalBundle(suite: GeneratedSuite, source: ResolvedEvalSource): Promise<EvalBundleDoc>;
+export declare function serializeEvalBundle(bundle: EvalBundleDoc): string;

package/lib/types/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/types/embedded-cli.d.ts

@@ -0,0 +1,2 @@
+export * from "./index.js";
+import "./cli.js";

package/lib/types/graders.d.ts

@@ -0,0 +1,22 @@
+import { type AuthoringGraderRef } from "./builder.js";
+export type Pattern = string | RegExp;
+export declare const qa: {
+    reportExists(): AuthoringGraderRef;
+    matches(pattern: Pattern): AuthoringGraderRef;
+    mentions(pattern: Pattern): AuthoringGraderRef;
+    mentionsBoth(patternA: Pattern, patternB: Pattern): AuthoringGraderRef;
+};
+export declare const trace: {
+    noAgentTtySession(): AuthoringGraderRef;
+};
+export interface JudgeSatisfiesOptions {
+    rubric: string;
+    criterion: string;
+    minScore: number;
+    model?: string;
+    validation?: Record<string, unknown>;
+}
+export declare const judge: {
+    satisfies(options: JudgeSatisfiesOptions): AuthoringGraderRef;
+};
+export declare function serializePattern(pattern: Pattern): string;

package/lib/types/index.d.ts

@@ -0,0 +1,4 @@
+export { CustomGraderHandle, ExperimentBuilder, OverlayFileSwapBuilder, OverlaySkillSwapBuilder, TestBuilder, VariantBuilder, assertCanonicalValue, defineExperiment, graderRef, inlineGrader, type AnalysisInput, type AuthoringGraderRef, type BaseRepoInput, type BudgetInput, type ExperimentConfig, type ExperimentSubjectInput, type SchedulingInput, type SuiteInput, type TestInput, } from "./builder.js";
+export { judge, qa, serializePattern, trace, type JudgeSatisfiesOptions, type Pattern } from "./graders.js";
+export { SUPPORTED_GRADER_TYPES } from "./schema.js";
+export type { AnalysisDoc, BaseRepoDoc, BudgetDoc, ContrastDoc, ExperimentDoc, ExperimentSubjectDoc, GeneratedSuite, GraderRegistryDoc, GraderSpecDoc, OverlayActionDoc, OverlayKindDoc, SchedulingDoc, SkillApplicable, SupportedGraderType, SuiteDoc, TaskDoc, TaskGraderDoc, TaskSeedDoc, VariantDoc, } from "./schema.js";

package/lib/types/layout.d.ts

@@ -0,0 +1,23 @@
+import type { GeneratedSuite, TaskDoc } from "./schema.js";
+export type ResolvedEvalSource = FileEvalSource | LayoutEvalSource;
+export interface FileEvalSource {
+    kind: "file";
+    sourcePath: string;
+}
+export interface LayoutEvalSource {
+    kind: "layout";
+    sourcePath: string;
+    layoutDir: string;
+    layoutName: string;
+    promptPath: string;
+    seedSrcDir: string;
+}
+export declare function resolveEvalSource(src: string): Promise<ResolvedEvalSource>;
+export declare function layoutTask(suite: GeneratedSuite, layout: LayoutEvalSource): TaskDoc;
+export declare function readLayoutPrompt(layout: LayoutEvalSource): Promise<string>;
+export interface LayoutSeedFile {
+    path: string;
+    content: Buffer;
+    executable: boolean;
+}
+export declare function collectLayoutSeedFiles(seedSrcDir: string): Promise<LayoutSeedFile[]>;

package/lib/types/schema.d.ts

@@ -0,0 +1,148 @@
+export type SkillApplicable = "applicable" | "not_applicable" | "ambiguous";
+export interface ExperimentDoc {
+    id: string;
+    backend: string;
+    backend_options?: BackendOptionsDoc;
+    hypothesis: string;
+    subject: ExperimentSubjectDoc;
+    suite: string;
+    grader_registry: string[];
+    suites: Record<string, SuiteDoc>;
+    base_repos: Record<string, BaseRepoDoc>;
+    variants: VariantDoc[];
+    repeats: number;
+    budget: BudgetDoc;
+    primary_metric: string;
+    efficacy_metrics: string[];
+    guardrail_metrics: string[];
+    scheduling: SchedulingDoc;
+    analysis: AnalysisDoc;
+}
+export interface BackendOptionsDoc {
+    docker?: DockerBackendOptionsDoc;
+    coder?: CoderBackendOptionsDoc;
+}
+export interface DockerBackendOptionsDoc {
+    image: string;
+}
+export interface CoderBackendOptionsDoc {
+    template: string;
+}
+export interface ExperimentSubjectDoc {
+    harness: string;
+    harness_version: string;
+    control_transport: string;
+    model: string;
+    allowed_tools: string[];
+    env: Record<string, string>;
+}
+export interface SuiteDoc {
+    default_graders: string[];
+    setup: string[];
+}
+export interface BaseRepoDoc {
+    git: GitRefDoc;
+    tools: string[];
+    env: Record<string, string>;
+}
+export interface GitRefDoc {
+    repo: string;
+    ref: string;
+}
+export interface VariantDoc {
+    id: string;
+    overlay: OverlayActionDoc[];
+}
+export type OverlayKindDoc = "file" | "skill" | "prompt/context-delta" | "tool-allowlist-delta" | "model-id-delta" | "harness-flag-delta";
+export interface OverlayActionDoc {
+    kind: OverlayKindDoc;
+    op: "swap" | "remove";
+    path?: string;
+    skill?: string;
+    file?: string;
+    value?: string;
+    key?: string;
+    tools?: string[];
+    model?: string;
+}
+export interface BudgetDoc {
+    max_usd: number;
+    confirm_required: boolean;
+    pilot_max_usd: number;
+}
+export interface SchedulingDoc {
+    order: string;
+    retry: string;
+}
+export interface AnalysisDoc {
+    pairing: string;
+    per_arm_ci: string;
+    significance: string;
+    contrasts: ContrastDoc[];
+}
+export interface ContrastDoc {
+    from: string;
+    to: string;
+}
+export interface TaskDoc {
+    id: string;
+    suite: string;
+    prompt: string;
+    seed: TaskSeedDoc;
+    skill_applicable: SkillApplicable;
+    golden_op_count?: number;
+    graders: TaskGraderDoc[];
+    tags: string[];
+}
+export interface TaskSeedDoc {
+    base_repo: string;
+    patches?: string[];
+}
+export interface TaskGraderDoc {
+    use?: string;
+    with?: Record<string, string | number | boolean>;
+    id?: string;
+    type?: SupportedGraderType;
+    run?: string;
+    timeout_s?: number;
+    assert?: string;
+    path?: string;
+    baseline?: string;
+    file?: string;
+    section?: string;
+    params?: string[];
+    match?: string;
+    regression?: boolean;
+    exploit?: boolean;
+}
+export type GraderRegistryDoc = Record<string, GraderSpecDoc>;
+export declare const SUPPORTED_GRADER_TYPES: readonly ["command", "state", "trace", "judge", "subprocess"];
+export type SupportedGraderType = (typeof SUPPORTED_GRADER_TYPES)[number];
+export interface GraderSpecDoc {
+    id?: string;
+    type: SupportedGraderType;
+    run?: string;
+    binary?: string;
+    env?: Record<string, string>;
+    timeout_s?: number;
+    assert?: string;
+    guardrail?: boolean;
+    metric?: boolean;
+    regression?: boolean;
+    exploit?: boolean;
+    path?: string;
+    baseline?: string;
+    file?: string;
+    section?: string;
+    params?: string[];
+    match?: string;
+    probe?: string;
+    rubric?: string;
+    model?: string;
+    args?: Record<string, unknown>;
+}
+export interface GeneratedSuite {
+    experiment: ExperimentDoc;
+    tasks: TaskDoc[];
+    graders: GraderRegistryDoc;
+}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@coder/aegis",
+  "version": "0.1.0",
+  "description": "aegis CLI and TypeScript eval-authoring facade (Go binary with embedded TS eval compiler and web UI, plus the @coder/aegis import surface)",
+  "type": "module",
+  "bin": {
+    "aegis": "bin/aegis.mjs"
+  },
+  "types": "./lib/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "import": "./lib/index.js",
+      "default": "./lib/index.js"
+    }
+  },
+  "files": [
+    "bin",
+    "lib"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/coder/aegis.git"
+  },
+  "optionalDependencies": {
+    "@coder/aegis-linux-x64": "0.1.0",
+    "@coder/aegis-linux-arm64": "0.1.0",
+    "@coder/aegis-darwin-x64": "0.1.0",
+    "@coder/aegis-darwin-arm64": "0.1.0",
+    "@coder/aegis-win32-x64": "0.1.0"
+  }
+}