@nexrall/code-core 1.4.74 → 1.4.75

package/README.md

@@ -12,6 +12,7 @@ npm install @nexrall/code-core
 |--------|-------------|
 | `@nexrall/code-core` | Everything via the root export |
 | `@nexrall/code-core/agent` | `runAgentLoop` — the main agentic loop |
+| `@nexrall/code-core/agent-types` | `loadAgentTypes`, `findAgentType` — sub-agent type definitions |
 | `@nexrall/code-core/tools` | `executeTool` — built-in tool executor (bash, file I/O, search…) |
 | `@nexrall/code-core/symbols` | `getSymbols`, `getWorkspaceSymbols` — regex-based LSP-lite scanner |
 | `@nexrall/code-core/checkpoint` | `CheckpointManager` — persistent rewind / rollback |
@@ -33,7 +34,7 @@ const messages = [
 ];
 
 const opts: AgentLoopOptions = {
-  model: 'turbo',          // 'turbo' (Sonnet) | 'pro' (Opus) | 'ultra' (Fable 5)
+  model: 'claude-sonnet-5-5',   // any catalogue model id; the tier aliases still work
   workDir: process.cwd(),
   env: { platform: 'node', cwd: process.cwd(), shell: 'bash' },
   clientType: 'cli',
@@ -53,13 +54,13 @@ const history = await runAgentLoop(messages, opts);
 console.log('Done —', history.length, 'messages');
 ```
 
-> **Requires authentication.** The agent streams through the Nexrall API — users must be logged in via `nexrall-code login` (or set `NEXRALL_TOKEN` env var).
+> **Requires authentication.** The agent streams through the Nexrall API — users must be logged in via `nex auth` (or set the `NEXRALL_TOKEN` env var).
 
 ## Agent loop options
 
 ```ts
 interface AgentLoopOptions {
-  model: 'turbo' | 'pro' | 'ultra';
+  model?: string;                // e.g. 'claude-sonnet-5-5', 'gpt-6-astra', 'deepseek-flash'
   workDir: string;
   env: EnvContext;
   clientType?: 'cli' | 'vscode';

package/dist/index.d.ts

@@ -36,6 +36,7 @@ export * from './plugins/index';
 export * from './plugins/installer';
 export * from './plugins/sources';
 export * from './plugins/data';
+export * from './plugins/eval';
 export * from './agent/trust';
 export * from './agent/crossProcessLock';
 export * from './agent/worktree';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC"}

package/dist/index.js

@@ -52,6 +52,7 @@ __exportStar(require("./plugins/index"), exports);
 __exportStar(require("./plugins/installer"), exports);
 __exportStar(require("./plugins/sources"), exports);
 __exportStar(require("./plugins/data"), exports);
+__exportStar(require("./plugins/eval"), exports);
 __exportStar(require("./agent/trust"), exports);
 __exportStar(require("./agent/crossProcessLock"), exports);
 __exportStar(require("./agent/worktree"), exports);

package/dist/plugins/eval.d.ts

@@ -0,0 +1,199 @@
+export declare const EVAL_SCHEMA_VERSION = 1;
+/** CC's default: each case runs three times. */
+export declare const EVAL_DEFAULT_RUNS = 3;
+export declare const EVAL_MAX_RUNS = 50;
+export declare const EVAL_DEFAULT_THRESHOLD = 1;
+export declare const EVAL_DEFAULT_MAX_TURNS = 10;
+export declare const EVAL_DEFAULT_TIMEOUT_S = 300;
+/** CC caps both; larger values are refused, not clamped silently. */
+export declare const EVAL_MAX_TURNS_CEILING = 200;
+export declare const EVAL_MAX_TIMEOUT_S = 3600;
+export type GraderType = 'regex' | 'tool_used' | 'tool_order' | 'file_exists' | 'llm' | 'baseline';
+/**
+ * Which arm a grader scores in.
+ *  · 'both' (default) — counted for the plugin arm AND the no-plugin baseline.
+ *  · 'with-only' — counted only when the plugin was loaded (CC's rule for the
+ *    two-arm runs: a grader that can only pass WITH the plugin must not drag
+ *    the baseline down).
+ */
+export type GraderArm = 'both' | 'with-only';
+export interface EvalGrader {
+    /** File name inside graders/, for reporting. */
+    file: string;
+    type: GraderType;
+    weight: number;
+    arm: GraderArm;
+    pattern?: string;
+    flags?: string;
+    /** 'contains' (default) | 'not_contains' | 'count:<n>' */
+    match?: string;
+    /** 'last_message' (default) | 'trace' | 'file' */
+    target?: string;
+    /** target 'file': project-relative path or glob to read. */
+    pathPattern?: string;
+    tool?: string;
+    inputMatch?: string;
+    min?: number;
+    max?: number;
+    before?: string;
+    after?: string;
+    path?: string;
+    exists?: boolean;
+    criteria?: string;
+    /** llm: 'last_message' (default) | 'trace'. */
+    focus?: string;
+    /** baseline: the reference transcript file (case-relative). */
+    baselineFile?: string;
+}
+export interface EvalCase {
+    name: string;
+    dir: string;
+    prompt: string;
+    runs: number;
+    maxTurns: number;
+    timeoutSeconds: number;
+    allowedTools: string[];
+    tags: string[];
+    expectedOutcome?: string;
+    graders: EvalGrader[];
+}
+export interface DiscoveredSuite {
+    cases: EvalCase[];
+    /** Case dirs that failed to parse — reported, never silently dropped. */
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+}
+/** One tool call as the child's stream reported it. */
+export interface EvalToolCall {
+    tool: string;
+    input: unknown;
+}
+/**
+ * What ONE run produced. `readFile` resolves a project-relative path or glob
+ * against the run's workspace — the caller owns the filesystem, so the graders
+ * stay pure.
+ */
+export interface EvalRunTrace {
+    lastMessage: string;
+    toolCalls: EvalToolCall[];
+    files: string[];
+    readFile: (relOrGlob: string) => string | null;
+    /** USD when the server reported it; null = unreported (never invented). */
+    costUsd: number | null;
+    tokens: number;
+    exitCode: number;
+    error?: string;
+}
+export interface GraderVerdict {
+    passed: boolean;
+    why: string;
+}
+/** The injected model call for llm/baseline graders. null = could not judge. */
+export type EvalJudge = (args: {
+    criteria: string;
+    focusText: string;
+    baselineText?: string;
+}) => Promise<boolean | null>;
+export interface CaseRunOutcome {
+    arm: 'with' | 'without';
+    run: number;
+    score: number;
+    graderResults: Array<{
+        file: string;
+        type: GraderType;
+        weight: number;
+        passed: boolean;
+        why: string;
+    }>;
+    costUsd: number | null;
+    costReported: boolean;
+    durationMs: number;
+    error?: string;
+}
+export interface CaseOutcome {
+    name: string;
+    with: CaseRunOutcome[];
+    without: CaseRunOutcome[];
+    withScore: number | null;
+    withoutScore: number | null;
+    delta: number | null;
+    passed: boolean;
+}
+export interface EvalReport {
+    schemaVersion: number;
+    plugin: string;
+    startedAt: number;
+    durationMs: number;
+    threshold: number;
+    runs: number;
+    judgeModel?: string;
+    partial?: boolean;
+    partialReason?: string;
+    cases: CaseOutcome[];
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+    aggregates: {
+        casesPassed: number;
+        casesTotal: number;
+        overallScore: number;
+        meanDelta: number | null;
+    };
+    costUsd: number;
+    costMissing: number;
+}
+/** `[a, b]` / `a, b` / `a` → ['a','b'] — the two shapes real files use. */
+export declare function parseList(value: string | undefined): string[];
+/**
+ * One grader file → EvalGrader. Throws on anything that would make the grader
+ * silently unscorable (unknown type, missing pattern/tool/path, bad count).
+ */
+export declare function parseGraderFile(file: string, raw: string): EvalGrader;
+/** One case directory → EvalCase. See the header for the layout. */
+export declare function parseEvalCase(dir: string): EvalCase;
+/** Every case under `<pluginDir>/evals` (the results dir is never a case). */
+export declare function discoverEvalCases(pluginDir: string): DiscoveredSuite;
+/** The text a grader's target names. Exposed for tests and the report's `why`. */
+export declare function targetText(g: EvalGrader, trace: EvalRunTrace): string | null;
+/**
+ * Score ONE grader over ONE run. Async because llm/baseline call the judge.
+ * A grader that cannot be evaluated FAILS (with the reason in `why`) — it must
+ * never pass by accident, or a broken suite would look green.
+ */
+export declare function evaluateGrader(g: EvalGrader, trace: EvalRunTrace, judge: EvalJudge | undefined): Promise<GraderVerdict>;
+/** Which graders count in an arm: with-only graders never score the baseline. */
+export declare function gradersForArm(graders: EvalGrader[], arm: 'with' | 'without'): EvalGrader[];
+/** Score one run: weighted fraction of its graders that passed. */
+export declare function scoreRun(graders: EvalGrader[], trace: EvalRunTrace, judge: EvalJudge | undefined, arm: 'with' | 'without'): Promise<{
+    score: number;
+    results: CaseRunOutcome['graderResults'];
+}>;
+/**
+ * Fold run outcomes into the report. Case score = mean of the plugin-arm run
+ * scores; `delta` = with − without (null when the baseline did not run).
+ * A case PASSES at ≥ threshold (CC's default: every grader, every run).
+ */
+export declare function aggregateReport(i: {
+    plugin: string;
+    startedAt: number;
+    durationMs: number;
+    threshold: number;
+    runs: number;
+    judgeModel?: string;
+    cases: Array<{
+        name: string;
+        with: CaseRunOutcome[];
+        without: CaseRunOutcome[];
+    }>;
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+    partial?: {
+        reason: string;
+    };
+}): EvalReport;
+//# sourceMappingURL=eval.d.ts.map

package/dist/plugins/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/plugins/eval.ts"],"names":[],"mappings":"AAiCA,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,gDAAgD;AAChD,eAAO,MAAM,iBAAiB,IAAI,CAAC;AACnC,eAAO,MAAM,aAAa,KAAK,CAAC;AAChC,eAAO,MAAM,sBAAsB,IAAM,CAAC;AAC1C,eAAO,MAAM,sBAAsB,KAAK,CAAC;AACzC,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAC1C,qEAAqE;AACrE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAC1C,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAMvC,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,WAAW,GAAG,YAAY,GAAG,aAAa,GAAG,KAAK,GAAG,UAAU,CAAC;AAEnG;;;;;;GAMG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,WAAW,CAAC;AAE7C,MAAM,WAAW,UAAU;IACzB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,SAAS,CAAC;IAEf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,yEAAyE;IACzE,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/C;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/C,2EAA2E;IAC3E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,gFAAgF;AAChF,MAAM,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,KAAK,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;AAE9B,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvG,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,cAAc,EAAE,CAAC;IACvB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,UAAU,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;KAC1B,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAID,2EAA2E;AAC3E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAO7D;AAUD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,CAoErE;AAED,oEAAoE;AACpE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,CAmDnD;AAED,8EAA8E;AAC9E,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe,CA6BpE;AAID,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,GAAG,MAAM,GAAG,IAAI,CAY5E;AAkBD;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,CAAC,EAAE,UAAU,EACb,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,SAAS,GAAG,SAAS,GAC3B,OAAO,CAAC,aAAa,CAAC,CAgFxB;AAED,iFAAiF;AACjF,wBAAgB,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,EAAE,CAE1F;AAED,mEAAmE;AACnE,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,UAAU,EAAE,EACrB,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,SAAS,GAAG,SAAS,EAC5B,GAAG,EAAE,MAAM,GAAG,SAAS,GACtB,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,cAAc,CAAC,eAAe,CAAC,CAAA;CAAE,CAAC,CAatE;AAID;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,cAAc,EAAE,CAAC;QACvB,OAAO,EAAE,cAAc,EAAE,CAAC;KAC3B,CAAC,CAAC;IACH,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B,GAAG,UAAU,CAsCb"}

package/dist/plugins/eval.js

@@ -0,0 +1,470 @@
+"use strict";
+// S21 — `nex plugin eval`: run a plugin against a suite of test cases and score
+// the results (Claude Code's `claude plugin eval`, shipped in their W37).
+//
+// This file is the PURE half: case discovery/parsing, the grader vocabulary,
+// the trace-based graders, and the scoring/aggregation. The caller owns ALL
+// I/O — spawning each run, reading the child's event log, and judging (the
+// `llm`/`baseline` graders call a model through an injected `judge` function so
+// this module stays unit-testable without a backend).
+//
+// Suite layout (CC's, minus what we do not support yet — see below):
+//
+//   my-plugin/
+//     evals/
+//       first-case/
+//         prompt.md          # frontmatter = case fields; body = the prompt
+//         graders/
+//           criteria.md      # frontmatter = grader; body = llm criteria
+//           skill-fired.md
+//       results/             # written by each run; never discovered as a case
+//
+// Deliberately NOT supported yet (refused loudly, never silently skipped):
+//   · case.yaml — needs a real YAML parser, which this repo does not carry
+//     (the frontmatter reader is "not a full YAML parser" by design). A case
+//     that ships one is an ERROR, because silently ignoring its fields would
+//     run a DIFFERENT suite than the author wrote.
+//   · mock MCP servers (`evals/mocks/`) — a follow-up; a case using them is
+//     likewise refused rather than run without the mocks.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVAL_MAX_TIMEOUT_S = exports.EVAL_MAX_TURNS_CEILING = exports.EVAL_DEFAULT_TIMEOUT_S = exports.EVAL_DEFAULT_MAX_TURNS = exports.EVAL_DEFAULT_THRESHOLD = exports.EVAL_MAX_RUNS = exports.EVAL_DEFAULT_RUNS = exports.EVAL_SCHEMA_VERSION = void 0;
+exports.parseList = parseList;
+exports.parseGraderFile = parseGraderFile;
+exports.parseEvalCase = parseEvalCase;
+exports.discoverEvalCases = discoverEvalCases;
+exports.targetText = targetText;
+exports.evaluateGrader = evaluateGrader;
+exports.gradersForArm = gradersForArm;
+exports.scoreRun = scoreRun;
+exports.aggregateReport = aggregateReport;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const frontmatter_1 = require("../util/frontmatter");
+const nestedInstructions_1 = require("../agent/nestedInstructions");
+exports.EVAL_SCHEMA_VERSION = 1;
+/** CC's default: each case runs three times. */
+exports.EVAL_DEFAULT_RUNS = 3;
+exports.EVAL_MAX_RUNS = 50;
+exports.EVAL_DEFAULT_THRESHOLD = 1.0;
+exports.EVAL_DEFAULT_MAX_TURNS = 10;
+exports.EVAL_DEFAULT_TIMEOUT_S = 300;
+/** CC caps both; larger values are refused, not clamped silently. */
+exports.EVAL_MAX_TURNS_CEILING = 200;
+exports.EVAL_MAX_TIMEOUT_S = 3600;
+/** `match: count:N` in a regex grader. */
+const COUNT_RE = /^count:(\d+)$/;
+// ─── Parsing helpers ─────────────────────────────────────────────────────────
+/** `[a, b]` / `a, b` / `a` → ['a','b'] — the two shapes real files use. */
+function parseList(value) {
+    if (value === undefined)
+        return [];
+    const inner = value.trim().replace(/^\[|\]$/g, '');
+    return inner
+        .split(',')
+        .map((s) => s.trim().replace(/^["']|["']$/g, ''))
+        .filter(Boolean);
+}
+function parsePositiveInt(value, dflt, ceiling, what) {
+    if (value === undefined || value.trim() === '')
+        return dflt;
+    const n = Number(value.trim());
+    if (!Number.isInteger(n) || n < 1)
+        throw new Error(`${what} must be a positive integer (got "${value}")`);
+    if (n > ceiling)
+        throw new Error(`${what} ${n} exceeds the ceiling of ${ceiling}`);
+    return n;
+}
+/**
+ * One grader file → EvalGrader. Throws on anything that would make the grader
+ * silently unscorable (unknown type, missing pattern/tool/path, bad count).
+ */
+function parseGraderFile(file, raw) {
+    const { meta, body } = (0, frontmatter_1.parseFrontmatter)(raw);
+    const type = (meta.type ?? '').trim();
+    if (!['regex', 'tool_used', 'tool_order', 'file_exists', 'llm', 'baseline'].includes(type)) {
+        throw new Error(`grader "${file}": type must be regex|tool_used|tool_order|file_exists|llm|baseline (got "${meta.type ?? ''}")`);
+    }
+    const weightRaw = (meta.weight ?? '1').trim();
+    const weight = Number(weightRaw);
+    if (!Number.isFinite(weight) || weight <= 0)
+        throw new Error(`grader "${file}": weight must be a positive number`);
+    const arm = (meta.arm ?? 'both').trim() === 'with-only' ? 'with-only' : 'both';
+    if (meta.arm !== undefined && !['both', 'with-only'].includes(meta.arm.trim())) {
+        throw new Error(`grader "${file}": arm must be both or with-only (got "${meta.arm}")`);
+    }
+    const g = { file, type, weight, arm };
+    if (type === 'regex') {
+        if (!meta.pattern)
+            throw new Error(`grader "${file}": regex grader needs a \`pattern\``);
+        g.pattern = meta.pattern;
+        g.flags = meta.flags?.trim() || '';
+        const match = (meta.match ?? 'contains').trim();
+        if (!(match === 'contains' || match === 'not_contains' || COUNT_RE.test(match))) {
+            throw new Error(`grader "${file}": match must be contains | not_contains | count:<n> (got "${match}")`);
+        }
+        g.match = match;
+        const target = (meta.target ?? 'last_message').trim();
+        if (!['last_message', 'trace', 'file'].includes(target)) {
+            throw new Error(`grader "${file}": target must be last_message | trace | file (got "${target}")`);
+        }
+        g.target = target;
+        if (target === 'file') {
+            if (!meta.path)
+                throw new Error(`grader "${file}": target "file" needs a \`path\``);
+            g.pathPattern = meta.path;
+        }
+    }
+    else if (type === 'tool_used') {
+        if (!meta.tool)
+            throw new Error(`grader "${file}": tool_used grader needs a \`tool\``);
+        g.tool = meta.tool.trim();
+        g.inputMatch = meta.input_match?.trim() || undefined;
+        g.min = parsePositiveInt(meta.min, 1, 10000, 'min');
+        g.max = meta.max !== undefined && meta.max.trim() !== '' ? parsePositiveInt(meta.max, 1, 10000, 'max') : undefined;
+        if (g.max !== undefined && g.max < (g.min ?? 1))
+            throw new Error(`grader "${file}": max < min`);
+    }
+    else if (type === 'tool_order') {
+        if (!meta.before || !meta.after)
+            throw new Error(`grader "${file}": tool_order needs \`before\` and \`after\``);
+        g.before = meta.before.trim();
+        g.after = meta.after.trim();
+    }
+    else if (type === 'file_exists') {
+        if (!meta.path)
+            throw new Error(`grader "${file}": file_exists needs a \`path\``);
+        g.path = meta.path.trim();
+        const ex = (meta.exists ?? 'true').trim().toLowerCase();
+        g.exists = !(ex === 'false' || ex === '0' || ex === 'no');
+    }
+    else {
+        // llm / baseline: criteria from the body (the file IS the criteria text).
+        const criteria = body.trim();
+        if (!criteria)
+            throw new Error(`grader "${file}": ${type} grader needs criteria in the file body`);
+        g.criteria = criteria;
+        if (type === 'llm') {
+            const focus = (meta.focus ?? 'last_message').trim();
+            if (!['last_message', 'trace'].includes(focus)) {
+                throw new Error(`grader "${file}": focus must be last_message | trace (got "${focus}")`);
+            }
+            g.focus = focus;
+        }
+        else {
+            if (!meta.baseline_file)
+                throw new Error(`grader "${file}": baseline grader needs \`baseline_file\``);
+            g.baselineFile = meta.baseline_file.trim();
+        }
+    }
+    return g;
+}
+/** One case directory → EvalCase. See the header for the layout. */
+function parseEvalCase(dir) {
+    const promptFile = path.join(dir, 'prompt.md');
+    const caseYaml = path.join(dir, 'case.yaml');
+    if (fs.existsSync(caseYaml)) {
+        throw new Error('case.yaml is not supported yet (this repo carries no YAML parser) — move its fields into prompt.md frontmatter');
+    }
+    if (fs.existsSync(path.join(dir, 'mocks'))) {
+        throw new Error('mock MCP servers (mocks/) are not supported yet');
+    }
+    let raw;
+    try {
+        raw = fs.readFileSync(promptFile, 'utf-8');
+    }
+    catch {
+        throw new Error('no prompt.md (a case needs one; case.yaml alone is not supported yet)');
+    }
+    const { meta, body } = (0, frontmatter_1.parseFrontmatter)(raw);
+    const prompt = body.trim();
+    if (!prompt)
+        throw new Error('prompt.md has an empty prompt body');
+    const gradersDir = path.join(dir, 'graders');
+    let graderFiles;
+    try {
+        graderFiles = fs
+            .readdirSync(gradersDir)
+            .filter((f) => f.endsWith('.md'))
+            .sort();
+    }
+    catch {
+        throw new Error('no graders/ directory (at least one grader is required)');
+    }
+    if (!graderFiles.length)
+        throw new Error('graders/ holds no .md files (at least one grader is required)');
+    const graders = graderFiles.map((f) => parseGraderFile(f, fs.readFileSync(path.join(gradersDir, f), 'utf-8')));
+    return {
+        name: meta.name?.trim() || path.basename(dir),
+        dir,
+        prompt,
+        runs: parsePositiveInt(meta.runs, exports.EVAL_DEFAULT_RUNS, exports.EVAL_MAX_RUNS, 'runs'),
+        maxTurns: parsePositiveInt(meta.max_turns, exports.EVAL_DEFAULT_MAX_TURNS, exports.EVAL_MAX_TURNS_CEILING, 'max_turns'),
+        timeoutSeconds: parsePositiveInt(meta.timeout_seconds, exports.EVAL_DEFAULT_TIMEOUT_S, exports.EVAL_MAX_TIMEOUT_S, 'timeout_seconds'),
+        allowedTools: parseList(meta.allowed_tools),
+        tags: parseList(meta.tags),
+        expectedOutcome: meta.expected_outcome?.trim() || undefined,
+        graders,
+    };
+}
+/** Every case under `<pluginDir>/evals` (the results dir is never a case). */
+function discoverEvalCases(pluginDir) {
+    const evalsDir = path.join(pluginDir, 'evals');
+    const errors = [];
+    let names;
+    try {
+        names = fs
+            .readdirSync(evalsDir)
+            .filter((n) => n !== 'results' && !n.startsWith('.'))
+            .sort();
+    }
+    catch {
+        return { cases: [], errors: [] };
+    }
+    const cases = [];
+    for (const n of names) {
+        const dir = path.join(evalsDir, n);
+        let isDir;
+        try {
+            isDir = fs.statSync(dir).isDirectory();
+        }
+        catch {
+            isDir = false;
+        }
+        if (!isDir)
+            continue;
+        try {
+            cases.push(parseEvalCase(dir));
+        }
+        catch (e) {
+            errors.push({ dir, error: e.message });
+        }
+    }
+    return { cases, errors };
+}
+// ─── The graders ─────────────────────────────────────────────────────────────
+/** The text a grader's target names. Exposed for tests and the report's `why`. */
+function targetText(g, trace) {
+    if (g.type === 'regex' && g.target === 'trace') {
+        return trace.toolCalls.map((c) => `${c.tool} ${JSON.stringify(c.input)}`).join('\n');
+    }
+    if (g.type === 'regex' && g.target === 'file') {
+        const content = trace.readFile(g.pathPattern ?? '');
+        return content; // null = no matching file; the grader fails with that reason
+    }
+    if (g.type === 'llm' && g.focus === 'trace') {
+        return trace.toolCalls.map((c) => `${c.tool} ${JSON.stringify(c.input)}`).join('\n');
+    }
+    return trace.lastMessage;
+}
+/** null = the pattern does not compile; the grader FAILS with that reason. */
+function compileRegex(g) {
+    try {
+        return new RegExp(g.pattern ?? '', g.flags ?? '');
+    }
+    catch {
+        return null;
+    }
+}
+function countMatches(re, text) {
+    // A global clone: a non-global regex .match() returns the first hit only, and
+    // count:N must count every occurrence.
+    const flags = re.flags.includes('g') ? re.flags : re.flags + 'g';
+    return (text.match(new RegExp(re.source, flags)) ?? []).length;
+}
+/**
+ * Score ONE grader over ONE run. Async because llm/baseline call the judge.
+ * A grader that cannot be evaluated FAILS (with the reason in `why`) — it must
+ * never pass by accident, or a broken suite would look green.
+ */
+async function evaluateGrader(g, trace, judge) {
+    try {
+        switch (g.type) {
+            case 'regex': {
+                const text = targetText(g, trace);
+                if (text === null)
+                    return { passed: false, why: `no file matches ${g.pathPattern} (regex ${g.pattern})` };
+                const re = compileRegex(g);
+                if (!re)
+                    return { passed: false, why: `invalid pattern: ${g.pattern}` };
+                const match = g.match ?? 'contains';
+                if (match === 'contains') {
+                    return { passed: re.test(text), why: re.test(text) ? 'found' : `not found in ${g.target ?? 'last_message'}` };
+                }
+                if (match === 'not_contains') {
+                    return {
+                        passed: !re.test(text),
+                        why: re.test(text) ? `unexpectedly found in ${g.target ?? 'last_message'}` : 'absent, as required',
+                    };
+                }
+                const cm = COUNT_RE.exec(match);
+                const want = Number(cm[1]);
+                const got = countMatches(re, text);
+                return { passed: got === want, why: `count ${got}, wanted ${want}` };
+            }
+            case 'tool_used': {
+                const n = trace.toolCalls.filter((c) => {
+                    if (c.tool !== g.tool)
+                        return false;
+                    if (!g.inputMatch)
+                        return true;
+                    try {
+                        return new RegExp(g.inputMatch).test(JSON.stringify(c.input));
+                    }
+                    catch {
+                        return false;
+                    }
+                }).length;
+                const min = g.min ?? 1;
+                const ok = n >= min && (g.max === undefined || n <= g.max);
+                return {
+                    passed: ok,
+                    why: `${g.tool} used ${n} time(s)` +
+                        (ok ? '' : ` (wanted ${g.max === undefined ? `>=${min}` : `${min}..${g.max}`})`),
+                };
+            }
+            case 'tool_order': {
+                const iBefore = trace.toolCalls.findIndex((c) => c.tool === g.before);
+                const iAfter = trace.toolCalls.findIndex((c) => c.tool === g.after);
+                if (iBefore === -1)
+                    return { passed: false, why: `${g.before} never called` };
+                if (iAfter === -1)
+                    return { passed: false, why: `${g.after} never called` };
+                return iBefore < iAfter
+                    ? { passed: true, why: `${g.before} before ${g.after}` }
+                    : { passed: false, why: `${g.after} came before ${g.before}` };
+            }
+            case 'file_exists': {
+                const re = (0, nestedInstructions_1.pathGlobToRegExp)(g.path ?? '');
+                const hit = re ? trace.files.some((f) => re.test(f)) : false;
+                const want = g.exists !== false;
+                return {
+                    passed: hit === want,
+                    why: want ? (hit ? 'present' : 'missing') : hit ? 'present, wanted absent' : 'absent, as required',
+                };
+            }
+            case 'llm': {
+                if (!judge)
+                    return { passed: false, why: 'no judge available (pass --judge-model or provide NEXRALL_EVAL_JUDGE_MODEL)' };
+                const focusText = targetText(g, trace) ?? '';
+                const verdict = await judge({ criteria: g.criteria ?? '', focusText });
+                if (verdict === null)
+                    return { passed: false, why: 'judge could not be reached' };
+                return { passed: verdict, why: verdict ? 'judge: pass' : 'judge: fail' };
+            }
+            case 'baseline': {
+                if (!judge)
+                    return { passed: false, why: 'no judge available (pass --judge-model)' };
+                const baselineText = trace.readFile(g.baselineFile ?? '');
+                if (baselineText === null)
+                    return { passed: false, why: `baseline file ${g.baselineFile} not found` };
+                const verdict = await judge({ criteria: g.criteria ?? '', focusText: trace.lastMessage, baselineText });
+                if (verdict === null)
+                    return { passed: false, why: 'judge could not be reached' };
+                return { passed: verdict, why: verdict ? 'at least as good as the baseline' : 'worse than the baseline' };
+            }
+        }
+    }
+    catch (e) {
+        return { passed: false, why: e.message };
+    }
+}
+/** Which graders count in an arm: with-only graders never score the baseline. */
+function gradersForArm(graders, arm) {
+    return arm === 'with' ? graders : graders.filter((g) => g.arm !== 'with-only');
+}
+/** Score one run: weighted fraction of its graders that passed. */
+async function scoreRun(graders, trace, judge, arm) {
+    const active = gradersForArm(graders, arm);
+    if (!active.length)
+        return { score: 0, results: [] };
+    const results = [];
+    let passedWeight = 0;
+    let totalWeight = 0;
+    for (const g of active) {
+        const v = await evaluateGrader(g, trace, judge);
+        totalWeight += g.weight;
+        if (v.passed)
+            passedWeight += g.weight;
+        results.push({ file: g.file, type: g.type, weight: g.weight, passed: v.passed, why: v.why });
+    }
+    return { score: totalWeight > 0 ? passedWeight / totalWeight : 0, results };
+}
+const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
+/**
+ * Fold run outcomes into the report. Case score = mean of the plugin-arm run
+ * scores; `delta` = with − without (null when the baseline did not run).
+ * A case PASSES at ≥ threshold (CC's default: every grader, every run).
+ */
+function aggregateReport(i) {
+    const cases = i.cases.map((c) => {
+        const withScore = mean(c.with.map((r) => r.score));
+        const withoutScore = mean(c.without.map((r) => r.score));
+        return {
+            name: c.name,
+            with: c.with,
+            without: c.without,
+            withScore,
+            withoutScore,
+            delta: withScore !== null && withoutScore !== null ? withScore - withoutScore : null,
+            passed: withScore !== null && withScore >= i.threshold,
+        };
+    });
+    const scores = cases.map((c) => c.withScore).filter((s) => s !== null);
+    const deltas = cases.map((c) => c.delta).filter((d) => d !== null);
+    const allRuns = cases.flatMap((c) => [...c.with, ...c.without]);
+    const reported = allRuns.filter((r) => r.costReported && r.costUsd !== null);
+    return {
+        schemaVersion: exports.EVAL_SCHEMA_VERSION,
+        plugin: i.plugin,
+        startedAt: i.startedAt,
+        durationMs: i.durationMs,
+        threshold: i.threshold,
+        runs: i.runs,
+        ...(i.judgeModel ? { judgeModel: i.judgeModel } : {}),
+        ...(i.partial ? { partial: true, partialReason: i.partial.reason } : {}),
+        cases,
+        errors: i.errors,
+        aggregates: {
+            casesPassed: cases.filter((c) => c.passed).length,
+            casesTotal: cases.length,
+            overallScore: scores.length ? scores.reduce((a, b) => a + b, 0) / scores.length : 0,
+            meanDelta: deltas.length ? deltas.reduce((a, b) => a + b, 0) / deltas.length : null,
+        },
+        costUsd: reported.reduce((a, r) => a + (r.costUsd ?? 0), 0),
+        costMissing: allRuns.length - reported.length,
+    };
+}
+//# sourceMappingURL=eval.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nexrall/code-core",
-  "version": "1.4.74",
+  "version": "1.4.75",
   "description": "Core agent loop, tools, and extension primitives for Nexrall Code — embed an AI coding agent in any Node.js application.",
   "license": "MIT",
   "author": "Nexrall <support@nexrall.com> (https://nexrall.com)",