@nexrall/code-core 1.4.74 → 1.4.76

package/dist/permissions/rules.js

@@ -33,15 +33,22 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.managedSettingsPath = managedSettingsPath;
 exports.setSessionRules = setSessionRules;
 exports.setSessionSettings = setSessionSettings;
 exports.getSessionSettings = getSessionSettings;
 exports.loadSettings = loadSettings;
+exports.matchedAllowRule = matchedAllowRule;
+exports.isBroadAllowRule = isBroadAllowRule;
 exports.evaluatePermission = evaluatePermission;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const os = __importStar(require("os"));
 function managedSettingsPath() {
+    // Tests (and a sandboxed doctor run) point this at a fixture. Checked first so it
+    // can never be shadowed by a real machine-level file.
+    if (process.env.NEXRALL_MANAGED_SETTINGS)
+        return process.env.NEXRALL_MANAGED_SETTINGS;
     if (process.platform === 'darwin')
         return '/Library/Application Support/Nexrall/managed-settings.json';
     if (process.platform === 'win32')
@@ -258,6 +265,32 @@ function ruleMatches(rule, tool, input, workDir, opts) {
         return true;
     return matchValues(tool, input, workDir).some((v) => matchPattern(pattern, v));
 }
+/** The first ALLOW rule that matches this call, or null. S20's classifier gate
+ * reads this to tell a NARROW pre-approval (wins, no review) from a BROAD one
+ * (suspended while the classifier is on — see isBroadAllowRule). */
+function matchedAllowRule(rules, tool, input, workDir) {
+    return rules.allow.find((r) => ruleMatches(r, tool, input, workDir, { forAllow: true })) ?? null;
+}
+/**
+ * Is this allow rule a BLANKET pre-approval ("anything this tool can do")?
+ * Those stop meaning anything once an auto-mode classifier exists: `Bash(*)`
+ * is exactly the rule a prompt-injected agent would wish for, and CC suspends
+ * the same set in their auto mode. Broad = bare tool name / `(*)`, or a bash
+ * rule that hands control to an interpreter (`Bash(python:*)`, `node -e`…).
+ * Narrow rules (`Bash(npm run test:*)`, `Read(src/**)`) keep resolving first.
+ */
+const INTERPRETER_HEADS = /^(?:\S*\/)?(python3?|node|bun|deno|ruby|perl|php|bash|sh|zsh|fish|pwsh|powershell|osascript|cmd)(\s|:|$)/i;
+function isBroadAllowRule(rule) {
+    const { tool, pattern } = parseRule(rule);
+    if (pattern === null || pattern === '' || pattern === '*')
+        return true;
+    const isBashRule = ['bash', 'run_command', 'execute_command', 'Bash'].includes(tool);
+    if (!isBashRule)
+        return false;
+    if (!/[*:]$/.test(pattern))
+        return false;
+    return INTERPRETER_HEADS.test(pattern.replace(/[:*]+$/, '').trim());
+}
 /**
  * Evaluate a tool call against merged rules. Returns the matching decision, or
  * null when no rule applies (caller decides the default, usually "ask").

package/dist/plugins/eval.d.ts

@@ -0,0 +1,199 @@
+export declare const EVAL_SCHEMA_VERSION = 1;
+/** CC's default: each case runs three times. */
+export declare const EVAL_DEFAULT_RUNS = 3;
+export declare const EVAL_MAX_RUNS = 50;
+export declare const EVAL_DEFAULT_THRESHOLD = 1;
+export declare const EVAL_DEFAULT_MAX_TURNS = 10;
+export declare const EVAL_DEFAULT_TIMEOUT_S = 300;
+/** CC caps both; larger values are refused, not clamped silently. */
+export declare const EVAL_MAX_TURNS_CEILING = 200;
+export declare const EVAL_MAX_TIMEOUT_S = 3600;
+export type GraderType = 'regex' | 'tool_used' | 'tool_order' | 'file_exists' | 'llm' | 'baseline';
+/**
+ * Which arm a grader scores in.
+ *  · 'both' (default) — counted for the plugin arm AND the no-plugin baseline.
+ *  · 'with-only' — counted only when the plugin was loaded (CC's rule for the
+ *    two-arm runs: a grader that can only pass WITH the plugin must not drag
+ *    the baseline down).
+ */
+export type GraderArm = 'both' | 'with-only';
+export interface EvalGrader {
+    /** File name inside graders/, for reporting. */
+    file: string;
+    type: GraderType;
+    weight: number;
+    arm: GraderArm;
+    pattern?: string;
+    flags?: string;
+    /** 'contains' (default) | 'not_contains' | 'count:<n>' */
+    match?: string;
+    /** 'last_message' (default) | 'trace' | 'file' */
+    target?: string;
+    /** target 'file': project-relative path or glob to read. */
+    pathPattern?: string;
+    tool?: string;
+    inputMatch?: string;
+    min?: number;
+    max?: number;
+    before?: string;
+    after?: string;
+    path?: string;
+    exists?: boolean;
+    criteria?: string;
+    /** llm: 'last_message' (default) | 'trace'. */
+    focus?: string;
+    /** baseline: the reference transcript file (case-relative). */
+    baselineFile?: string;
+}
+export interface EvalCase {
+    name: string;
+    dir: string;
+    prompt: string;
+    runs: number;
+    maxTurns: number;
+    timeoutSeconds: number;
+    allowedTools: string[];
+    tags: string[];
+    expectedOutcome?: string;
+    graders: EvalGrader[];
+}
+export interface DiscoveredSuite {
+    cases: EvalCase[];
+    /** Case dirs that failed to parse — reported, never silently dropped. */
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+}
+/** One tool call as the child's stream reported it. */
+export interface EvalToolCall {
+    tool: string;
+    input: unknown;
+}
+/**
+ * What ONE run produced. `readFile` resolves a project-relative path or glob
+ * against the run's workspace — the caller owns the filesystem, so the graders
+ * stay pure.
+ */
+export interface EvalRunTrace {
+    lastMessage: string;
+    toolCalls: EvalToolCall[];
+    files: string[];
+    readFile: (relOrGlob: string) => string | null;
+    /** USD when the server reported it; null = unreported (never invented). */
+    costUsd: number | null;
+    tokens: number;
+    exitCode: number;
+    error?: string;
+}
+export interface GraderVerdict {
+    passed: boolean;
+    why: string;
+}
+/** The injected model call for llm/baseline graders. null = could not judge. */
+export type EvalJudge = (args: {
+    criteria: string;
+    focusText: string;
+    baselineText?: string;
+}) => Promise<boolean | null>;
+export interface CaseRunOutcome {
+    arm: 'with' | 'without';
+    run: number;
+    score: number;
+    graderResults: Array<{
+        file: string;
+        type: GraderType;
+        weight: number;
+        passed: boolean;
+        why: string;
+    }>;
+    costUsd: number | null;
+    costReported: boolean;
+    durationMs: number;
+    error?: string;
+}
+export interface CaseOutcome {
+    name: string;
+    with: CaseRunOutcome[];
+    without: CaseRunOutcome[];
+    withScore: number | null;
+    withoutScore: number | null;
+    delta: number | null;
+    passed: boolean;
+}
+export interface EvalReport {
+    schemaVersion: number;
+    plugin: string;
+    startedAt: number;
+    durationMs: number;
+    threshold: number;
+    runs: number;
+    judgeModel?: string;
+    partial?: boolean;
+    partialReason?: string;
+    cases: CaseOutcome[];
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+    aggregates: {
+        casesPassed: number;
+        casesTotal: number;
+        overallScore: number;
+        meanDelta: number | null;
+    };
+    costUsd: number;
+    costMissing: number;
+}
+/** `[a, b]` / `a, b` / `a` → ['a','b'] — the two shapes real files use. */
+export declare function parseList(value: string | undefined): string[];
+/**
+ * One grader file → EvalGrader. Throws on anything that would make the grader
+ * silently unscorable (unknown type, missing pattern/tool/path, bad count).
+ */
+export declare function parseGraderFile(file: string, raw: string): EvalGrader;
+/** One case directory → EvalCase. See the header for the layout. */
+export declare function parseEvalCase(dir: string): EvalCase;
+/** Every case under `<pluginDir>/evals` (the results dir is never a case). */
+export declare function discoverEvalCases(pluginDir: string): DiscoveredSuite;
+/** The text a grader's target names. Exposed for tests and the report's `why`. */
+export declare function targetText(g: EvalGrader, trace: EvalRunTrace): string | null;
+/**
+ * Score ONE grader over ONE run. Async because llm/baseline call the judge.
+ * A grader that cannot be evaluated FAILS (with the reason in `why`) — it must
+ * never pass by accident, or a broken suite would look green.
+ */
+export declare function evaluateGrader(g: EvalGrader, trace: EvalRunTrace, judge: EvalJudge | undefined): Promise<GraderVerdict>;
+/** Which graders count in an arm: with-only graders never score the baseline. */
+export declare function gradersForArm(graders: EvalGrader[], arm: 'with' | 'without'): EvalGrader[];
+/** Score one run: weighted fraction of its graders that passed. */
+export declare function scoreRun(graders: EvalGrader[], trace: EvalRunTrace, judge: EvalJudge | undefined, arm: 'with' | 'without'): Promise<{
+    score: number;
+    results: CaseRunOutcome['graderResults'];
+}>;
+/**
+ * Fold run outcomes into the report. Case score = mean of the plugin-arm run
+ * scores; `delta` = with − without (null when the baseline did not run).
+ * A case PASSES at ≥ threshold (CC's default: every grader, every run).
+ */
+export declare function aggregateReport(i: {
+    plugin: string;
+    startedAt: number;
+    durationMs: number;
+    threshold: number;
+    runs: number;
+    judgeModel?: string;
+    cases: Array<{
+        name: string;
+        with: CaseRunOutcome[];
+        without: CaseRunOutcome[];
+    }>;
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+    partial?: {
+        reason: string;
+    };
+}): EvalReport;
+//# sourceMappingURL=eval.d.ts.map

package/dist/plugins/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/plugins/eval.ts"],"names":[],"mappings":"AAiCA,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,gDAAgD;AAChD,eAAO,MAAM,iBAAiB,IAAI,CAAC;AACnC,eAAO,MAAM,aAAa,KAAK,CAAC;AAChC,eAAO,MAAM,sBAAsB,IAAM,CAAC;AAC1C,eAAO,MAAM,sBAAsB,KAAK,CAAC;AACzC,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAC1C,qEAAqE;AACrE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAC1C,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAMvC,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,WAAW,GAAG,YAAY,GAAG,aAAa,GAAG,KAAK,GAAG,UAAU,CAAC;AAEnG;;;;;;GAMG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,WAAW,CAAC;AAE7C,MAAM,WAAW,UAAU;IACzB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,SAAS,CAAC;IAEf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,yEAAyE;IACzE,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/C;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/C,2EAA2E;IAC3E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,gFAAgF;AAChF,MAAM,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,KAAK,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;AAE9B,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvG,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,cAAc,EAAE,CAAC;IACvB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,UAAU,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;KAC1B,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAID,2EAA2E;AAC3E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAO7D;AAUD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,CAoErE;AAED,oEAAoE;AACpE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,CAmDnD;AAED,8EAA8E;AAC9E,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe,CA6BpE;AAID,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,GAAG,MAAM,GAAG,IAAI,CAY5E;AAkBD;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,CAAC,EAAE,UAAU,EACb,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,SAAS,GAAG,SAAS,GAC3B,OAAO,CAAC,aAAa,CAAC,CAgFxB;AAED,iFAAiF;AACjF,wBAAgB,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,EAAE,CAE1F;AAED,mEAAmE;AACnE,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,UAAU,EAAE,EACrB,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,SAAS,GAAG,SAAS,EAC5B,GAAG,EAAE,MAAM,GAAG,SAAS,GACtB,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,cAAc,CAAC,eAAe,CAAC,CAAA;CAAE,CAAC,CAatE;AAID;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,cAAc,EAAE,CAAC;QACvB,OAAO,EAAE,cAAc,EAAE,CAAC;KAC3B,CAAC,CAAC;IACH,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B,GAAG,UAAU,CAsCb"}