@wix/pathgrade 0.34.0 → 0.36.0

package/dist/affected/config.d.ts

@@ -18,8 +18,11 @@
  */
 export interface AffectedConfig {
     global: string[];
+    include?: string[];
+    exclude?: string[];
 }
 export interface LoadOptions {
+    configPath?: string;
     onWarning?: (message: string) => void;
 }
 export declare function loadAffectedConfig(repoRoot: string, options?: LoadOptions): Promise<AffectedConfig>;

package/dist/affected/config.js

@@ -41,9 +41,11 @@ export async function loadAffectedConfig(repoRoot, options = {}) {
             return { global: [] };
         }
     }
-    const configPath = VITEST_CONFIG_CANDIDATES
-        .map(c => path.join(repoRoot, c))
-        .find(p => fs.existsSync(p));
+    const configPath = options.configPath
+        ? path.resolve(repoRoot, options.configPath)
+        : VITEST_CONFIG_CANDIDATES
+            .map(c => path.join(repoRoot, c))
+            .find(p => fs.existsSync(p));
     if (!configPath)
         return { global: [] };
     let loaded;
@@ -52,7 +54,11 @@ export async function loadAffectedConfig(repoRoot, options = {}) {
         loaded = await jiti.import(configPath, { default: true });
     }
     catch (err) {
-        warn(`pathgrade: failed to load ${path.relative(repoRoot, configPath)}: ${errMsg(err)}`);
+        const message = `pathgrade: failed to load ${path.relative(repoRoot, configPath)}: ${errMsg(err)}`;
+        if (options.configPath) {
+            throw new Error(message);
+        }
+        warn(message);
         return { global: [] };
     }
     const plugins = findPluginsList(loaded);
@@ -68,7 +74,13 @@ export async function loadAffectedConfig(repoRoot, options = {}) {
     }
     const opts = pathgradePlugin.__pathgradeOptions ?? {};
     const global = opts.affected?.global;
-    return { global: Array.isArray(global) ? global : [] };
+    const include = opts.include;
+    const exclude = opts.exclude;
+    return {
+        global: Array.isArray(global) ? global : [],
+        ...(Array.isArray(include) ? { include } : {}),
+        ...(Array.isArray(exclude) ? { exclude } : {}),
+    };
 }
 /**
  * Given a loaded vitest config (either the raw export or a `defineConfig()`

package/dist/agents/codex-app-server/agent.js

@@ -1,12 +1,13 @@
 import { BaseAgent, getRuntimeEnv, getWorkspacePath, } from '../../types.js';
 import { mountMcpForCodexAppServer } from '../../providers/mcp-runtime-mounting.js';
 import { assertMcpSecretReferencesReady } from '../../providers/mcp-config.js';
+import { buildSummary, enrichSkillEvents, extractSkillNameFromPath, inferCodexExecAction, } from '../../tool-events.js';
 import { requireAskBusForLiveBatches, } from '../../sdk/ask-bus/bus.js';
 import { toAskUserToolEvent } from '../../sdk/ask-bus/projection.js';
 import { decideMcpToolCall, redactMcpSecrets, } from '../../sdk/mcp-safety.js';
 import { spawnAppServerTransport, } from './transport.js';
 import { normalizeUpstreamQuestion, toWireAnswerMap, } from './wire-translators.js';
-const DEFAULT_MODEL = 'gpt-5.3-codex';
+const DEFAULT_MODEL = 'gpt-5.4';
 const TURN_COMPLETED_METHOD = 'turn/completed';
 function recordFromUnknown(value) {
     if (value && typeof value === 'object' && !Array.isArray(value)) {
@@ -40,6 +41,27 @@ function extractMcpToolApprovalRequest(params) {
 function parseToolNameFromApprovalMessage(message) {
     return message.match(/tool\s+"([^"]+)"/i)?.[1];
 }
+function extractCommandActionSkillName(action) {
+    if (typeof action.path === 'string') {
+        const direct = extractSkillNameFromPath(action.path);
+        if (direct)
+            return direct;
+        const embedded = extractSkillNameFromText(action.path);
+        if (embedded)
+            return embedded;
+    }
+    return typeof action.command === 'string' ? extractSkillNameFromText(action.command) : undefined;
+}
+function extractSkillNameFromText(value) {
+    if (!value)
+        return undefined;
+    return value.match(/(?:^|[/\s"'])\.(?:agents|claude)\/skills\/([^/\s"']+)\/SKILL\.md(?:$|[\s"'])/)?.[1];
+}
+function extractSkillPathFromText(value) {
+    if (!value)
+        return undefined;
+    return value.match(/(?:^|[\s"'])(?<path>(?:\/|\.{1,2}\/)?[^\s"']*(?:\.agents|\.claude)\/skills\/[^/\s"']+\/SKILL\.md)(?:$|[\s"'])/)?.groups?.path;
+}
 function projectItemIntoTurn(item, turn) {
     if (item.type === 'agentMessage') {
         const msg = item;
@@ -50,16 +72,40 @@ function projectItemIntoTurn(item, turn) {
     }
     if (item.type === 'commandExecution') {
         const cmd = item;
+        const action = inferCodexExecAction(cmd.command);
+        const skillPath = extractSkillPathFromText(cmd.command);
+        const args = {
+            command: cmd.command,
+            ...(skillPath ? { path: skillPath } : {}),
+        };
         turn.nonAskToolEvents.push({
-            action: 'run_shell',
+            action,
             provider: 'codex',
             providerToolName: 'commandExecution',
             turnNumber: turn.turnNumber,
-            arguments: { command: cmd.command },
-            summary: `run_shell: ${cmd.command}`,
+            arguments: args,
+            summary: buildSummary(action, 'commandExecution', args),
             confidence: 'high',
             rawSnippet: JSON.stringify(cmd),
         });
+        const recordedSkills = new Set();
+        for (const action of cmd.commandActions ?? []) {
+            const skillName = extractCommandActionSkillName(action) ?? extractSkillNameFromText(cmd.command);
+            if (!skillName || recordedSkills.has(skillName))
+                continue;
+            recordedSkills.add(skillName);
+            turn.nonAskToolEvents.push({
+                action: 'use_skill',
+                provider: 'codex',
+                providerToolName: `commandExecution.commandActions.${action.type ?? 'unknown'}`,
+                turnNumber: turn.turnNumber,
+                arguments: { path: action.path, name: action.name },
+                summary: `use_skill ${skillName}`,
+                confidence: 'high',
+                rawSnippet: JSON.stringify(action),
+                skillName,
+            });
+        }
         return;
     }
     if (item.type === 'fileChange') {
@@ -515,7 +561,7 @@ function assembleTurnResult(args) {
         .snapshot()
         .filter((s) => askBatchIds.has(s.batchId))
         .map((s) => toAskUserToolEvent(s));
-    const toolEvents = [...askEvents, ...activeTurn.nonAskToolEvents];
+    const toolEvents = enrichSkillEvents([...askEvents, ...activeTurn.nonAskToolEvents]);
     const rawOutput = exitCode === 0
         ? message
         : [

package/dist/agents/codex-app-server/transport.js

@@ -142,9 +142,11 @@ function quoteCodexConfigString(value) {
 }
 export function buildAppServerSpawnArgs(args = [], env = process.env) {
     const baseUrl = env.OPENAI_BASE_URL?.trim();
+    const requestUserInputConfig = ['-c', 'features.default_mode_request_user_input=true'];
     if (!baseUrl)
-        return [...args, 'app-server'];
+        return [...requestUserInputConfig, ...args, 'app-server'];
     return [
+        ...requestUserInputConfig,
         '-c', `model_provider=${quoteCodexConfigString(CODEX_PROXY_PROVIDER_ID)}`,
         '-c', `model_providers.${CODEX_PROXY_PROVIDER_ID}.name=${quoteCodexConfigString('PathGrade OpenAI Proxy')}`,
         '-c', `model_providers.${CODEX_PROXY_PROVIDER_ID}.base_url=${quoteCodexConfigString(baseUrl)}`,

package/dist/agents/codex.js

@@ -26,7 +26,7 @@ export class CodexAgent extends TranscriptAgent {
         };
     }
 }
-const DEFAULT_CODEX_MODEL = 'gpt-5.3-codex';
+const DEFAULT_CODEX_MODEL = 'gpt-5.4';
 const CODEX_PROXY_PROVIDER_ID = 'pathgrade_openai_proxy';
 function buildCodexExecCommand(promptPath, model = DEFAULT_CODEX_MODEL) {
     const quotedPromptPath = JSON.stringify(promptPath);

package/dist/commands/run-changed.js

@@ -13,6 +13,7 @@
  * only producer of the file list here.
  */
 import * as fs from 'fs';
+import picomatch from 'picomatch';
 import { selectAffected } from '../affected/select.js';
 import { resolveBaseRef, computeChangedFiles } from '../affected/git.js';
 import { loadAffectedConfig } from '../affected/config.js';
@@ -52,13 +53,22 @@ export async function runChanged(opts) {
         baseRefLine = `pathgrade: base = ${baseRef} (merge-base with HEAD)`;
     }
     // 2. Selection
-    const evalFiles = discoverEvalFiles(cwd);
-    const config = await loadAffectedConfig(cwd, {
-        onWarning: w => {
-            if (!parsed.quiet)
-                process.stderr.write(`${w}\n`);
-        },
-    });
+    const configPath = findVitestConfigArg(parsed.vitestArgs);
+    let config;
+    try {
+        config = await loadAffectedConfig(cwd, {
+            configPath,
+            onWarning: w => {
+                if (!parsed.quiet)
+                    process.stderr.write(`${w}\n`);
+            },
+        });
+    }
+    catch (err) {
+        process.stderr.write(`${errMsg(err)}\n`);
+        return 1;
+    }
+    const evalFiles = filterEvalFilesForConfig(discoverEvalFiles(cwd), config);
     let result;
     try {
         result = selectAffected({
@@ -93,6 +103,11 @@ export async function runChanged(opts) {
         return 0;
     }
     const selectedFiles = result.selected.map(s => s.file);
+    if (hasPassWithNoTests(parsed.vitestArgs)) {
+        process.stderr.write('pathgrade run: --passWithNoTests cannot be used with pathgrade run --changed. ' +
+            'The command already exits 0 when no evals are selected; if selected evals resolve to no Vitest files, CI must fail.\n');
+        return 1;
+    }
     const argv = ['run', ...selectedFiles, ...parsed.vitestArgs];
     if (!parsed.quiet) {
         process.stderr.write(`→ vitest run ${selectedFiles.join(' ')}\n`);
@@ -126,6 +141,37 @@ function readChangedFilesList(filePath) {
 function errMsg(err) {
     return err instanceof Error ? err.message : String(err);
 }
+function hasPassWithNoTests(args) {
+    return args.some(arg => {
+        if (arg === '--passWithNoTests')
+            return true;
+        if (!arg.startsWith('--passWithNoTests='))
+            return false;
+        return arg.slice('--passWithNoTests='.length).toLowerCase() !== 'false';
+    });
+}
+function findVitestConfigArg(args) {
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg === '--config' || arg === '-c')
+            return args[i + 1];
+        if (arg.startsWith('--config='))
+            return arg.slice('--config='.length);
+        if (arg.startsWith('-c='))
+            return arg.slice('-c='.length);
+    }
+    return undefined;
+}
+function filterEvalFilesForConfig(evalFiles, config) {
+    if (!config.include && !config.exclude)
+        return evalFiles;
+    const includeMatchers = config.include?.map(g => picomatch(g, { dot: true }));
+    const excludeMatchers = config.exclude?.map(g => picomatch(g, { dot: true })) ?? [];
+    return evalFiles.filter(file => {
+        const included = includeMatchers ? includeMatchers.some(m => m(file)) : true;
+        return included && !excludeMatchers.some(m => m(file));
+    });
+}
 async function defaultSpawnVitest(req) {
     const { spawn } = await import('child_process');
     return await new Promise(resolve => {

package/dist/plugin/reporter.d.ts

@@ -15,6 +15,7 @@ export declare class PathgradeReporter implements Reporter {
      */
     private getGroupKey;
     private toTestEntry;
+    private isReportableEntry;
     private printCliSummary;
     private writeJsonResults;
     private buildEvalReport;

package/dist/plugin/reporter.js

@@ -45,6 +45,8 @@ export class PathgradeReporter {
             for (const testCase of mod.children.allTests()) {
                 const groupKey = this.getGroupKey(testCase);
                 const entry = this.toTestEntry(testCase);
+                if (!this.isReportableEntry(entry))
+                    continue;
                 if (!groupMap.has(groupKey)) {
                     groupMap.set(groupKey, []);
                 }
@@ -100,6 +102,9 @@ export class PathgradeReporter {
             diagnostics,
         };
     }
+    isReportableEntry(entry) {
+        return entry.state !== 'skipped' && entry.state !== 'pending';
+    }
     printCliSummary(groups) {
         console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
         const forceVerbose = this.opts.diagnostics === true || process.env.PATHGRADE_DIAGNOSTICS === '1';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "0.34.0",
+  "version": "0.36.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "main": "./dist/sdk/index.js",
   "types": "./dist/sdk/index.d.ts",
@@ -97,5 +97,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "77ac8633dc5bff7dea81e34b2bad4228699443f0788ce2eb56a2bc36"
+  "falconPackageHash": "85d2875d7e92c76ffad2391c7756cbca44f843842a3d1b700b3e11f5"
 }