@wix/pathgrade 1.0.4 → 1.0.6

package/dist/agents/codex-app-server/agent.js

@@ -8,7 +8,7 @@ import { decideMcpToolCall, redactMcpSecrets, } from '../../sdk/mcp-safety.js';
 import { spawnAppServerTransport, } from './transport.js';
 import { normalizeUpstreamQuestion, toWireAnswerMap, } from './wire-translators.js';
 import { extractTurnCompletionFailure } from './turn-completion.js';
-const DEFAULT_MODEL = 'gpt-5.4';
+import { resolveCodexModel } from '../codex-model.js';
 const TURN_COMPLETED_METHOD = 'turn/completed';
 function recordFromUnknown(value) {
     if (value && typeof value === 'object' && !Array.isArray(value)) {
@@ -212,7 +212,7 @@ export class CodexAppServerAgent extends BaseAgent {
         const askBus = requireAskBusForLiveBatches(options, 'CodexAppServerAgent');
         const workspacePath = getWorkspacePath(runtime);
         const runtimeEnv = getRuntimeEnv(runtime);
-        const model = options?.model ?? DEFAULT_MODEL;
+        const model = resolveCodexModel(options?.model);
         const sandboxMode = this.deps.sandboxMode ?? 'workspace-write';
         let handle = null;
         let threadId = null;

package/dist/agents/codex-model.d.ts

@@ -0,0 +1 @@
+export declare function resolveCodexModel(explicitModel?: string): string;

package/dist/agents/codex-model.js

@@ -0,0 +1,4 @@
+const DEFAULT_CODEX_MODEL = 'gpt-5.6-luna';
+export function resolveCodexModel(explicitModel) {
+    return explicitModel ?? DEFAULT_CODEX_MODEL;
+}

package/dist/agents/codex.js

@@ -1,11 +1,13 @@
 import { TOOL_NAME_MAP, buildSummary, inferCodexExecAction, enrichSkillEvents } from '../tool-events.js';
 import { TranscriptAgent } from './transcript-agent.js';
 import { assertMcpRuntimeMountingSupportedForCodexExec } from '../providers/mcp-runtime-mounting.js';
+import { resolveCodexModel } from './codex-model.js';
 export class CodexAgent extends TranscriptAgent {
     async runTurn(instruction, runCommand, options) {
         await assertMcpRuntimeMountingSupportedForCodexExec({
             mcpConfigPath: options?.mcpConfigPath,
         });
+        await assertVerifiedCodexCli(runCommand);
         const promptPath = await this.writePromptFile(instruction, runCommand);
         const command = buildCodexExecCommand(promptPath, options?.model);
         const result = await runCommand(command);
@@ -26,17 +28,26 @@ export class CodexAgent extends TranscriptAgent {
         };
     }
 }
-const DEFAULT_CODEX_MODEL = 'gpt-5.4';
+const VERIFIED_CODEX_CLI_VERSION = '0.148.0-alpha.9';
+async function assertVerifiedCodexCli(runCommand) {
+    const result = await runCommand('codex --version');
+    const reportedVersion = (result.stdout || result.stderr).trim();
+    const expected = `codex-cli ${VERIFIED_CODEX_CLI_VERSION}`;
+    if (result.exitCode !== 0 || reportedVersion !== expected) {
+        throw new Error(`Codex CLI ${VERIFIED_CODEX_CLI_VERSION} required for PathGrade exec; received ${reportedVersion || `exit code ${result.exitCode}`}. Install @openai/codex@${VERIFIED_CODEX_CLI_VERSION}.`);
+    }
+}
 const CODEX_PROXY_PROVIDER_ID = 'pathgrade_openai_proxy';
-function buildCodexExecCommand(promptPath, model = DEFAULT_CODEX_MODEL) {
+function buildCodexExecCommand(promptPath, model) {
     const quotedPromptPath = JSON.stringify(promptPath);
-    const quotedModel = JSON.stringify(model);
-    const execArgs = `--full-auto --skip-git-repo-check -m ${quotedModel} - < ${quotedPromptPath}`;
+    const quotedModel = JSON.stringify(resolveCodexModel(model));
+    const globalArgs = '-a never -s workspace-write';
+    const execArgs = `--skip-git-repo-check -m ${quotedModel} - < ${quotedPromptPath}`;
     return [
         'if [ -n "${OPENAI_BASE_URL:-}" ]; then',
-        `codex exec ${buildCodexProxyConfigArgs()} ${execArgs};`,
+        `codex ${globalArgs} exec ${buildCodexProxyConfigArgs()} ${execArgs};`,
         'else',
-        `codex exec ${execArgs};`,
+        `codex ${globalArgs} exec ${execArgs};`,
         'fi',
     ].join(' ');
 }

package/dist/sdk/agent-resolution.d.ts

@@ -1,4 +1,4 @@
-import type { AgentName, AgentTransport, AgentOptions } from './types.js';
+import type { AgentExecutionTransport, AgentName, AgentTransport, AgentOptions } from './types.js';
 export declare class InvalidTransportEnvError extends Error {
     constructor(value: string);
 }
@@ -10,3 +10,4 @@ export declare function resolveCodexTransport(opts: {
 }, env: {
     PATHGRADE_CODEX_TRANSPORT?: string;
 }): AgentTransport;
+export declare function resolveExecutionTransport(agentName: AgentName, codexTransport?: AgentTransport): AgentExecutionTransport;

package/dist/sdk/agent-resolution.js

@@ -19,3 +19,10 @@ export function resolveCodexTransport(opts, env) {
     }
     return 'app-server';
 }
+export function resolveExecutionTransport(agentName, codexTransport) {
+    if (agentName === 'claude')
+        return 'claude-agent-sdk';
+    if (agentName === 'cursor')
+        return 'cursor-agent';
+    return codexTransport ?? 'app-server';
+}

package/dist/sdk/agent.js

@@ -1,5 +1,5 @@
 import { prepareWorkspace } from '../providers/workspace.js';
-import { resolveAgentName, resolveCodexTransport } from './agent-resolution.js';
+import { resolveAgentName, resolveCodexTransport, resolveExecutionTransport, } from './agent-resolution.js';
 import { lifecycleCore } from './lifecycle.js';
 import { ChatSessionImpl } from './chat.js';
 import { runConversation } from './converse.js';
@@ -73,7 +73,7 @@ class AgentImpl {
         return {
             name: this.agentName,
             ...(this.modelOpt ? { requestedModel: this.modelOpt } : {}),
-            ...(this.transport ? { transport: this.transport } : {}),
+            transport: resolveExecutionTransport(this.agentName, this.transport),
             ...(interactionMode ? { interactionMode } : {}),
         };
     }

package/dist/sdk/index.d.ts

@@ -21,7 +21,7 @@ export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObserve
 export { getAgentCapabilities } from './types.js';
 export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
 export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
-export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentInteractionMode, } from './types.js';
+export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentExecutionTransport, AgentInteractionMode, } from './types.js';
 export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
 export type { JudgePipelineOptions } from './judge-pipeline.js';
 export type { RunScorerOptions } from './run-scorer.js';

package/dist/sdk/types.d.ts

@@ -7,12 +7,14 @@ import type { LLMPort } from '../utils/llm-types.js';
 import type { McpSafetyOptions } from './mcp-safety.js';
 export type AgentName = 'claude' | 'codex' | 'cursor';
 export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
+/** Runtime channel that actually executed the agent. */
+export type AgentExecutionTransport = AgentTransport | 'claude-agent-sdk' | 'cursor-agent';
 /** Privacy-safe execution dimensions exposed to result observers. */
 export interface AgentExecutionMetadata {
     name: AgentName;
     /** Model override requested by the caller; not necessarily the provider-resolved model. */
     requestedModel?: string;
-    transport?: AgentTransport;
+    transport?: AgentExecutionTransport;
     interactionMode?: AgentInteractionMode;
 }
 export interface AgentOptions {
@@ -70,7 +72,7 @@ export interface Agent {
     readonly messages: Message[];
     readonly log: LogEntry[];
     readonly workspace: string;
-    /** Provider/configuration metadata only; never contains prompts or paths. */
+    /** Provider/runtime metadata only; never contains prompts or paths. */
     readonly executionMetadata?: AgentExecutionMetadata;
     dispose(): Promise<void>;
 }

package/dist/utils/llm-providers/anthropic.js

@@ -27,6 +27,35 @@ function sumInputTokens(usage) {
         + (usage?.cache_creation_input_tokens ?? 0)
         + (usage?.cache_read_input_tokens ?? 0);
 }
+const DEFAULT_ANTHROPIC_MODEL = 'claude-sonnet-5';
+const DEFAULT_ANTHROPIC_MODEL_CONFIG = {
+    transport: 'raw-messages',
+    temperature: 'default-zero',
+};
+const ANTHROPIC_MODEL_CONFIGS = {
+    'claude-haiku-4-5-20251001': { transport: 'raw-messages', temperature: 'default-zero' },
+    'claude-opus-4-7': { transport: 'raw-messages', temperature: 'unsupported' },
+    'claude-opus-5': { transport: 'raw-messages', temperature: 'unsupported' },
+    'claude-sonnet-5': { transport: 'raw-messages', temperature: 'unsupported' },
+    'claude-opus-5[1m]': { transport: 'claude-agent-sdk' },
+};
+function getAnthropicModelConfig(model) {
+    return ANTHROPIC_MODEL_CONFIGS[model] ?? DEFAULT_ANTHROPIC_MODEL_CONFIG;
+}
+function assertRawAnthropicModel(model, config) {
+    if (config.transport !== 'raw-messages') {
+        throw new Error(`Anthropic model ${model} uses Claude Agent SDK transport syntax and cannot be sent to the raw Messages API`);
+    }
+}
+function resolveTemperature(model, config, temperature) {
+    if (config.temperature === 'unsupported') {
+        if (temperature !== undefined) {
+            throw new Error(`Anthropic model ${model} does not accept temperature; omit the option`);
+        }
+        return undefined;
+    }
+    return temperature ?? 0;
+}
 async function postAnthropic(apiKey, useCache, body, env) {
     const response = await fetch(`${resolveBaseUrl(env)}/v1/messages`, {
         method: 'POST',
@@ -52,8 +81,10 @@ export const anthropicProvider = {
         if (!apiKey) {
             throw new Error('No ANTHROPIC_API_KEY available');
         }
-        const model = opts.model || 'claude-sonnet-4-20250514';
-        const temperature = opts.temperature ?? 0;
+        const model = opts.model || DEFAULT_ANTHROPIC_MODEL;
+        const modelConfig = getAnthropicModelConfig(model);
+        assertRawAnthropicModel(model, modelConfig);
+        const temperature = resolveTemperature(model, modelConfig, opts.temperature);
         const useCache = opts.cacheControl ?? false;
         const content = useCache
             ? [{ type: 'text', text: prompt, cache_control: { type: 'ephemeral' } }]
@@ -62,7 +93,7 @@ export const anthropicProvider = {
             const data = await postAnthropic(apiKey, useCache, {
                 model,
                 max_tokens: 4096,
-                temperature,
+                ...(temperature === undefined ? {} : { temperature }),
                 messages: [{ role: 'user', content }],
             }, opts.env);
             const inputTokens = sumInputTokens(data?.usage);
@@ -84,8 +115,10 @@ export const anthropicProvider = {
         if (!apiKey) {
             throw new Error('No ANTHROPIC_API_KEY available');
         }
-        const model = opts.model || 'claude-sonnet-4-20250514';
-        const temperature = opts.temperature ?? 0;
+        const model = opts.model || DEFAULT_ANTHROPIC_MODEL;
+        const modelConfig = getAnthropicModelConfig(model);
+        assertRawAnthropicModel(model, modelConfig);
+        const temperature = resolveTemperature(model, modelConfig, opts.temperature);
         const maxTokens = opts.maxTokens ?? 4096;
         const useCache = opts.cacheControl === true;
         // Tool schemas: only the last gets cache_control, serving as a cache
@@ -96,7 +129,7 @@ export const anthropicProvider = {
         const body = {
             model,
             max_tokens: maxTokens,
-            temperature,
+            ...(temperature === undefined ? {} : { temperature }),
             messages,
             tools,
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -62,6 +62,7 @@
     "test": "yarn build && vitest run && yarn test:runner-cli-smoke",
     "test:runner-cli-smoke": "node tests/runner-cli-smoke/run-smokes.mjs",
     "test:evals": "vitest run --config evals/vitest.config.mts",
+    "smoke:models": "tsx scripts/model-smoke.ts",
     "test:coverage": "vitest run --coverage",
     "dev": "tsx src/pathgrade.ts",
     "build": "tsc -p tsconfig.build.json && cp src/viewer.html dist/viewer.html && cp src/adapters/jest/reporter.cjs dist/adapters/jest/reporter.cjs"
@@ -126,5 +127,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "112cde138733c678a01778b225c1b3891d22417d98bbf2c479f8734c"
+  "falconPackageHash": "b61abd1d02054fc169ad426379b1be872d282675641bfd6012d5bbe7"
 }