@agentv/core 0.18.0 → 0.20.0

package/dist/index.d.cts

@@ -29,28 +29,28 @@ type TestMessageContent = string | readonly JsonObject[];
  * System-authored instruction message.
  */
 type SystemTestMessage = {
-    readonly role: "system";
+    readonly role: 'system';
     readonly content: TestMessageContent;
 };
 /**
  * User-authored prompt message.
  */
 type UserTestMessage = {
-    readonly role: "user";
+    readonly role: 'user';
     readonly content: TestMessageContent;
 };
 /**
  * Assistant response message.
  */
 type AssistantTestMessage = {
-    readonly role: "assistant";
+    readonly role: 'assistant';
     readonly content: TestMessageContent;
 };
 /**
  * Tool invocation message.
  */
 type ToolTestMessage = {
-    readonly role: "tool";
+    readonly role: 'tool';
     readonly content: TestMessageContent;
 };
 /**
@@ -78,7 +78,7 @@ type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
     readonly name: string;
-    readonly type: "code";
+    readonly type: 'code';
     readonly script: string;
     readonly resolvedScriptPath?: string;
     readonly cwd?: string;
@@ -86,7 +86,7 @@ type CodeEvaluatorConfig = {
 };
 type LlmJudgeEvaluatorConfig = {
     readonly name: string;
-    readonly type: "llm_judge";
+    readonly type: 'llm_judge';
     readonly prompt?: string;
     readonly promptPath?: string;
 };
@@ -145,16 +145,16 @@ interface EvaluatorResult {
 /**
  * Convenience accessor matching the Python hit_count property.
  */
-declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
+declare function getHitCount(result: Pick<EvaluationResult, 'hits'>): number;
 
-type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
+type ChatMessageRole = 'system' | 'user' | 'assistant' | 'tool' | 'function';
 interface ChatMessage {
     readonly role: ChatMessageRole;
     readonly content: string;
     readonly name?: string;
 }
 type ChatPrompt = readonly ChatMessage[];
-type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
+type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
 interface ProviderRequest {
     readonly question: string;
     readonly systemPrompt?: string;
@@ -267,7 +267,7 @@ interface TargetDefinition {
  * - 'agent': File references only (for providers with filesystem access)
  * - 'lm': Embedded file content with XML tags (for language model providers)
  */
-type FormattingMode = "agent" | "lm";
+type FormattingMode = 'agent' | 'lm';
 /**
  * Extract fenced code blocks from AgentV user segments.
  */
@@ -391,7 +391,7 @@ interface CodexResolvedConfig {
     readonly cwd?: string;
     readonly timeoutMs?: number;
     readonly logDir?: string;
-    readonly logFormat?: "summary" | "json";
+    readonly logFormat?: 'summary' | 'json';
 }
 interface MockResolvedConfig {
     readonly response?: string;
@@ -407,11 +407,11 @@ interface VSCodeResolvedConfig {
     readonly workspaceTemplate?: string;
 }
 type CliHealthcheck = {
-    readonly type: "http";
+    readonly type: 'http';
     readonly url: string;
     readonly timeoutMs?: number;
 } | {
-    readonly type: "command";
+    readonly type: 'command';
     readonly commandTemplate: string;
     readonly timeoutMs?: number;
     readonly cwd?: string;
@@ -425,49 +425,49 @@ interface CliResolvedConfig {
     readonly verbose?: boolean;
 }
 type ResolvedTarget = {
-    readonly kind: "azure";
+    readonly kind: 'azure';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: AzureResolvedConfig;
 } | {
-    readonly kind: "anthropic";
+    readonly kind: 'anthropic';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: AnthropicResolvedConfig;
 } | {
-    readonly kind: "gemini";
+    readonly kind: 'gemini';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: GeminiResolvedConfig;
 } | {
-    readonly kind: "codex";
+    readonly kind: 'codex';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
 } | {
-    readonly kind: "mock";
+    readonly kind: 'mock';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: MockResolvedConfig;
 } | {
-    readonly kind: "vscode" | "vscode-insiders";
+    readonly kind: 'vscode' | 'vscode-insiders';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: VSCodeResolvedConfig;
 } | {
-    readonly kind: "cli";
+    readonly kind: 'cli';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
@@ -480,7 +480,7 @@ declare function readTargetDefinitions(filePath: string): Promise<readonly Targe
 declare function listTargetNames(definitions: readonly TargetDefinition[]): readonly string[];
 
 interface EnsureSubagentsOptions {
-    readonly kind: "vscode" | "vscode-insiders";
+    readonly kind: 'vscode' | 'vscode-insiders';
     readonly count: number;
     readonly verbose?: boolean;
 }
@@ -595,7 +595,7 @@ interface RunEvalCaseOptions {
 interface ProgressEvent {
     readonly workerId: number;
     readonly evalId: string;
-    readonly status: "pending" | "running" | "completed" | "failed";
+    readonly status: 'pending' | 'running' | 'completed' | 'failed';
     readonly startedAt?: number;
     readonly completedAt?: number;
     readonly error?: string;

package/dist/index.d.ts

@@ -29,28 +29,28 @@ type TestMessageContent = string | readonly JsonObject[];
  * System-authored instruction message.
  */
 type SystemTestMessage = {
-    readonly role: "system";
+    readonly role: 'system';
     readonly content: TestMessageContent;
 };
 /**
  * User-authored prompt message.
  */
 type UserTestMessage = {
-    readonly role: "user";
+    readonly role: 'user';
     readonly content: TestMessageContent;
 };
 /**
  * Assistant response message.
  */
 type AssistantTestMessage = {
-    readonly role: "assistant";
+    readonly role: 'assistant';
     readonly content: TestMessageContent;
 };
 /**
  * Tool invocation message.
  */
 type ToolTestMessage = {
-    readonly role: "tool";
+    readonly role: 'tool';
     readonly content: TestMessageContent;
 };
 /**
@@ -78,7 +78,7 @@ type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
     readonly name: string;
-    readonly type: "code";
+    readonly type: 'code';
     readonly script: string;
     readonly resolvedScriptPath?: string;
     readonly cwd?: string;
@@ -86,7 +86,7 @@ type CodeEvaluatorConfig = {
 };
 type LlmJudgeEvaluatorConfig = {
     readonly name: string;
-    readonly type: "llm_judge";
+    readonly type: 'llm_judge';
     readonly prompt?: string;
     readonly promptPath?: string;
 };
@@ -145,16 +145,16 @@ interface EvaluatorResult {
 /**
  * Convenience accessor matching the Python hit_count property.
  */
-declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
+declare function getHitCount(result: Pick<EvaluationResult, 'hits'>): number;
 
-type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
+type ChatMessageRole = 'system' | 'user' | 'assistant' | 'tool' | 'function';
 interface ChatMessage {
     readonly role: ChatMessageRole;
     readonly content: string;
     readonly name?: string;
 }
 type ChatPrompt = readonly ChatMessage[];
-type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
+type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
 interface ProviderRequest {
     readonly question: string;
     readonly systemPrompt?: string;
@@ -267,7 +267,7 @@ interface TargetDefinition {
  * - 'agent': File references only (for providers with filesystem access)
  * - 'lm': Embedded file content with XML tags (for language model providers)
  */
-type FormattingMode = "agent" | "lm";
+type FormattingMode = 'agent' | 'lm';
 /**
  * Extract fenced code blocks from AgentV user segments.
  */
@@ -391,7 +391,7 @@ interface CodexResolvedConfig {
     readonly cwd?: string;
     readonly timeoutMs?: number;
     readonly logDir?: string;
-    readonly logFormat?: "summary" | "json";
+    readonly logFormat?: 'summary' | 'json';
 }
 interface MockResolvedConfig {
     readonly response?: string;
@@ -407,11 +407,11 @@ interface VSCodeResolvedConfig {
     readonly workspaceTemplate?: string;
 }
 type CliHealthcheck = {
-    readonly type: "http";
+    readonly type: 'http';
     readonly url: string;
     readonly timeoutMs?: number;
 } | {
-    readonly type: "command";
+    readonly type: 'command';
     readonly commandTemplate: string;
     readonly timeoutMs?: number;
     readonly cwd?: string;
@@ -425,49 +425,49 @@ interface CliResolvedConfig {
     readonly verbose?: boolean;
 }
 type ResolvedTarget = {
-    readonly kind: "azure";
+    readonly kind: 'azure';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: AzureResolvedConfig;
 } | {
-    readonly kind: "anthropic";
+    readonly kind: 'anthropic';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: AnthropicResolvedConfig;
 } | {
-    readonly kind: "gemini";
+    readonly kind: 'gemini';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: GeminiResolvedConfig;
 } | {
-    readonly kind: "codex";
+    readonly kind: 'codex';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
 } | {
-    readonly kind: "mock";
+    readonly kind: 'mock';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: MockResolvedConfig;
 } | {
-    readonly kind: "vscode" | "vscode-insiders";
+    readonly kind: 'vscode' | 'vscode-insiders';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: VSCodeResolvedConfig;
 } | {
-    readonly kind: "cli";
+    readonly kind: 'cli';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
@@ -480,7 +480,7 @@ declare function readTargetDefinitions(filePath: string): Promise<readonly Targe
 declare function listTargetNames(definitions: readonly TargetDefinition[]): readonly string[];
 
 interface EnsureSubagentsOptions {
-    readonly kind: "vscode" | "vscode-insiders";
+    readonly kind: 'vscode' | 'vscode-insiders';
     readonly count: number;
     readonly verbose?: boolean;
 }
@@ -595,7 +595,7 @@ interface RunEvalCaseOptions {
 interface ProgressEvent {
     readonly workerId: number;
     readonly evalId: string;
-    readonly status: "pending" | "running" | "completed" | "failed";
+    readonly status: 'pending' | 'running' | 'completed' | 'failed';
     readonly startedAt?: number;
     readonly completedAt?: number;
     readonly error?: string;

package/dist/index.js

@@ -8,7 +8,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-QHEZJRTU.js";
+} from "./chunk-SVY324GN.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -2496,6 +2496,43 @@ import {
   getSubagentRoot,
   provisionSubagents
 } from "subagent";
+
+// src/evaluation/providers/vscode-templates.ts
+var AGENTV_REQUEST_TEMPLATE = `[[ ## system_instructions ## ]]
+
+**IMPORTANT**: You are processing an evaluation request from AgentV. Follow these exact steps:
+
+1. Read and understand all guideline files and attachments provided above
+2. Process the user query below according to the guidelines
+3. Write your complete response to: {{responseFileTmp}}
+4. When completely finished, run these PowerShell commands to signal completion:
+\`\`\`powershell
+Move-Item -LiteralPath '{{responseFileTmp}}' -Destination '{{responseFileFinal}}'
+if (Test-Path subagent.lock) { Remove-Item subagent.lock }
+\`\`\`
+
+Do not proceed to step 4 until your response is completely written to the temporary file.
+
+[[ ## task ## ]]
+
+{{userQuery}}
+`;
+var AGENTV_BATCH_REQUEST_TEMPLATE = `[[ ## system_instructions ## ]]
+
+**IMPORTANT**: You are processing a batch evaluation request from AgentV. Follow these exact steps:
+
+1. Read and understand all guideline files and attachments provided above
+2. Process the user query below according to the guidelines
+3. Write your complete response to: {{responseFileTmp}}
+4. When completely finished and the response is stable, rename it to: {{responseFileFinal}}
+5. Do not unlock the workspace from this request; batch orchestration will handle unlocking after all responses are ready.
+
+[[ ## task ## ]]
+
+{{userQuery}}
+`;
+
+// src/evaluation/providers/vscode.ts
 var VSCodeProvider = class {
   id;
   kind;
@@ -2517,6 +2554,7 @@ var VSCodeProvider = class {
     const session = await dispatchAgentSession({
       userQuery: promptContent,
       extraAttachments: inputFiles,
+      requestTemplate: AGENTV_REQUEST_TEMPLATE,
       wait: this.config.waitForResponse,
       dryRun: this.config.dryRun,
       vscodeCmd: this.config.command,
@@ -2563,6 +2601,7 @@ var VSCodeProvider = class {
     const session = await dispatchBatchAgent({
       userQueries,
       extraAttachments: combinedInputFiles,
+      requestTemplate: AGENTV_BATCH_REQUEST_TEMPLATE,
       wait: this.config.waitForResponse,
       dryRun: this.config.dryRun,
       vscodeCmd: this.config.command,