create-agentmark 0.10.0 → 0.10.2

package/dist/index.js

@@ -34,7 +34,7 @@ var createAdapterConfig = (provider) => {
       }
     },
     "claude-agent-sdk": {
-      package: "@agentmark-ai/claude-agent-sdk-adapter",
+      package: "@agentmark-ai/claude-agent-sdk-v0-adapter",
       dependencies: ["@anthropic-ai/claude-agent-sdk@^0.1.0"],
       classes: {
         modelRegistry: "ClaudeAgentModelRegistry",
@@ -84,7 +84,7 @@ sdk.initTracing({ disableBatch: true });
   if (adapter === "claude-agent-sdk") {
     return `import "dotenv/config";
 import { query } from "@anthropic-ai/claude-agent-sdk";
-import { withTracing } from "@agentmark-ai/claude-agent-sdk-adapter";
+import { withTracing } from "@agentmark-ai/claude-agent-sdk-v0-adapter";
 ${tracingImport}import { client } from "./agentmark.client";
 ${tracingInit}
 const telemetry = {
@@ -598,15 +598,7 @@ text_config:
   model_name: ${model}
   max_calls: 2
   tools:
-    search_knowledgebase:
-      description: Search the company knowledgebase for information about shipping, warranty, and returns policies.
-      parameters:
-        type: object
-        properties:
-          query:
-            type: string
-            description: The search query to find relevant information
-        required: [query]
+    - search_knowledgebase
 test_settings:
   dataset: customer-query.jsonl
   props:
@@ -649,7 +641,7 @@ object_config:
       - names
 test_settings:
   dataset: party.jsonl
-  evals:
+  scores:
     - exact_match_json
   props:
     party_text: "We're having a party with Alice, Bob, and Carol."
@@ -823,7 +815,7 @@ import type { ToolsInput } from '@mastra/core/agent';
 import { z } from 'zod';` : `import { tool } from 'ai';
 import type { Tool } from 'ai';
 import { z } from 'zod';`;
-  const createClientCall = isClaudeAgentSdk ? `return createAgentMarkClient<AgentMarkTypes>({ loader, modelRegistry, evalRegistry, adapterOptions, mcpServers: { 'customer-support': customerSupportTools } });` : `return createAgentMarkClient<AgentMarkTypes>({ loader, modelRegistry, tools, evalRegistry });`;
+  const createClientCall = isClaudeAgentSdk ? `return createAgentMarkClient<AgentMarkTypes>({ loader, modelRegistry, scores, adapterOptions, mcpServers: { 'customer-support': customerSupportTools } });` : `return createAgentMarkClient<AgentMarkTypes>({ loader, modelRegistry, tools, scores });`;
   const toolSchemaField = isMastra ? `parameters: z.object({ query: z.string().describe('The search query') })` : `inputSchema: z.object({ query: z.string().describe('The search query') })`;
   const toolsReturnType = isMastra ? "ToolsInput" : "Record<string, Tool>";
   const toolsSetup = isClaudeAgentSdk ? `
@@ -876,7 +868,7 @@ import path from 'node:path';
 import dotenv from 'dotenv';
 dotenv.config({ path: path.resolve(__dirname, '.env') });
 import { createAgentMarkClient, ${modelRegistry} } from "${adapterConfig.package}";
-import type { EvalRegistry } from "${adapterConfig.package}";
+import type { ScoreRegistry } from "@agentmark-ai/prompt-core";
 ${loaderImport}
 import AgentMarkTypes from './agentmark.types';
 ${providerImport}
@@ -886,22 +878,26 @@ ${adapterOptionsImport}
 ${modelRegistrySetup}
 ${toolsSetup}
 
-const evalRegistry: EvalRegistry = {
-  exact_match_json: ({ output, expectedOutput }) => {
-    if (!expectedOutput) {
-      return { score: 0, label: 'error', reason: 'No expected output provided', passed: false };
-    }
-    try {
-      const ok = JSON.stringify(output) === JSON.stringify(JSON.parse(expectedOutput));
-      return {
-        score: ok ? 1 : 0,
-        label: ok ? 'correct' : 'incorrect',
-        reason: ok ? 'Exact match' : 'Mismatch',
-        passed: ok
-      };
-    } catch (e) {
-      return { score: 0, label: 'error', reason: 'Failed to parse expected output as JSON', passed: false };
-    }
+const scores: ScoreRegistry = {
+  exact_match_json: {
+    schema: { type: 'boolean' },
+    description: 'Whether output matches expected JSON exactly',
+    eval: ({ output, expectedOutput }) => {
+      if (!expectedOutput) {
+        return { score: 0, label: 'error', reason: 'No expected output provided', passed: false };
+      }
+      try {
+        const ok = JSON.stringify(output) === JSON.stringify(JSON.parse(expectedOutput));
+        return {
+          score: ok ? 1 : 0,
+          label: ok ? 'correct' : 'incorrect',
+          reason: ok ? 'Exact match' : 'Mismatch',
+          passed: ok
+        };
+      } catch (e) {
+        return { score: 0, label: 'error', reason: 'Failed to parse expected output as JSON', passed: false };
+      }
+    },
   },
 };
 
@@ -1229,7 +1225,7 @@ const adapter = new ${handlerClass}(client as any);
 export default async function handler(request: {
   type: 'prompt-run' | 'dataset-run';
   data: {
-    ast: unknown;
+    ast: any;
     customProps?: Record<string, unknown>;
     options?: { shouldStream?: boolean };
     experimentId?: string;
@@ -1484,7 +1480,7 @@ description = "An AgentMark application using Claude Agent SDK"
 requires-python = ">=3.12"
 dependencies = [
     "agentmark-sdk>=0.1.0",
-    "agentmark-claude-agent-sdk>=0.1.0",
+    "agentmark-claude-agent-sdk-v0>=0.1.0",
     "agentmark-prompt-core>=0.1.0",
     "python-dotenv>=1.0.0",
     "claude-agent-sdk>=0.1.0",
@@ -1515,7 +1511,7 @@ description = "An AgentMark application using Pydantic AI"
 requires-python = ">=3.12"
 dependencies = [
     "agentmark-sdk>=0.1.0",
-    "agentmark-pydantic-ai>=0.1.0",
+    "agentmark-pydantic-ai-v0>=0.1.0",
     "agentmark-prompt-core>=0.1.0",
     "python-dotenv>=1.0.0",
     "pydantic-ai[openai]>=0.1.0",
@@ -1541,40 +1537,122 @@ asyncio_mode = "auto"
 strict = true
 `;
 };
-var getAgentmarkClientContent = (_deploymentMode, adapter) => {
+var getHandlerPyContent = (adapter) => {
+  const webhookClass = adapter === "claude-agent-sdk" ? "ClaudeAgentSDKWebhookHandler" : "PydanticAIWebhookHandler";
+  const webhookImport = adapter === "claude-agent-sdk" ? "from agentmark_claude_agent_sdk import ClaudeAgentSDKWebhookHandler" : "from agentmark_pydantic_ai_v0 import PydanticAIWebhookHandler";
+  return `"""AgentMark handler for managed cloud deployments.
+
+This file is used by the AgentMark platform to execute prompts and experiments
+on deployed infrastructure. It mirrors the TypeScript handler.ts pattern.
+"""
+
+import os
+
+from agentmark_sdk import AgentMarkSDK
+${webhookImport}
+from agentmark_client import client
+
+# Initialize tracing
+sdk = AgentMarkSDK(
+    api_key=os.environ.get("AGENTMARK_API_KEY", ""),
+    app_id=os.environ.get("AGENTMARK_APP_ID", ""),
+    base_url=os.environ.get("AGENTMARK_BASE_URL"),
+)
+sdk.init_tracing(disable_batch=True)
+
+adapter = ${webhookClass}(client)
+
+
+async def handler(request: dict):
+    """Handle prompt-run and dataset-run requests from the platform."""
+    req_type = request.get("type")
+    data = request.get("data", {})
+
+    if req_type == "prompt-run":
+        return await adapter.run_prompt(data["ast"], {
+            "shouldStream": data.get("options", {}).get("shouldStream", True),
+            "customProps": data.get("customProps"),
+        })
+
+    if req_type == "dataset-run":
+        return await adapter.run_experiment(
+            data["ast"],
+            data.get("experimentId", ""),
+            data.get("datasetPath"),
+        )
+
+    raise ValueError(f"Unknown request type: {req_type}")
+`;
+};
+var getAgentmarkClientContent = (deploymentMode, adapter) => {
+  const isCloud = deploymentMode === "cloud";
+  const loaderImport = isCloud ? `from agentmark.prompt_core import ApiLoader` : `from agentmark.prompt_core import FileLoader`;
+  const loaderSetup = isCloud ? `# API loader for cloud deployment \u2014 fetches datasets from the AgentMark gateway
+loader = ApiLoader.cloud()` : `# File loader for local development
+project_root = Path(__file__).parent.resolve()
+loader = FileLoader(base_dir=str(project_root))`;
   if (adapter === "claude-agent-sdk") {
     return `"""AgentMark client configuration.
 
 This file configures the AgentMark client with Claude Agent SDK adapter.
-Customize the model registry as needed.
+Customize the model registry and eval registry as needed.
 """
 
+import json
 import os
 from pathlib import Path
 from dotenv import load_dotenv
 
-from agentmark.prompt_core import FileLoader
+${loaderImport}
 from agentmark_claude_agent_sdk import (
     create_claude_agent_client,
-    create_default_model_registry,
+    ClaudeAgentModelRegistry,
 )
 
 # Load environment variables
 load_dotenv()
 
-# Configure model registry with default mappings
-# Supports: claude-* models
-model_registry = create_default_model_registry()
+# Register the model providers your prompts use.
+# This maps "anthropic/claude-sonnet-4-20250514" in prompt files to the Claude Agent SDK.
+model_registry = ClaudeAgentModelRegistry()
+model_registry.register_providers({
+    "anthropic": "anthropic",
+})
+
+
+# Eval registry \u2014 define evaluation functions for experiments
+# TODO: Update to use scores (ScoreRegistry) once the Python SDK supports it.
+# The TypeScript SDK already uses scores with schema definitions.
+def exact_match_json(params):
+    """Check if output matches expected output exactly."""
+    output = params.get("output")
+    expected_output = params.get("expectedOutput")
+    if not expected_output:
+        return {"score": 0, "label": "error", "reason": "No expected output provided", "passed": False}
+    try:
+        actual = json.loads(output) if isinstance(output, str) else output
+        expected = json.loads(expected_output) if isinstance(expected_output, str) else expected_output
+        ok = actual == expected
+        return {
+            "score": 1 if ok else 0,
+            "label": "correct" if ok else "incorrect",
+            "reason": "Exact match" if ok else "Mismatch",
+            "passed": ok,
+        }
+    except (json.JSONDecodeError, TypeError):
+        return {"score": 0, "label": "error", "reason": "Failed to parse JSON", "passed": False}
 
-# Create file loader for local development
-# Uses the project root as base directory for resolving relative paths
-project_root = Path(__file__).parent.resolve()
-loader = FileLoader(base_dir=str(project_root))
+eval_registry = {
+    "exact_match_json": exact_match_json,
+}
+
+${loaderSetup}
 
 # Create the client
 # Claude Agent SDK handles tools natively through the SDK
 client = create_claude_agent_client(
     model_registry=model_registry,
+    eval_registry=eval_registry,
     loader=loader,
 )
 
@@ -1584,25 +1662,30 @@ __all__ = ["client"]
   return `"""AgentMark client configuration.
 
 This file configures the AgentMark client with Pydantic AI adapter.
-Customize the model registry and tools as needed.
+Customize the model registry, tools, and eval registry as needed.
 """
 
+import json
 import os
 from pathlib import Path
 from dotenv import load_dotenv
 
-from agentmark.prompt_core import FileLoader
+${loaderImport}
 from agentmark_pydantic_ai_v0 import (
     create_pydantic_ai_client,
-    create_default_model_registry,
+    PydanticAIModelRegistry,
 )
 
 # Load environment variables
 load_dotenv()
 
-# Configure model registry with default mappings
-# Supports: gpt-*, claude-*, gemini-*, etc.
-model_registry = create_default_model_registry()
+# Register the model providers your prompts use.
+# This maps "openai/gpt-4o" in prompt files to "openai:gpt-4o" for Pydantic AI.
+model_registry = PydanticAIModelRegistry()
+model_registry.register_providers({
+    "openai": "openai",
+    "anthropic": "anthropic",
+})
 
 # Define tools as native pydantic-ai Tool objects or callables
 # Example:
@@ -1611,15 +1694,40 @@ model_registry = create_default_model_registry()
 # tools = [search]
 tools = []
 
-# Create file loader for local development
-# Uses the project root as base directory for resolving relative paths
-project_root = Path(__file__).parent.resolve()
-loader = FileLoader(base_dir=str(project_root))
+
+# Eval registry \u2014 define evaluation functions for experiments
+# TODO: Update to use scores (ScoreRegistry) once the Python SDK supports it.
+# The TypeScript SDK already uses scores with schema definitions.
+def exact_match_json(params):
+    """Check if output matches expected output exactly."""
+    output = params.get("output")
+    expected_output = params.get("expectedOutput")
+    if not expected_output:
+        return {"score": 0, "label": "error", "reason": "No expected output provided", "passed": False}
+    try:
+        actual = json.loads(output) if isinstance(output, str) else output
+        expected = json.loads(expected_output) if isinstance(expected_output, str) else expected_output
+        ok = actual == expected
+        return {
+            "score": 1 if ok else 0,
+            "label": "correct" if ok else "incorrect",
+            "reason": "Exact match" if ok else "Mismatch",
+            "passed": ok,
+        }
+    except (json.JSONDecodeError, TypeError):
+        return {"score": 0, "label": "error", "reason": "Failed to parse JSON", "passed": False}
+
+eval_registry = {
+    "exact_match_json": exact_match_json,
+}
+
+${loaderSetup}
 
 # Create the client
 client = create_pydantic_ai_client(
     model_registry=model_registry,
     tools=tools,
+    eval_registry=eval_registry,
     loader=loader,
 )
 
@@ -1847,6 +1955,15 @@ var createPythonApp = async (client, targetPath = ".", apiKey = "", deploymentMo
       console.log("\u23ED\uFE0F  Skipped pyproject.toml (existing project)");
     }
     fs5.writeFileSync(`${targetPath}/agentmark_client.py`, getAgentmarkClientContent(deploymentMode, adapter));
+    if (deploymentMode === "cloud") {
+      const handlerPath = path3.join(targetPath, "handler.py");
+      if (fs5.existsSync(handlerPath)) {
+        console.log("\u23ED\uFE0F  Skipped handler.py (already exists - preserving customizations)");
+      } else {
+        fs5.writeFileSync(handlerPath, getHandlerPyContent(adapter));
+        console.log(`\u2705 Created handler.py for cloud deployment`);
+      }
+    }
     if (!isExistingProject) {
       fs5.writeFileSync(`${targetPath}/main.py`, getMainPyContent(adapter, deploymentMode));
     } else {
@@ -1935,7 +2052,7 @@ var createPythonApp = async (client, targetPath = ".", apiKey = "", deploymentMo
     if (pythonVenv) {
       const activateCmd = process.platform === "win32" ? `${pythonVenv.name}\\Scripts\\activate` : `source ${pythonVenv.name}/bin/activate`;
       console.log(`  $ ${activateCmd}`);
-      console.log('  $ pip install agentmark-pydantic-ai agentmark-prompt-core python-dotenv "pydantic-ai[openai]"');
+      console.log('  $ pip install agentmark-pydantic-ai-v0 agentmark-prompt-core python-dotenv "pydantic-ai[openai]"');
     } else {
       console.log("  $ python -m venv .venv");
       console.log("  $ source .venv/bin/activate  # On Windows: .venv\\Scripts\\activate");
@@ -2097,22 +2214,45 @@ function initGitRepo(targetPath) {
 }
 
 // src/index.ts
+var VALID_ADAPTERS_TS = ["ai-sdk", "claude-agent-sdk", "mastra"];
+var VALID_ADAPTERS_PY = ["pydantic-ai", "claude-agent-sdk"];
+var VALID_CLIENTS = ["claude-code", "cursor", "vscode", "zed", "skip"];
 var parseArgs = () => {
   const args = process.argv.slice(2);
-  let deploymentMode;
-  let language;
-  for (const arg of args) {
-    if (arg === "--cloud") {
-      deploymentMode = "cloud";
-    } else if (arg === "--self-host") {
-      deploymentMode = "static";
-    } else if (arg === "--python") {
-      language = "python";
-    } else if (arg === "--typescript") {
-      language = "typescript";
+  const result = {};
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    switch (arg) {
+      case "--cloud":
+        result.deploymentMode = "cloud";
+        break;
+      case "--self-host":
+        result.deploymentMode = "static";
+        break;
+      case "--python":
+        result.language = "python";
+        break;
+      case "--typescript":
+        result.language = "typescript";
+        break;
+      case "--overwrite":
+        result.overwrite = true;
+        break;
+      case "--path":
+        result.path = args[++i];
+        break;
+      case "--adapter":
+        result.adapter = args[++i];
+        break;
+      case "--api-key":
+        result.apiKey = args[++i];
+        break;
+      case "--client":
+        result.client = args[++i];
+        break;
     }
   }
-  return { deploymentMode, language };
+  return result;
 };
 var main = async () => {
   const cliArgs = parseArgs();
@@ -2123,12 +2263,16 @@ var main = async () => {
     agentmarkPath: "."
   };
   console.log("Initializing project.");
-  const { folderName } = await prompts2({
-    name: "folderName",
-    type: "text",
-    message: "Where would you like to create your AgentMark app?",
-    initial: "my-agentmark-app"
-  });
+  let folderName = cliArgs.path;
+  if (!folderName) {
+    const response = await prompts2({
+      name: "folderName",
+      type: "text",
+      message: "Where would you like to create your AgentMark app?",
+      initial: "my-agentmark-app"
+    });
+    folderName = response.folderName;
+  }
   const isCurrentDir = isCurrentDirectory(folderName);
   const targetPath = isCurrentDir ? process.cwd() : path6.resolve(folderName);
   if (!isCurrentDir) {
@@ -2138,7 +2282,15 @@ var main = async () => {
   if (projectInfo.isExistingProject) {
     displayProjectDetectionSummary(projectInfo);
   }
-  const resolutions = await promptForResolutions(projectInfo.conflictingFiles);
+  let resolutions;
+  if (cliArgs.overwrite) {
+    resolutions = projectInfo.conflictingFiles.map((f) => ({
+      path: f.path,
+      action: "overwrite"
+    }));
+  } else {
+    resolutions = await promptForResolutions(projectInfo.conflictingFiles);
+  }
   let language = cliArgs.language;
   if (!language) {
     const response = await prompts2({
@@ -2152,40 +2304,49 @@ var main = async () => {
     });
     language = response.language;
   }
-  let adapter;
-  if (language === "python") {
-    const response = await prompts2({
-      name: "adapter",
-      type: "select",
-      message: "Which adapter would you like to use?",
-      choices: [
-        { title: "Pydantic AI", value: "pydantic-ai" },
-        { title: "Claude Agent SDK", value: "claude-agent-sdk" }
-      ]
-    });
-    adapter = response.adapter;
-  } else {
-    const response = await prompts2({
-      name: "adapter",
-      type: "select",
-      message: "Which adapter would you like to use?",
-      choices: [
-        { title: "AI SDK (Vercel)", value: "ai-sdk" },
-        { title: "Claude Agent SDK", value: "claude-agent-sdk" },
-        { title: "Mastra", value: "mastra" }
-      ]
+  let adapter = cliArgs.adapter;
+  if (!adapter) {
+    if (language === "python") {
+      const response = await prompts2({
+        name: "adapter",
+        type: "select",
+        message: "Which adapter would you like to use?",
+        choices: [
+          { title: "Pydantic AI", value: "pydantic-ai" },
+          { title: "Claude Agent SDK", value: "claude-agent-sdk" }
+        ]
+      });
+      adapter = response.adapter;
+    } else {
+      const response = await prompts2({
+        name: "adapter",
+        type: "select",
+        message: "Which adapter would you like to use?",
+        choices: [
+          { title: "AI SDK (Vercel)", value: "ai-sdk" },
+          { title: "Claude Agent SDK", value: "claude-agent-sdk" },
+          { title: "Mastra", value: "mastra" }
+        ]
+      });
+      adapter = response.adapter;
+    }
+  }
+  const validAdapters = language === "python" ? VALID_ADAPTERS_PY : VALID_ADAPTERS_TS;
+  if (!validAdapters.includes(adapter)) {
+    console.error(`Invalid adapter "${adapter}" for ${language}. Valid: ${validAdapters.join(", ")}`);
+    process.exit(1);
+  }
+  let apiKey = cliArgs.apiKey ?? "";
+  if (!cliArgs.apiKey && cliArgs.apiKey !== "") {
+    const apiKeyName = adapter === "claude-agent-sdk" ? "Anthropic" : "OpenAI";
+    const { providedApiKey } = await prompts2({
+      name: "providedApiKey",
+      type: "password",
+      message: `Enter your ${apiKeyName} API key (or press Enter to skip):`,
+      initial: ""
     });
-    adapter = response.adapter;
-  }
-  const apiKeyName = adapter === "claude-agent-sdk" ? "Anthropic" : "OpenAI";
-  let apiKey = "";
-  const { providedApiKey } = await prompts2({
-    name: "providedApiKey",
-    type: "password",
-    message: `Enter your ${apiKeyName} API key (or press Enter to skip):`,
-    initial: ""
-  });
-  apiKey = providedApiKey || "";
+    apiKey = providedApiKey || "";
+  }
   let deploymentMode = cliArgs.deploymentMode;
   if (!deploymentMode) {
     const response = await prompts2({
@@ -2207,18 +2368,26 @@ var main = async () => {
     });
     deploymentMode = response.deploymentMode;
   }
-  const { client } = await prompts2({
-    name: "client",
-    type: "select",
-    message: "Make your IDE an AgentMark expert",
-    choices: [
-      { title: "Claude Code", value: "claude-code" },
-      { title: "Cursor", value: "cursor" },
-      { title: "VS Code", value: "vscode" },
-      { title: "Zed", value: "zed" },
-      { title: "Skip", value: "skip" }
-    ]
-  });
+  let client = cliArgs.client;
+  if (!client) {
+    const response = await prompts2({
+      name: "client",
+      type: "select",
+      message: "Make your IDE an AgentMark expert",
+      choices: [
+        { title: "Claude Code", value: "claude-code" },
+        { title: "Cursor", value: "cursor" },
+        { title: "VS Code", value: "vscode" },
+        { title: "Zed", value: "zed" },
+        { title: "Skip", value: "skip" }
+      ]
+    });
+    client = response.client;
+  }
+  if (!VALID_CLIENTS.includes(client)) {
+    console.error(`Invalid client "${client}". Valid: ${VALID_CLIENTS.join(", ")}`);
+    process.exit(1);
+  }
   let usedModels;
   if (language === "python") {
     usedModels = await createPythonApp(client, targetPath, apiKey, deploymentMode, adapter, projectInfo, resolutions);
@@ -2227,7 +2396,7 @@ var main = async () => {
   }
   config.builtInModels = usedModels;
   if (deploymentMode === "cloud") {
-    config.handler = "handler.ts";
+    config.handler = language === "python" ? "handler.py" : "handler.ts";
   }
   const agentmarkJsonPath = path6.join(targetPath, "agentmark.json");
   const agentmarkJsonResolution = resolutions.find((r) => r.path === "agentmark.json");