@agentmark-ai/cli 0.1.0

package/dist/cost-mapping/cost-mapping.js

@@ -0,0 +1,36 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCostFormula = exports.getModelCostMappings = void 0;
+let prices = {};
+const pricesUrl = "https://raw.githubusercontent.com/agentmark-ai/agentmark/refs/heads/main/packages/cli/cli-src/cost-mapping/pricing.json";
+const getModelCostMappings = () => __awaiter(void 0, void 0, void 0, function* () {
+    if (Object.keys(prices).length > 0) {
+        return prices;
+    }
+    // If no prices URL configured, return empty mappings
+    if (!pricesUrl) {
+        return prices;
+    }
+    const res = yield fetch(pricesUrl);
+    const data = yield res.json();
+    prices = data;
+    return prices;
+});
+exports.getModelCostMappings = getModelCostMappings;
+const getCostFormula = (inputCost, outputCost, unitScale) => {
+    return (inputTokens, outputTokens) => {
+        const cost = inputCost * inputTokens + outputCost * outputTokens;
+        return cost / unitScale;
+    };
+};
+exports.getCostFormula = getCostFormula;
+//# sourceMappingURL=cost-mapping.js.map

package/dist/cost-mapping/cost-mapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost-mapping.js","sourceRoot":"","sources":["../../cli-src/cost-mapping/cost-mapping.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,IAAI,MAAM,GAAqE,EAAE,CAAC;AAElF,MAAM,SAAS,GAAG,yHAAyH,CAAA;AACpI,MAAM,oBAAoB,GAAG,GAEjC,EAAE;IACH,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,qDAAqD;IACrD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAA;IAClC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAC7B,MAAM,GAAG,IAAI,CAAA;IACb,OAAO,MAAM,CAAA;AACf,CAAC,CAAA,CAAA;AAdY,QAAA,oBAAoB,wBAchC;AAEM,MAAM,cAAc,GAAG,CAC5B,SAAiB,EACjB,UAAkB,EAClB,SAAiB,EACjB,EAAE;IACF,OAAO,CAAC,WAAmB,EAAE,YAAoB,EAAE,EAAE;QACnD,MAAM,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,UAAU,GAAG,YAAY,CAAC;QACjE,OAAO,IAAI,GAAG,SAAS,CAAC;IAC1B,CAAC,CAAC;AACJ,CAAC,CAAC;AATW,QAAA,cAAc,kBASzB"}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/index.js

@@ -0,0 +1,127 @@
+#!/usr/bin/env node
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// Load .env file from current working directory
+const dotenv_1 = require("dotenv");
+(0, dotenv_1.config)();
+const commander_1 = require("commander");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const dev_1 = __importDefault(require("./commands/dev"));
+const generate_types_1 = __importDefault(require("./commands/generate-types"));
+const pull_models_1 = __importDefault(require("./commands/pull-models"));
+const run_prompt_1 = __importDefault(require("./commands/run-prompt"));
+const run_experiment_1 = __importDefault(require("./commands/run-experiment"));
+const build_1 = __importDefault(require("./commands/build"));
+// Read version from package.json
+const packageJson = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../package.json'), 'utf-8'));
+commander_1.program
+    .version(packageJson.version, '-v, --version', 'Output the current version')
+    .name('agentmark')
+    .description('AgentMark CLI - Build, test, and deploy AI agents');
+commander_1.program
+    .command("dev")
+    .option("--api-port <number>", "API server port (default: 9418)")
+    .option("--webhook-port <number>", "Webhook server port (default: 9417)")
+    .option("--app-port <number>", "AgentMark UI app port (default: 3000)")
+    .option("-t, --tunnel", "Expose webhook server publicly via tunnel")
+    .description("Start development servers (API server + webhook + UI app)")
+    .action((options) => __awaiter(void 0, void 0, void 0, function* () {
+    yield dev_1.default({
+        apiPort: options.apiPort ? parseInt(options.apiPort, 10) : undefined,
+        webhookPort: options.webhookPort ? parseInt(options.webhookPort, 10) : undefined,
+        appPort: options.appPort ? parseInt(options.appPort, 10) : undefined,
+        tunnel: options.tunnel || false
+    });
+}));
+commander_1.program
+    .command('generate-types')
+    .option('-l, --language <language>', 'Language to generate types for', 'typescript')
+    .option('--local <port>', 'Local server port number')
+    .option('--root-dir <path>', 'Root directory containing agentmark files')
+    .action((options) => __awaiter(void 0, void 0, void 0, function* () {
+    const localPort = options.local ? parseInt(options.local, 10) : undefined;
+    yield generate_types_1.default({
+        language: options.language,
+        local: localPort,
+        rootDir: options.rootDir
+    });
+}));
+commander_1.program
+    .command("pull-models")
+    .description('Pull models from a provider')
+    .action(() => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        yield pull_models_1.default();
+    }
+    catch (error) {
+        commander_1.program.error(error.message);
+    }
+}));
+commander_1.program
+    .command("run-prompt <filepath>")
+    .description('Run a prompt with test props')
+    .option('--server <url>', 'URL of an AgentMark webhook server (e.g., http://localhost:9417)')
+    .option('--props <json>', 'Props as JSON string (e.g., \'{"key": "value"}\')')
+    .option('--props-file <path>', 'Path to JSON or YAML file containing props')
+    .action((filepath, options) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        yield run_prompt_1.default(filepath, options);
+    }
+    catch (error) {
+        commander_1.program.error(error.message);
+    }
+}));
+commander_1.program
+    .command("run-experiment <filepath>")
+    .description('Run an experiment against its dataset, with evals by default')
+    .option('--server <url>', 'URL of an AgentMark webhook server (e.g., http://localhost:9417)')
+    .option('--skip-eval', 'Skip running evals even if they exist')
+    .option('--format <format>', 'Output format: table, csv, json, or jsonl (default: table)')
+    .option('--threshold <percent>', 'Fail if pass percentage is below threshold (0-100)', (v) => {
+    const n = parseInt(v, 10);
+    if (!Number.isFinite(n))
+        throw new Error('Threshold must be a number');
+    if (n < 0 || n > 100)
+        throw new Error('Threshold must be between 0 and 100');
+    return n;
+})
+    .action((filepath, options) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        const format = options.format || 'table';
+        if (!['table', 'csv', 'json', 'jsonl'].includes(format)) {
+            throw new Error('Format must be one of: table, csv, json, jsonl');
+        }
+        const thresholdPercent = typeof options.threshold === 'number' ? options.threshold : undefined;
+        yield run_experiment_1.default(filepath, { skipEval: !!options.skipEval, format, thresholdPercent, server: options.server });
+    }
+    catch (error) {
+        commander_1.program.error(error.message);
+    }
+}));
+commander_1.program
+    .command("build")
+    .description('Build prompts and datasets into pre-compiled JSON files for static loading')
+    .option('-o, --out <directory>', 'Output directory (default: dist/agentmark)')
+    .action((options) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        yield build_1.default({ outDir: options.out });
+    }
+    catch (error) {
+        commander_1.program.error(error.message);
+    }
+}));
+commander_1.program.parse(process.argv);
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../cli-src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAEA,gDAAgD;AAChD,mCAAgC;AAChC,IAAA,eAAM,GAAE,CAAC;AAET,yCAAoC;AACpC,2BAAkC;AAClC,+BAA4B;AAC5B,yDAAiC;AACjC,+EAAsD;AACtD,yEAAgD;AAChD,uEAA8C;AAC9C,+EAAsD;AACtD,6DAAqC;AAErC,iCAAiC;AACjC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAA,iBAAY,EAAC,IAAA,WAAI,EAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AAE1F,mBAAO;KACJ,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,eAAe,EAAE,4BAA4B,CAAC;KAC3E,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,mDAAmD,CAAC,CAAC;AAEpE,mBAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,MAAM,CAAC,qBAAqB,EAAE,iCAAiC,CAAC;KAChE,MAAM,CAAC,yBAAyB,EAAE,qCAAqC,CAAC;KACxE,MAAM,CAAC,qBAAqB,EAAE,uCAAuC,CAAC;KACtE,MAAM,CAAC,cAAc,EAAE,2CAA2C,CAAC;KACnE,WAAW,CAAC,2DAA2D,CAAC;KACxE,MAAM,CAAC,CAAO,OAAO,EAAE,EAAE;IACxB,MAAO,aAAW,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QACpE,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QAChF,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QACpE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAChC,CAAC,CAAC;AACL,CAAC,CAAA,CAAC,CAAC;AAEL,mBAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,MAAM,CAAC,2BAA2B,EAAE,gCAAgC,EAAE,YAAY,CAAC;KACnF,MAAM,CAAC,gBAAgB,EAAE,0BAA0B,CAAC;KACpD,MAAM,CAAC,mBAAmB,EAAE,2CAA2C,CAAC;KACxE,MAAM,CAAC,CAAO,OAAO,EAAE,EAAE;IACxB,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,MAAO,wBAAqB,CAAC;QAC3B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;AACL,CAAC,CAAA,CAAC,CAAC;AAEL,mBAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,6BAA6B,CAAC;KAC1C,MAAM,CAAC,GAAS,EAAE;IACjB,IAAI,CAAC;QACH,MAAO,qBAAkB,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mBAAO,CAAC,KAAK,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC,CAAA,CAAC,CAAC;AAEL,mBAAO;KACJ,OAAO,CAAC,uBAAuB,CAAC;KAChC,WAAW,CAAC,8BAA8B,CAAC;KAC3C,MAAM,CAAC,gBAAgB,EAAE,kEAAkE,CAAC;KAC5F,MAAM,CAAC,gBAAgB,EAAE,mDAAmD,CAAC;KAC7E,MAAM,CAAC,qBAAqB,EAAE,4CAA4C,CAAC;KAC3E,MAAM,CAAC,CAAO,QAAgB,EAAE,OAAgE,EAAE,EAAE;IACnG,IAAI,CAAC;QACH,MAAO,oBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mBAAO,CAAC,KAAK,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC,CAAA,CAAC,CAAC;AAEL,mBAAO;KACJ,OAAO,CAAC,2BAA2B,CAAC;KACpC,WAAW,CAAC,8DAA8D,CAAC;KAC3E,MAAM,CAAC,gBAAgB,EAAE,kEAAkE,CAAC;KAC5F,MAAM,CAAC,aAAa,EAAE,uCAAuC,CAAC;KAC9D,MAAM,CAAC,mBAAmB,EAAE,4DAA4D,CAAC;KACzF,MAAM,CAAC,uBAAuB,EAAE,oDAAoD,EAAE,CAAC,CAAC,EAAE,EAAE;IAC3F,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC7E,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;KACD,MAAM,CAAC,CAAO,QAAgB,EAAE,OAAqF,EAAE,EAAE;IACxH,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC;QACzC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,gBAAgB,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/F,MAAO,wBAAqB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7H,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mBAAO,CAAC,KAAK,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC,CAAA,CAAC,CAAC;AAEL,mBAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,4EAA4E,CAAC;KACzF,MAAM,CAAC,uBAAuB,EAAE,4CAA4C,CAAC;KAC7E,MAAM,CAAC,CAAO,OAAyB,EAAE,EAAE;IAC1C,IAAI,CAAC;QACH,MAAO,eAAa,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mBAAO,CAAC,KAAK,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC,CAAA,CAAC,CAAC;AAEL,mBAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}

package/dist/runner-server/core.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Core platform-agnostic handler for AgentMark webhook requests.
+ * This handler processes requests from any platform adapter and returns
+ * a standardized response that adapters can translate to platform-specific formats.
+ */
+import type { WebhookHandler, WebhookRequest, WebhookResponse } from './types';
+/**
+ * Handles a webhook request and returns a platform-agnostic response.
+ * This is the core business logic that all platform adapters use.
+ *
+ * @param request - The standardized webhook request
+ * @param handler - The webhook handler instance (e.g., VercelAdapterWebhookHandler)
+ * @returns A standardized response that adapters translate to platform formats
+ */
+export declare function handleWebhookRequest(request: WebhookRequest, handler: WebhookHandler): Promise<WebhookResponse>;

package/dist/runner-server/core.js

@@ -0,0 +1,180 @@
+"use strict";
+/**
+ * Core platform-agnostic handler for AgentMark webhook requests.
+ * This handler processes requests from any platform adapter and returns
+ * a standardized response that adapters can translate to platform-specific formats.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleWebhookRequest = handleWebhookRequest;
+/**
+ * Handles a webhook request and returns a platform-agnostic response.
+ * This is the core business logic that all platform adapters use.
+ *
+ * @param request - The standardized webhook request
+ * @param handler - The webhook handler instance (e.g., VercelAdapterWebhookHandler)
+ * @returns A standardized response that adapters translate to platform formats
+ */
+function handleWebhookRequest(request, handler) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a, _b;
+        try {
+            const { type, data } = request;
+            // Log the event type with more detail
+            if (!type) {
+                console.log(`   ⚠️  Event: missing 'type' field`);
+                // Check for common mistakes
+                const anyRequest = request;
+                if (anyRequest.event) {
+                    console.log(`   💡 Found nested 'event' wrapper`);
+                    console.log(`   ❌ Incorrect: { "event": { "type": "...", "data": {...} } }`);
+                    console.log(`   ✅ Correct:   { "type": "...", "data": {...} }`);
+                }
+                else {
+                    console.log(`   📦 Request body:`, JSON.stringify(request, null, 2));
+                }
+                return {
+                    type: 'error',
+                    error: 'Missing event type',
+                    details: 'Request must include a "type" field at the top level: {"type": "prompt-run", "data": {...}}',
+                    status: 400
+                };
+            }
+            console.log(`   📝 Event: ${type}`);
+            // Validate known event types
+            if (type !== 'prompt-run' && type !== 'dataset-run') {
+                console.log(`   ⚠️  Unknown event type: ${type}`);
+                console.log(`   Valid types: "prompt-run", "dataset-run"`);
+                return {
+                    type: 'error',
+                    error: 'Unknown event type',
+                    details: `Expected event.type to be 'prompt-run' or 'dataset-run', got: ${type}`,
+                    status: 400
+                };
+            }
+            // Validate request structure
+            if (!data) {
+                console.log(`   ⚠️  Missing 'data' field in request`);
+                return {
+                    type: 'error',
+                    error: 'Missing data object',
+                    details: 'Request must include a "data" object with the prompt AST and options',
+                    status: 400
+                };
+            }
+            // Validate AST field
+            if (!data.ast) {
+                console.log(`   ⚠️  Missing 'data.ast' field`);
+                console.log(`   📦 Available fields in data:`, Object.keys(data).join(', '));
+                // Check for common mistakes
+                const anyData = data;
+                if (anyData.prompt) {
+                    console.log(`   💡 Found 'data.prompt' but expected 'data.ast'`);
+                    console.log(`   ❌ Incorrect: { "data": { "prompt": {...} } }`);
+                    console.log(`   ✅ Correct:   { "data": { "ast": {...} } }`);
+                }
+                return {
+                    type: 'error',
+                    error: 'Missing AST object',
+                    details: 'The request must include the prompt AST in data.ast (not data.prompt)',
+                    status: 400
+                };
+            }
+            if (typeof data.ast !== 'object') {
+                console.log(`   ⚠️  Invalid AST type: ${typeof data.ast} (expected object)`);
+                return {
+                    type: 'error',
+                    error: 'Invalid AST type',
+                    details: `The AST must be an object, got ${typeof data.ast}`,
+                    status: 400
+                };
+            }
+            // Handle prompt execution
+            if (type === 'prompt-run') {
+                console.log('   🤖 Executing prompt...');
+                const options = {
+                    shouldStream: (_a = data.options) === null || _a === void 0 ? void 0 : _a.shouldStream,
+                    customProps: data.customProps,
+                };
+                const response = yield handler.runPrompt(data.ast, options);
+                // Handle streaming response
+                if (response.type === 'stream') {
+                    console.log('   ✓ Prompt executed successfully (streaming)');
+                    return {
+                        type: 'stream',
+                        stream: response.stream,
+                        headers: response.streamHeader || { 'AgentMark-Streaming': 'true' },
+                        traceId: response.traceId
+                    };
+                }
+                // Handle regular JSON response (text, object, image, speech)
+                console.log('   ✓ Prompt executed successfully');
+                return {
+                    type: 'json',
+                    data: response,
+                    status: 200
+                };
+            }
+            // Handle dataset/experiment execution
+            if (type === 'dataset-run') {
+                console.log('   🧪 Running experiment with dataset...');
+                const experimentId = (_b = data.experimentId) !== null && _b !== void 0 ? _b : 'local-experiment';
+                let response;
+                try {
+                    response = yield handler.runExperiment(data.ast, experimentId, data.datasetPath);
+                }
+                catch (e) {
+                    const errorMessage = (e === null || e === void 0 ? void 0 : e.message) || String(e);
+                    console.log(`   ❌ Experiment failed: ${errorMessage}`);
+                    return {
+                        type: 'error',
+                        error: errorMessage,
+                        details: 'An error occurred while running the experiment. Check that your prompt and dataset are valid.',
+                        status: 500
+                    };
+                }
+                // Dataset runs always return streams
+                if (response === null || response === void 0 ? void 0 : response.stream) {
+                    console.log('   ✓ Experiment started successfully (streaming)');
+                    return {
+                        type: 'stream',
+                        stream: response.stream,
+                        headers: response.streamHeaders || { 'AgentMark-Streaming': 'true' }
+                    };
+                }
+                console.log('   ❌ Experiment failed: No stream returned');
+                return {
+                    type: 'error',
+                    error: 'Expected stream from dataset-run',
+                    details: 'Dataset execution should return a streaming response',
+                    status: 500
+                };
+            }
+            // Unknown event type
+            return {
+                type: 'error',
+                error: 'Unknown event type',
+                details: `Expected event.type to be 'prompt-run' or 'dataset-run', got: ${type || 'undefined'}`,
+                status: 400
+            };
+        }
+        catch (e) {
+            const errorMessage = (e === null || e === void 0 ? void 0 : e.message) || String(e);
+            return {
+                type: 'error',
+                error: 'Webhook handler error',
+                details: errorMessage,
+                status: 500
+            };
+        }
+    });
+}
+//# sourceMappingURL=core.js.map

package/dist/runner-server/core.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"core.js","sourceRoot":"","sources":["../../cli-src/runner-server/core.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;AAYH,oDA4KC;AApLD;;;;;;;GAOG;AACH,SAAsB,oBAAoB,CACxC,OAAuB,EACvB,OAAuB;;;QAEvB,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;YAE/B,sCAAsC;YACtC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;gBAElD,4BAA4B;gBAC5B,MAAM,UAAU,GAAG,OAAc,CAAC;gBAClC,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;oBAClD,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;oBAC7E,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;gBAClE,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACvE,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,oBAAoB;oBAC3B,OAAO,EAAE,6FAA6F;oBACtG,MAAM,EAAE,GAAG;iBACZ,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;YAEpC,6BAA6B;YAC7B,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;gBACpD,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,EAAE,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,oBAAoB;oBAC3B,OAAO,EAAE,iEAAiE,IAAI,EAAE;oBAChF,MAAM,EAAE,GAAG;iBACZ,CAAC;YACJ,CAAC;YAED,6BAA6B;YAC7B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;gBACtD,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,sEAAsE;oBAC/E,MAAM,EAAE,GAAG;iBACZ,CAAC;YACJ,CAAC;YAED,qBAAqB;YACrB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE7E,4BAA4B;gBAC5B,MAAM,OAAO,GAAG,IAAW,CAAC;gBAC5B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;oBACjE,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;oBAC/D,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;gBAC9D,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,oBAAoB;oBAC3B,OAAO,EAAE,uEAAuE;oBAChF,MAAM,EAAE,GAAG;iBACZ,CAAC;YACJ,CAAC;YAED,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC;gBAC7E,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,kBAAkB;oBACzB,OAAO,EAAE,kCAAkC,OAAO,IAAI,CAAC,GAAG,EAAE;oBAC5D,MAAM,EAAE,GAAG;iBACZ,CAAC;YACJ,CAAC;YAED,0BAA0B;YAC1B,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;gBAEzC,MAAM,OAAO,GAAG;oBACd,YAAY,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,YAAY;oBACxC,WAAW,EAAE,IAAI,CAAC,WAAW;iBAC9B,CAAC;gBAEF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAE5D,4BAA4B;gBAC5B,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC/B,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;oBAC7D,OAAO;wBACL,IAAI,EAAE,QAAQ;wBACd,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,OAAO,EAAE,QAAQ,CAAC,YAAY,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE;wBACnE,OAAO,EAAE,QAAQ,CAAC,OAAO;qBAC1B,CAAC;gBACJ,CAAC;gBAED,6DAA6D;gBAC7D,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;gBACjD,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,GAAG;iBACZ,CAAC;YACJ,CAAC;YAED,sCAAsC;YACtC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;gBACxD,MAAM,YAAY,GAAG,MAAA,IAAI,CAAC,YAAY,mCAAI,kBAAkB,CAAC;gBAE7D,IAAI,QAAQ,CAAC;gBACb,IAAI,CAAC;oBACH,QAAQ,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnF,CAAC;gBAAC,OAAO,CAAM,EAAE,CAAC;oBAChB,MAAM,YAAY,GAAG,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,OAAO,KAAI,MAAM,CAAC,CAAC,CAAC,CAAC;oBAC7C,OAAO,CAAC,GAAG,CAAC,2BAA2B,YAAY,EAAE,CAAC,CAAC;oBACvD,OAAO;wBACL,IAAI,EAAE,OAAO;wBACb,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,+FAA+F;wBACxG,MAAM,EAAE,GAAG;qBACZ,CAAC;gBACJ,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,MAAM,EAAE,CAAC;oBACrB,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;oBAChE,OAAO;wBACL,IAAI,EAAE,QAAQ;wBACd,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,OAAO,EAAE,QAAQ,CAAC,aAAa,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE;qBACrE,CAAC;gBACJ,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;gBAC1D,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,kCAAkC;oBACzC,OAAO,EAAE,sDAAsD;oBAC/D,MAAM,EAAE,GAAG;iBACZ,CAAC;YACJ,CAAC;YAED,qBAAqB;YACrB,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,oBAAoB;gBAC3B,OAAO,EAAE,iEAAiE,IAAI,IAAI,WAAW,EAAE;gBAC/F,MAAM,EAAE,GAAG;aACZ,CAAC;QAEJ,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,YAAY,GAAG,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,OAAO,KAAI,MAAM,CAAC,CAAC,CAAC,CAAC;YAE7C,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,uBAAuB;gBAC9B,OAAO,EAAE,YAAY;gBACrB,MAAM,EAAE,GAAG;aACZ,CAAC;QACJ,CAAC;IACH,CAAC;CAAA"}

package/dist/runner-server/middleware/signature-verification.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Webhook signature verification middleware for securing webhook endpoints.
+ * Uses HMAC-SHA256 signatures to verify requests are from trusted sources.
+ */
+export interface SignatureVerificationOptions {
+    /**
+     * The secret key used to verify webhook signatures.
+     * Should match the secret used by the sender to create the signature.
+     */
+    secret: string;
+    /**
+     * The name of the header containing the signature.
+     * Defaults to 'x-agentmark-signature-256'
+     */
+    headerName?: string;
+    /**
+     * Whether to skip verification (useful for local development).
+     * Defaults to false.
+     */
+    skipVerification?: boolean;
+}
+/**
+ * Verifies the webhook signature from the request.
+ *
+ * @param body - The request body as a string
+ * @param signature - The signature from the request header
+ * @param secret - The webhook secret
+ * @returns True if signature is valid, false otherwise
+ */
+export declare function verifyWebhookSignature(body: string, signature: string, secret: string): Promise<boolean>;
+/**
+ * Checks if signature verification should be enforced based on environment.
+ *
+ * @param options - Signature verification options
+ * @returns True if verification should be skipped
+ */
+export declare function shouldSkipVerification(options: SignatureVerificationOptions): boolean;

package/dist/runner-server/middleware/signature-verification.js

@@ -0,0 +1,55 @@
+"use strict";
+/**
+ * Webhook signature verification middleware for securing webhook endpoints.
+ * Uses HMAC-SHA256 signatures to verify requests are from trusted sources.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.verifyWebhookSignature = verifyWebhookSignature;
+exports.shouldSkipVerification = shouldSkipVerification;
+const shared_utils_1 = require("@agentmark-ai/shared-utils");
+/**
+ * Verifies the webhook signature from the request.
+ *
+ * @param body - The request body as a string
+ * @param signature - The signature from the request header
+ * @param secret - The webhook secret
+ * @returns True if signature is valid, false otherwise
+ */
+function verifyWebhookSignature(body, signature, secret) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            return yield (0, shared_utils_1.verifySignature)(secret, signature, body);
+        }
+        catch (error) {
+            console.error('Signature verification error:', error);
+            return false;
+        }
+    });
+}
+/**
+ * Checks if signature verification should be enforced based on environment.
+ *
+ * @param options - Signature verification options
+ * @returns True if verification should be skipped
+ */
+function shouldSkipVerification(options) {
+    // Skip if explicitly disabled
+    if (options.skipVerification) {
+        return true;
+    }
+    // Skip if no secret is provided (local dev without secret)
+    if (!options.secret || options.secret === 'DEFAULT') {
+        return true;
+    }
+    return false;
+}
+//# sourceMappingURL=signature-verification.js.map

package/dist/runner-server/middleware/signature-verification.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"signature-verification.js","sourceRoot":"","sources":["../../../cli-src/runner-server/middleware/signature-verification.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;AAgCH,wDAWC;AAQD,wDAYC;AA7DD,6DAA6D;AAsB7D;;;;;;;GAOG;AACH,SAAsB,sBAAsB,CAC1C,IAAY,EACZ,SAAiB,EACjB,MAAc;;QAEd,IAAI,CAAC;YACH,OAAO,MAAM,IAAA,8BAAe,EAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;YACtD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CAAA;AAED;;;;;GAKG;AACH,SAAgB,sBAAsB,CAAC,OAAqC;IAC1E,8BAA8B;IAC9B,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,2DAA2D;IAC3D,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/runner-server/types.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Platform-agnostic types for the AgentMark webhook server.
+ * These types are used by the core handler and all platform adapters.
+ */
+import type { WebhookPromptResponse, WebhookDatasetResponse } from '@agentmark-ai/prompt-core';
+/**
+ * Telemetry options for tracking prompt execution.
+ */
+export interface TelemetryOptions {
+    isEnabled: boolean;
+    metadata?: {
+        traceId?: string;
+        traceName?: string;
+        sessionId?: string;
+        sessionName?: string;
+        [key: string]: any;
+    };
+}
+/**
+ * Generic webhook handler interface that any adapter can implement.
+ * This is the contract that adapters (e.g., VercelAdapterWebhookHandler) must fulfill.
+ */
+export interface WebhookHandler {
+    runPrompt(promptAst: any, options?: {
+        shouldStream?: boolean;
+        customProps?: Record<string, any>;
+        telemetry?: TelemetryOptions;
+    }): Promise<WebhookPromptResponse>;
+    runExperiment(promptAst: any, datasetRunName: string, datasetPath?: string): Promise<WebhookDatasetResponse>;
+}
+/**
+ * Standardized request format for all platform adapters.
+ * Platform adapters translate their specific request formats into this structure.
+ */
+export interface WebhookRequest {
+    type: 'prompt-run' | 'dataset-run';
+    data: {
+        ast: any;
+        customProps?: Record<string, any>;
+        options?: {
+            shouldStream?: boolean;
+        };
+        experimentId?: string;
+        datasetPath?: string;
+        promptPath?: string;
+    };
+}
+/**
+ * Standardized response format from the core handler.
+ * Platform adapters translate this into their specific response formats.
+ */
+export type WebhookResponse = {
+    type: 'json';
+    data: any;
+    status?: number;
+} | {
+    type: 'stream';
+    stream: ReadableStream;
+    headers: Record<string, string>;
+    traceId?: string;
+} | {
+    type: 'error';
+    error: string;
+    details?: string;
+    status: number;
+};
+export type { WebhookPromptResponse, WebhookDatasetResponse } from '@agentmark-ai/prompt-core';

package/dist/runner-server/types.js

@@ -0,0 +1,7 @@
+"use strict";
+/**
+ * Platform-agnostic types for the AgentMark webhook server.
+ * These types are used by the core handler and all platform adapters.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/runner-server/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../cli-src/runner-server/types.ts"],"names":[],"mappings":";AAAA;;;GAGG"}

package/dist/runner-server.d.ts

@@ -0,0 +1,32 @@
+/**
+ * AgentMark Webhook Server
+ *
+ * This file provides the webhook server for local development,
+ * used to execute prompts and experiments via Express.
+ */
+import { Server } from 'node:http';
+import type { WebhookHandler } from './runner-server/types';
+import { type SignatureVerificationOptions } from './runner-server/middleware/signature-verification';
+export type { WebhookHandler } from './runner-server/types';
+export interface WebhookServerOptions {
+    port?: number;
+    handler: WebhookHandler;
+    apiServerUrl?: string;
+    templatesDirectory?: string;
+    /**
+     * Webhook signature verification options.
+     * If not provided, will check AGENTMARK_WEBHOOK_SECRET env var.
+     */
+    signatureVerification?: SignatureVerificationOptions;
+}
+/**
+ * Creates an HTTP server that wraps a webhook handler instance.
+ * This server provides endpoints for executing prompts and experiments via HTTP.
+ * Used by the CLI and local development workflows.
+ *
+ * @param options - Server configuration options
+ * @returns HTTP server instance
+ */
+export declare function createWebhookServer(options: WebhookServerOptions): Promise<Server>;
+export { handleWebhookRequest } from './runner-server/core';
+export type { WebhookRequest, WebhookResponse } from './runner-server/types';