@chanl/eval-cli 0.4.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 chanl.ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/agent-loader.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Parsed agent definition from a YAML file.
+ */
+export interface AgentDefinition {
+    name: string;
+    model: string;
+    provider: string;
+    systemPrompt: string;
+    temperature?: number;
+    maxTokens?: number;
+    /** For http provider: the endpoint URL */
+    httpEndpoint?: string;
+}
+/**
+ * Parse a model string into provider + model name.
+ *
+ * Formats:
+ *   "gpt-4o"                 -> { provider: "openai", model: "gpt-4o" }
+ *   "openai:gpt-4o"          -> { provider: "openai", model: "gpt-4o" }
+ *   "anthropic:claude-sonnet" -> { provider: "anthropic", model: "claude-sonnet" }
+ *   "http:localhost:8080"     -> { provider: "http", endpoint: "localhost:8080" }
+ *   "http:http://my-agent:8080" -> { provider: "http", endpoint: "http://my-agent:8080" }
+ */
+export declare function parseModelString(modelStr: string, fallbackProvider?: string): {
+    provider: string;
+    model: string;
+    httpEndpoint?: string;
+};
+/**
+ * Load and parse an agent YAML file.
+ *
+ * @param filePath - Path to the agent YAML file
+ * @param fallbackProvider - Provider to use when model string has no prefix and
+ *                           auto-detection fails (typically from CLI config)
+ * @returns Parsed agent definition
+ * @throws Error if file is missing, YAML is invalid, or required fields are absent
+ */
+export declare function loadAgentYaml(filePath: string, fallbackProvider?: string): AgentDefinition;
+//# sourceMappingURL=agent-loader.d.ts.map

package/dist/agent-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-loader.d.ts","sourceRoot":"","sources":["../src/agent-loader.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA0BD;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,MAAM,EAChB,gBAAgB,CAAC,EAAE,MAAM,GACxB;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,CA2B5D;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,MAAM,EAChB,gBAAgB,CAAC,EAAE,MAAM,GACxB,eAAe,CA0GjB"}

package/dist/agent-loader.js

@@ -0,0 +1,166 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseModelString = parseModelString;
+exports.loadAgentYaml = loadAgentYaml;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const yaml = __importStar(require("js-yaml"));
+/**
+ * Well-known model prefixes and their providers.
+ * Used to auto-detect provider when no explicit prefix is given.
+ */
+const MODEL_PROVIDER_MAP = {
+    'gpt-': 'openai',
+    'o1': 'openai',
+    'o3': 'openai',
+    'o4': 'openai',
+    'claude-': 'anthropic',
+};
+/**
+ * Parse a model string into provider + model name.
+ *
+ * Formats:
+ *   "gpt-4o"                 -> { provider: "openai", model: "gpt-4o" }
+ *   "openai:gpt-4o"          -> { provider: "openai", model: "gpt-4o" }
+ *   "anthropic:claude-sonnet" -> { provider: "anthropic", model: "claude-sonnet" }
+ *   "http:localhost:8080"     -> { provider: "http", endpoint: "localhost:8080" }
+ *   "http:http://my-agent:8080" -> { provider: "http", endpoint: "http://my-agent:8080" }
+ */
+function parseModelString(modelStr, fallbackProvider) {
+    // Check for explicit provider prefix
+    const colonIdx = modelStr.indexOf(':');
+    if (colonIdx > 0) {
+        const prefix = modelStr.slice(0, colonIdx).toLowerCase();
+        if (prefix === 'openai') {
+            return { provider: 'openai', model: modelStr.slice(colonIdx + 1) };
+        }
+        if (prefix === 'anthropic') {
+            return { provider: 'anthropic', model: modelStr.slice(colonIdx + 1) };
+        }
+        if (prefix === 'http') {
+            const endpoint = modelStr.slice(colonIdx + 1);
+            return { provider: 'http', model: '', httpEndpoint: endpoint };
+        }
+    }
+    // Auto-detect from model name
+    for (const [prefix, provider] of Object.entries(MODEL_PROVIDER_MAP)) {
+        if (modelStr.toLowerCase().startsWith(prefix)) {
+            return { provider, model: modelStr };
+        }
+    }
+    // Fall back to config's provider or default to openai
+    return { provider: fallbackProvider || 'openai', model: modelStr };
+}
+/**
+ * Load and parse an agent YAML file.
+ *
+ * @param filePath - Path to the agent YAML file
+ * @param fallbackProvider - Provider to use when model string has no prefix and
+ *                           auto-detection fails (typically from CLI config)
+ * @returns Parsed agent definition
+ * @throws Error if file is missing, YAML is invalid, or required fields are absent
+ */
+function loadAgentYaml(filePath, fallbackProvider) {
+    const resolvedPath = path.resolve(filePath);
+    if (!fs.existsSync(resolvedPath)) {
+        throw new Error(`Agent YAML file not found: ${resolvedPath}`);
+    }
+    const raw = fs.readFileSync(resolvedPath, 'utf-8');
+    let parsed;
+    try {
+        parsed = yaml.load(raw);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Invalid YAML in agent file ${resolvedPath}: ${message}`);
+    }
+    if (!parsed || typeof parsed !== 'object') {
+        throw new Error(`Agent YAML file is empty or not an object: ${resolvedPath}`);
+    }
+    // Validate required fields
+    if (!parsed.name || typeof parsed.name !== 'string') {
+        throw new Error(`Agent YAML missing required field "name" in ${resolvedPath}`);
+    }
+    if (!parsed.model || typeof parsed.model !== 'string') {
+        throw new Error(`Agent YAML missing required field "model" in ${resolvedPath}`);
+    }
+    // Resolve system prompt: inline or from file
+    let systemPrompt;
+    if (parsed.system_prompt && parsed.system_prompt_file) {
+        throw new Error(`Agent YAML has both "system_prompt" and "system_prompt_file" — use only one in ${resolvedPath}`);
+    }
+    if (parsed.system_prompt_file) {
+        const promptPath = path.resolve(path.dirname(resolvedPath), parsed.system_prompt_file);
+        if (!fs.existsSync(promptPath)) {
+            throw new Error(`system_prompt_file not found: ${promptPath} (referenced from ${resolvedPath})`);
+        }
+        systemPrompt = fs.readFileSync(promptPath, 'utf-8').trim();
+    }
+    else if (parsed.system_prompt) {
+        systemPrompt = parsed.system_prompt.trim();
+    }
+    else {
+        throw new Error(`Agent YAML missing "system_prompt" or "system_prompt_file" in ${resolvedPath}`);
+    }
+    if (!systemPrompt) {
+        throw new Error(`Agent YAML has empty system prompt in ${resolvedPath}`);
+    }
+    // Parse model string into provider + model
+    const { provider, model, httpEndpoint } = parseModelString(parsed.model, fallbackProvider);
+    const result = {
+        name: parsed.name,
+        model,
+        provider,
+        systemPrompt,
+    };
+    if (parsed.temperature !== undefined) {
+        if (typeof parsed.temperature !== 'number' || parsed.temperature < 0 || parsed.temperature > 2) {
+            throw new Error(`Agent YAML "temperature" must be a number between 0 and 2 in ${resolvedPath}`);
+        }
+        result.temperature = parsed.temperature;
+    }
+    if (parsed.max_tokens !== undefined) {
+        if (typeof parsed.max_tokens !== 'number' || parsed.max_tokens < 1) {
+            throw new Error(`Agent YAML "max_tokens" must be a positive number in ${resolvedPath}`);
+        }
+        result.maxTokens = parsed.max_tokens;
+    }
+    if (httpEndpoint) {
+        result.httpEndpoint = httpEndpoint;
+    }
+    return result;
+}
+//# sourceMappingURL=agent-loader.js.map

package/dist/agent-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-loader.js","sourceRoot":"","sources":["../src/agent-loader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,4CA8BC;AAWD,sCA6GC;AA1MD,uCAAyB;AACzB,2CAA6B;AAC7B,8CAAgC;AA4BhC;;;GAGG;AACH,MAAM,kBAAkB,GAA2B;IACjD,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF;;;;;;;;;GASG;AACH,SAAgB,gBAAgB,CAC9B,QAAgB,EAChB,gBAAyB;IAEzB,qCAAqC;IACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QAEzD,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;QACrE,CAAC;QACD,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3B,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;QACxE,CAAC;QACD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;YAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;QACjE,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACpE,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QACvC,CAAC;IACH,CAAC;IAED,sDAAsD;IACtD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,IAAI,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,aAAa,CAC3B,QAAgB,EAChB,gBAAyB;IAEzB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE5C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACnD,IAAI,MAAoB,CAAC;IAEzB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAiB,CAAC;IAC1C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,KAAK,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,8CAA8C,YAAY,EAAE,CAC7D,CAAC;IACJ,CAAC;IAED,2BAA2B;IAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CACb,+CAA+C,YAAY,EAAE,CAC9D,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,gDAAgD,YAAY,EAAE,CAC/D,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,IAAI,YAAoB,CAAC;IAEzB,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,kFAAkF,YAAY,EAAE,CACjG,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAC7B,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAC1B,MAAM,CAAC,kBAAkB,CAC1B,CAAC;QACF,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,qBAAqB,YAAY,GAAG,CAChF,CAAC;QACJ,CAAC;QACD,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7D,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QAChC,YAAY,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IAC7C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CACb,iEAAiE,YAAY,EAAE,CAChF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACb,yCAAyC,YAAY,EAAE,CACxD,CAAC;IACJ,CAAC;IAED,2CAA2C;IAC3C,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,gBAAgB,CACxD,MAAM,CAAC,KAAK,EACZ,gBAAgB,CACjB,CAAC;IAEF,MAAM,MAAM,GAAoB;QAC9B,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK;QACL,QAAQ;QACR,YAAY;KACb,CAAC;IAEF,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACrC,IAAI,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,GAAG,CAAC,IAAI,MAAM,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YAC/F,MAAM,IAAI,KAAK,CACb,gEAAgE,YAAY,EAAE,CAC/E,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACpC,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACb,wDAAwD,YAAY,EAAE,CACvE,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC;IACvC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/analytics.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Initialize analytics from config.
+ * No-op if analytics are disabled.
+ */
+export declare function initAnalytics(): void;
+/**
+ * Track an analytics event. No-op if disabled.
+ * Never throws — errors are silently swallowed.
+ */
+export declare function track(event: string, properties?: Record<string, any>): void;
+/**
+ * Get the current analytics distinct ID.
+ */
+export declare function getAnalyticsId(): string;
+//# sourceMappingURL=analytics.d.ts.map

package/dist/analytics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analytics.d.ts","sourceRoot":"","sources":["../src/analytics.ts"],"names":[],"mappings":"AAiBA;;;GAGG;AACH,wBAAgB,aAAa,IAAI,IAAI,CAgBpC;AAED;;;GAGG;AACH,wBAAgB,KAAK,CACnB,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC/B,IAAI,CAqDN;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAKvC"}

package/dist/analytics.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initAnalytics = initAnalytics;
+exports.track = track;
+exports.getAnalyticsId = getAnalyticsId;
+const config_1 = require("./config");
+const crypto_1 = require("crypto");
+const POSTHOG_KEY = 'phc_chanl_eval_placeholder';
+const POSTHOG_HOST = 'https://app.posthog.com';
+let initialized = false;
+let analyticsEnabled = false;
+let distinctId = '';
+/**
+ * Initialize analytics from config.
+ * No-op if analytics are disabled.
+ */
+function initAnalytics() {
+    try {
+        const config = (0, config_1.loadConfig)();
+        analyticsEnabled = config.analytics !== false;
+        distinctId = config.analyticsId || '';
+        if (!distinctId && analyticsEnabled) {
+            // Generate a new anonymous ID — it'll be persisted on next config save
+            distinctId = (0, crypto_1.randomUUID)();
+        }
+        initialized = true;
+    }
+    catch {
+        initialized = true;
+        analyticsEnabled = false;
+    }
+}
+/**
+ * Track an analytics event. No-op if disabled.
+ * Never throws — errors are silently swallowed.
+ */
+function track(event, properties) {
+    if (!initialized) {
+        initAnalytics();
+    }
+    if (!analyticsEnabled || !distinctId)
+        return;
+    try {
+        const payload = {
+            event,
+            distinctId,
+            properties: {
+                ...properties,
+                $lib: 'chanl-cli',
+                $lib_version: '0.1.0',
+            },
+            timestamp: new Date().toISOString(),
+        };
+        // Fire-and-forget POST to PostHog
+        // Using dynamic import to avoid adding posthog-node as hard dependency
+        const https = require('https');
+        const data = JSON.stringify({
+            api_key: POSTHOG_KEY,
+            batch: [payload],
+        });
+        const url = new URL(`${POSTHOG_HOST}/batch/`);
+        const req = https.request({
+            hostname: url.hostname,
+            port: 443,
+            path: url.pathname,
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Content-Length': Buffer.byteLength(data),
+            },
+            timeout: 3000,
+        }, () => {
+            // Response ignored — fire and forget
+        });
+        req.on('error', () => {
+            // Silently ignore network errors
+        });
+        req.write(data);
+        req.end();
+    }
+    catch {
+        // Never throw from analytics
+    }
+}
+/**
+ * Get the current analytics distinct ID.
+ */
+function getAnalyticsId() {
+    if (!initialized) {
+        initAnalytics();
+    }
+    return distinctId;
+}
+//# sourceMappingURL=analytics.js.map

package/dist/analytics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analytics.js","sourceRoot":"","sources":["../src/analytics.ts"],"names":[],"mappings":";;AAqBA,sCAgBC;AAMD,sBAwDC;AAKD,wCAKC;AA7GD,qCAAsC;AACtC,mCAAoC;AASpC,MAAM,WAAW,GAAG,4BAA4B,CAAC;AACjD,MAAM,YAAY,GAAG,yBAAyB,CAAC;AAE/C,IAAI,WAAW,GAAG,KAAK,CAAC;AACxB,IAAI,gBAAgB,GAAG,KAAK,CAAC;AAC7B,IAAI,UAAU,GAAG,EAAE,CAAC;AAEpB;;;GAGG;AACH,SAAgB,aAAa;IAC3B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;QAC5B,gBAAgB,GAAG,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC;QAC9C,UAAU,GAAG,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC;QAEtC,IAAI,CAAC,UAAU,IAAI,gBAAgB,EAAE,CAAC;YACpC,uEAAuE;YACvE,UAAU,GAAG,IAAA,mBAAU,GAAE,CAAC;QAC5B,CAAC;QAED,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,WAAW,GAAG,IAAI,CAAC;QACnB,gBAAgB,GAAG,KAAK,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAgB,KAAK,CACnB,KAAa,EACb,UAAgC;IAEhC,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU;QAAE,OAAO;IAE7C,IAAI,CAAC;QACH,MAAM,OAAO,GAAmB;YAC9B,KAAK;YACL,UAAU;YACV,UAAU,EAAE;gBACV,GAAG,UAAU;gBACb,IAAI,EAAE,WAAW;gBACjB,YAAY,EAAE,OAAO;aACtB;YACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;QAEF,kCAAkC;QAClC,uEAAuE;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;YAC1B,OAAO,EAAE,WAAW;YACpB,KAAK,EAAE,CAAC,OAAO,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,YAAY,SAAS,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CACvB;YACE,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG,CAAC,QAAQ;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;aAC1C;YACD,OAAO,EAAE,IAAI;SACd,EACD,GAAG,EAAE;YACH,qCAAqC;QACvC,CAAC,CACF,CAAC;QAEF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,iCAAiC;QACnC,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChB,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,6BAA6B;IAC/B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc;IAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;IAClB,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/assertions.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Assertion engine for `chanl test` — evaluates pass/fail assertions
+ * against scenario execution results.
+ */
+export interface KeywordAssertion {
+    type: 'keyword';
+    must_include?: string[];
+    must_not_include?: string[];
+    description?: string;
+}
+export interface ResponseTimeAssertion {
+    type: 'response_time';
+    max_seconds: number;
+    description?: string;
+}
+export interface PromptAssertion {
+    type: 'prompt';
+    rubric: string;
+    min_score: number;
+    description?: string;
+}
+export interface ToolCallAssertion {
+    type: 'tool_call';
+    expected: string[];
+    description?: string;
+}
+export interface ScoreAssertion {
+    type: 'score';
+    min_score: number;
+    description?: string;
+}
+export type Assertion = KeywordAssertion | ResponseTimeAssertion | PromptAssertion | ToolCallAssertion | ScoreAssertion;
+export interface AssertionResult {
+    type: string;
+    description: string;
+    passed: boolean;
+    actual: string;
+    expected: string;
+    reason: string;
+}
+/** Shape of a parsed test YAML file. */
+export interface TestDefinition {
+    scenario: string;
+    /** Prompt entity ID — defines the agent under test */
+    promptId?: string;
+    /** @deprecated Use promptId instead. Kept for backwards compat parsing. */
+    agent?: string;
+    assertions: Assertion[];
+}
+/**
+ * Extract the full transcript text from an execution's stepResults.
+ * Concatenates all actualResponse fields.
+ */
+export declare function extractTranscriptText(execution: any): string;
+/**
+ * Extract average agent response time in seconds from execution data.
+ * Checks execution.metrics first, then computes from stepResults durations.
+ */
+export declare function extractAvgResponseTimeSecs(execution: any): number | null;
+/**
+ * Extract tool names mentioned in an execution.
+ * Checks stepResults metadata and falls back to transcript keyword search.
+ */
+export declare function extractToolCalls(execution: any): string[];
+/**
+ * Extract the overall score from an execution, if present.
+ */
+export declare function extractOverallScore(execution: any): number | null;
+/**
+ * Evaluate all assertions against an execution result.
+ */
+export declare function evaluateAssertions(assertions: Assertion[], execution: any): AssertionResult[];
+//# sourceMappingURL=assertions.d.ts.map

package/dist/assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,SAAS,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,eAAe,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,SAAS,GACjB,gBAAgB,GAChB,qBAAqB,GACrB,eAAe,GACf,iBAAiB,GACjB,cAAc,CAAC;AAEnB,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wCAAwC;AACxC,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,SAAS,EAAE,CAAC;CACzB;AAMD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,GAAG,GAAG,MAAM,CAQ5D;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CA6BxE;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,GAAG,GAAG,MAAM,EAAE,CA4BzD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CAQjE;AAwMD;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,SAAS,EAAE,EACvB,SAAS,EAAE,GAAG,GACb,eAAe,EAAE,CA8BnB"}