callkaro-cli-unofficial 0.9.5 → 0.10.0

package/README.md

@@ -200,8 +200,21 @@ npm run catalog:check    # fail if catalog.json disagrees with the snapshot
 
 `catalog:check` runs offline against a committed snapshot, so it works in CI with no credentials. On 1 October 2026 it caught shipped tables that listed withdrawn models *and* missed live ones in both directions. A wrong voice model otherwise persists silently and only surfaces as a broken call.
 
+### Live test calls, with you or an LLM as the caller
+
+```bash
+cku test start <agentId> --yes          # a coding agent (or you) plays the caller, one turn per command
+cku test say <sessionId> "Kitne ka hai?"
+cku test end <sessionId>                # transcript with function calls and node switches
+
+cku test run <agentId> --test <testCaseId> --llm anthropic --yes   # the CLI plays the caller and grades the call
+```
+
+This is the agent editor's **Test Chat** tab from a terminal: a real test call on the real call runtime, over the same LiveKit room the dashboard joins, with the caller typing instead of speaking. Unlike a simulation, nothing is mocked, so node switching, function calls and post-call analysis are what production does, and every function runs for real. `run` uses any OpenAI-compatible LLM (OpenAI, Anthropic, Gemini, DeepSeek, Groq, OpenRouter, Ollama, or `--llm-url`), with the key read from the environment, and exits 1 when the judge fails the call. It needs `@livekit/rtc-node`, an optional dependency with prebuilt binaries for Linux, macOS and Windows.
+
 ### Smaller things
 
+- `cku sim create --real-functions` — every function on the version runs for real in the simulation, like switching every Mock off in the dashboard
 - `cku agents chat` — text chat with an agent's real prompt and tools, no phone call. Scriptable with `--say`, replayable with `--history`. Custom in-call functions run for real
 - `cku scheduled list|queued|get` — scheduled calls and the queue of what will actually dial (dashboard login)
 - `cku sim update` — edit a test case in place

package/dist/commands/liveTest.js

@@ -0,0 +1,330 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.testStartCommand = testStartCommand;
+exports.testSayCommand = testSayCommand;
+exports.testEndCommand = testEndCommand;
+exports.testServeCommand = testServeCommand;
+exports.testRunCommand = testRunCommand;
+const child_process_1 = require("child_process");
+const crypto_1 = require("crypto");
+const http_1 = require("http");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const chalk_1 = __importDefault(require("chalk"));
+const session_1 = require("../lib/session");
+const config_1 = require("../lib/config");
+const ui_1 = require("../lib/ui");
+const versions_1 = require("../lib/versions");
+const liveTest_1 = require("../lib/liveTest");
+const jsonInput_1 = require("./jsonInput");
+/**
+ * `cku test`: live test calls over the dashboard's Test Chat path (lib/liveTest.ts).
+ *
+ * Two ways to play the caller:
+ *  - start / say / end: a coding agent (or a person) plays the caller one turn per command.
+ *    A detached background process holds the LiveKit room between commands and serves them on
+ *    127.0.0.1, guarded by a per-session secret.
+ *  - run: the CLI plays the caller with an LLM and grades the call against criteria.
+ */
+const SESSIONS_DIR = (0, path_1.join)(config_1.CONFIG_DIR, 'test-sessions');
+const IDLE_MS = 10 * 60_000;
+const WARNING = 'This places a real test call: every agent function runs for real (CRM writes, WhatsApp, webhooks), '
+    + 'credits are used, and the call appears in call history with post-call analysis.';
+const sessionPath = (id) => (0, path_1.join)(SESSIONS_DIR, `${id}.json`);
+function printEvents(events) {
+    for (const e of events) {
+        if (e.type === 'agent')
+            (0, ui_1.info)(`${chalk_1.default.cyan('agent>')} ${e.text}`);
+        else if (e.type === 'user')
+            (0, ui_1.info)(`${chalk_1.default.green('caller>')} ${e.text}`);
+        else if (e.type === 'function')
+            (0, ui_1.info)(chalk_1.default.dim(`  → ${e.name} ${JSON.stringify(e.arguments ?? {})}`));
+        else if (e.type === 'capability')
+            (0, ui_1.info)(chalk_1.default.dim(`  [node: ${e.capability}]`));
+        else if (e.type === 'ended')
+            (0, ui_1.info)(chalk_1.default.dim('  (call ended)'));
+    }
+}
+async function readMetadata(opts) {
+    if (!opts.variables)
+        return {};
+    const v = await (0, jsonInput_1.readJsonFlag)(opts.variables, '--variables');
+    if (!v || typeof v !== 'object' || Array.isArray(v))
+        throw new Error('--variables must be a JSON object, e.g. {"customer_name":"Ravi"}');
+    return v;
+}
+/** The test call is created under the agent's owner, which is also what the dashboard sends. */
+async function startToken(agentId, version, metadata) {
+    const res = await (0, session_1.requireApi)().getAgent(agentId, version);
+    const agent = res.agent || res;
+    const userId = agent?.userId;
+    if (!userId)
+        throw new Error(`Agent ${agentId} came back without a userId, so the test-call host cannot be chosen.`);
+    return (0, liveTest_1.requestTestToken)({ userId: String(userId), agentId, version, metadata });
+}
+// --- start / say / end ----------------------------------------------------------------------
+async function testStartCommand(agentId, opts) {
+    try {
+        if (!(await (0, ui_1.confirm)(`${WARNING} Start it?`, opts.yes)))
+            return;
+        const version = (0, versions_1.firstVersion)(opts.versions);
+        const { token, callSid } = await startToken(agentId, version, await readMetadata(opts));
+        (0, fs_1.mkdirSync)(SESSIONS_DIR, { recursive: true, mode: 0o700 });
+        const id = (0, crypto_1.randomBytes)(4).toString('hex');
+        (0, fs_1.writeFileSync)((0, path_1.join)(SESSIONS_DIR, `${id}.req.json`), JSON.stringify({ token, callSid, agentId }), { mode: 0o600 });
+        (0, child_process_1.spawn)(process.execPath, [process.argv[1], 'test', '_serve', id], {
+            detached: true, stdio: 'ignore', windowsHide: true, env: { ...process.env, CK_NO_UPDATE_CHECK: '1' },
+        }).unref();
+        const deadline = Date.now() + 60_000;
+        while (!(0, fs_1.existsSync)(sessionPath(id))) {
+            if (Date.now() > deadline)
+                throw new Error('The test session did not come up within 60s. If the call started, find it with: cku calls list --sid ${callSid}');
+            await new Promise((r) => setTimeout(r, 250));
+        }
+        const s = JSON.parse((0, fs_1.readFileSync)(sessionPath(id), 'utf8'));
+        if (s.error) {
+            (0, fs_1.unlinkSync)(sessionPath(id));
+            throw new Error(s.error);
+        }
+        if (opts.json)
+            return void console.log(JSON.stringify({ sessionId: id, callSid, events: s.greeting }, null, 2));
+        (0, ui_1.info)(chalk_1.default.dim(`session ${id}  call ${callSid}`));
+        if (s.greeting.length)
+            printEvents(s.greeting);
+        else
+            (0, ui_1.info)(chalk_1.default.dim('(the agent is waiting for the caller to speak first)'));
+        (0, ui_1.info)(chalk_1.default.dim(`Next: cku test say ${id} "<what the caller says>"   ·   cku test end ${id}`));
+    }
+    catch (err) {
+        (0, ui_1.handleError)(err);
+    }
+}
+async function callSession(id, path, body) {
+    if (!/^[0-9a-f]{8}$/.test(id) || !(0, fs_1.existsSync)(sessionPath(id))) {
+        throw new Error(`No test session "${id}". Start one with: cku test start <agentId>`);
+    }
+    const s = JSON.parse((0, fs_1.readFileSync)(sessionPath(id), 'utf8'));
+    try {
+        const res = await fetch(`http://127.0.0.1:${s.port}${path}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json', 'x-session-secret': s.secret },
+            body: JSON.stringify(body),
+        });
+        const data = await res.json();
+        if (!res.ok)
+            throw new Error(data.error || `Session answered HTTP ${res.status}`);
+        return data;
+    }
+    catch (err) {
+        if (err?.cause?.code === 'ECONNREFUSED') {
+            try {
+                (0, fs_1.unlinkSync)(sessionPath(id));
+            }
+            catch { /* gone */ }
+            throw new Error(`Test session ${id} is no longer running (it ends after 10 idle minutes). Find the call with: cku calls list --sid ${s.callSid}`);
+        }
+        throw err;
+    }
+}
+async function testSayCommand(id, words, opts) {
+    try {
+        const text = words.join(' ').trim();
+        if (!text)
+            throw new Error('Nothing to say. Usage: cku test say <sessionId> "<what the caller says>"');
+        const timeoutMs = (0, ui_1.intFlag)(opts.timeout, '--timeout', 60) * 1000;
+        const data = await callSession(id, '/say', { text, timeoutMs });
+        if (opts.json)
+            return void console.log(JSON.stringify(data, null, 2));
+        printEvents(data.events.filter((e) => e.type !== 'user'));
+        if (!data.events.some((e) => e.type === 'agent') && !data.ended)
+            (0, ui_1.info)(chalk_1.default.dim('(no agent reply before the timeout)'));
+        if (data.ended)
+            (0, ui_1.info)(chalk_1.default.dim(`The call is over. Collect it with: cku test end ${id}`));
+    }
+    catch (err) {
+        (0, ui_1.handleError)(err);
+    }
+}
+async function testEndCommand(id, opts) {
+    try {
+        const data = await callSession(id, '/end', {});
+        if (opts.json)
+            return void console.log(JSON.stringify(data, null, 2));
+        (0, ui_1.info)((0, liveTest_1.formatTranscript)(data.events));
+        (0, ui_1.info)(chalk_1.default.dim(`call ${data.callSid}: post-call results: cku calls list --sid ${data.callSid}`));
+    }
+    catch (err) {
+        (0, ui_1.handleError)(err);
+    }
+}
+/** The background half of `start`: owns the room, answers say/end on localhost, exits when done or idle. */
+async function testServeCommand(id) {
+    const reqPath = (0, path_1.join)(SESSIONS_DIR, `${id}.req.json`);
+    const { token, callSid, agentId } = JSON.parse((0, fs_1.readFileSync)(reqPath, 'utf8'));
+    (0, fs_1.unlinkSync)(reqPath);
+    const write = (s) => (0, fs_1.writeFileSync)(sessionPath(id), JSON.stringify(s), { mode: 0o600 });
+    let session;
+    let greeting;
+    try {
+        ({ session, greeting } = await liveTest_1.LiveSession.open(token, callSid));
+    }
+    catch (err) {
+        write({ error: `Could not join the test call room: ${err.message}` });
+        return;
+    }
+    const secret = (0, crypto_1.randomBytes)(16).toString('hex');
+    let idle;
+    let busy = false;
+    const shutdown = async () => {
+        server.close();
+        try {
+            (0, fs_1.unlinkSync)(sessionPath(id));
+        }
+        catch { /* gone */ }
+        await session.close();
+        process.exit(0);
+    };
+    const touch = () => { clearTimeout(idle); idle = setTimeout(shutdown, IDLE_MS); };
+    const server = (0, http_1.createServer)((req, res) => {
+        const reply = (status, body) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(body)); };
+        if (req.headers['x-session-secret'] !== secret || req.method !== 'POST')
+            return reply(403, { error: 'forbidden' });
+        let raw = '';
+        req.on('data', (c) => { raw += c; if (raw.length > 100_000)
+            req.destroy(); });
+        req.on('end', async () => {
+            touch();
+            try {
+                const body = raw ? JSON.parse(raw) : {};
+                if (req.url === '/say') {
+                    if (busy)
+                        return reply(409, { error: 'The agent is still answering the previous turn. Wait for that command to finish.' });
+                    busy = true;
+                    try {
+                        const events = await session.say(String(body.text || ''), Number(body.timeoutMs) || 60_000);
+                        return reply(200, { sessionId: id, callSid, events, ended: session.ended });
+                    }
+                    finally {
+                        busy = false;
+                    }
+                }
+                if (req.url === '/end') {
+                    reply(200, { sessionId: id, callSid, agentId, events: session.events, ended: session.ended });
+                    return void setTimeout(shutdown, 100);
+                }
+                reply(404, { error: 'unknown route' });
+            }
+            catch (err) {
+                reply(500, { error: err.message });
+            }
+        });
+    });
+    server.listen(0, '127.0.0.1', () => {
+        const port = server.address().port;
+        write({ pid: process.pid, port, secret, callSid, agentId, greeting });
+        touch();
+    });
+}
+async function scenario(agentId, opts) {
+    let s = { name: '', persona: opts.persona, prompt: opts.prompt, criteria: opts.criteria, variables: {} };
+    if (opts.test) {
+        const res = await (0, session_1.requireApi)().listTestCases(agentId);
+        const t = (res.testCases || res.data || []).find((x) => x._id === opts.test);
+        if (!t)
+            throw new Error(`Test case ${opts.test} is not one of this agent's. List them with: cku sim tests ${agentId}`);
+        s = { name: t.testcaseName || '', persona: t.userPersona, prompt: t.userPrompt, criteria: t.successCriteria, variables: t.testVariables || {} };
+        if (opts.persona)
+            s.persona = opts.persona;
+        if (opts.prompt)
+            s.prompt = opts.prompt;
+        if (opts.criteria)
+            s.criteria = opts.criteria;
+    }
+    s.variables = { ...s.variables, ...(await readMetadata(opts)) };
+    if (!s.prompt)
+        throw new Error('Say what the caller wants: --prompt "<goal and beats>" (or --test <testCaseId> to reuse a saved simulation test case).');
+    return { ...s, prompt: s.prompt };
+}
+async function testRunCommand(agentId, opts) {
+    let session;
+    try {
+        const s = await scenario(agentId, opts);
+        const llm = (0, liveTest_1.resolveLlm)(opts);
+        const judge = { ...llm, model: opts.judgeModel || llm.model };
+        const maxTurns = (0, ui_1.intFlag)(opts.maxTurns, '--max-turns', 20);
+        const turnMs = (0, ui_1.intFlag)(opts.timeout, '--timeout', 60) * 1000;
+        if (!(await (0, ui_1.confirm)(`${WARNING} Run it?`, opts.yes)))
+            return;
+        const log = (events) => { if (!opts.json)
+            printEvents(events); };
+        const { token, callSid } = await startToken(agentId, (0, versions_1.firstVersion)(opts.versions), s.variables);
+        if (!opts.json)
+            (0, ui_1.info)(chalk_1.default.dim(`call ${callSid}  caller: ${llm.name}/${llm.model}`));
+        const opened = await liveTest_1.LiveSession.open(token, callSid);
+        session = opened.session;
+        log(opened.greeting);
+        const history = [{ role: 'system', content: (0, liveTest_1.callerSystemPrompt)(s) }];
+        const agentSaid = (events) => events.filter((e) => e.type === 'agent').map((e) => e.text).join(' ');
+        let heard = agentSaid(opened.greeting);
+        let endedBy = 'max-turns';
+        for (let turn = 0; turn < maxTurns; turn++) {
+            if (session.ended) {
+                endedBy = 'agent';
+                break;
+            }
+            history.push({ role: 'user', content: heard || '(The call is connected. The agent has not said anything yet.)' });
+            const line = (0, liveTest_1.parseCallerLine)(await (0, liveTest_1.chatComplete)(llm, history));
+            history.push({ role: 'assistant', content: line.text || '(hangs up)' });
+            if (line.text) {
+                if (session.ended) {
+                    endedBy = 'agent';
+                    break;
+                }
+                const events = await session.say(line.text, turnMs);
+                log([{ type: 'user', text: line.text, t: Date.now() }, ...events]);
+                heard = agentSaid(events);
+                if (session.ended) {
+                    endedBy = 'agent';
+                    break;
+                }
+                if (!events.length) {
+                    endedBy = 'timeout';
+                    break;
+                }
+            }
+            if (line.end) {
+                endedBy = 'caller';
+                break;
+            }
+        }
+        const events = session.events;
+        await session.close();
+        session = undefined;
+        let verdict;
+        if (s.criteria)
+            verdict = (0, liveTest_1.parseVerdict)(await (0, liveTest_1.chatComplete)(judge, (0, liveTest_1.judgeMessages)(s.criteria, events), 0));
+        const out = { callSid, agentId, test: opts.test, endedBy, caller: `${llm.name}/${llm.model}`, events, verdict };
+        if (opts.json)
+            console.log(JSON.stringify(out, null, 2));
+        else {
+            (0, ui_1.info)(chalk_1.default.dim(`ended by: ${endedBy}`));
+            if (verdict) {
+                const label = verdict.invalid ? chalk_1.default.yellow('GRADING FAILED') : verdict.pass ? chalk_1.default.green('PASS') : chalk_1.default.red('FAIL');
+                (0, ui_1.info)(`${label}  ${verdict.summary}`);
+                for (const c of verdict.criteria)
+                    (0, ui_1.info)(`  ${c.pass ? chalk_1.default.green('✔') : chalk_1.default.red('✖')} ${c.criterion}${c.evidence ? chalk_1.default.dim(` (${c.evidence})`) : ''}${c.reason && !c.pass ? ` — ${c.reason}` : ''}`);
+            }
+            (0, ui_1.info)(chalk_1.default.dim(`post-call results: cku calls list --sid ${callSid}`));
+        }
+        if (verdict && !verdict.pass)
+            process.exitCode = 1;
+    }
+    catch (err) {
+        if (session)
+            await session.close();
+        (0, ui_1.handleError)(err);
+    }
+}

package/dist/commands/simulations.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.sleep = void 0;
 exports.toTestVariables = toTestVariables;
 exports.toTestFunctions = toTestFunctions;
+exports.realFunctionList = realFunctionList;
 exports.simTestsListCommand = simTestsListCommand;
 exports.simCreateCommand = simCreateCommand;
 exports.simUpdateCommand = simUpdateCommand;
@@ -58,6 +59,21 @@ function toTestFunctions(input) {
         return { name: f.name, ...(f.mock === undefined ? {} : { mock: f.mock }), ...('response' in f ? { response: f.response } : {}) };
     });
 }
+/**
+ * `--real-functions`: every function on the agent version (version-level + node/capability), as
+ * `{name, mock:false}`, the CLI twin of switching every Mock toggle off in the dashboard. Entries
+ * from `--functions` win by name. WhatsApp functions stay out (they must never run for real).
+ */
+function realFunctionList(version, explicit = []) {
+    const all = [...(version.functions || []), ...(version.capabilities || []).flatMap((c) => c.functions || [])];
+    const names = [...new Set(all.filter((f) => f?.name && !/whatsapp/i.test(String(f.type || ''))).map((f) => f.name))];
+    const byName = new Map(explicit.map((f) => [f.name, f]));
+    return [...names.map((n) => byName.get(n) || { name: n, mock: false }), ...explicit.filter((f) => !names.includes(f.name))];
+}
+async function withRealFunctions(api, agentId, version, explicit) {
+    const res = await api.getAgent(agentId, version);
+    return realFunctionList(res.agent || res, explicit);
+}
 async function simTestsListCommand(agentId, opts) {
     const api = (0, session_1.requireApi)();
     try {
@@ -78,7 +94,9 @@ async function simCreateCommand(agentId, opts) {
     try {
         // Parse the JSON flags before prompting, so a typo fails before any typing.
         const testVariables = opts.variables ? toTestVariables(await (0, jsonInput_1.readJsonFlag)(opts.variables, '--variables')) : {};
-        const testFunctions = opts.functions ? toTestFunctions(await (0, jsonInput_1.readJsonFlag)(opts.functions, '--functions')) : [];
+        let testFunctions = opts.functions ? toTestFunctions(await (0, jsonInput_1.readJsonFlag)(opts.functions, '--functions')) : [];
+        if (opts.realFunctions)
+            testFunctions = await withRealFunctions(api, agentId, opts.versions, testFunctions);
         let { name, persona, prompt, criteria } = opts;
         if (!name || !prompt || !criteria) {
             (0, ui_1.needTty)('Pass --name, --prompt and --criteria.');
@@ -143,8 +161,13 @@ async function simUpdateCommand(testCaseId, opts) {
             body.testVariables = toTestVariables(await (0, jsonInput_1.readJsonFlag)(opts.variables, '--variables'));
         if (opts.functions !== undefined)
             body.testFunctions = toTestFunctions(await (0, jsonInput_1.readJsonFlag)(opts.functions, '--functions'));
+        if (opts.realFunctions) {
+            if (!opts.agent)
+                return (0, ui_1.handleError)(new Error('--real-functions on update needs --agent <agentId> (the test case does not carry it).'));
+            body.testFunctions = await withRealFunctions(api, opts.agent, opts.versions, body.testFunctions || []);
+        }
         if (!Object.keys(body).length) {
-            return (0, ui_1.handleError)(new Error('Nothing to update. Pass at least one of --name, --persona, --prompt, --criteria, --model, --variables, --functions.'));
+            return (0, ui_1.handleError)(new Error('Nothing to update. Pass at least one of --name, --persona, --prompt, --criteria, --model, --variables, --functions, --real-functions.'));
         }
         const res = await api.updateTestCase(testCaseId, body);
         (0, ui_1.success)(`Updated test case ${res.testCase?._id || testCaseId}: ${Object.keys(body).join(', ')}.`);

package/dist/index.js

@@ -76,6 +76,7 @@ const knowledge_1 = require("./commands/knowledge");
 const aiTools_1 = require("./commands/aiTools");
 const batches_1 = require("./commands/batches");
 const simulations_1 = require("./commands/simulations");
+const liveTest_1 = require("./commands/liveTest");
 const simVariables_1 = require("./commands/simVariables");
 const dashboardApiCmd_1 = require("./commands/dashboardApiCmd");
 const adminAuth_1 = require("./lib/adminAuth");
@@ -1397,6 +1398,8 @@ sim.command('create <agentId>').description('create a test case')
     .option('--model <model>', 'test LLM model (default: platform default; see `cku sim models`)')
     .option('--variables <json|@file>', 'testVariables: a JSON object of name -> value (non-strings are stringified)')
     .option('--functions <json|@file>', 'a JSON array of {"name":"fn","mock":true,"response":{}}; "mock":false runs the REAL function (never for WhatsApp functions)')
+    .option('--real-functions', 'run every agent function for real (mock:false), like switching every Mock off in the dashboard; --functions entries win by name')
+    .option('--versions <versionId>', 'with --real-functions: read the functions from this version (default: the agent\'s default version)')
     .action(simulations_1.simCreateCommand);
 sim.command('update <testCaseId>').description('edit a test case (only the fields you pass are sent)')
     .option('--name <name>', 'test case name')
@@ -1406,6 +1409,9 @@ sim.command('update <testCaseId>').description('edit a test case (only the field
     .option('--model <model>', 'test LLM model')
     .option('--variables <json|@file>', 'testVariables: replaces the whole map')
     .option('--functions <json|@file>', 'function list, replaces the whole list; entries are {name, mock?, response?}, "mock":false runs the REAL function')
+    .option('--real-functions', 'replace the list with every agent function run for real (mock:false); --functions entries win by name; needs --agent')
+    .option('--agent <agentId>', 'agent to read functions from (for --real-functions)')
+    .option('--versions <versionId>', 'with --real-functions: read the functions from this version')
     .action(simulations_1.simUpdateCommand);
 sim.command('models').description('models that can play the simulated caller').option('--json', 'raw JSON').action(simulations_1.simModelsCommand);
 sim.command('delete <testCaseId>').description('delete a test case').option('--yes', 'skip confirmation')
@@ -1449,6 +1455,47 @@ simVars.command('create <agentId> <name>').description('save a variable set')
     .action(simVariables_1.simVarsCreateCommand);
 simVars.command('delete <agentId> <nameOrId>').description('delete a saved set').option('--yes', 'skip confirmation')
     .action(simVariables_1.simVarsDeleteCommand);
+// --- live test calls (the editor's Test Chat, from a terminal) -------------
+const liveTest = program.command('test').description('live test calls against the real call runtime; you or an LLM play the caller (functions RUN FOR REAL)');
+liveTest.command('start <agentId>').description('start a test call and print the agent\'s opening turn; then `test say` / `test end`')
+    .option('--versions <versionId>', 'version to test (default: the published one)')
+    .option('--variables <json|@file>', 'call metadata as a JSON object')
+    .option('--yes', 'confirm the real test call without asking')
+    .option('--json', 'print {sessionId, callSid, events} as JSON')
+    .action(liveTest_1.testStartCommand);
+liveTest.command('say <sessionId> <text...>').description('send one caller turn and print the agent\'s reply')
+    .option('--timeout <seconds>', 'wait this long for the agent to finish its turn (default 60)')
+    .option('--json', 'print {events, ended} as JSON')
+    .action(liveTest_1.testSayCommand);
+liveTest.command('end <sessionId>').description('hang up and print the full transcript with function calls')
+    .option('--json', 'print {callSid, events, ended} as JSON')
+    .action(liveTest_1.testEndCommand);
+liveTest.command('_serve <sessionId>', { hidden: true }).action(liveTest_1.testServeCommand);
+liveTest.command('run <agentId>').description('the CLI plays the caller with an LLM, then grades the call against --criteria')
+    .option('--test <testCaseId>', 'reuse a saved simulation test case (persona, prompt, criteria, variables)')
+    .option('--persona <text>', 'who the caller is')
+    .option('--prompt <text>', 'what the caller wants and does in this call')
+    .option('--criteria <text>', 'success criteria for the judge; without it the call is not graded')
+    .option('--variables <json|@file>', 'call metadata as a JSON object (merged over the test case\'s)')
+    .option('--versions <versionId>', 'version to test (default: the published one)')
+    .option('--llm <provider>', 'openai | anthropic | gemini | deepseek | groq | openrouter | ollama | custom (default: first key found in the environment)')
+    .option('--llm-model <model>', 'caller model (default per provider)')
+    .option('--llm-url <url>', 'OpenAI-compatible base URL (custom provider; key in CKU_LLM_API_KEY)')
+    .option('--judge-model <model>', 'judge model on the same provider (default: the caller model)')
+    .option('--max-turns <n>', 'caller turns before giving up (default 20)')
+    .option('--timeout <seconds>', 'per agent turn (default 60)')
+    .option('--yes', 'confirm the real test call without asking')
+    .option('--json', 'print {callSid, endedBy, events, verdict} as JSON')
+    .addHelpText('after', `
+Keys come from the environment only: OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY,
+DEEPSEEK_API_KEY, GROQ_API_KEY, OPENROUTER_API_KEY, or CKU_LLM_API_KEY for any provider.
+Exit 1 when the judge fails the call (or cannot grade it).
+
+Examples:
+  cku test run <agentId> --test <testCaseId> --yes
+  cku test run <agentId> --persona "Busy dealer in Pune" --prompt "Ask for the price, then say you will think about it" \\
+    --criteria "1. Agent states the price from call data. 2. Agent ends politely with end_call." --llm anthropic --yes`)
+    .action(liveTest_1.testRunCommand);
 // --- audit strategies ------------------------------------------------------
 const chatSim = program.command('chat-sim').description('chat agent simulations & test cases');
 chatSim.command('tests <chatAgentId>').description('list chat test cases').option('--json', 'raw JSON').action(chatSimulations_1.chatSimTestsListCommand);

package/dist/lib/liveTest.js

@@ -0,0 +1,361 @@
+"use strict";
+/**
+ * Live test calls: the agent editor's "Test Chat" tab, from a terminal.
+ *
+ * Read out of the dashboard bundle (no upstream CLI has this):
+ *  1. POST {base}/outbound_call?is_test=true&chat=true[&version=] with {userId, agent_id, metadata}
+ *     returns {token, call_sid}. The request carries no credential: the server only checks that
+ *     `Origin` is the dashboard's (403 "Caller not allowed" otherwise), so it is sent here too.
+ *  2. Join the LiveKit room at wss://livekit-inbound.callkaro.ai with that token.
+ *  3. User turns go out as text on topic `lk.chat`. Agent text comes back as `lk.transcription`
+ *     text streams (LiveKit Agents 1.x), function calls as data packets on `lk.history`, the
+ *     active node as the agent participant's `current_capability` attribute, and the agent's
+ *     turn state as `lk.agent.state`. `lk.history` also carries a `{type:"message"}` item per
+ *     finished message, which is what marks an agent turn complete: seen live, `lk.agent.state`
+ *     returns to `listening` up to 1.2s before the transcript of that turn arrives.
+ *
+ * This is the real call runtime, not a simulation: every function runs for real and the call
+ * lands in call history with post-call analysis.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.END_MARK = exports.LLM_PRESETS = exports.LiveSession = exports.deps = exports.TEST_IDENTITY = exports.LIVEKIT_URL = void 0;
+exports.testCallBase = testCallBase;
+exports.requestTestToken = requestTestToken;
+exports.formatTranscript = formatTranscript;
+exports.resolveLlm = resolveLlm;
+exports.chatComplete = chatComplete;
+exports.callerSystemPrompt = callerSystemPrompt;
+exports.parseCallerLine = parseCallerLine;
+exports.judgeMessages = judgeMessages;
+exports.parseVerdict = parseVerdict;
+const fs_1 = require("fs");
+const config_1 = require("./config");
+exports.LIVEKIT_URL = 'wss://livekit-inbound.callkaro.ai';
+exports.TEST_IDENTITY = 'test_participant';
+// ponytail: copied from the dashboard bundle's host picker; re-read the bundle if a Spinny
+// account's test call 404s. CK_TEST_CALL_URL overrides the host for every account.
+const SPINNY_USER_IDS = new Set([
+    '681b6d70d73a629e2f9f9b5b', '683761f141599a2ac50570c7', '684980f1ad348c30b0155cc3',
+    '686eb5e262c7062950b41a23', '686eb58c62c7062950b41a20', '68e4e4da4a8835a2360e8aa6',
+    '6915a152a205df5fd2b576d7', '691c3d20409942e2d0492f76', '698dfcedbbb4b9d3496cb860',
+    '6a06f48caabb1c14cb9311b4',
+]);
+function testCallBase(userId) {
+    if (process.env.CK_TEST_CALL_URL)
+        return process.env.CK_TEST_CALL_URL.replace(/\/+$/, '');
+    return SPINNY_USER_IDS.has(userId) ? 'https://spinny.callkaro.ai' : 'https://api.callkaro.ai';
+}
+async function requestTestToken(input) {
+    let url = `${testCallBase(input.userId)}/outbound_call?is_test=true&chat=true`;
+    if (input.version)
+        url += `&version=${encodeURIComponent(input.version)}`;
+    const res = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Origin: (0, config_1.webUrl)(), Referer: `${(0, config_1.webUrl)()}/` },
+        body: JSON.stringify({ userId: input.userId, agent_id: input.agentId, metadata: input.metadata || {} }),
+        signal: AbortSignal.timeout(30_000),
+    });
+    const body = await res.text();
+    let data = {};
+    try {
+        data = JSON.parse(body);
+    }
+    catch { /* not JSON */ }
+    if (res.status !== 200) {
+        throw new Error(`Starting the test call failed (HTTP ${res.status}): ${data.message || body.slice(0, 300) || 'no body'}`);
+    }
+    if (!data.token)
+        throw new Error('The test-call server answered without a LiveKit token, so there is no room to join.');
+    return { token: data.token, callSid: data.call_sid };
+}
+/** Swappable for tests: the native LiveKit module is loaded only when a test call starts. */
+exports.deps = {
+    rtc: () => {
+        try {
+            return require('@livekit/rtc-node');
+        }
+        catch {
+            throw new Error('Live tests need @livekit/rtc-node, which did not install on this machine. '
+                + 'Run: npm install --global @livekit/rtc-node, or reinstall cku on a supported platform '
+                + '(Linux, macOS or Windows, x64 or arm64).');
+        }
+    },
+};
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+class LiveSession {
+    callSid;
+    events = [];
+    ended = false;
+    room;
+    agentState = '';
+    sawState = false;
+    sawHistory = false;
+    agentMessages = 0;
+    openStreams = 0;
+    lastActivity = Date.now();
+    lastSent = '';
+    segments = new Map();
+    constructor(callSid) {
+        this.callSid = callSid;
+    }
+    /** Join the room and collect the agent's opening turn (empty when the agent waits for the caller). */
+    static async open(token, callSid, opts = {}) {
+        const rtc = exports.deps.rtc();
+        const s = new LiveSession(callSid);
+        const room = new rtc.Room();
+        s.room = room;
+        const touch = () => { s.lastActivity = Date.now(); };
+        const push = (e) => { const ev = { ...e, t: Date.now() }; s.events.push(ev); touch(); return ev; };
+        const readAttrs = (attrs) => {
+            if (!attrs)
+                return;
+            if ('lk.agent.state' in attrs) {
+                s.agentState = attrs['lk.agent.state'];
+                s.sawState = true;
+                touch();
+            }
+            if (attrs.current_capability)
+                push({ type: 'capability', capability: attrs.current_capability });
+        };
+        // CK_TEST_DEBUG=<file> appends every raw room event, for when the worker's protocol drifts.
+        const debug = (kind, data) => {
+            if (process.env.CK_TEST_DEBUG)
+                (0, fs_1.appendFileSync)(process.env.CK_TEST_DEBUG, `${new Date().toISOString()} ${kind} ${JSON.stringify(data)}\n`);
+        };
+        room.registerTextStreamHandler('lk.transcription', async (reader, from) => {
+            s.openStreams++;
+            touch();
+            try {
+                const text = String(await reader.readAll()).trim();
+                debug('transcription', { from: from.identity, attributes: reader.info?.attributes, text });
+                if (!text || from.identity === exports.TEST_IDENTITY || text === s.lastSent)
+                    return;
+                const seg = reader.info?.attributes?.['lk.segment_id'];
+                const prior = seg && s.segments.get(seg);
+                if (prior) {
+                    prior.text = text;
+                    touch();
+                    return;
+                }
+                const ev = push({ type: 'agent', text });
+                if (seg)
+                    s.segments.set(seg, ev);
+            }
+            finally {
+                s.openStreams--;
+            }
+        });
+        room.on('dataReceived', (payload, p, _k, topic) => {
+            debug('data', { from: p?.identity, topic, payload: new TextDecoder().decode(payload).slice(0, 2000) });
+            if (topic !== 'lk.history')
+                return;
+            try {
+                const data = JSON.parse(new TextDecoder().decode(payload));
+                if (data.type === 'message') {
+                    // Duplicates the transcript text; only the count matters (see the header).
+                    s.sawHistory = true;
+                    if (data.role === 'assistant')
+                        s.agentMessages++;
+                    touch();
+                }
+                else if (data.type === 'function') {
+                    let args = data.arguments;
+                    try {
+                        if (typeof args === 'string')
+                            args = JSON.parse(args || '{}');
+                    }
+                    catch { /* keep the string */ }
+                    push({ type: 'function', name: data.name, arguments: args });
+                }
+                else {
+                    push({ type: 'history', data });
+                }
+            }
+            catch { /* not JSON */ }
+        });
+        room.on('participantConnected', (p) => { debug('joined', { identity: p.identity, kind: p.kind, attributes: p.attributes }); readAttrs(p.attributes); });
+        room.on('participantAttributesChanged', (changed, p) => {
+            debug('attributes', { from: p?.identity, changed });
+            if (p?.identity !== exports.TEST_IDENTITY)
+                readAttrs(changed);
+        });
+        const end = () => { if (!s.ended) {
+            s.ended = true;
+            push({ type: 'ended' });
+        } };
+        room.on('participantDisconnected', (p) => { if (p?.identity !== exports.TEST_IDENTITY)
+            end(); });
+        room.on('disconnected', (reason) => { debug('disconnected', { reason }); end(); });
+        await room.connect(exports.LIVEKIT_URL, token, { autoSubscribe: false });
+        for (const p of room.remoteParticipants?.values?.() || [])
+            readAttrs(p.attributes);
+        const greeting = await s.waitTurn(opts.greetingTimeoutMs ?? 20_000, 0);
+        return { session: s, greeting };
+    }
+    /**
+     * Wait until the agent has finished its turn: it spoke or called a function since `from` (and,
+     * once the worker is known to send them, a finished-message item arrived on `lk.history`), no transcript
+     * stream is still open, the agent reports `listening` (or never reports a state), and nothing
+     * new has arrived for a short settle window. `lk.agent.state` makes the window short; without
+     * it a longer quiet period has to stand in.
+     */
+    async waitTurn(timeoutMs, from = this.events.length, messagesBefore = this.agentMessages) {
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline && !this.ended) {
+            const spoke = this.events.slice(from).some((e) => e.type === 'agent' || e.type === 'function');
+            const got = spoke && (!this.sawHistory || this.agentMessages > messagesBefore);
+            const idle = this.openStreams === 0 && (!this.sawState || this.agentState === 'listening');
+            const quiet = Date.now() - this.lastActivity >= (this.sawState ? 700 : 2500);
+            if (got && idle && quiet)
+                break;
+            await sleep(100);
+        }
+        return this.events.slice(from);
+    }
+    async say(text, timeoutMs = 60_000) {
+        if (this.ended)
+            return [{ type: 'ended', t: Date.now() }];
+        const from = this.events.length;
+        const messagesBefore = this.agentMessages;
+        this.events.push({ type: 'user', text, t: Date.now() });
+        this.lastSent = text.trim();
+        this.lastActivity = Date.now();
+        await this.room.localParticipant.sendText(text, { topic: 'lk.chat' });
+        // Give the agent a moment to leave `listening`, so a still-idle state is not read as done.
+        if (this.sawState) {
+            const until = Date.now() + 3000;
+            while (this.agentState === 'listening' && Date.now() < until && !this.ended)
+                await sleep(50);
+        }
+        return this.waitTurn(timeoutMs, from + 1, messagesBefore);
+    }
+    async close() {
+        try {
+            await this.room?.disconnect();
+        }
+        catch { /* already gone */ }
+        try {
+            exports.deps.rtc().dispose?.();
+        }
+        catch { /* best effort */ }
+    }
+}
+exports.LiveSession = LiveSession;
+/** One line per event, numbered by turn, for people and for the judge. */
+function formatTranscript(events) {
+    const lines = [];
+    let turn = 0;
+    for (const e of events) {
+        if (e.type === 'agent')
+            lines.push(`T${++turn} agent: ${e.text}`);
+        else if (e.type === 'user')
+            lines.push(`T${++turn} caller: ${e.text}`);
+        else if (e.type === 'function')
+            lines.push(`   [function ${e.name} ${JSON.stringify(e.arguments ?? {})}]`);
+        else if (e.type === 'capability')
+            lines.push(`   [node: ${e.capability}]`);
+        else if (e.type === 'ended')
+            lines.push('   [call ended]');
+    }
+    return lines.join('\n');
+}
+// --- LLM caller and judge -------------------------------------------------------------------
+/** Every provider here speaks the OpenAI chat-completions shape, so one client covers them all. */
+exports.LLM_PRESETS = {
+    openai: { url: 'https://api.openai.com/v1', env: 'OPENAI_API_KEY', model: 'gpt-4.1-mini' },
+    anthropic: { url: 'https://api.anthropic.com/v1', env: 'ANTHROPIC_API_KEY', model: 'claude-haiku-5-5' },
+    gemini: { url: 'https://generativelanguage.googleapis.com/v1beta/openai', env: 'GEMINI_API_KEY', model: 'gemini-2.5-flash' },
+    deepseek: { url: 'https://api.deepseek.com/v1', env: 'DEEPSEEK_API_KEY', model: 'deepseek-chat' },
+    groq: { url: 'https://api.groq.com/openai/v1', env: 'GROQ_API_KEY', model: 'llama-3.3-70b-versatile' },
+    openrouter: { url: 'https://openrouter.ai/api/v1', env: 'OPENROUTER_API_KEY', model: 'openai/gpt-4.1-mini' },
+    ollama: { url: 'http://localhost:11434/v1', model: 'llama3.1' },
+};
+/**
+ * Pick the provider: --llm, else --llm-url (custom, key in CKU_LLM_API_KEY), else the first
+ * preset whose key is in the environment. Keys only come from the environment, never a flag,
+ * so they stay out of shell history and process listings.
+ */
+function resolveLlm(opts) {
+    const names = Object.keys(exports.LLM_PRESETS);
+    let name = opts.llm;
+    if (!name && opts.llmUrl)
+        name = 'custom';
+    if (!name)
+        name = names.find((n) => exports.LLM_PRESETS[n].env && process.env[exports.LLM_PRESETS[n].env]);
+    if (!name) {
+        throw new Error(`No LLM for the simulated caller. Set one of ${names.filter((n) => exports.LLM_PRESETS[n].env).map((n) => exports.LLM_PRESETS[n].env).join(', ')}, `
+            + 'or pass --llm ollama, or --llm-url <OpenAI-compatible base URL> with the key in CKU_LLM_API_KEY. '
+            + 'To play the caller yourself instead, use: cku test start <agentId>');
+    }
+    if (name === 'custom') {
+        if (!opts.llmUrl)
+            throw new Error('--llm custom needs --llm-url <OpenAI-compatible base URL>.');
+        if (!opts.llmModel)
+            throw new Error('--llm-url needs --llm-model <model id>.');
+        return { name, url: opts.llmUrl.replace(/\/+$/, ''), key: process.env.CKU_LLM_API_KEY, model: opts.llmModel };
+    }
+    const p = exports.LLM_PRESETS[name];
+    if (!p)
+        throw new Error(`Unknown --llm "${name}". Use one of: ${names.join(', ')}, custom.`);
+    const key = process.env.CKU_LLM_API_KEY || (p.env ? process.env[p.env] : undefined);
+    if (p.env && !key)
+        throw new Error(`--llm ${name} needs ${p.env} (or CKU_LLM_API_KEY) set in the environment.`);
+    return { name, url: (opts.llmUrl || p.url).replace(/\/+$/, ''), key, model: opts.llmModel || p.model };
+}
+async function chatComplete(cfg, messages, temperature = 0.7) {
+    const res = await fetch(`${cfg.url}/chat/completions`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', ...(cfg.key ? { Authorization: `Bearer ${cfg.key}` } : {}) },
+        body: JSON.stringify({ model: cfg.model, messages, temperature }),
+        signal: AbortSignal.timeout(120_000),
+    });
+    const body = await res.text();
+    if (!res.ok)
+        throw new Error(`${cfg.name} (${cfg.model}) returned HTTP ${res.status}: ${body.slice(0, 300)}`);
+    const content = JSON.parse(body)?.choices?.[0]?.message?.content;
+    return Array.isArray(content) ? content.map((c) => c.text ?? '').join('') : String(content ?? '');
+}
+exports.END_MARK = '[END]';
+function callerSystemPrompt(s) {
+    return [
+        'You are role-playing a person on a phone call with a voice agent. Stay in character the whole call.',
+        s.persona ? `Who you are: ${s.persona}` : '',
+        `What happens in this call: ${s.prompt}`,
+        'Rules:',
+        '- Reply with only the words you say out loud: one short spoken turn, usually one or two sentences. No stage directions, no quotes, no labels.',
+        '- Speak the way this person would, in their language and register (Hindi, Hinglish, English, ...).',
+        '- Never say you are a test, a simulation or an AI.',
+        `- When the call is over for you (goal done, you give up, or the agent said goodbye), end your reply with ${exports.END_MARK}. Reply with only ${exports.END_MARK} if there is nothing left to say.`,
+    ].filter(Boolean).join('\n');
+}
+/** The caller's next line, and whether it hangs up after it. */
+function parseCallerLine(raw) {
+    const end = raw.includes(exports.END_MARK);
+    const text = raw.replace(exports.END_MARK, '').replace(/^\s*(caller|me|you)\s*:\s*/i, '').trim().replace(/^"(.*)"$/s, '$1');
+    return { text, end };
+}
+function judgeMessages(criteria, events) {
+    return [
+        {
+            role: 'system',
+            content: 'You grade a test call between a simulated caller and a voice agent. Judge only from the transcript. '
+                + 'Split the success criteria into separate checks. A check passes only if the transcript shows it; anything the '
+                + 'criteria forbid fails. Cite the turn (T3) or function line that decides each check. Answer with JSON only: '
+                + '{"pass": boolean, "criteria": [{"criterion": string, "pass": boolean, "evidence": string, "reason": string}], "summary": string}. '
+                + '"pass" is true only if every check passes.',
+        },
+        { role: 'user', content: `Success criteria:\n${criteria}\n\nTranscript:\n${formatTranscript(events)}` },
+    ];
+}
+/** The judge's JSON, tolerating a code fence or prose around it. A reply with no JSON is INVALID, not FAIL. */
+function parseVerdict(raw) {
+    const m = raw.match(/\{[\s\S]*\}/);
+    try {
+        const v = JSON.parse(m ? m[0] : raw);
+        const criteria = Array.isArray(v.criteria) ? v.criteria : [];
+        return { pass: v.pass === true && criteria.every((c) => c.pass !== false), criteria, summary: String(v.summary ?? '') };
+    }
+    catch {
+        return { pass: false, criteria: [], summary: `Judge did not return JSON: ${raw.slice(0, 200)}`, invalid: true };
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "callkaro-cli-unofficial",
-  "version": "0.9.5",
+  "version": "0.10.0",
   "license": "Apache-2.0",
   "description": "Unofficial fork of the CallKaro CLI — manage your voice AI agents from the terminal",
   "bin": {
@@ -48,5 +48,8 @@
     "cli",
     "agents",
     "telephony"
-  ]
+  ],
+  "optionalDependencies": {
+    "@livekit/rtc-node": "^1.2.2"
+  }
 }