@nac3/forge-cli 0.2.0-alpha.9 → 0.2.1-alpha.60

package/dist/telemetry/usage.js

@@ -0,0 +1,208 @@
+/**
+ * Yujin Forge -- usage + cost telemetry.
+ *
+ * Append-only event log at ~/.yujin-forge/usage.jsonl. Every
+ * Claude API call (or other priced operation) records one JSONL
+ * line with { ts, kind, model?, tokens_in?, tokens_out?, units?,
+ * label?, project_root? }.
+ *
+ * The reader (`yf operate costs`) aggregates by day + by kind +
+ * by model and applies a pricing table to produce a cost report.
+ *
+ * Append is best-effort: telemetry failures NEVER throw to the
+ * caller -- a broken usage file should not break user work.
+ *
+ * Pricing is hard-coded for the 2026 model lineup. Users can
+ * override via ~/.yujin-forge/pricing.json:
+ *   {
+ *     "models": {
+ *        "claude-opus-4-7":  { "in_per_mtok": 15, "out_per_mtok": 75 },
+ *        "claude-sonnet-4-6": { "in_per_mtok":  3, "out_per_mtok": 15 }
+ *     },
+ *     "voice": {
+ *        "tts_per_million_chars": 16,
+ *        "stt_per_minute":        0.006
+ *     }
+ *   }
+ *
+ * ASCII-only.
+ */
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { configDir } from '../license/index.js';
+/* Defaults reflect early-2026 Anthropic + Google Cloud pricing.
+ * Users can override via ~/.yujin-forge/pricing.json. */
+export const DEFAULT_PRICING = {
+    models: {
+        'claude-opus-4-7': { in_per_mtok: 15, out_per_mtok: 75 },
+        'claude-opus-4': { in_per_mtok: 15, out_per_mtok: 75 },
+        'claude-sonnet-4-6': { in_per_mtok: 3, out_per_mtok: 15 },
+        'claude-sonnet-4': { in_per_mtok: 3, out_per_mtok: 15 },
+        'claude-haiku-4-5': { in_per_mtok: 1, out_per_mtok: 5 },
+        /* Older lineup kept for back-compat. */
+        'claude-3-5-sonnet': { in_per_mtok: 3, out_per_mtok: 15 },
+        'claude-3-5-haiku': { in_per_mtok: 0.8, out_per_mtok: 4 },
+    },
+    voice: {
+        tts_per_million_chars: 16, /* Google Cloud TTS Neural2 */
+        stt_per_minute: 0.006, /* Google Cloud STT enhanced */
+    },
+};
+function usageFile() {
+    return path.join(configDir(), 'usage.jsonl');
+}
+function pricingFile() {
+    return path.join(configDir(), 'pricing.json');
+}
+async function ensureDir() {
+    try {
+        await fs.mkdir(configDir(), { recursive: true });
+    }
+    catch { /* best effort */ }
+}
+/**
+ * Append a usage event. Never throws -- failures swallowed.
+ */
+export async function recordUsage(ev) {
+    try {
+        await ensureDir();
+        const line = { ts: ev.ts ?? new Date().toISOString(), ...ev };
+        await fs.appendFile(usageFile(), JSON.stringify(line) + '\n', 'utf-8');
+    }
+    catch {
+        /* swallow -- telemetry must never break user flow */
+    }
+}
+/**
+ * Load pricing table (defaults overlaid with user overrides).
+ */
+export async function loadPricing() {
+    try {
+        const raw = await fs.readFile(pricingFile(), 'utf-8');
+        const override = JSON.parse(raw);
+        return {
+            models: { ...(DEFAULT_PRICING.models ?? {}), ...(override.models ?? {}) },
+            voice: { ...(DEFAULT_PRICING.voice ?? {}), ...(override.voice ?? {}) },
+        };
+    }
+    catch {
+        return DEFAULT_PRICING;
+    }
+}
+/**
+ * Read all usage events. Tolerates malformed lines.
+ */
+export async function readUsage() {
+    try {
+        const raw = await fs.readFile(usageFile(), 'utf-8');
+        return raw.split('\n')
+            .filter((l) => l.trim().length > 0)
+            .map((l) => {
+            try {
+                return JSON.parse(l);
+            }
+            catch {
+                return null;
+            }
+        })
+            .filter((x) => x !== null);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Compute cost breakdown over a time window. `sinceDays` defaults
+ * to 30. Pass 0 to mean "all-time".
+ */
+export async function computeCosts(sinceDays = 30) {
+    const events = await readUsage();
+    const pricing = await loadPricing();
+    const now = Date.now();
+    const cutoffMs = sinceDays > 0 ? now - sinceDays * 24 * 60 * 60 * 1000 : 0;
+    const inWindow = events.filter((e) => Date.parse(e.ts) >= cutoffMs);
+    const breakdown = {
+        total_usd: 0,
+        by_kind: {},
+        by_model: {},
+        by_day: {},
+        events_total: events.length,
+        events_window: inWindow.length,
+        window_label: sinceDays === 0 ? 'all-time' : 'last ' + sinceDays + ' days',
+    };
+    for (const ev of inWindow) {
+        let usd = 0;
+        if (ev.kind.startsWith('claude.')) {
+            const m = ev.model ?? 'unknown';
+            const p = pricing.models?.[m];
+            if (p) {
+                usd = ((ev.tokens_in ?? 0) / 1_000_000) * p.in_per_mtok
+                    + ((ev.tokens_out ?? 0) / 1_000_000) * p.out_per_mtok;
+            }
+            if (!breakdown.by_model[m]) {
+                breakdown.by_model[m] = { count: 0, tokens_in: 0, tokens_out: 0, usd: 0 };
+            }
+            const slot = breakdown.by_model[m];
+            slot.count++;
+            slot.tokens_in += ev.tokens_in ?? 0;
+            slot.tokens_out += ev.tokens_out ?? 0;
+            slot.usd += usd;
+        }
+        else if (ev.kind === 'voice.tts') {
+            usd = ((ev.units ?? 0) / 1_000_000) * (pricing.voice?.tts_per_million_chars ?? 0);
+        }
+        else if (ev.kind === 'voice.stt') {
+            usd = (ev.units ?? 0) * (pricing.voice?.stt_per_minute ?? 0);
+        }
+        breakdown.total_usd += usd;
+        if (!breakdown.by_kind[ev.kind])
+            breakdown.by_kind[ev.kind] = { count: 0, usd: 0 };
+        breakdown.by_kind[ev.kind].count++;
+        breakdown.by_kind[ev.kind].usd += usd;
+        const day = ev.ts.slice(0, 10);
+        if (!breakdown.by_day[day])
+            breakdown.by_day[day] = { count: 0, usd: 0 };
+        breakdown.by_day[day].count++;
+        breakdown.by_day[day].usd += usd;
+    }
+    return breakdown;
+}
+/**
+ * Pretty-format a CostBreakdown for terminal output. Returns
+ * lines; caller does the printing.
+ */
+export function formatCosts(b) {
+    const lines = [];
+    lines.push('Window:          ' + b.window_label);
+    lines.push('Events total:    ' + b.events_total);
+    lines.push('Events in window:' + b.events_window);
+    lines.push('Total USD:       $' + b.total_usd.toFixed(4));
+    lines.push('');
+    lines.push('By kind:');
+    const kinds = Object.entries(b.by_kind).sort((a, c) => c[1].usd - a[1].usd);
+    for (const [k, v] of kinds) {
+        lines.push('  ' + k.padEnd(18) + 'count=' + String(v.count).padStart(5) + '  usd=$' + v.usd.toFixed(4));
+    }
+    if (Object.keys(b.by_model).length > 0) {
+        lines.push('');
+        lines.push('By model:');
+        const models = Object.entries(b.by_model).sort((a, c) => c[1].usd - a[1].usd);
+        for (const [m, v] of models) {
+            lines.push('  ' + m.padEnd(24)
+                + 'count=' + String(v.count).padStart(5)
+                + '  in=' + String(v.tokens_in).padStart(8)
+                + '  out=' + String(v.tokens_out).padStart(7)
+                + '  usd=$' + v.usd.toFixed(4));
+        }
+    }
+    if (Object.keys(b.by_day).length > 0) {
+        lines.push('');
+        lines.push('By day:');
+        const days = Object.entries(b.by_day).sort((a, c) => (a[0] < c[0] ? 1 : -1)).slice(0, 14);
+        for (const [d, v] of days) {
+            lines.push('  ' + d + '  count=' + String(v.count).padStart(5) + '  usd=$' + v.usd.toFixed(4));
+        }
+    }
+    return lines;
+}
+//# sourceMappingURL=usage.js.map

package/dist/telemetry/usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.js","sourceRoot":"","sources":["../../src/telemetry/usage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AA8BhD;yDACyD;AACzD,MAAM,CAAC,MAAM,eAAe,GAAiB;IAC3C,MAAM,EAAE;QACN,iBAAiB,EAAY,EAAE,WAAW,EAAE,EAAE,EAAG,YAAY,EAAE,EAAE,EAAE;QACnE,eAAe,EAAc,EAAE,WAAW,EAAE,EAAE,EAAG,YAAY,EAAE,EAAE,EAAE;QACnE,mBAAmB,EAAU,EAAE,WAAW,EAAG,CAAC,EAAG,YAAY,EAAE,EAAE,EAAE;QACnE,iBAAiB,EAAY,EAAE,WAAW,EAAG,CAAC,EAAG,YAAY,EAAE,EAAE,EAAE;QACnE,kBAAkB,EAAW,EAAE,WAAW,EAAG,CAAC,EAAG,YAAY,EAAG,CAAC,EAAE;QACnE,wCAAwC;QACxC,mBAAmB,EAAU,EAAE,WAAW,EAAG,CAAC,EAAG,YAAY,EAAE,EAAE,EAAE;QACnE,kBAAkB,EAAW,EAAE,WAAW,EAAG,GAAG,EAAC,YAAY,EAAG,CAAC,EAAE;KACpE;IACD,KAAK,EAAE;QACL,qBAAqB,EAAE,EAAE,EAAI,8BAA8B;QAC3D,cAAc,EAAS,KAAK,EAAC,+BAA+B;KAC7D;CACF,CAAC;AAEF,SAAS,SAAS;IAChB,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,aAAa,CAAC,CAAC;AAC/C,CAAC;AACD,SAAS,WAAW;IAClB,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,cAAc,CAAC,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,SAAS;IACtB,IAAI,CAAC;QAAC,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;AACvF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAA4C;IAC5E,IAAI,CAAC;QACH,MAAM,SAAS,EAAE,CAAC;QAClB,MAAM,IAAI,GAAe,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC;QAC1E,MAAM,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,qDAAqD;IACvD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAiB,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,EAAE,GAAG,CAAC,eAAe,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE;YACzE,KAAK,EAAG,EAAE,GAAG,CAAC,eAAe,CAAC,KAAK,IAAK,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAK,EAAE,CAAC,EAAE;SAC1E,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,eAAe,CAAC;IACzB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;QACpD,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,IAAI,CAAC;gBAAC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAe,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,IAAI,CAAC;YAAC,CAAC;QACpE,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAmB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAYD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,SAAS,GAAG,EAAE;IAC/C,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,CAAC;IAEpE,MAAM,SAAS,GAAkB;QAC/B,SAAS,EAAK,CAAC;QACf,OAAO,EAAO,EAAE;QAChB,QAAQ,EAAM,EAAE;QAChB,MAAM,EAAQ,EAAE;QAChB,YAAY,EAAE,MAAM,CAAC,MAAM;QAC3B,aAAa,EAAC,QAAQ,CAAC,MAAM;QAC7B,YAAY,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,OAAO;KAC3E,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,SAAS,CAAC;YAChC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9B,IAAI,CAAC,EAAE,CAAC;gBACN,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,SAAS,IAAK,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,WAAW;sBAClD,CAAC,CAAC,EAAE,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC;YAC5D,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3B,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;YAC5E,CAAC;YACD,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC;YACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,IAAK,EAAE,CAAC,SAAS,IAAK,CAAC,CAAC;YACtC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,UAAU,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,GAAG,IAAW,GAAG,CAAC;QACzB,CAAC;aAAM,IAAI,EAAE,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,qBAAqB,IAAI,CAAC,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,EAAE,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,GAAG,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,cAAc,IAAI,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,SAAS,CAAC,SAAS,IAAI,GAAG,CAAC;QAE3B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC;YAAE,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACnF,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAE,CAAC,KAAK,EAAE,CAAC;QACpC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAE,CAAC,GAAG,IAAI,GAAG,CAAC;QAEvC,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACzE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,KAAK,EAAE,CAAC;QAC/B,SAAS,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,GAAG,IAAI,GAAG,CAAC;IACpC,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,CAAgB;IAC1C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACvB,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5E,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9E,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;kBACjB,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;kBACtC,OAAO,GAAI,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;kBAC1C,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;kBAC3C,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1F,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/test_agent/gemini_user.d.ts

@@ -0,0 +1,9 @@
+import type { SyntheticUserClient } from './runner.js';
+export interface GeminiSyntheticUserOpts {
+    apiKey: string;
+    /** Default 'gemini-2.5-flash-lite' -- cheapest valid model for
+     *  multi-turn persona simulation. */
+    model?: string;
+}
+export declare function makeGeminiSyntheticUser(opts: GeminiSyntheticUserOpts): SyntheticUserClient;
+//# sourceMappingURL=gemini_user.d.ts.map

package/dist/test_agent/gemini_user.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini_user.d.ts","sourceRoot":"","sources":["../../src/test_agent/gemini_user.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvD,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf;yCACqC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA4BD,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,uBAAuB,GAAG,mBAAmB,CA8B1F"}

package/dist/test_agent/gemini_user.js

@@ -0,0 +1,58 @@
+import { GeminiClient } from '../llm/gemini_client.js';
+function buildSystemPrompt(profile, turnIndex) {
+    const lines = [
+        'You are a SYNTHETIC USER in a test harness. You are NOT the assistant.',
+        'You drive a multi-turn conversation against a target Forge chat server.',
+        'On each turn you produce the SINGLE next user message and nothing else.',
+        '',
+        '--- Persona ---',
+        profile.persona,
+        '',
+        '--- Goal ---',
+        profile.goal,
+        '',
+        '--- Constraints ---',
+        ...profile.constraints.map((c) => '- ' + c),
+        '',
+        '--- Output rules ---',
+        'Output ONLY the next user message in plain text. No JSON, no markdown',
+        'fences, no role tags, no commentary. Reply with what a real user would',
+        'type. Keep it conversational and natural. Length 1-4 sentences typical;',
+        'longer only when the persona would write a longer message.',
+        '',
+        'This is turn ' + turnIndex + '.',
+    ];
+    return lines.join('\n');
+}
+export function makeGeminiSyntheticUser(opts) {
+    const gemini = new GeminiClient({
+        apiKey: opts.apiKey,
+        defaultModel: opts.model ?? 'gemini-2.5-flash-lite',
+    });
+    return {
+        async nextTurn({ profile, history, turn_index }) {
+            const messages = history.map((h) => ({
+                /* Flip the perspective: the conversation history is from the
+                 * REAL chat; for the synthetic user, what the real chat said
+                 * (assistant) is what the synthetic user is RESPONDING to,
+                 * and what the synthetic user previously said (user) is its
+                 * own prior turn. */
+                role: (h.role === 'user' ? 'assistant' : 'user'),
+                content: h.content,
+            }));
+            /* If this is the first turn, kick the conversation off with a
+             * single system-only call. Gemini complains when there are no
+             * user messages, so emit a synthetic seed. */
+            if (messages.length === 0) {
+                messages.push({ role: 'user', content: '(begin the conversation in line with your persona + goal)' });
+            }
+            const r = await gemini.chat({
+                messages,
+                system: buildSystemPrompt(profile, turn_index),
+                maxTokens: 400,
+            });
+            return { text: (r.text ?? '').trim() };
+        },
+    };
+}
+//# sourceMappingURL=gemini_user.js.map

package/dist/test_agent/gemini_user.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini_user.js","sourceRoot":"","sources":["../../src/test_agent/gemini_user.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAWvD,SAAS,iBAAiB,CAAC,OAAgB,EAAE,SAAiB;IAC5D,MAAM,KAAK,GAAG;QACZ,wEAAwE;QACxE,yEAAyE;QACzE,yEAAyE;QACzE,EAAE;QACF,iBAAiB;QACjB,OAAO,CAAC,OAAO;QACf,EAAE;QACF,cAAc;QACd,OAAO,CAAC,IAAI;QACZ,EAAE;QACF,qBAAqB;QACrB,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC;QAC3C,EAAE;QACF,sBAAsB;QACtB,uEAAuE;QACvE,wEAAwE;QACxE,yEAAyE;QACzE,4DAA4D;QAC5D,EAAE;QACF,eAAe,GAAG,SAAS,GAAG,GAAG;KAClC,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAA6B;IACnE,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;QAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,IAAI,uBAAuB;KACpD,CAAC,CAAC;IACH,OAAO;QACL,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE;YAC7C,MAAM,QAAQ,GAAkB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClD;;;;qCAIqB;gBACrB,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAyB;gBACxE,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC,CAAC;YACJ;;0DAE8C;YAC9C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,2DAA2D,EAAE,CAAC,CAAC;YACxG,CAAC;YACD,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;gBAC1B,QAAQ;gBACR,MAAM,EAAE,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAAC;gBAC9C,SAAS,EAAE,GAAG;aACf,CAAC,CAAC;YACH,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACzC,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/test_agent/profiles.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Agente de Test profile library (PND-024 Etapa 1).
+ *
+ * Three baseline profiles cover the common Forge entry points:
+ *
+ *  - greenfield-spec-writer: user opens Forge against an empty
+ *    directory with a partial spec idea. Expects Forge to ingest
+ *    the spec and propose next steps.
+ *  - brownfield-bug-fixer: user opens Forge against an existing
+ *    repo to fix a small bug. Expects Forge to read manifest, find
+ *    relevant files, propose changes.
+ *  - casual-out-of-scope: user opens Forge and asks for a song /
+ *    poem / chat. Expects Forge to skip the warmup (PND-022) and
+ *    respond conversationally instead of asking target/mode.
+ *
+ * Each profile is an immutable export -- the runner reads them via
+ * getProfile(slug) and never mutates. Adding profiles is a single
+ * declaration here; the CLI surface picks them up automatically.
+ */
+import type { Profile } from './types.js';
+export declare function listProfiles(): readonly Profile[];
+export declare function getProfile(slug: string): Profile | null;
+//# sourceMappingURL=profiles.d.ts.map

package/dist/test_agent/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../src/test_agent/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AA2H1C,wBAAgB,YAAY,IAAI,SAAS,OAAO,EAAE,CAEjD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAEvD"}

package/dist/test_agent/profiles.js

@@ -0,0 +1,117 @@
+const GREENFIELD_SPEC_WRITER = {
+    slug: 'greenfield-spec-writer',
+    name: 'Greenfield spec writer',
+    persona: 'You are a startup founder with a half-baked product idea. You '
+        + 'just installed Yujin Forge to scaffold a new project. You have '
+        + 'a rough sense of what you want but you write in plain Spanish, '
+        + 'not technical jargon.',
+    goal: 'Get Forge to ingest your spec idea and propose a concrete '
+        + 'next step (e.g. specific files to create, a target stack to use, '
+        + 'or a clarification question that nails the ambiguity).',
+    constraints: [
+        'Reply in plain Spanish.',
+        'Do not paste code -- you are not a developer.',
+        'When Forge asks the target/mode warmup, answer briefly and move on.',
+        'If Forge proposes a concrete next step (file scaffold, stack choice, '
+            + 'clarifying question), the goal is met -- thank them.',
+    ],
+    max_turns: 6,
+    assertions: [
+        {
+            kind: 'reply_not_contains',
+            label: 'Forge does not paste raw code into the first reply',
+            payload: { text: 'function ', case_insensitive: true },
+        },
+        {
+            kind: 'turn_count_at_most',
+            label: 'Forge proposes a next step within 4 turns',
+            payload: { max: 4 },
+        },
+    ],
+    success_when: {
+        kind: 'reply_matches_regex',
+        label: 'Forge proposes a scaffold / stack / clarifying question',
+        payload: {
+            regex: '(scaffold|estructura inicial|stack|template|que tipo de|precisas|necesitas|cu[aá]l|escribir(e|ia) el|propongo)',
+            flags: 'i',
+        },
+    },
+};
+const BROWNFIELD_BUG_FIXER = {
+    slug: 'brownfield-bug-fixer',
+    name: 'Brownfield bug fixer',
+    persona: 'You are an experienced developer who just cloned an existing '
+        + 'project and opened Forge in its directory. You have a bug '
+        + 'report and want Forge to help you locate the relevant files.',
+    goal: 'Have Forge invoke a manifest-read / file-read / fs-find verb '
+        + 'so you can confirm it is grounding itself in the actual code '
+        + 'instead of inventing context.',
+    constraints: [
+        'Write in technical English.',
+        'When Forge asks clarification, give a concrete file path + symptom.',
+        'Do not accept hand-waved answers -- ask for the actual file.',
+    ],
+    max_turns: 6,
+    assertions: [
+        {
+            kind: 'turn_count_at_most',
+            label: 'Forge grounds itself in actual code within 3 turns',
+            payload: { max: 3 },
+            blocker: false,
+        },
+    ],
+    success_when: {
+        kind: 'tool_invoked',
+        label: 'Forge invokes a file-read or manifest verb',
+        payload: { verb_id: 'yujin.fs.read-file' },
+    },
+};
+const CASUAL_OUT_OF_SCOPE = {
+    slug: 'casual-out-of-scope',
+    name: 'Casual / out-of-scope conversation',
+    persona: 'You are someone who opened Forge by accident and want to chat '
+        + 'casually -- ask for a song, a poem, or just say hi. You have '
+        + 'no plans to build software in this session.',
+    goal: 'See whether Forge respects out-of-scope intent (PND-022) or '
+        + 'whether it intrusively asks the target/mode warmup despite '
+        + 'your clearly non-dev prompt.',
+    constraints: [
+        'Open with a clearly non-dev request (song, poem, greeting).',
+        'Do NOT mention any dev keywords (deploy, react, code, etc).',
+        'If Forge asks the warmup anyway, push back politely and continue '
+            + 'with your original request.',
+    ],
+    max_turns: 4,
+    assertions: [
+        {
+            kind: 'reply_not_contains',
+            label: 'Forge does not ask the target/mode warmup',
+            payload: { text: 'responsive', case_insensitive: true },
+            blocker: false,
+        },
+        {
+            kind: 'reply_not_contains',
+            label: 'Forge does not ask didactico/tecnico mode warmup',
+            payload: { text: 'didactico', case_insensitive: true },
+            blocker: false,
+        },
+    ],
+    success_when: {
+        kind: 'reply_matches_regex',
+        label: 'Forge engages with the non-dev intent at least once',
+        payload: { regex: '(canci[oó]n|cancion|poema|hola|saludo|m[uú]sica|verso|haiku)', flags: 'i' },
+    },
+};
+const PROFILES = [
+    GREENFIELD_SPEC_WRITER,
+    BROWNFIELD_BUG_FIXER,
+    CASUAL_OUT_OF_SCOPE,
+];
+const PROFILE_BY_SLUG = new Map(PROFILES.map((p) => [p.slug, p]));
+export function listProfiles() {
+    return PROFILES;
+}
+export function getProfile(slug) {
+    return PROFILE_BY_SLUG.get(slug) ?? null;
+}
+//# sourceMappingURL=profiles.js.map

package/dist/test_agent/profiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../../src/test_agent/profiles.ts"],"names":[],"mappings":"AAqBA,MAAM,sBAAsB,GAAY;IACtC,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,wBAAwB;IAC9B,OAAO,EACL,gEAAgE;UAC9D,iEAAiE;UACjE,iEAAiE;UACjE,uBAAuB;IAC3B,IAAI,EACF,4DAA4D;UAC1D,mEAAmE;UACnE,wDAAwD;IAC5D,WAAW,EAAE;QACX,yBAAyB;QACzB,+CAA+C;QAC/C,qEAAqE;QACrE,uEAAuE;cACnE,sDAAsD;KAC3D;IACD,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE;QACV;YACE,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,oDAAoD;YAC3D,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,IAAI,EAAE;SACvD;QACD;YACE,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,2CAA2C;YAClD,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;SACpB;KACF;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,qBAAqB;QAC3B,KAAK,EAAE,yDAAyD;QAChE,OAAO,EAAE;YACP,KAAK,EAAE,gHAAgH;YACvH,KAAK,EAAE,GAAG;SACX;KACF;CACF,CAAC;AAEF,MAAM,oBAAoB,GAAY;IACpC,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,sBAAsB;IAC5B,OAAO,EACL,+DAA+D;UAC7D,4DAA4D;UAC5D,8DAA8D;IAClE,IAAI,EACF,+DAA+D;UAC7D,+DAA+D;UAC/D,+BAA+B;IACnC,WAAW,EAAE;QACX,6BAA6B;QAC7B,qEAAqE;QACrE,8DAA8D;KAC/D;IACD,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE;QACV;YACE,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,oDAAoD;YAC3D,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;YACnB,OAAO,EAAE,KAAK;SACf;KACF;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,cAAc;QACpB,KAAK,EAAE,4CAA4C;QACnD,OAAO,EAAE,EAAE,OAAO,EAAE,oBAAoB,EAAE;KAC3C;CACF,CAAC;AAEF,MAAM,mBAAmB,GAAY;IACnC,IAAI,EAAE,qBAAqB;IAC3B,IAAI,EAAE,oCAAoC;IAC1C,OAAO,EACL,gEAAgE;UAC9D,+DAA+D;UAC/D,6CAA6C;IACjD,IAAI,EACF,8DAA8D;UAC5D,6DAA6D;UAC7D,8BAA8B;IAClC,WAAW,EAAE;QACX,6DAA6D;QAC7D,6DAA6D;QAC7D,mEAAmE;cAC/D,6BAA6B;KAClC;IACD,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE;QACV;YACE,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,2CAA2C;YAClD,OAAO,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,gBAAgB,EAAE,IAAI,EAAE;YACvD,OAAO,EAAE,KAAK;SACf;QACD;YACE,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,kDAAkD;YACzD,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,IAAI,EAAE;YACtD,OAAO,EAAE,KAAK;SACf;KACF;IACD,YAAY,EAAE;QACZ,IAAI,EAAE,qBAAqB;QAC3B,KAAK,EAAE,qDAAqD;QAC5D,OAAO,EAAE,EAAE,KAAK,EAAE,8DAA8D,EAAE,KAAK,EAAE,GAAG,EAAE;KAC/F;CACF,CAAC;AAEF,MAAM,QAAQ,GAAc;IAC1B,sBAAsB;IACtB,oBAAoB;IACpB,mBAAmB;CACpB,CAAC;AAEF,MAAM,eAAe,GAAyB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAExF,MAAM,UAAU,YAAY;IAC1B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AAC3C,CAAC"}

package/dist/test_agent/runner.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Agente de Test multi-turn runner (PND-024 Etapa 1).
+ *
+ * Drives a synthetic conversation between a Profile-driven user
+ * (Gemini Flash Lite by default, anything ChatLLMClient-compatible
+ * via dependency injection) and a target Forge chat endpoint
+ * (any HTTP-reachable Forge `/api/chat`).
+ *
+ * The runner is profile-agnostic -- it loads the assertion + success
+ * criterion logic from src/test_agent/types.ts and walks the profile
+ * through max_turns iterations, stopping early when success_when
+ * matches or a blocker assertion fails.
+ *
+ * ASCII-only.
+ */
+import type { Profile, ToolRoundLite, ProfileRunReport } from './types.js';
+export interface SyntheticUserClient {
+    /** Compose the next user message given the conversation so far +
+     *  the profile guidance. Implementations call Gemini / Anthropic /
+     *  whatever to actually generate the text. */
+    nextTurn(opts: {
+        profile: Profile;
+        history: Array<{
+            role: 'user' | 'assistant';
+            content: string;
+        }>;
+        turn_index: number;
+    }): Promise<{
+        text: string;
+    }>;
+}
+export interface TargetChatClient {
+    /** POST to the target Forge's /api/chat endpoint (or any
+     *  surface that takes the conversation history + returns a
+     *  reply + tool rounds). */
+    chat(opts: {
+        messages: Array<{
+            role: 'user' | 'assistant';
+            content: string;
+        }>;
+    }): Promise<{
+        reply: string;
+        tool_rounds: ToolRoundLite[];
+    }>;
+}
+export interface RunProfileOpts {
+    profile: Profile;
+    synthetic: SyntheticUserClient;
+    target: TargetChatClient;
+    /** Optional cap override on profile.max_turns. */
+    max_turns?: number;
+    /** Optional clock injection (tests). Defaults to Date.now. */
+    now?: () => number;
+}
+export declare function runProfile(opts: RunProfileOpts): Promise<ProfileRunReport>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/test_agent/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/test_agent/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,KAAK,EACV,OAAO,EAA0C,aAAa,EAC9D,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,mBAAmB;IAClC;;kDAE8C;IAC9C,QAAQ,CAAC,IAAI,EAAE;QACb,OAAO,EAAM,OAAO,CAAC;QACrB,OAAO,EAAM,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACpE,UAAU,EAAG,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B;;gCAE4B;IAC5B,IAAI,CAAC,IAAI,EAAE;QAAE,QAAQ,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;QACxF,KAAK,EAAQ,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,EAAE,CAAC;KAC9B,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAS,OAAO,CAAC;IACxB,SAAS,EAAO,mBAAmB,CAAC;IACpC,MAAM,EAAU,gBAAgB,CAAC;IACjC,kDAAkD;IAClD,SAAS,CAAC,EAAM,MAAM,CAAC;IACvB,8DAA8D;IAC9D,GAAG,CAAC,EAAY,MAAM,MAAM,CAAC;CAC9B;AA8DD,wBAAsB,UAAU,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA8EhF"}

package/dist/test_agent/runner.js

@@ -0,0 +1,134 @@
+const DEFAULT_MAX_TURNS = 8;
+function evaluateAssertion(a, ctx, index) {
+    switch (a.kind) {
+        case 'reply_contains': {
+            const text = String(a.payload['text'] ?? '');
+            const ci = a.payload['case_insensitive'] === true;
+            const needle = ci ? text.toLowerCase() : text;
+            const hay = ci ? ctx.reply.toLowerCase() : ctx.reply;
+            const passed = hay.includes(needle);
+            return { index, kind: a.kind, label: a.label, passed,
+                ...(passed ? {} : { detail: 'reply missing "' + text + '"' }) };
+        }
+        case 'reply_not_contains': {
+            const text = String(a.payload['text'] ?? '');
+            const ci = a.payload['case_insensitive'] === true;
+            const needle = ci ? text.toLowerCase() : text;
+            const hay = ci ? ctx.reply.toLowerCase() : ctx.reply;
+            const passed = !hay.includes(needle);
+            return { index, kind: a.kind, label: a.label, passed,
+                ...(passed ? {} : { detail: 'reply contains "' + text + '"' }) };
+        }
+        case 'reply_matches_regex': {
+            const regex = String(a.payload['regex'] ?? '');
+            const flags = String(a.payload['flags'] ?? '');
+            let re;
+            try {
+                re = new RegExp(regex, flags);
+            }
+            catch {
+                return { index, kind: a.kind, label: a.label, passed: false,
+                    detail: 'invalid regex ' + JSON.stringify(regex) };
+            }
+            const passed = re.test(ctx.reply);
+            return { index, kind: a.kind, label: a.label, passed,
+                ...(passed ? {} : { detail: 'reply did not match ' + JSON.stringify(regex) }) };
+        }
+        case 'tool_invoked': {
+            const verb = String(a.payload['verb_id'] ?? '');
+            const passed = ctx.tool_rounds.some((r) => r.verb_id === verb || r.tool === verb);
+            return { index, kind: a.kind, label: a.label, passed,
+                ...(passed ? {} : { detail: 'verb ' + verb + ' not invoked' }) };
+        }
+        case 'tool_not_invoked': {
+            const verb = String(a.payload['verb_id'] ?? '');
+            const passed = !ctx.tool_rounds.some((r) => r.verb_id === verb || r.tool === verb);
+            return { index, kind: a.kind, label: a.label, passed,
+                ...(passed ? {} : { detail: 'verb ' + verb + ' invoked' }) };
+        }
+        case 'turn_count_at_most': {
+            const max = Number(a.payload['max'] ?? Number.MAX_SAFE_INTEGER);
+            const passed = ctx.turn <= max;
+            return { index, kind: a.kind, label: a.label, passed,
+                ...(passed ? {} : { detail: 'turn ' + ctx.turn + ' > max ' + max }) };
+        }
+    }
+}
+export async function runProfile(opts) {
+    const profile = opts.profile;
+    const max = opts.max_turns ?? profile.max_turns ?? DEFAULT_MAX_TURNS;
+    const now = opts.now ?? Date.now;
+    const startedMs = now();
+    const startedAt = new Date(startedMs).toISOString();
+    const history = [];
+    const turns = [];
+    let successMet = false;
+    let stoppedByBlocker = false;
+    for (let i = 1; i <= max; i++) {
+        const turnStart = now();
+        const userTurn = await opts.synthetic.nextTurn({
+            profile, history: history.slice(), turn_index: i,
+        });
+        const userMessage = userTurn.text.trim();
+        history.push({ role: 'user', content: userMessage });
+        const targetReply = await opts.target.chat({ messages: history.slice() });
+        history.push({ role: 'assistant', content: targetReply.reply });
+        const evalCtx = {
+            reply: targetReply.reply,
+            tool_rounds: targetReply.tool_rounds,
+            turn: i,
+        };
+        const assertion_results = profile.assertions.map((a, idx) => evaluateAssertion(a, evalCtx, idx));
+        let recordedSuccess = false;
+        if (profile.success_when) {
+            const r = evaluateAssertion(profile.success_when, evalCtx, -1);
+            assertion_results.push(r);
+            if (r.passed) {
+                successMet = true;
+                recordedSuccess = true;
+            }
+        }
+        void recordedSuccess;
+        const turnEnd = now();
+        turns.push({
+            turn: i,
+            user_message: userMessage,
+            assistant_reply: targetReply.reply,
+            tool_rounds: targetReply.tool_rounds,
+            latency_ms: turnEnd - turnStart,
+            assertion_results,
+        });
+        if (successMet)
+            break;
+        const blockerFailed = assertion_results.some((r) => !r.passed && profile.assertions[r.index]?.blocker === true);
+        if (blockerFailed) {
+            stoppedByBlocker = true;
+            break;
+        }
+    }
+    const endedMs = now();
+    const allResults = turns.flatMap((t) => t.assertion_results);
+    const passes = allResults.filter((r) => r.passed).length;
+    const failures = allResults.filter((r) => !r.passed).length;
+    let status;
+    if (successMet)
+        status = 'passed';
+    else if (stoppedByBlocker || failures > 0)
+        status = 'failed';
+    else
+        status = 'incomplete';
+    return {
+        profile_slug: profile.slug,
+        profile_name: profile.name,
+        status,
+        turns_used: turns.length,
+        success_criterion_met: successMet,
+        failures,
+        passes,
+        duration_ms: endedMs - startedMs,
+        turns,
+        started_at: startedAt,
+        ended_at: new Date(endedMs).toISOString(),
+    };
+}
+//# sourceMappingURL=runner.js.map