@caupulican/pi-adaptative 0.81.27 → 0.81.30

package/dist/core/tool-recovery-stats.js

@@ -0,0 +1,206 @@
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, } from "node:fs";
+import { dirname, join } from "node:path";
+import { isToolArgumentValidationLogRecord, } from "./tool-recovery-log-records.js";
+const MAX_DETAIL_KEYS = 1_000;
+const MAX_STATS_FILES = 1_000;
+const MAX_STATS_FILE_AGE_MS = 90 * 24 * 60 * 60 * 1000;
+const OTHER_DETAIL_KEY = "__other__";
+export function createEmptyToolArgumentValidationStats() {
+    return {
+        clean: 0,
+        repaired: 0,
+        bounced: 0,
+        failureModes: {},
+        repairsApplied: {},
+        taught: { none: 0, note: 0, rule: 0 },
+        executionOutcome: { not_run: 0, succeeded: 0, failed: 0 },
+        teachEfficacy: {},
+    };
+}
+function boundedDetailKey(record, key) {
+    if (record[key] !== undefined || Object.keys(record).length < MAX_DETAIL_KEYS - 1)
+        return key;
+    return OTHER_DETAIL_KEY;
+}
+export function consumeToolArgumentValidationRecord(stats, record) {
+    stats[record.outcome] += 1;
+    const taught = record.taught ?? "none";
+    const executionOutcome = record.executionOutcome ?? "not_run";
+    stats.taught[taught] += 1;
+    stats.executionOutcome[executionOutcome] += 1;
+    const modes = new Set([...record.failureModes, ...record.repairsApplied]);
+    for (const mode of record.failureModes) {
+        const key = boundedDetailKey(stats.failureModes, mode);
+        stats.failureModes[key] = (stats.failureModes[key] ?? 0) + 1;
+    }
+    for (const repair of record.repairsApplied) {
+        const key = boundedDetailKey(stats.repairsApplied, repair);
+        stats.repairsApplied[key] = (stats.repairsApplied[key] ?? 0) + 1;
+    }
+    for (const mode of modes) {
+        const key = boundedDetailKey(stats.teachEfficacy, `${record.provider ?? "unknown"}/${record.model ?? "unknown"}:${mode}`);
+        let efficacy = stats.teachEfficacy[key];
+        if (!efficacy) {
+            efficacy = {
+                recurrenceBefore: 0,
+                recurrenceAfter: 0,
+                repairedThenSucceeded: 0,
+                repairedThenFailed: 0,
+                repairedThenNotRun: 0,
+            };
+            stats.teachEfficacy[key] = efficacy;
+        }
+        if (taught === "none")
+            efficacy.recurrenceBefore++;
+        else
+            efficacy.recurrenceAfter++;
+    }
+    if (record.outcome === "repaired") {
+        for (const repair of record.repairsApplied) {
+            const key = boundedDetailKey(stats.teachEfficacy, `${record.provider ?? "unknown"}/${record.model ?? "unknown"}:${repair}`);
+            let efficacy = stats.teachEfficacy[key];
+            if (!efficacy) {
+                efficacy = {
+                    recurrenceBefore: 0,
+                    recurrenceAfter: 0,
+                    repairedThenSucceeded: 0,
+                    repairedThenFailed: 0,
+                    repairedThenNotRun: 0,
+                };
+                stats.teachEfficacy[key] = efficacy;
+            }
+            if (executionOutcome === "succeeded")
+                efficacy.repairedThenSucceeded++;
+            if (executionOutcome === "failed")
+                efficacy.repairedThenFailed++;
+            if (executionOutcome === "not_run")
+                efficacy.repairedThenNotRun++;
+        }
+    }
+}
+export function getToolRecoveryRecordSequence(record) {
+    const prefix = `${record.sessionId}:`;
+    if (!record.recordId.startsWith(prefix))
+        return -1;
+    const sequence = Number(record.recordId.slice(prefix.length));
+    return Number.isSafeInteger(sequence) && sequence >= 0 ? sequence : -1;
+}
+function statsPath(eventLogPath, sessionId) {
+    const id = createHash("sha256").update(sessionId).digest("hex").slice(0, 24);
+    return join(dirname(eventLogPath), "tool-recovery-stats", `${id}.json`);
+}
+function isFiniteCount(value) {
+    return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+function isCountRecord(value) {
+    return !!value && typeof value === "object" && Object.values(value).every(isFiniteCount);
+}
+function isStatsShape(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const stats = value;
+    if (!isFiniteCount(stats.clean) || !isFiniteCount(stats.repaired) || !isFiniteCount(stats.bounced))
+        return false;
+    if (!isCountRecord(stats.failureModes) || !isCountRecord(stats.repairsApplied))
+        return false;
+    if (!isCountRecord(stats.taught) || !isCountRecord(stats.executionOutcome))
+        return false;
+    if (!stats.teachEfficacy || typeof stats.teachEfficacy !== "object")
+        return false;
+    return Object.values(stats.teachEfficacy).every((value) => !!value &&
+        isFiniteCount(value.recurrenceBefore) &&
+        isFiniteCount(value.recurrenceAfter) &&
+        isFiniteCount(value.repairedThenSucceeded) &&
+        isFiniteCount(value.repairedThenFailed) &&
+        isFiniteCount(value.repairedThenNotRun));
+}
+function isPersistedStats(value, sessionId) {
+    if (!value || typeof value !== "object")
+        return false;
+    const record = value;
+    return (record.version === 1 &&
+        record.sessionId === sessionId &&
+        isFiniteCount(record.lastRecordSequence) &&
+        isStatsShape(record.stats));
+}
+export function readPersistedToolRecoveryStats(eventLogPath, sessionId) {
+    const path = statsPath(eventLogPath, sessionId);
+    if (!existsSync(path))
+        return undefined;
+    try {
+        const parsed = JSON.parse(readFileSync(path, "utf-8"));
+        return isPersistedStats(parsed, sessionId) ? parsed : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function pruneStatsFiles(directory, currentPath) {
+    const now = Date.now();
+    const files = readdirSync(directory)
+        .filter((name) => name.endsWith(".json"))
+        .map((name) => {
+        const path = join(directory, name);
+        return { path, mtimeMs: statSync(path).mtimeMs };
+    })
+        .sort((left, right) => left.mtimeMs - right.mtimeMs);
+    let retained = files.length;
+    for (const file of files) {
+        if (file.path === currentPath)
+            continue;
+        if (retained <= MAX_STATS_FILES && now - file.mtimeMs <= MAX_STATS_FILE_AGE_MS)
+            continue;
+        try {
+            unlinkSync(file.path);
+            retained--;
+        }
+        catch { }
+    }
+}
+function writePersistedToolRecoveryStats(eventLogPath, summary) {
+    const path = statsPath(eventLogPath, summary.sessionId);
+    const directory = dirname(path);
+    mkdirSync(directory, { recursive: true });
+    const isNewSummary = !existsSync(path);
+    const temporaryPath = `${path}.${process.pid}.tmp`;
+    writeFileSync(temporaryPath, JSON.stringify(summary), "utf-8");
+    renameSync(temporaryPath, path);
+    if (isNewSummary)
+        pruneStatsFiles(directory, path);
+}
+export function updatePersistedToolRecoveryStats(eventLogPath, record) {
+    let summary = readPersistedToolRecoveryStats(eventLogPath, record.sessionId);
+    if (!summary) {
+        summary = {
+            version: 1,
+            sessionId: record.sessionId,
+            lastRecordSequence: -1,
+            stats: createEmptyToolArgumentValidationStats(),
+        };
+        try {
+            for (const line of readFileSync(eventLogPath, "utf-8").split("\n")) {
+                if (line.trim().length === 0)
+                    continue;
+                const parsed = JSON.parse(line);
+                if (!isToolArgumentValidationLogRecord(parsed) || parsed.sessionId !== record.sessionId)
+                    continue;
+                const sequence = getToolRecoveryRecordSequence(parsed);
+                if (sequence <= summary.lastRecordSequence)
+                    continue;
+                consumeToolArgumentValidationRecord(summary.stats, parsed);
+                summary.lastRecordSequence = sequence;
+            }
+        }
+        catch {
+            // Fall through and at least account for the current record.
+        }
+    }
+    const sequence = getToolRecoveryRecordSequence(record);
+    if (sequence > summary.lastRecordSequence) {
+        consumeToolArgumentValidationRecord(summary.stats, record);
+        summary.lastRecordSequence = sequence;
+    }
+    writePersistedToolRecoveryStats(eventLogPath, summary);
+}
+//# sourceMappingURL=tool-recovery-stats.js.map

package/dist/core/tool-recovery-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-recovery-stats.js","sourceRoot":"","sources":["../../src/core/tool-recovery-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACN,UAAU,EACV,SAAS,EACT,WAAW,EACX,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,EACV,aAAa,GACb,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EACN,iCAAiC,GAEjC,MAAM,gCAAgC,CAAC;AAExC,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACvD,MAAM,gBAAgB,GAAG,WAAW,CAAC;AASrC,MAAM,UAAU,sCAAsC,GAAgC;IACrF,OAAO;QACN,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;QACV,YAAY,EAAE,EAAE;QAChB,cAAc,EAAE,EAAE;QAClB,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;QACrC,gBAAgB,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACzD,aAAa,EAAE,EAAE;KACjB,CAAC;AAAA,CACF;AAED,SAAS,gBAAgB,CAAC,MAA+B,EAAE,GAAW,EAAU;IAC/E,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,eAAe,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IAC9F,OAAO,gBAAgB,CAAC;AAAA,CACxB;AAED,MAAM,UAAU,mCAAmC,CAClD,KAAkC,EAClC,MAAoC,EAC7B;IACP,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC;IACvC,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,SAAS,CAAC;IAC9D,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1B,KAAK,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,YAAY,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;IAC1E,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QACvD,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9D,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAC3D,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAClE,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,gBAAgB,CAC3B,KAAK,CAAC,aAAa,EACnB,GAAG,MAAM,CAAC,QAAQ,IAAI,SAAS,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,IAAI,IAAI,EAAE,CACtE,CAAC;QACF,IAAI,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,QAAQ,GAAG;gBACV,gBAAgB,EAAE,CAAC;gBACnB,eAAe,EAAE,CAAC;gBAClB,qBAAqB,EAAE,CAAC;gBACxB,kBAAkB,EAAE,CAAC;gBACrB,kBAAkB,EAAE,CAAC;aACrB,CAAC;YACF,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;QACrC,CAAC;QACD,IAAI,MAAM,KAAK,MAAM;YAAE,QAAQ,CAAC,gBAAgB,EAAE,CAAC;;YAC9C,QAAQ,CAAC,eAAe,EAAE,CAAC;IACjC,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QACnC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,gBAAgB,CAC3B,KAAK,CAAC,aAAa,EACnB,GAAG,MAAM,CAAC,QAAQ,IAAI,SAAS,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,IAAI,MAAM,EAAE,CACxE,CAAC;YACF,IAAI,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,QAAQ,GAAG;oBACV,gBAAgB,EAAE,CAAC;oBACnB,eAAe,EAAE,CAAC;oBAClB,qBAAqB,EAAE,CAAC;oBACxB,kBAAkB,EAAE,CAAC;oBACrB,kBAAkB,EAAE,CAAC;iBACrB,CAAC;gBACF,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;YACrC,CAAC;YACD,IAAI,gBAAgB,KAAK,WAAW;gBAAE,QAAQ,CAAC,qBAAqB,EAAE,CAAC;YACvE,IAAI,gBAAgB,KAAK,QAAQ;gBAAE,QAAQ,CAAC,kBAAkB,EAAE,CAAC;YACjE,IAAI,gBAAgB,KAAK,SAAS;gBAAE,QAAQ,CAAC,kBAAkB,EAAE,CAAC;QACnE,CAAC;IACF,CAAC;AAAA,CACD;AAED,MAAM,UAAU,6BAA6B,CAC5C,MAAuE,EAC9D;IACT,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,GAAG,CAAC;IACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC9D,OAAO,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAAA,CACvE;AAED,SAAS,SAAS,CAAC,YAAoB,EAAE,SAAiB,EAAU;IACnE,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,qBAAqB,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAAA,CACxE;AAED,SAAS,aAAa,CAAC,KAAc,EAAmB;IACvD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AAAA,CACzE;AAED,SAAS,aAAa,CAAC,KAAc,EAAmC;IACvE,OAAO,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;AAAA,CACzF;AAED,SAAS,YAAY,CAAC,KAAc,EAAwC;IAC3E,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,KAAK,GAAG,KAA6C,CAAC;IAC5D,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACjH,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7F,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,gBAAgB,CAAC;QAAE,OAAO,KAAK,CAAC;IACzF,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClF,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,KAAK,CAC9C,CAAC,KAAK,EAAE,EAAE,CACT,CAAC,CAAC,KAAK;QACP,aAAa,CAAC,KAAK,CAAC,gBAAgB,CAAC;QACrC,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC;QACpC,aAAa,CAAC,KAAK,CAAC,qBAAqB,CAAC;QAC1C,aAAa,CAAC,KAAK,CAAC,kBAAkB,CAAC;QACvC,aAAa,CAAC,KAAK,CAAC,kBAAkB,CAAC,CACxC,CAAC;AAAA,CACF;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,SAAiB,EAAuC;IACjG,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,MAAM,GAAG,KAA4C,CAAC;IAC5D,OAAO,CACN,MAAM,CAAC,OAAO,KAAK,CAAC;QACpB,MAAM,CAAC,SAAS,KAAK,SAAS;QAC9B,aAAa,CAAC,MAAM,CAAC,kBAAkB,CAAC;QACxC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAC1B,CAAC;AAAA,CACF;AAED,MAAM,UAAU,8BAA8B,CAC7C,YAAoB,EACpB,SAAiB,EACwB;IACzC,MAAM,IAAI,GAAG,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QAChE,OAAO,gBAAgB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,SAAS,eAAe,CAAC,SAAiB,EAAE,WAAmB,EAAQ;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC;SAClC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;IAAA,CACjD,CAAC;SACD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;IACtD,IAAI,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;YAAE,SAAS;QACxC,IAAI,QAAQ,IAAI,eAAe,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,qBAAqB;YAAE,SAAS;QACzF,IAAI,CAAC;YACJ,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,QAAQ,EAAE,CAAC;QACZ,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACX,CAAC;AAAA,CACD;AAED,SAAS,+BAA+B,CAAC,YAAoB,EAAE,OAAmC,EAAQ;IACzG,MAAM,IAAI,GAAG,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;IACnD,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IAC/D,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IAChC,IAAI,YAAY;QAAE,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAAA,CACnD;AAED,MAAM,UAAU,gCAAgC,CAAC,YAAoB,EAAE,MAAuC,EAAQ;IACrH,IAAI,OAAO,GAAG,8BAA8B,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG;YACT,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,kBAAkB,EAAE,CAAC,CAAC;YACtB,KAAK,EAAE,sCAAsC,EAAE;SAC/C,CAAC;QACF,IAAI,CAAC;YACJ,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBACvC,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,iCAAiC,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS;oBAAE,SAAS;gBAClG,MAAM,QAAQ,GAAG,6BAA6B,CAAC,MAAM,CAAC,CAAC;gBACvD,IAAI,QAAQ,IAAI,OAAO,CAAC,kBAAkB;oBAAE,SAAS;gBACrD,mCAAmC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC3D,OAAO,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACvC,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,4DAA4D;QAC7D,CAAC;IACF,CAAC;IACD,MAAM,QAAQ,GAAG,6BAA6B,CAAC,MAAM,CAAC,CAAC;IACvD,IAAI,QAAQ,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC3C,mCAAmC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IACD,+BAA+B,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;AAAA,CACvD","sourcesContent":["import { createHash } from \"node:crypto\";\nimport {\n\texistsSync,\n\tmkdirSync,\n\treaddirSync,\n\treadFileSync,\n\trenameSync,\n\tstatSync,\n\tunlinkSync,\n\twriteFileSync,\n} from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport type { ToolArgumentValidationRecord, ToolArgumentValidationStats } from \"./session-analytics.ts\";\nimport {\n\tisToolArgumentValidationLogRecord,\n\ttype ToolArgumentValidationLogRecord,\n} from \"./tool-recovery-log-records.ts\";\n\nconst MAX_DETAIL_KEYS = 1_000;\nconst MAX_STATS_FILES = 1_000;\nconst MAX_STATS_FILE_AGE_MS = 90 * 24 * 60 * 60 * 1000;\nconst OTHER_DETAIL_KEY = \"__other__\";\n\ninterface PersistedToolRecoveryStats {\n\tversion: 1;\n\tsessionId: string;\n\tlastRecordSequence: number;\n\tstats: ToolArgumentValidationStats;\n}\n\nexport function createEmptyToolArgumentValidationStats(): ToolArgumentValidationStats {\n\treturn {\n\t\tclean: 0,\n\t\trepaired: 0,\n\t\tbounced: 0,\n\t\tfailureModes: {},\n\t\trepairsApplied: {},\n\t\ttaught: { none: 0, note: 0, rule: 0 },\n\t\texecutionOutcome: { not_run: 0, succeeded: 0, failed: 0 },\n\t\tteachEfficacy: {},\n\t};\n}\n\nfunction boundedDetailKey(record: Record<string, unknown>, key: string): string {\n\tif (record[key] !== undefined || Object.keys(record).length < MAX_DETAIL_KEYS - 1) return key;\n\treturn OTHER_DETAIL_KEY;\n}\n\nexport function consumeToolArgumentValidationRecord(\n\tstats: ToolArgumentValidationStats,\n\trecord: ToolArgumentValidationRecord,\n): void {\n\tstats[record.outcome] += 1;\n\tconst taught = record.taught ?? \"none\";\n\tconst executionOutcome = record.executionOutcome ?? \"not_run\";\n\tstats.taught[taught] += 1;\n\tstats.executionOutcome[executionOutcome] += 1;\n\tconst modes = new Set([...record.failureModes, ...record.repairsApplied]);\n\tfor (const mode of record.failureModes) {\n\t\tconst key = boundedDetailKey(stats.failureModes, mode);\n\t\tstats.failureModes[key] = (stats.failureModes[key] ?? 0) + 1;\n\t}\n\tfor (const repair of record.repairsApplied) {\n\t\tconst key = boundedDetailKey(stats.repairsApplied, repair);\n\t\tstats.repairsApplied[key] = (stats.repairsApplied[key] ?? 0) + 1;\n\t}\n\tfor (const mode of modes) {\n\t\tconst key = boundedDetailKey(\n\t\t\tstats.teachEfficacy,\n\t\t\t`${record.provider ?? \"unknown\"}/${record.model ?? \"unknown\"}:${mode}`,\n\t\t);\n\t\tlet efficacy = stats.teachEfficacy[key];\n\t\tif (!efficacy) {\n\t\t\tefficacy = {\n\t\t\t\trecurrenceBefore: 0,\n\t\t\t\trecurrenceAfter: 0,\n\t\t\t\trepairedThenSucceeded: 0,\n\t\t\t\trepairedThenFailed: 0,\n\t\t\t\trepairedThenNotRun: 0,\n\t\t\t};\n\t\t\tstats.teachEfficacy[key] = efficacy;\n\t\t}\n\t\tif (taught === \"none\") efficacy.recurrenceBefore++;\n\t\telse efficacy.recurrenceAfter++;\n\t}\n\tif (record.outcome === \"repaired\") {\n\t\tfor (const repair of record.repairsApplied) {\n\t\t\tconst key = boundedDetailKey(\n\t\t\t\tstats.teachEfficacy,\n\t\t\t\t`${record.provider ?? \"unknown\"}/${record.model ?? \"unknown\"}:${repair}`,\n\t\t\t);\n\t\t\tlet efficacy = stats.teachEfficacy[key];\n\t\t\tif (!efficacy) {\n\t\t\t\tefficacy = {\n\t\t\t\t\trecurrenceBefore: 0,\n\t\t\t\t\trecurrenceAfter: 0,\n\t\t\t\t\trepairedThenSucceeded: 0,\n\t\t\t\t\trepairedThenFailed: 0,\n\t\t\t\t\trepairedThenNotRun: 0,\n\t\t\t\t};\n\t\t\t\tstats.teachEfficacy[key] = efficacy;\n\t\t\t}\n\t\t\tif (executionOutcome === \"succeeded\") efficacy.repairedThenSucceeded++;\n\t\t\tif (executionOutcome === \"failed\") efficacy.repairedThenFailed++;\n\t\t\tif (executionOutcome === \"not_run\") efficacy.repairedThenNotRun++;\n\t\t}\n\t}\n}\n\nexport function getToolRecoveryRecordSequence(\n\trecord: Pick<ToolArgumentValidationLogRecord, \"recordId\" | \"sessionId\">,\n): number {\n\tconst prefix = `${record.sessionId}:`;\n\tif (!record.recordId.startsWith(prefix)) return -1;\n\tconst sequence = Number(record.recordId.slice(prefix.length));\n\treturn Number.isSafeInteger(sequence) && sequence >= 0 ? sequence : -1;\n}\n\nfunction statsPath(eventLogPath: string, sessionId: string): string {\n\tconst id = createHash(\"sha256\").update(sessionId).digest(\"hex\").slice(0, 24);\n\treturn join(dirname(eventLogPath), \"tool-recovery-stats\", `${id}.json`);\n}\n\nfunction isFiniteCount(value: unknown): value is number {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value >= 0;\n}\n\nfunction isCountRecord(value: unknown): value is Record<string, number> {\n\treturn !!value && typeof value === \"object\" && Object.values(value).every(isFiniteCount);\n}\n\nfunction isStatsShape(value: unknown): value is ToolArgumentValidationStats {\n\tif (!value || typeof value !== \"object\") return false;\n\tconst stats = value as Partial<ToolArgumentValidationStats>;\n\tif (!isFiniteCount(stats.clean) || !isFiniteCount(stats.repaired) || !isFiniteCount(stats.bounced)) return false;\n\tif (!isCountRecord(stats.failureModes) || !isCountRecord(stats.repairsApplied)) return false;\n\tif (!isCountRecord(stats.taught) || !isCountRecord(stats.executionOutcome)) return false;\n\tif (!stats.teachEfficacy || typeof stats.teachEfficacy !== \"object\") return false;\n\treturn Object.values(stats.teachEfficacy).every(\n\t\t(value) =>\n\t\t\t!!value &&\n\t\t\tisFiniteCount(value.recurrenceBefore) &&\n\t\t\tisFiniteCount(value.recurrenceAfter) &&\n\t\t\tisFiniteCount(value.repairedThenSucceeded) &&\n\t\t\tisFiniteCount(value.repairedThenFailed) &&\n\t\t\tisFiniteCount(value.repairedThenNotRun),\n\t);\n}\n\nfunction isPersistedStats(value: unknown, sessionId: string): value is PersistedToolRecoveryStats {\n\tif (!value || typeof value !== \"object\") return false;\n\tconst record = value as Partial<PersistedToolRecoveryStats>;\n\treturn (\n\t\trecord.version === 1 &&\n\t\trecord.sessionId === sessionId &&\n\t\tisFiniteCount(record.lastRecordSequence) &&\n\t\tisStatsShape(record.stats)\n\t);\n}\n\nexport function readPersistedToolRecoveryStats(\n\teventLogPath: string,\n\tsessionId: string,\n): PersistedToolRecoveryStats | undefined {\n\tconst path = statsPath(eventLogPath, sessionId);\n\tif (!existsSync(path)) return undefined;\n\ttry {\n\t\tconst parsed: unknown = JSON.parse(readFileSync(path, \"utf-8\"));\n\t\treturn isPersistedStats(parsed, sessionId) ? parsed : undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nfunction pruneStatsFiles(directory: string, currentPath: string): void {\n\tconst now = Date.now();\n\tconst files = readdirSync(directory)\n\t\t.filter((name) => name.endsWith(\".json\"))\n\t\t.map((name) => {\n\t\t\tconst path = join(directory, name);\n\t\t\treturn { path, mtimeMs: statSync(path).mtimeMs };\n\t\t})\n\t\t.sort((left, right) => left.mtimeMs - right.mtimeMs);\n\tlet retained = files.length;\n\tfor (const file of files) {\n\t\tif (file.path === currentPath) continue;\n\t\tif (retained <= MAX_STATS_FILES && now - file.mtimeMs <= MAX_STATS_FILE_AGE_MS) continue;\n\t\ttry {\n\t\t\tunlinkSync(file.path);\n\t\t\tretained--;\n\t\t} catch {}\n\t}\n}\n\nfunction writePersistedToolRecoveryStats(eventLogPath: string, summary: PersistedToolRecoveryStats): void {\n\tconst path = statsPath(eventLogPath, summary.sessionId);\n\tconst directory = dirname(path);\n\tmkdirSync(directory, { recursive: true });\n\tconst isNewSummary = !existsSync(path);\n\tconst temporaryPath = `${path}.${process.pid}.tmp`;\n\twriteFileSync(temporaryPath, JSON.stringify(summary), \"utf-8\");\n\trenameSync(temporaryPath, path);\n\tif (isNewSummary) pruneStatsFiles(directory, path);\n}\n\nexport function updatePersistedToolRecoveryStats(eventLogPath: string, record: ToolArgumentValidationLogRecord): void {\n\tlet summary = readPersistedToolRecoveryStats(eventLogPath, record.sessionId);\n\tif (!summary) {\n\t\tsummary = {\n\t\t\tversion: 1,\n\t\t\tsessionId: record.sessionId,\n\t\t\tlastRecordSequence: -1,\n\t\t\tstats: createEmptyToolArgumentValidationStats(),\n\t\t};\n\t\ttry {\n\t\t\tfor (const line of readFileSync(eventLogPath, \"utf-8\").split(\"\\n\")) {\n\t\t\t\tif (line.trim().length === 0) continue;\n\t\t\t\tconst parsed: unknown = JSON.parse(line);\n\t\t\t\tif (!isToolArgumentValidationLogRecord(parsed) || parsed.sessionId !== record.sessionId) continue;\n\t\t\t\tconst sequence = getToolRecoveryRecordSequence(parsed);\n\t\t\t\tif (sequence <= summary.lastRecordSequence) continue;\n\t\t\t\tconsumeToolArgumentValidationRecord(summary.stats, parsed);\n\t\t\t\tsummary.lastRecordSequence = sequence;\n\t\t\t}\n\t\t} catch {\n\t\t\t// Fall through and at least account for the current record.\n\t\t}\n\t}\n\tconst sequence = getToolRecoveryRecordSequence(record);\n\tif (sequence > summary.lastRecordSequence) {\n\t\tconsumeToolArgumentValidationRecord(summary.stats, record);\n\t\tsummary.lastRecordSequence = sequence;\n\t}\n\twritePersistedToolRecoveryStats(eventLogPath, summary);\n}\n"]}

package/dist/core/tool-selection/expected-utility.d.ts

@@ -0,0 +1,51 @@
+export interface ExpectedUtilityWeights {
+    latency: number;
+    tokens: number;
+    risk: number;
+    context: number;
+    latencyScaleMs: number;
+    tokenScale: number;
+    minimumEvidence: number;
+    minimumMargin: number;
+    highEntropy: number;
+}
+export declare const DEFAULT_EXPECTED_UTILITY_WEIGHTS: ExpectedUtilityWeights;
+export interface ExpectedUtilityCandidate {
+    tool: string;
+    value: number;
+    alpha: number;
+    beta: number;
+    sampleCount: number;
+    latencyMs?: number;
+    tokenEstimate?: number;
+    riskCost?: number;
+    contextCost?: number;
+    deterministicMatch?: boolean;
+    /** Candidates with unresolved paths may be observed/shortlisted but never recommended. */
+    automaticEligible?: boolean;
+    /** `no_tool` uses a known probability rather than a learned posterior. */
+    successProbability?: number;
+}
+export interface RankedToolCandidate extends ExpectedUtilityCandidate {
+    successProbability: number;
+    latencyCost: number;
+    tokenCost: number;
+    riskCost: number;
+    contextCost: number;
+    utility: number;
+    probability: number;
+}
+export type ToolSelectionDisposition = "recommend" | "shortlist" | "abstain";
+export interface ToolSelectionDecision {
+    disposition: ToolSelectionDisposition;
+    recommendation?: string;
+    shortlist: string[];
+    entropy: number;
+    margin: number;
+    ranked: RankedToolCandidate[];
+}
+export declare function betaSuccessProbability(alpha: number, beta: number): number;
+export declare function normalizedEntropy(probabilities: readonly number[]): number;
+export declare function rankExpectedUtilityCandidates(candidates: readonly ExpectedUtilityCandidate[], weights?: ExpectedUtilityWeights): RankedToolCandidate[];
+export declare function decideExpectedUtility(candidates: readonly ExpectedUtilityCandidate[], weights?: ExpectedUtilityWeights): ToolSelectionDecision;
+//# sourceMappingURL=expected-utility.d.ts.map

package/dist/core/tool-selection/expected-utility.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"expected-utility.d.ts","sourceRoot":"","sources":["../../../src/core/tool-selection/expected-utility.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,sBAAsB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,gCAAgC,EAAE,sBAU9C,CAAC;AAEF,MAAM,WAAW,wBAAwB;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,0FAA0F;IAC1F,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,0EAA0E;IAC1E,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAoB,SAAQ,wBAAwB;IACpE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,wBAAwB,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC;AAE7E,MAAM,WAAW,qBAAqB;IACrC,WAAW,EAAE,wBAAwB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,mBAAmB,EAAE,CAAC;CAC9B;AAUD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAK1E;AAED,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAM1E;AAUD,wBAAgB,6BAA6B,CAC5C,UAAU,EAAE,SAAS,wBAAwB,EAAE,EAC/C,OAAO,GAAE,sBAAyD,GAChE,mBAAmB,EAAE,CAqBvB;AAED,wBAAgB,qBAAqB,CACpC,UAAU,EAAE,SAAS,wBAAwB,EAAE,EAC/C,OAAO,GAAE,sBAAyD,GAChE,qBAAqB,CAwBvB","sourcesContent":["export interface ExpectedUtilityWeights {\n\tlatency: number;\n\ttokens: number;\n\trisk: number;\n\tcontext: number;\n\tlatencyScaleMs: number;\n\ttokenScale: number;\n\tminimumEvidence: number;\n\tminimumMargin: number;\n\thighEntropy: number;\n}\n\nexport const DEFAULT_EXPECTED_UTILITY_WEIGHTS: ExpectedUtilityWeights = {\n\tlatency: 0.15,\n\ttokens: 0.1,\n\trisk: 0.2,\n\tcontext: 0.1,\n\tlatencyScaleMs: 5_000,\n\ttokenScale: 1_000,\n\tminimumEvidence: 3,\n\tminimumMargin: 0.1,\n\thighEntropy: 0.85,\n};\n\nexport interface ExpectedUtilityCandidate {\n\ttool: string;\n\tvalue: number;\n\talpha: number;\n\tbeta: number;\n\tsampleCount: number;\n\tlatencyMs?: number;\n\ttokenEstimate?: number;\n\triskCost?: number;\n\tcontextCost?: number;\n\tdeterministicMatch?: boolean;\n\t/** Candidates with unresolved paths may be observed/shortlisted but never recommended. */\n\tautomaticEligible?: boolean;\n\t/** `no_tool` uses a known probability rather than a learned posterior. */\n\tsuccessProbability?: number;\n}\n\nexport interface RankedToolCandidate extends ExpectedUtilityCandidate {\n\tsuccessProbability: number;\n\tlatencyCost: number;\n\ttokenCost: number;\n\triskCost: number;\n\tcontextCost: number;\n\tutility: number;\n\tprobability: number;\n}\n\nexport type ToolSelectionDisposition = \"recommend\" | \"shortlist\" | \"abstain\";\n\nexport interface ToolSelectionDecision {\n\tdisposition: ToolSelectionDisposition;\n\trecommendation?: string;\n\tshortlist: string[];\n\tentropy: number;\n\tmargin: number;\n\tranked: RankedToolCandidate[];\n}\n\nfunction clampUnit(value: number): number {\n\treturn Math.max(0, Math.min(1, value));\n}\n\nfunction finiteOr(value: number | undefined, fallback: number): number {\n\treturn typeof value === \"number\" && Number.isFinite(value) ? value : fallback;\n}\n\nexport function betaSuccessProbability(alpha: number, beta: number): number {\n\tconst positiveAlpha = Math.max(0, finiteOr(alpha, 0));\n\tconst positiveBeta = Math.max(0, finiteOr(beta, 0));\n\tconst total = positiveAlpha + positiveBeta;\n\treturn total > 0 ? positiveAlpha / total : 0.5;\n}\n\nexport function normalizedEntropy(probabilities: readonly number[]): number {\n\tif (probabilities.length <= 1) return 0;\n\tconst positive = probabilities.filter((probability) => probability > 0 && Number.isFinite(probability));\n\tif (positive.length === 0) return 0;\n\tconst entropy = -positive.reduce((total, probability) => total + probability * Math.log(probability), 0);\n\treturn entropy <= 0 ? 0 : entropy / Math.log(probabilities.length);\n}\n\nfunction softmax(utilities: readonly number[]): number[] {\n\tif (utilities.length === 0) return [];\n\tconst maximum = Math.max(...utilities);\n\tconst exponentials = utilities.map((utility) => Math.exp(Math.max(-60, utility - maximum)));\n\tconst total = exponentials.reduce((sum, value) => sum + value, 0);\n\treturn total > 0 ? exponentials.map((value) => value / total) : utilities.map(() => 1 / utilities.length);\n}\n\nexport function rankExpectedUtilityCandidates(\n\tcandidates: readonly ExpectedUtilityCandidate[],\n\tweights: ExpectedUtilityWeights = DEFAULT_EXPECTED_UTILITY_WEIGHTS,\n): RankedToolCandidate[] {\n\tconst unranked = candidates.map((candidate) => {\n\t\tconst successProbability = clampUnit(\n\t\t\tcandidate.successProbability ?? betaSuccessProbability(candidate.alpha, candidate.beta),\n\t\t);\n\t\tconst latencyCost = clampUnit(finiteOr(candidate.latencyMs, 0) / Math.max(1, weights.latencyScaleMs));\n\t\tconst tokenCost = clampUnit(finiteOr(candidate.tokenEstimate, 0) / Math.max(1, weights.tokenScale));\n\t\tconst riskCost = clampUnit(finiteOr(candidate.riskCost, 0));\n\t\tconst contextCost = clampUnit(finiteOr(candidate.contextCost, 0));\n\t\tconst utility =\n\t\t\tsuccessProbability * clampUnit(candidate.value) -\n\t\t\tweights.latency * latencyCost -\n\t\t\tweights.tokens * tokenCost -\n\t\t\tweights.risk * riskCost -\n\t\t\tweights.context * contextCost;\n\t\treturn { ...candidate, successProbability, latencyCost, tokenCost, riskCost, contextCost, utility };\n\t});\n\tconst probabilities = softmax(unranked.map((candidate) => candidate.utility));\n\treturn unranked\n\t\t.map((candidate, index) => ({ ...candidate, probability: probabilities[index] ?? 0 }))\n\t\t.sort((left, right) => right.utility - left.utility || left.tool.localeCompare(right.tool));\n}\n\nexport function decideExpectedUtility(\n\tcandidates: readonly ExpectedUtilityCandidate[],\n\tweights: ExpectedUtilityWeights = DEFAULT_EXPECTED_UTILITY_WEIGHTS,\n): ToolSelectionDecision {\n\tconst ranked = rankExpectedUtilityCandidates(candidates, weights);\n\tconst best = ranked[0];\n\tconst runnerUp = ranked[1];\n\tconst entropy = normalizedEntropy(ranked.map((candidate) => candidate.probability));\n\tconst margin = best && runnerUp ? best.utility - runnerUp.utility : (best?.utility ?? 0);\n\tconst shortlist = ranked.slice(0, 3).map((candidate) => candidate.tool);\n\tif (!best || best.utility <= 0) {\n\t\treturn { disposition: \"abstain\", shortlist: [], entropy, margin, ranked };\n\t}\n\tif (entropy >= weights.highEntropy || margin < weights.minimumMargin) {\n\t\treturn { disposition: \"shortlist\", shortlist, entropy, margin, ranked };\n\t}\n\tif (best.automaticEligible !== false && (best.deterministicMatch || best.sampleCount >= weights.minimumEvidence)) {\n\t\treturn {\n\t\t\tdisposition: \"recommend\",\n\t\t\trecommendation: best.tool,\n\t\t\tshortlist: [],\n\t\t\tentropy,\n\t\t\tmargin,\n\t\t\tranked,\n\t\t};\n\t}\n\treturn { disposition: \"abstain\", shortlist: [], entropy, margin, ranked };\n}\n"]}

package/dist/core/tool-selection/expected-utility.js

@@ -0,0 +1,85 @@
+export const DEFAULT_EXPECTED_UTILITY_WEIGHTS = {
+    latency: 0.15,
+    tokens: 0.1,
+    risk: 0.2,
+    context: 0.1,
+    latencyScaleMs: 5_000,
+    tokenScale: 1_000,
+    minimumEvidence: 3,
+    minimumMargin: 0.1,
+    highEntropy: 0.85,
+};
+function clampUnit(value) {
+    return Math.max(0, Math.min(1, value));
+}
+function finiteOr(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+export function betaSuccessProbability(alpha, beta) {
+    const positiveAlpha = Math.max(0, finiteOr(alpha, 0));
+    const positiveBeta = Math.max(0, finiteOr(beta, 0));
+    const total = positiveAlpha + positiveBeta;
+    return total > 0 ? positiveAlpha / total : 0.5;
+}
+export function normalizedEntropy(probabilities) {
+    if (probabilities.length <= 1)
+        return 0;
+    const positive = probabilities.filter((probability) => probability > 0 && Number.isFinite(probability));
+    if (positive.length === 0)
+        return 0;
+    const entropy = -positive.reduce((total, probability) => total + probability * Math.log(probability), 0);
+    return entropy <= 0 ? 0 : entropy / Math.log(probabilities.length);
+}
+function softmax(utilities) {
+    if (utilities.length === 0)
+        return [];
+    const maximum = Math.max(...utilities);
+    const exponentials = utilities.map((utility) => Math.exp(Math.max(-60, utility - maximum)));
+    const total = exponentials.reduce((sum, value) => sum + value, 0);
+    return total > 0 ? exponentials.map((value) => value / total) : utilities.map(() => 1 / utilities.length);
+}
+export function rankExpectedUtilityCandidates(candidates, weights = DEFAULT_EXPECTED_UTILITY_WEIGHTS) {
+    const unranked = candidates.map((candidate) => {
+        const successProbability = clampUnit(candidate.successProbability ?? betaSuccessProbability(candidate.alpha, candidate.beta));
+        const latencyCost = clampUnit(finiteOr(candidate.latencyMs, 0) / Math.max(1, weights.latencyScaleMs));
+        const tokenCost = clampUnit(finiteOr(candidate.tokenEstimate, 0) / Math.max(1, weights.tokenScale));
+        const riskCost = clampUnit(finiteOr(candidate.riskCost, 0));
+        const contextCost = clampUnit(finiteOr(candidate.contextCost, 0));
+        const utility = successProbability * clampUnit(candidate.value) -
+            weights.latency * latencyCost -
+            weights.tokens * tokenCost -
+            weights.risk * riskCost -
+            weights.context * contextCost;
+        return { ...candidate, successProbability, latencyCost, tokenCost, riskCost, contextCost, utility };
+    });
+    const probabilities = softmax(unranked.map((candidate) => candidate.utility));
+    return unranked
+        .map((candidate, index) => ({ ...candidate, probability: probabilities[index] ?? 0 }))
+        .sort((left, right) => right.utility - left.utility || left.tool.localeCompare(right.tool));
+}
+export function decideExpectedUtility(candidates, weights = DEFAULT_EXPECTED_UTILITY_WEIGHTS) {
+    const ranked = rankExpectedUtilityCandidates(candidates, weights);
+    const best = ranked[0];
+    const runnerUp = ranked[1];
+    const entropy = normalizedEntropy(ranked.map((candidate) => candidate.probability));
+    const margin = best && runnerUp ? best.utility - runnerUp.utility : (best?.utility ?? 0);
+    const shortlist = ranked.slice(0, 3).map((candidate) => candidate.tool);
+    if (!best || best.utility <= 0) {
+        return { disposition: "abstain", shortlist: [], entropy, margin, ranked };
+    }
+    if (entropy >= weights.highEntropy || margin < weights.minimumMargin) {
+        return { disposition: "shortlist", shortlist, entropy, margin, ranked };
+    }
+    if (best.automaticEligible !== false && (best.deterministicMatch || best.sampleCount >= weights.minimumEvidence)) {
+        return {
+            disposition: "recommend",
+            recommendation: best.tool,
+            shortlist: [],
+            entropy,
+            margin,
+            ranked,
+        };
+    }
+    return { disposition: "abstain", shortlist: [], entropy, margin, ranked };
+}
+//# sourceMappingURL=expected-utility.js.map

package/dist/core/tool-selection/expected-utility.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"expected-utility.js","sourceRoot":"","sources":["../../../src/core/tool-selection/expected-utility.ts"],"names":[],"mappings":"AAYA,MAAM,CAAC,MAAM,gCAAgC,GAA2B;IACvE,OAAO,EAAE,IAAI;IACb,MAAM,EAAE,GAAG;IACX,IAAI,EAAE,GAAG;IACT,OAAO,EAAE,GAAG;IACZ,cAAc,EAAE,KAAK;IACrB,UAAU,EAAE,KAAK;IACjB,eAAe,EAAE,CAAC;IAClB,aAAa,EAAE,GAAG;IAClB,WAAW,EAAE,IAAI;CACjB,CAAC;AAwCF,SAAS,SAAS,CAAC,KAAa,EAAU;IACzC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAAA,CACvC;AAED,SAAS,QAAQ,CAAC,KAAyB,EAAE,QAAgB,EAAU;IACtE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAAA,CAC9E;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa,EAAE,IAAY,EAAU;IAC3E,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,aAAa,GAAG,YAAY,CAAC;IAC3C,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAAA,CAC/C;AAED,MAAM,UAAU,iBAAiB,CAAC,aAAgC,EAAU;IAC3E,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IACxG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IACzG,OAAO,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;AAAA,CACnE;AAED,SAAS,OAAO,CAAC,SAA4B,EAAY;IACxD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;IACvC,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAClE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;AAAA,CAC1G;AAED,MAAM,UAAU,6BAA6B,CAC5C,UAA+C,EAC/C,OAAO,GAA2B,gCAAgC,EAC1C;IACxB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC;QAC9C,MAAM,kBAAkB,GAAG,SAAS,CACnC,SAAS,CAAC,kBAAkB,IAAI,sBAAsB,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CACvF,CAAC;QACF,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;QACtG,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GACZ,kBAAkB,GAAG,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC;YAC/C,OAAO,CAAC,OAAO,GAAG,WAAW;YAC7B,OAAO,CAAC,MAAM,GAAG,SAAS;YAC1B,OAAO,CAAC,IAAI,GAAG,QAAQ;YACvB,OAAO,CAAC,OAAO,GAAG,WAAW,CAAC;QAC/B,OAAO,EAAE,GAAG,SAAS,EAAE,kBAAkB,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;IAAA,CACpG,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9E,OAAO,QAAQ;SACb,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACrF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAAA,CAC7F;AAED,MAAM,UAAU,qBAAqB,CACpC,UAA+C,EAC/C,OAAO,GAA2B,gCAAgC,EAC1C;IACxB,MAAM,MAAM,GAAG,6BAA6B,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACvB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;IACpF,MAAM,MAAM,GAAG,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC;IACzF,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;QAChC,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,OAAO,IAAI,OAAO,CAAC,WAAW,IAAI,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;QACtE,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IACzE,CAAC;IACD,IAAI,IAAI,CAAC,iBAAiB,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QAClH,OAAO;YACN,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,IAAI,CAAC,IAAI;YACzB,SAAS,EAAE,EAAE;YACb,OAAO;YACP,MAAM;YACN,MAAM;SACN,CAAC;IACH,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAAA,CAC1E","sourcesContent":["export interface ExpectedUtilityWeights {\n\tlatency: number;\n\ttokens: number;\n\trisk: number;\n\tcontext: number;\n\tlatencyScaleMs: number;\n\ttokenScale: number;\n\tminimumEvidence: number;\n\tminimumMargin: number;\n\thighEntropy: number;\n}\n\nexport const DEFAULT_EXPECTED_UTILITY_WEIGHTS: ExpectedUtilityWeights = {\n\tlatency: 0.15,\n\ttokens: 0.1,\n\trisk: 0.2,\n\tcontext: 0.1,\n\tlatencyScaleMs: 5_000,\n\ttokenScale: 1_000,\n\tminimumEvidence: 3,\n\tminimumMargin: 0.1,\n\thighEntropy: 0.85,\n};\n\nexport interface ExpectedUtilityCandidate {\n\ttool: string;\n\tvalue: number;\n\talpha: number;\n\tbeta: number;\n\tsampleCount: number;\n\tlatencyMs?: number;\n\ttokenEstimate?: number;\n\triskCost?: number;\n\tcontextCost?: number;\n\tdeterministicMatch?: boolean;\n\t/** Candidates with unresolved paths may be observed/shortlisted but never recommended. */\n\tautomaticEligible?: boolean;\n\t/** `no_tool` uses a known probability rather than a learned posterior. */\n\tsuccessProbability?: number;\n}\n\nexport interface RankedToolCandidate extends ExpectedUtilityCandidate {\n\tsuccessProbability: number;\n\tlatencyCost: number;\n\ttokenCost: number;\n\triskCost: number;\n\tcontextCost: number;\n\tutility: number;\n\tprobability: number;\n}\n\nexport type ToolSelectionDisposition = \"recommend\" | \"shortlist\" | \"abstain\";\n\nexport interface ToolSelectionDecision {\n\tdisposition: ToolSelectionDisposition;\n\trecommendation?: string;\n\tshortlist: string[];\n\tentropy: number;\n\tmargin: number;\n\tranked: RankedToolCandidate[];\n}\n\nfunction clampUnit(value: number): number {\n\treturn Math.max(0, Math.min(1, value));\n}\n\nfunction finiteOr(value: number | undefined, fallback: number): number {\n\treturn typeof value === \"number\" && Number.isFinite(value) ? value : fallback;\n}\n\nexport function betaSuccessProbability(alpha: number, beta: number): number {\n\tconst positiveAlpha = Math.max(0, finiteOr(alpha, 0));\n\tconst positiveBeta = Math.max(0, finiteOr(beta, 0));\n\tconst total = positiveAlpha + positiveBeta;\n\treturn total > 0 ? positiveAlpha / total : 0.5;\n}\n\nexport function normalizedEntropy(probabilities: readonly number[]): number {\n\tif (probabilities.length <= 1) return 0;\n\tconst positive = probabilities.filter((probability) => probability > 0 && Number.isFinite(probability));\n\tif (positive.length === 0) return 0;\n\tconst entropy = -positive.reduce((total, probability) => total + probability * Math.log(probability), 0);\n\treturn entropy <= 0 ? 0 : entropy / Math.log(probabilities.length);\n}\n\nfunction softmax(utilities: readonly number[]): number[] {\n\tif (utilities.length === 0) return [];\n\tconst maximum = Math.max(...utilities);\n\tconst exponentials = utilities.map((utility) => Math.exp(Math.max(-60, utility - maximum)));\n\tconst total = exponentials.reduce((sum, value) => sum + value, 0);\n\treturn total > 0 ? exponentials.map((value) => value / total) : utilities.map(() => 1 / utilities.length);\n}\n\nexport function rankExpectedUtilityCandidates(\n\tcandidates: readonly ExpectedUtilityCandidate[],\n\tweights: ExpectedUtilityWeights = DEFAULT_EXPECTED_UTILITY_WEIGHTS,\n): RankedToolCandidate[] {\n\tconst unranked = candidates.map((candidate) => {\n\t\tconst successProbability = clampUnit(\n\t\t\tcandidate.successProbability ?? betaSuccessProbability(candidate.alpha, candidate.beta),\n\t\t);\n\t\tconst latencyCost = clampUnit(finiteOr(candidate.latencyMs, 0) / Math.max(1, weights.latencyScaleMs));\n\t\tconst tokenCost = clampUnit(finiteOr(candidate.tokenEstimate, 0) / Math.max(1, weights.tokenScale));\n\t\tconst riskCost = clampUnit(finiteOr(candidate.riskCost, 0));\n\t\tconst contextCost = clampUnit(finiteOr(candidate.contextCost, 0));\n\t\tconst utility =\n\t\t\tsuccessProbability * clampUnit(candidate.value) -\n\t\t\tweights.latency * latencyCost -\n\t\t\tweights.tokens * tokenCost -\n\t\t\tweights.risk * riskCost -\n\t\t\tweights.context * contextCost;\n\t\treturn { ...candidate, successProbability, latencyCost, tokenCost, riskCost, contextCost, utility };\n\t});\n\tconst probabilities = softmax(unranked.map((candidate) => candidate.utility));\n\treturn unranked\n\t\t.map((candidate, index) => ({ ...candidate, probability: probabilities[index] ?? 0 }))\n\t\t.sort((left, right) => right.utility - left.utility || left.tool.localeCompare(right.tool));\n}\n\nexport function decideExpectedUtility(\n\tcandidates: readonly ExpectedUtilityCandidate[],\n\tweights: ExpectedUtilityWeights = DEFAULT_EXPECTED_UTILITY_WEIGHTS,\n): ToolSelectionDecision {\n\tconst ranked = rankExpectedUtilityCandidates(candidates, weights);\n\tconst best = ranked[0];\n\tconst runnerUp = ranked[1];\n\tconst entropy = normalizedEntropy(ranked.map((candidate) => candidate.probability));\n\tconst margin = best && runnerUp ? best.utility - runnerUp.utility : (best?.utility ?? 0);\n\tconst shortlist = ranked.slice(0, 3).map((candidate) => candidate.tool);\n\tif (!best || best.utility <= 0) {\n\t\treturn { disposition: \"abstain\", shortlist: [], entropy, margin, ranked };\n\t}\n\tif (entropy >= weights.highEntropy || margin < weights.minimumMargin) {\n\t\treturn { disposition: \"shortlist\", shortlist, entropy, margin, ranked };\n\t}\n\tif (best.automaticEligible !== false && (best.deterministicMatch || best.sampleCount >= weights.minimumEvidence)) {\n\t\treturn {\n\t\t\tdisposition: \"recommend\",\n\t\t\trecommendation: best.tool,\n\t\t\tshortlist: [],\n\t\t\tentropy,\n\t\t\tmargin,\n\t\t\tranked,\n\t\t};\n\t}\n\treturn { disposition: \"abstain\", shortlist: [], entropy, margin, ranked };\n}\n"]}

package/dist/core/tool-selection/tool-performance-store.d.ts

@@ -0,0 +1,82 @@
+import { type HostFingerprint } from "../models/fitness-store.ts";
+export type ToolSelectionIntentClass = "read" | "search" | "execute" | "write" | "retrieve" | "explain" | "other";
+export interface ToolPerformanceKey {
+    modelRef: string;
+    intentClass: ToolSelectionIntentClass;
+    tool: string;
+}
+export interface ToolPerformanceStats extends ToolPerformanceKey {
+    alpha: number;
+    beta: number;
+    sampleCount: number;
+    latencyEwmaMs?: number;
+    latencyDeviationEwmaMs?: number;
+    inputTokenEstimateEwma?: number;
+    outputTokenEstimateEwma?: number;
+    repairCount: number;
+    bounceCount: number;
+    failureCount: number;
+    lastUsedAt: string;
+}
+export interface ToolSelectionObservation {
+    at: string;
+    modelRef: string;
+    intentClass: ToolSelectionIntentClass;
+    actualTool: string;
+    firstTool: boolean;
+    succeeded: boolean;
+    disposition: "recommend" | "shortlist" | "abstain";
+    recommendation?: string;
+    shortlist: string[];
+    entropy: number;
+    margin: number;
+    /** Redacted ranking: names and numeric scores only, never prompts, arguments, paths, or output. */
+    ranked: Array<{
+        tool: string;
+        utility: number;
+        probability: number;
+    }>;
+    latencyMs?: number;
+    inputTokenEstimate?: number;
+    outputTokenEstimate?: number;
+}
+export interface ToolSelectionMetrics {
+    firstToolAttempts: number;
+    firstToolSuccesses: number;
+    wrongToolOrFailureCount: number;
+    recommendationCount: number;
+    recommendationMatchedCount: number;
+    shortlistCount: number;
+    abstentionCount: number;
+    averageLatencyMs?: number;
+    averageInputTokenEstimate?: number;
+    averageOutputTokenEstimate?: number;
+}
+export interface ToolExecutionObservation {
+    key: ToolPerformanceKey;
+    success: boolean;
+    latencyMs: number;
+    inputTokenEstimate?: number;
+    outputTokenEstimate?: number;
+    selection: Omit<ToolSelectionObservation, "at" | "modelRef" | "intentClass" | "actualTool" | "succeeded">;
+    at?: string;
+}
+export declare class ToolPerformanceStore {
+    private readonly filePath;
+    private readonly fingerprint;
+    constructor(filePath: string, options?: {
+        fingerprint?: () => HostFingerprint;
+    });
+    static forAgentDir(agentDir: string, options?: {
+        fingerprint?: () => HostFingerprint;
+    }): ToolPerformanceStore;
+    private load;
+    private save;
+    private hostData;
+    get(key: ToolPerformanceKey): ToolPerformanceStats;
+    recordValidation(key: ToolPerformanceKey, outcome: "repaired" | "bounced", at?: string): ToolPerformanceStats;
+    recordExecution(observation: ToolExecutionObservation): ToolPerformanceStats;
+    getMetrics(modelRef?: string): ToolSelectionMetrics;
+    getObservations(modelRef?: string): ToolSelectionObservation[];
+}
+//# sourceMappingURL=tool-performance-store.d.ts.map

package/dist/core/tool-selection/tool-performance-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-performance-store.d.ts","sourceRoot":"","sources":["../../../src/core/tool-selection/tool-performance-store.ts"],"names":[],"mappings":"AAEA,OAAO,EAA0B,KAAK,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAO1F,MAAM,MAAM,wBAAwB,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC;AAElH,MAAM,WAAW,kBAAkB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,wBAAwB,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,oBAAqB,SAAQ,kBAAkB;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,wBAAwB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,wBAAwB,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,WAAW,EAAE,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC;IACnD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,mGAAmG;IACnG,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,oBAAoB;IACpC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,uBAAuB,EAAE,MAAM,CAAC;IAChC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,0BAA0B,EAAE,MAAM,CAAC;IACnC,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACpC;AAaD,MAAM,WAAW,wBAAwB;IACxC,GAAG,EAAE,kBAAkB,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,YAAY,GAAG,WAAW,CAAC,CAAC;IAC1G,EAAE,CAAC,EAAE,MAAM,CAAC;CACZ;AA8ID,qBAAa,oBAAoB;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAwB;IAEpD,YAAY,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,eAAe,CAAA;KAAO,EAGlF;IAED,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,eAAe,CAAA;KAAO,GAAG,oBAAoB,CAEhH;IAED,OAAO,CAAC,IAAI;IASZ,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ;IAShB,GAAG,CAAC,GAAG,EAAE,kBAAkB,GAAG,oBAAoB,CAKjD;IAED,gBAAgB,CACf,GAAG,EAAE,kBAAkB,EACvB,OAAO,EAAE,UAAU,GAAG,SAAS,EAC/B,EAAE,SAA2B,GAC3B,oBAAoB,CAetB;IAED,eAAe,CAAC,WAAW,EAAE,wBAAwB,GAAG,oBAAoB,CAyC3E;IAED,UAAU,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,oBAAoB,CA6BlD;IAED,eAAe,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,wBAAwB,EAAE,CAS7D;CACD","sourcesContent":["import { existsSync, mkdirSync, readFileSync, writeFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { currentHostFingerprint, type HostFingerprint } from \"../models/fitness-store.ts\";\n\nconst STORE_VERSION = 1;\nconst MAX_STATS_PER_HOST = 500;\nconst MAX_OBSERVATIONS_PER_HOST = 1_000;\nconst EWMA_ALPHA = 0.25;\n\nexport type ToolSelectionIntentClass = \"read\" | \"search\" | \"execute\" | \"write\" | \"retrieve\" | \"explain\" | \"other\";\n\nexport interface ToolPerformanceKey {\n\tmodelRef: string;\n\tintentClass: ToolSelectionIntentClass;\n\ttool: string;\n}\n\nexport interface ToolPerformanceStats extends ToolPerformanceKey {\n\talpha: number;\n\tbeta: number;\n\tsampleCount: number;\n\tlatencyEwmaMs?: number;\n\tlatencyDeviationEwmaMs?: number;\n\tinputTokenEstimateEwma?: number;\n\toutputTokenEstimateEwma?: number;\n\trepairCount: number;\n\tbounceCount: number;\n\tfailureCount: number;\n\tlastUsedAt: string;\n}\n\nexport interface ToolSelectionObservation {\n\tat: string;\n\tmodelRef: string;\n\tintentClass: ToolSelectionIntentClass;\n\tactualTool: string;\n\tfirstTool: boolean;\n\tsucceeded: boolean;\n\tdisposition: \"recommend\" | \"shortlist\" | \"abstain\";\n\trecommendation?: string;\n\tshortlist: string[];\n\tentropy: number;\n\tmargin: number;\n\t/** Redacted ranking: names and numeric scores only, never prompts, arguments, paths, or output. */\n\tranked: Array<{ tool: string; utility: number; probability: number }>;\n\tlatencyMs?: number;\n\tinputTokenEstimate?: number;\n\toutputTokenEstimate?: number;\n}\n\nexport interface ToolSelectionMetrics {\n\tfirstToolAttempts: number;\n\tfirstToolSuccesses: number;\n\twrongToolOrFailureCount: number;\n\trecommendationCount: number;\n\trecommendationMatchedCount: number;\n\tshortlistCount: number;\n\tabstentionCount: number;\n\taverageLatencyMs?: number;\n\taverageInputTokenEstimate?: number;\n\taverageOutputTokenEstimate?: number;\n}\n\ninterface HostToolPerformanceData {\n\thost: HostFingerprint;\n\tstats: Record<string, ToolPerformanceStats>;\n\tobservations: ToolSelectionObservation[];\n}\n\ninterface ToolPerformanceStoreFile {\n\tversion: 1;\n\thosts: Record<string, HostToolPerformanceData>;\n}\n\nexport interface ToolExecutionObservation {\n\tkey: ToolPerformanceKey;\n\tsuccess: boolean;\n\tlatencyMs: number;\n\tinputTokenEstimate?: number;\n\toutputTokenEstimate?: number;\n\tselection: Omit<ToolSelectionObservation, \"at\" | \"modelRef\" | \"intentClass\" | \"actualTool\" | \"succeeded\">;\n\tat?: string;\n}\n\nfunction statKey(key: ToolPerformanceKey): string {\n\treturn `${key.modelRef}\\0${key.intentClass}\\0${key.tool}`;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n\treturn typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction isIntentClass(value: unknown): value is ToolSelectionIntentClass {\n\treturn (\n\t\tvalue === \"read\" ||\n\t\tvalue === \"search\" ||\n\t\tvalue === \"execute\" ||\n\t\tvalue === \"write\" ||\n\t\tvalue === \"retrieve\" ||\n\t\tvalue === \"explain\" ||\n\t\tvalue === \"other\"\n\t);\n}\n\nfunction isStats(value: unknown): value is ToolPerformanceStats {\n\treturn (\n\t\tisRecord(value) &&\n\t\ttypeof value.modelRef === \"string\" &&\n\t\tisIntentClass(value.intentClass) &&\n\t\ttypeof value.tool === \"string\" &&\n\t\ttypeof value.alpha === \"number\" &&\n\t\ttypeof value.beta === \"number\" &&\n\t\ttypeof value.sampleCount === \"number\" &&\n\t\ttypeof value.repairCount === \"number\" &&\n\t\ttypeof value.bounceCount === \"number\" &&\n\t\ttypeof value.failureCount === \"number\" &&\n\t\ttypeof value.lastUsedAt === \"string\"\n\t);\n}\n\nfunction isObservation(value: unknown): value is ToolSelectionObservation {\n\treturn (\n\t\tisRecord(value) &&\n\t\ttypeof value.at === \"string\" &&\n\t\ttypeof value.modelRef === \"string\" &&\n\t\tisIntentClass(value.intentClass) &&\n\t\ttypeof value.actualTool === \"string\" &&\n\t\ttypeof value.firstTool === \"boolean\" &&\n\t\ttypeof value.succeeded === \"boolean\" &&\n\t\t(value.disposition === \"recommend\" || value.disposition === \"shortlist\" || value.disposition === \"abstain\") &&\n\t\tArray.isArray(value.shortlist) &&\n\t\tvalue.shortlist.every((tool) => typeof tool === \"string\") &&\n\t\ttypeof value.entropy === \"number\" &&\n\t\ttypeof value.margin === \"number\" &&\n\t\tArray.isArray(value.ranked) &&\n\t\tvalue.ranked.every(\n\t\t\t(entry) =>\n\t\t\t\tisRecord(entry) &&\n\t\t\t\ttypeof entry.tool === \"string\" &&\n\t\t\t\ttypeof entry.utility === \"number\" &&\n\t\t\t\ttypeof entry.probability === \"number\",\n\t\t) &&\n\t\t(value.latencyMs === undefined || typeof value.latencyMs === \"number\") &&\n\t\t(value.inputTokenEstimate === undefined || typeof value.inputTokenEstimate === \"number\") &&\n\t\t(value.outputTokenEstimate === undefined || typeof value.outputTokenEstimate === \"number\")\n\t);\n}\n\nfunction emptyStats(key: ToolPerformanceKey, at: string): ToolPerformanceStats {\n\treturn {\n\t\t...key,\n\t\talpha: 1,\n\t\tbeta: 1,\n\t\tsampleCount: 0,\n\t\trepairCount: 0,\n\t\tbounceCount: 0,\n\t\tfailureCount: 0,\n\t\tlastUsedAt: at,\n\t};\n}\n\nfunction finiteNonNegative(value: number | undefined): number | undefined {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value >= 0 ? value : undefined;\n}\n\nfunction updateEwma(previous: number | undefined, next: number | undefined): number | undefined {\n\tif (next === undefined) return previous;\n\treturn previous === undefined ? next : previous + EWMA_ALPHA * (next - previous);\n}\n\nfunction updateDeviation(\n\tpreviousValue: number | undefined,\n\tpreviousDeviation: number | undefined,\n\tnext: number | undefined,\n): number | undefined {\n\tif (next === undefined) return previousDeviation;\n\tconst deviation = previousValue === undefined ? 0 : Math.abs(next - previousValue);\n\treturn updateEwma(previousDeviation, deviation);\n}\n\nfunction parseHost(value: unknown): HostToolPerformanceData | undefined {\n\tif (!isRecord(value) || !isRecord(value.host) || !isRecord(value.stats) || !Array.isArray(value.observations))\n\t\treturn undefined;\n\tconst host = value.host;\n\tif (\n\t\ttypeof host.id !== \"string\" ||\n\t\ttypeof host.cpu !== \"string\" ||\n\t\ttypeof host.cores !== \"number\" ||\n\t\ttypeof host.totalMemGb !== \"number\"\n\t)\n\t\treturn undefined;\n\treturn {\n\t\thost: { id: host.id, cpu: host.cpu, cores: host.cores, totalMemGb: host.totalMemGb },\n\t\tstats: Object.fromEntries(\n\t\t\tObject.entries(value.stats).filter((entry): entry is [string, ToolPerformanceStats] => isStats(entry[1])),\n\t\t),\n\t\tobservations: value.observations.filter(isObservation),\n\t};\n}\n\nfunction parseFile(value: unknown): ToolPerformanceStoreFile {\n\tif (!isRecord(value) || value.version !== STORE_VERSION || !isRecord(value.hosts)) {\n\t\treturn { version: STORE_VERSION, hosts: {} };\n\t}\n\treturn {\n\t\tversion: STORE_VERSION,\n\t\thosts: Object.fromEntries(\n\t\t\tObject.entries(value.hosts)\n\t\t\t\t.map(([hostId, host]) => [hostId, parseHost(host)] as const)\n\t\t\t\t.filter((entry): entry is readonly [string, HostToolPerformanceData] => entry[1] !== undefined),\n\t\t),\n\t};\n}\n\nfunction trimStats(stats: Record<string, ToolPerformanceStats>): Record<string, ToolPerformanceStats> {\n\tconst entries = Object.entries(stats);\n\tif (entries.length <= MAX_STATS_PER_HOST) return stats;\n\treturn Object.fromEntries(\n\t\tentries\n\t\t\t.sort(([, left], [, right]) => Date.parse(right.lastUsedAt) - Date.parse(left.lastUsedAt))\n\t\t\t.slice(0, MAX_STATS_PER_HOST),\n\t);\n}\n\nexport class ToolPerformanceStore {\n\tprivate readonly filePath: string;\n\tprivate readonly fingerprint: () => HostFingerprint;\n\n\tconstructor(filePath: string, options: { fingerprint?: () => HostFingerprint } = {}) {\n\t\tthis.filePath = filePath;\n\t\tthis.fingerprint = options.fingerprint ?? currentHostFingerprint;\n\t}\n\n\tstatic forAgentDir(agentDir: string, options: { fingerprint?: () => HostFingerprint } = {}): ToolPerformanceStore {\n\t\treturn new ToolPerformanceStore(join(agentDir, \"state\", \"tool-performance.json\"), options);\n\t}\n\n\tprivate load(): ToolPerformanceStoreFile {\n\t\ttry {\n\t\t\tif (!existsSync(this.filePath)) return { version: STORE_VERSION, hosts: {} };\n\t\t\treturn parseFile(JSON.parse(readFileSync(this.filePath, \"utf8\")));\n\t\t} catch {\n\t\t\treturn { version: STORE_VERSION, hosts: {} };\n\t\t}\n\t}\n\n\tprivate save(file: ToolPerformanceStoreFile): void {\n\t\tmkdirSync(dirname(this.filePath), { recursive: true });\n\t\twriteFileSync(this.filePath, `${JSON.stringify(file, null, \"\\t\")}\\n`, \"utf8\");\n\t}\n\n\tprivate hostData(file: ToolPerformanceStoreFile): HostToolPerformanceData {\n\t\tconst fingerprint = this.fingerprint();\n\t\tconst existing = file.hosts[fingerprint.id];\n\t\tif (existing) return existing;\n\t\tconst created: HostToolPerformanceData = { host: fingerprint, stats: {}, observations: [] };\n\t\tfile.hosts[fingerprint.id] = created;\n\t\treturn created;\n\t}\n\n\tget(key: ToolPerformanceKey): ToolPerformanceStats {\n\t\tconst file = this.load();\n\t\tconst host = file.hosts[this.fingerprint().id];\n\t\tconst stats = host?.stats[statKey(key)];\n\t\treturn stats ? { ...stats } : emptyStats(key, new Date(0).toISOString());\n\t}\n\n\trecordValidation(\n\t\tkey: ToolPerformanceKey,\n\t\toutcome: \"repaired\" | \"bounced\",\n\t\tat = new Date().toISOString(),\n\t): ToolPerformanceStats {\n\t\tconst file = this.load();\n\t\tconst host = this.hostData(file);\n\t\tconst storageKey = statKey(key);\n\t\tconst current = host.stats[storageKey] ?? emptyStats(key, at);\n\t\tconst next: ToolPerformanceStats = {\n\t\t\t...current,\n\t\t\trepairCount: current.repairCount + (outcome === \"repaired\" ? 1 : 0),\n\t\t\tbounceCount: current.bounceCount + (outcome === \"bounced\" ? 1 : 0),\n\t\t\tlastUsedAt: at,\n\t\t};\n\t\thost.stats[storageKey] = next;\n\t\thost.stats = trimStats(host.stats);\n\t\tthis.save(file);\n\t\treturn { ...next };\n\t}\n\n\trecordExecution(observation: ToolExecutionObservation): ToolPerformanceStats {\n\t\tconst at = observation.at ?? new Date().toISOString();\n\t\tconst file = this.load();\n\t\tconst host = this.hostData(file);\n\t\tconst storageKey = statKey(observation.key);\n\t\tconst current = host.stats[storageKey] ?? emptyStats(observation.key, at);\n\t\tconst latencyMs = finiteNonNegative(observation.latencyMs);\n\t\tconst inputTokenEstimate = finiteNonNegative(observation.inputTokenEstimate);\n\t\tconst outputTokenEstimate = finiteNonNegative(observation.outputTokenEstimate);\n\t\tconst next: ToolPerformanceStats = {\n\t\t\t...current,\n\t\t\talpha: current.alpha + (observation.success ? 1 : 0),\n\t\t\tbeta: current.beta + (observation.success ? 0 : 1),\n\t\t\tsampleCount: current.sampleCount + 1,\n\t\t\tlatencyEwmaMs: updateEwma(current.latencyEwmaMs, latencyMs),\n\t\t\tlatencyDeviationEwmaMs: updateDeviation(current.latencyEwmaMs, current.latencyDeviationEwmaMs, latencyMs),\n\t\t\tinputTokenEstimateEwma: updateEwma(current.inputTokenEstimateEwma, inputTokenEstimate),\n\t\t\toutputTokenEstimateEwma: updateEwma(current.outputTokenEstimateEwma, outputTokenEstimate),\n\t\t\tfailureCount: current.failureCount + (observation.success ? 0 : 1),\n\t\t\tlastUsedAt: at,\n\t\t};\n\t\thost.stats[storageKey] = next;\n\t\thost.stats = trimStats(host.stats);\n\t\thost.observations.push({\n\t\t\t...observation.selection,\n\t\t\tat,\n\t\t\tmodelRef: observation.key.modelRef,\n\t\t\tintentClass: observation.key.intentClass,\n\t\t\tactualTool: observation.key.tool,\n\t\t\tsucceeded: observation.success,\n\t\t\tranked: observation.selection.ranked.slice(0, 6),\n\t\t\tshortlist: observation.selection.shortlist.slice(0, 3),\n\t\t\tlatencyMs,\n\t\t\tinputTokenEstimate,\n\t\t\toutputTokenEstimate,\n\t\t});\n\t\tif (host.observations.length > MAX_OBSERVATIONS_PER_HOST) {\n\t\t\thost.observations = host.observations.slice(-MAX_OBSERVATIONS_PER_HOST);\n\t\t}\n\t\tthis.save(file);\n\t\treturn { ...next };\n\t}\n\n\tgetMetrics(modelRef?: string): ToolSelectionMetrics {\n\t\tconst host = this.load().hosts[this.fingerprint().id];\n\t\tconst observations = (host?.observations ?? []).filter(\n\t\t\t(observation) => modelRef === undefined || observation.modelRef === modelRef,\n\t\t);\n\t\tconst firstTools = observations.filter((observation) => observation.firstTool);\n\t\tconst recommended = observations.filter((observation) => observation.disposition === \"recommend\");\n\t\tconst average = (values: Array<number | undefined>): number | undefined => {\n\t\t\tconst present = values.filter((value): value is number => value !== undefined);\n\t\t\treturn present.length > 0 ? present.reduce((sum, value) => sum + value, 0) / present.length : undefined;\n\t\t};\n\t\treturn {\n\t\t\tfirstToolAttempts: firstTools.length,\n\t\t\tfirstToolSuccesses: firstTools.filter((observation) => observation.succeeded).length,\n\t\t\twrongToolOrFailureCount: observations.filter(\n\t\t\t\t(observation) =>\n\t\t\t\t\t!observation.succeeded ||\n\t\t\t\t\t(observation.disposition === \"recommend\" && observation.recommendation !== observation.actualTool),\n\t\t\t).length,\n\t\t\trecommendationCount: recommended.length,\n\t\t\trecommendationMatchedCount: recommended.filter(\n\t\t\t\t(observation) => observation.recommendation === observation.actualTool,\n\t\t\t).length,\n\t\t\tshortlistCount: observations.filter((observation) => observation.disposition === \"shortlist\").length,\n\t\t\tabstentionCount: observations.filter((observation) => observation.disposition === \"abstain\").length,\n\t\t\taverageLatencyMs: average(observations.map((observation) => observation.latencyMs)),\n\t\t\taverageInputTokenEstimate: average(observations.map((observation) => observation.inputTokenEstimate)),\n\t\t\taverageOutputTokenEstimate: average(observations.map((observation) => observation.outputTokenEstimate)),\n\t\t};\n\t}\n\n\tgetObservations(modelRef?: string): ToolSelectionObservation[] {\n\t\tconst host = this.load().hosts[this.fingerprint().id];\n\t\treturn (host?.observations ?? [])\n\t\t\t.filter((observation) => modelRef === undefined || observation.modelRef === modelRef)\n\t\t\t.map((observation) => ({\n\t\t\t\t...observation,\n\t\t\t\tshortlist: [...observation.shortlist],\n\t\t\t\tranked: observation.ranked.map((candidate) => ({ ...candidate })),\n\t\t\t}));\n\t}\n}\n"]}