openclaw-llm-verifier 0.1.0

package/dist/index.js

@@ -0,0 +1,106 @@
+/**
+ * OpenClaw plugin entry point (constitution IV: Protocol Compliance).
+ *
+ * Uses definePluginEntry from openclaw/plugin-sdk/plugin-entry per the
+ * building-plugins specification (https://docs.openclaw.ai/zh-CN/plugins/building-plugins).
+ * Registers two tools: `score` (US1/US2) and `select` (US3).
+ */
+import { Type } from "typebox";
+import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
+import { createGeminiClient, selectBest, scoreAllTrials, scorePairCriterion, } from "./verifier.js";
+const ScoreParams = Type.Object({
+    problem: Type.String({ description: "Task description" }),
+    traceA: Type.String({ description: "Trajectory A text" }),
+    traceB: Type.String({ description: "Trajectory B text" }),
+    criterion: Type.Object({
+        id: Type.String(),
+        name: Type.String(),
+        description: Type.String(),
+    }),
+    groundTruthNote: Type.String(),
+});
+const SelectParams = Type.Object({
+    taskName: Type.String(),
+    trials: Type.Array(Type.Object({
+        trialName: Type.String(),
+        reward: Type.Number(),
+        problem: Type.String(),
+        trace: Type.String(),
+    })),
+    criteria: Type.Array(Type.Object({
+        id: Type.String(),
+        name: Type.String(),
+        description: Type.String(),
+    })),
+    groundTruthNote: Type.String(),
+    nReps: Type.Optional(Type.Number()),
+});
+export default definePluginEntry({
+    id: "openclaw-llm-verifier",
+    name: "LLM-as-a-Verifier",
+    description: "Call-chain verification with logprob-based scoring, per-call " +
+        "tracking, and cached re-scoring.",
+    register(api) {
+        let client = null;
+        function getClient() {
+            if (!client) {
+                client = createGeminiClient();
+            }
+            return client;
+        }
+        api.registerTool({
+            name: "score",
+            label: "Score",
+            description: "Score a trajectory pair on one criterion. Returns normalized " +
+                "scoreA, scoreB in [0,1] derived from Gemini logprobs.",
+            parameters: ScoreParams,
+            async execute(_toolCallId, params, _signal, _onUpdate) {
+                const c = getClient();
+                const [scoreA, scoreB] = await scorePairCriterion(c, params.problem, params.traceA, params.traceB, params.criterion, params.groundTruthNote);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({ scoreA, scoreB }),
+                        },
+                    ],
+                    details: { scoreA, scoreB },
+                };
+            },
+        });
+        api.registerTool({
+            name: "select",
+            label: "Select",
+            description: "Select the best trial via round-robin tournament scoring. " +
+                "Batch-scores all pairs on all criteria, then picks the trial " +
+                "with the most wins.",
+            parameters: SelectParams,
+            async execute(_toolCallId, params, _signal, _onUpdate) {
+                const c = getClient();
+                const nReps = params.nReps ?? 4;
+                const tasks = {
+                    [params.taskName]: params.trials,
+                };
+                const criteria = params.criteria;
+                const scores = await scoreAllTrials(c, tasks, [params.taskName], criteria, params.groundTruthNote, nReps);
+                const sel = selectBest(tasks, [params.taskName], scores, criteria.map((cr) => cr.id), nReps);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                bestTrial: sel[params.taskName].trial,
+                                scores,
+                            }),
+                        },
+                    ],
+                    details: {
+                        bestTrial: sel[params.taskName].trial,
+                        scores,
+                    },
+                };
+            },
+        });
+    },
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EACL,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,kBAAkB,GAGnB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC;IAC9B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC;IACzD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACzD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACzD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC;QACrB,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE;QACjB,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE;QACnB,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE;KAC3B,CAAC;IACF,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE;CAC/B,CAAC,CAAC;AAGH,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/B,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE;IACvB,MAAM,EAAE,IAAI,CAAC,KAAK,CAChB,IAAI,CAAC,MAAM,CAAC;QACV,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE;QACxB,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;QACrB,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;QACtB,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE;KACrB,CAAC,CACH;IACD,QAAQ,EAAE,IAAI,CAAC,KAAK,CAClB,IAAI,CAAC,MAAM,CAAC;QACV,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE;QACjB,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE;QACnB,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE;KAC3B,CAAC,CACH;IACD,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE;IAC9B,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;CACpC,CAAC,CAAC;AAGH,eAAe,iBAAiB,CAAC;IAC/B,EAAE,EAAE,uBAAuB;IAC3B,IAAI,EAAE,mBAAmB;IACzB,WAAW,EACT,+DAA+D;QAC/D,kCAAkC;IAEpC,QAAQ,CAAC,GAAG;QACV,IAAI,MAAM,GAAiD,IAAI,CAAC;QAEhE,SAAS,SAAS;YAChB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,kBAAkB,EAAE,CAAC;YAChC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,GAAG,CAAC,YAAY,CAAC;YACf,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,OAAO;YACd,WAAW,EACT,+DAA+D;gBAC/D,uDAAuD;YACzD,UAAU,EAAE,WAAW;YACvB,KAAK,CAAC,OAAO,CACX,WAAmB,EACnB,MAAmB,EACnB,OAAqB,EACrB,SAAmB;gBAEnB,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;gBACtB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,kBAAkB,CAC/C,CAAC,EACD,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,SAAsB,EAC7B,MAAM,CAAC,eAAe,CACvB,CAAC;gBACF,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;yBACzC;qBACF;oBACD,OAAO,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;iBAC5B,CAAC;YACJ,CAAC;SACF,CAAC,CAAC;QAEH,GAAG,CAAC,YAAY,CAAC;YACf,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,QAAQ;YACf,WAAW,EACT,4DAA4D;gBAC5D,+DAA+D;gBAC/D,qBAAqB;YACvB,UAAU,EAAE,YAAY;YACxB,KAAK,CAAC,OAAO,CACX,WAAmB,EACnB,MAAoB,EACpB,OAAqB,EACrB,SAAmB;gBAEnB,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;gBAChC,MAAM,KAAK,GAA4B;oBACrC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAiB;iBAC5C,CAAC;gBACF,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAuB,CAAC;gBAEhD,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC,CAAC,EACD,KAAK,EACL,CAAC,MAAM,CAAC,QAAQ,CAAC,EACjB,QAAQ,EACR,MAAM,CAAC,eAAe,EACtB,KAAK,CACN,CAAC;gBAEF,MAAM,GAAG,GAAG,UAAU,CACpB,KAAK,EACL,CAAC,MAAM,CAAC,QAAQ,CAAC,EACjB,MAAM,EACN,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAC3B,KAAK,CACN,CAAC;gBAEF,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK;gCACrC,MAAM;6BACP,CAAC;yBACH;qBACF;oBACD,OAAO,EAAE;wBACP,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK;wBACrC,MAAM;qBACP;iBACF,CAAC;YACJ,CAAC;SACF,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC"}

package/dist/verifier.js

@@ -0,0 +1,246 @@
+/**
+ * Verifier core logic, ported from scripts/verifier_core.py (Python).
+ *
+ * Constitution III (Verifier Parity, NON-NEGOTIABLE): this TypeScript
+ * port MUST reproduce the Python source's numeric behavior exactly.
+ * Constitution VI (Source-Faithful Port): naming, constants, and data
+ * shapes carry over verbatim (snake_case -> camelCase only where TS
+ * convention requires).
+ *
+ * DEVIATION (constitution VI scope note): the Python source's fallback
+ * patterns (extract_score returning 0.5 on parse failure, score_all_trials
+ * writing 0.5/0.5 on exception) are NOT carried over. Errors propagate.
+ */
+import { GoogleGenAI } from "@google/genai";
+// ---------------------------------------------------------------------------
+// g=20 scoring scale (mirrors source GRANULARITY + SCALE)
+// ---------------------------------------------------------------------------
+export const GRANULARITY = 20;
+const VALID_TOKENS = {};
+for (let i = 0; i < GRANULARITY; i++) {
+    VALID_TOKENS[String.fromCharCode(65 + i)] = GRANULARITY - i; // A-T
+    VALID_TOKENS[String.fromCharCode(97 + i)] = GRANULARITY - i; // a-t
+}
+export const SCALE = {
+    scaleDescription: "Rate how likely the agent correctly solved the task on a " +
+        "20-point scale using letters A through T:\n" +
+        "  A = clearly and completely succeeded with verified output (best)\n" +
+        "  B-D = succeeded with only minor issues\n" +
+        "  E-G = above average, mostly correct with some issues\n" +
+        "  H-J = uncertain, leans toward success\n" +
+        "  K-M = uncertain, leans toward failure\n" +
+        "  N-P = below average, significant issues remain\n" +
+        "  Q-S = failed with some partial progress\n" +
+        "  T = clearly and completely failed (worst)",
+    scoreFormat: "LETTER_A_TO_T",
+    validTokens: VALID_TOKENS,
+};
+// ---------------------------------------------------------------------------
+// Errors
+// ---------------------------------------------------------------------------
+export class ScoreExtractionError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ScoreExtractionError";
+    }
+}
+// ---------------------------------------------------------------------------
+// Gemini client
+// ---------------------------------------------------------------------------
+export function createGeminiClient() {
+    const vertexKey = process.env.VERTEX_API_KEY;
+    if (vertexKey) {
+        return new GoogleGenAI({ vertexai: true, apiKey: vertexKey });
+    }
+    const apiKey = process.env.GEMINI_API_KEY;
+    if (apiKey) {
+        return new GoogleGenAI({ apiKey });
+    }
+    throw new Error("set GEMINI_API_KEY or VERTEX_API_KEY in environment");
+}
+export async function callGemini(client, prompt, topLogprobs = 20) {
+    const response = await client.models.generateContent({
+        model: "gemini-2.5-flash",
+        contents: [{ role: "user", parts: [{ text: prompt }] }],
+        config: {
+            maxOutputTokens: 4096,
+            temperature: 1.0,
+            responseLogprobs: true,
+            logprobs: topLogprobs,
+            thinkingConfig: { thinkingBudget: 0 },
+        },
+    });
+    const text = response.text ?? "";
+    const candidate = response.candidates?.[0];
+    const logprobsResult = candidate?.logprobsResult;
+    let tokens = null;
+    let positionLogprobs = null;
+    if (logprobsResult?.topCandidates) {
+        positionLogprobs = logprobsResult.topCandidates.map((pos) => (pos.candidates ?? []).map((lp) => [lp.token ?? "", lp.logProbability ?? 0]));
+        if (logprobsResult.chosenCandidates) {
+            tokens = logprobsResult.chosenCandidates.map((c) => c.token ?? "");
+        }
+    }
+    return { text, tokens, positionLogprobs };
+}
+// ---------------------------------------------------------------------------
+// Score extraction (logprob path, NO fallback)
+// ---------------------------------------------------------------------------
+export function findTagLogprobs(tokens, positionLogprobs, tag) {
+    if (!tokens || !positionLogprobs)
+        return null;
+    let textSoFar = "";
+    for (let i = 0; i < tokens.length; i++) {
+        textSoFar += tokens[i];
+        if (textSoFar.trimEnd().endsWith(tag)) {
+            if (i + 1 < positionLogprobs.length) {
+                return positionLogprobs[i + 1];
+            }
+        }
+    }
+    return null;
+}
+export function extractScoreStrict(text, tokens, positionLogprobs, tag) {
+    const validTokens = SCALE.validTokens;
+    const tagLp = findTagLogprobs(tokens, positionLogprobs, tag);
+    const probs = {};
+    if (tagLp) {
+        for (const [tokStr, logprob] of tagLp) {
+            const tok = tokStr.trim();
+            if (tok in validTokens) {
+                const val = validTokens[tok];
+                const p = Math.exp(logprob);
+                probs[val] = Math.max(probs[val] ?? 0, p);
+            }
+        }
+    }
+    const probEntries = Object.entries(probs);
+    if (probEntries.length > 0) {
+        const uniqueVals = [...new Set(Object.values(validTokens))].sort((a, b) => a - b);
+        const minVal = uniqueVals[0];
+        const maxVal = uniqueVals[uniqueVals.length - 1];
+        const totalP = Object.values(probs).reduce((s, p) => s + p, 0);
+        const expected = probEntries.reduce((s, [v, p]) => s + Number(v) * p, 0) / totalP;
+        return maxVal > minVal
+            ? (expected - minVal) / (maxVal - minVal)
+            : 0.5;
+    }
+    // DEVIATION (constitution VI scope note): source returns 0.5 here;
+    // we throw instead.
+    throw new ScoreExtractionError(`no valid score tokens at tag ${tag} (logprob path empty)`);
+}
+// ---------------------------------------------------------------------------
+// Pairwise prompt + scoring
+// ---------------------------------------------------------------------------
+export function createPromptForCriterion(problem, traceA, traceB, criterion, groundTruthNote) {
+    return ("You are an expert evaluator of AI coding agents. " +
+        "You will see a task description and two agent trajectories. " +
+        `Your job is to evaluate them on ONE specific criterion: ` +
+        `**${criterion.name}**.\n\n` +
+        `${groundTruthNote}\n\n` +
+        `**Task:**\n${problem}\n\n` +
+        `**Trajectory A:**\n${traceA}\n\n` +
+        `**Trajectory B:**\n${traceB}\n\n` +
+        `**Evaluation Guideline — ${criterion.name}:**\n` +
+        `${criterion.description}\n\n` +
+        `Score each trajectory ONLY on this specific criterion. Ignore other ` +
+        `aspects of the trajectory that are not relevant to ` +
+        `"${criterion.name}".\n\n` +
+        `**Rating Scale:**\n${SCALE.scaleDescription}\n\n` +
+        "Then output your final scores:\n" +
+        `<score_A>${SCALE.scoreFormat}</score_A>\n` +
+        `<score_B>${SCALE.scoreFormat}</score_B>\n\n` +
+        "Begin your analysis now.");
+}
+export async function scorePairCriterion(client, problem, traceA, traceB, criterion, groundTruthNote) {
+    const prompt = createPromptForCriterion(problem, traceA, traceB, criterion, groundTruthNote);
+    const { text, tokens, positionLogprobs } = await callGemini(client, prompt);
+    const sa = extractScoreStrict(text, tokens, positionLogprobs, "<score_A>");
+    const sb = extractScoreStrict(text, tokens, positionLogprobs, "<score_B>");
+    return [sa, sb];
+}
+// ---------------------------------------------------------------------------
+// Batch scoring (no try/catch -> 0.5 fallback; errors propagate)
+// ---------------------------------------------------------------------------
+export async function scoreAllTrials(client, tasks, swingTasks, criteria, groundTruthNote, nReps, cache = {}) {
+    const cached = { ...cache };
+    const jobs = [];
+    for (const taskName of swingTasks) {
+        const trials = tasks[taskName];
+        const n = trials.length;
+        for (let i = 0; i < n; i++) {
+            for (let j = i + 1; j < n; j++) {
+                for (const crit of criteria) {
+                    for (let rep = 0; rep < nReps; rep++) {
+                        const key = `${crit.id}|${taskName}|${i},${j}|${rep}`;
+                        if (!(key in cached)) {
+                            jobs.push({ key, trialI: trials[i], trialJ: trials[j], crit });
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // DEVIATION: source catches per-pair exceptions and writes 0.5/0.5.
+    // We let errors propagate (constitution no-fallback).
+    const results = await Promise.all(jobs.map(async (job) => {
+        const [sa, sb] = await scorePairCriterion(client, job.trialI.problem, job.trialI.trace, job.trialJ.trace, job.crit, groundTruthNote);
+        return { key: job.key, scoreI: sa, scoreJ: sb };
+    }));
+    for (const r of results) {
+        cached[r.key] = { scoreI: r.scoreI, scoreJ: r.scoreJ };
+    }
+    return cached;
+}
+// ---------------------------------------------------------------------------
+// Tournament selection
+// ---------------------------------------------------------------------------
+export function selectBest(tasks, swingTasks, scores, criteriaIds, nReps = 1, repIdx = null) {
+    const selections = {};
+    for (const taskName of swingTasks) {
+        const trials = tasks[taskName];
+        const n = trials.length;
+        const wins = new Array(n).fill(0);
+        for (let i = 0; i < n; i++) {
+            for (let j = i + 1; j < n; j++) {
+                let siSum = 0;
+                let sjSum = 0;
+                let count = 0;
+                for (const cid of criteriaIds) {
+                    const reps = repIdx !== null
+                        ? [repIdx]
+                        : Array.from({ length: nReps }, (_, k) => k);
+                    for (const rep of reps) {
+                        const key = `${cid}|${taskName}|${i},${j}|${rep}`;
+                        const entry = scores[key] ?? { scoreI: 0.5, scoreJ: 0.5 };
+                        siSum += entry.scoreI;
+                        sjSum += entry.scoreJ;
+                        count++;
+                    }
+                }
+                const si = count ? siSum / count : 0.5;
+                const sj = count ? sjSum / count : 0.5;
+                if (si > sj)
+                    wins[i] += 1;
+                else if (sj > si)
+                    wins[j] += 1;
+                else {
+                    wins[i] += 0.5;
+                    wins[j] += 0.5;
+                }
+            }
+        }
+        let bestIdx = 0;
+        for (let t = 1; t < n; t++) {
+            if (wins[t] > wins[bestIdx])
+                bestIdx = t;
+        }
+        selections[taskName] = {
+            idx: bestIdx,
+            trial: trials[bestIdx].trialName,
+            reward: trials[bestIdx].reward,
+        };
+    }
+    return selections;
+}
+//# sourceMappingURL=verifier.js.map

package/dist/verifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verifier.js","sourceRoot":"","sources":["../verifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,8EAA8E;AAC9E,0DAA0D;AAC1D,8EAA8E;AAE9E,MAAM,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAE9B,MAAM,YAAY,GAA2B,EAAE,CAAC;AAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;IACrC,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,MAAM;IACnE,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,MAAM;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,gBAAgB,EACd,2DAA2D;QAC3D,6CAA6C;QAC7C,sEAAsE;QACtE,4CAA4C;QAC5C,0DAA0D;QAC1D,2CAA2C;QAC3C,2CAA2C;QAC3C,oDAAoD;QACpD,6CAA6C;QAC7C,6CAA6C;IAC/C,WAAW,EAAE,eAAe;IAC5B,WAAW,EAAE,YAAY;CAC1B,CAAC;AAyBF,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,MAAM,UAAU,kBAAkB;IAChC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC7C,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,qDAAqD,CACtD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAmB,EACnB,MAAc,EACd,WAAW,GAAG,EAAE;IAEhB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC;QACnD,KAAK,EAAE,kBAAkB;QACzB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QACvD,MAAM,EAAE;YACN,eAAe,EAAE,IAAI;YACrB,WAAW,EAAE,GAAG;YAChB,gBAAgB,EAAE,IAAI;YACtB,QAAQ,EAAE,WAAW;YACrB,cAAc,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE;SACtC;KACF,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,SAAS,EAAE,cAAc,CAAC;IAEjD,IAAI,MAAM,GAAoB,IAAI,CAAC;IACnC,IAAI,gBAAgB,GAA0C,IAAI,CAAC;IAEnE,IAAI,cAAc,EAAE,aAAa,EAAE,CAAC;QAClC,gBAAgB,GAAG,cAAc,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC1D,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CACxB,CAAC,EAAE,EAAoB,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,cAAc,IAAI,CAAC,CAAC,CACnE,CACF,CAAC;QACF,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;YACpC,MAAM,GAAG,cAAc,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;AAC5C,CAAC;AAED,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,MAAM,UAAU,eAAe,CAC7B,MAAuB,EACvB,gBAAuD,EACvD,GAAW;IAEX,IAAI,CAAC,MAAM,IAAI,CAAC,gBAAgB;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBACpC,OAAO,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,MAAuB,EACvB,gBAAuD,EACvD,GAAW;IAEX,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;IACtC,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,CAAC,CAAC;IAC7D,MAAM,KAAK,GAA2B,EAAE,CAAC;IAEzC,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;gBACvB,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC5B,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAC9D,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAChB,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,MAAM,QAAQ,GACZ,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;QACnE,OAAO,MAAM,GAAG,MAAM;YACpB,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;YACzC,CAAC,CAAC,GAAG,CAAC;IACV,CAAC;IAED,mEAAmE;IACnE,oBAAoB;IACpB,MAAM,IAAI,oBAAoB,CAC5B,gCAAgC,GAAG,uBAAuB,CAC3D,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,UAAU,wBAAwB,CACtC,OAAe,EACf,MAAc,EACd,MAAc,EACd,SAAoB,EACpB,eAAuB;IAEvB,OAAO,CACL,mDAAmD;QACnD,8DAA8D;QAC9D,0DAA0D;QAC1D,KAAK,SAAS,CAAC,IAAI,SAAS;QAC5B,GAAG,eAAe,MAAM;QACxB,cAAc,OAAO,MAAM;QAC3B,sBAAsB,MAAM,MAAM;QAClC,sBAAsB,MAAM,MAAM;QAClC,4BAA4B,SAAS,CAAC,IAAI,OAAO;QACjD,GAAG,SAAS,CAAC,WAAW,MAAM;QAC9B,sEAAsE;QACtE,qDAAqD;QACrD,IAAI,SAAS,CAAC,IAAI,QAAQ;QAC1B,sBAAsB,KAAK,CAAC,gBAAgB,MAAM;QAClD,kCAAkC;QAClC,YAAY,KAAK,CAAC,WAAW,cAAc;QAC3C,YAAY,KAAK,CAAC,WAAW,gBAAgB;QAC7C,0BAA0B,CAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAmB,EACnB,OAAe,EACf,MAAc,EACd,MAAc,EACd,SAAoB,EACpB,eAAuB;IAEvB,MAAM,MAAM,GAAG,wBAAwB,CACrC,OAAO,EACP,MAAM,EACN,MAAM,EACN,SAAS,EACT,eAAe,CAChB,CAAC;IACF,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5E,MAAM,EAAE,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC;IAC3E,MAAM,EAAE,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC;IAC3E,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAmB,EACnB,KAA8B,EAC9B,UAAoB,EACpB,QAAqB,EACrB,eAAuB,EACvB,KAAa,EACb,QAA4D,EAAE;IAE9D,MAAM,MAAM,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAE5B,MAAM,IAAI,GAKL,EAAE,CAAC;IAER,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC5B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC;wBACrC,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,EAAE,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;wBACtD,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC;4BACrB,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;wBACjE,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,sDAAsD;IACtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,MAAM,kBAAkB,CACvC,MAAM,EACN,GAAG,CAAC,MAAM,CAAC,OAAO,EAClB,GAAG,CAAC,MAAM,CAAC,KAAK,EAChB,GAAG,CAAC,MAAM,CAAC,KAAK,EAChB,GAAG,CAAC,IAAI,EACR,eAAe,CAChB,CAAC;QACF,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAClD,CAAC,CAAC,CACH,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,UAAU,UAAU,CACxB,KAA8B,EAC9B,UAAoB,EACpB,MAA0D,EAC1D,WAAqB,EACrB,KAAK,GAAG,CAAC,EACT,SAAwB,IAAI;IAE5B,MAAM,UAAU,GAAmE,EAAE,CAAC;IAEtF,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,KAAK,GAAG,CAAC,CAAC;gBAEd,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;oBAC9B,MAAM,IAAI,GACR,MAAM,KAAK,IAAI;wBACb,CAAC,CAAC,CAAC,MAAM,CAAC;wBACV,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;oBACjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;wBAClD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;wBAC1D,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;wBACtB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;wBACtB,KAAK,EAAE,CAAC;oBACV,CAAC;gBACH,CAAC;gBAED,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;gBACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;gBAEvC,IAAI,EAAE,GAAG,EAAE;oBAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;qBACrB,IAAI,EAAE,GAAG,EAAE;oBAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;qBAC1B,CAAC;oBACJ,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;oBACf,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;gBAAE,OAAO,GAAG,CAAC,CAAC;QAC3C,CAAC;QAED,UAAU,CAAC,QAAQ,CAAC,GAAG;YACrB,GAAG,EAAE,OAAO;YACZ,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS;YAChC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;SAC/B,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/openclaw.plugin.json

@@ -0,0 +1,31 @@
+{
+  "id": "openclaw-llm-verifier",
+  "name": "LLM-as-a-Verifier",
+  "description": "Call-chain verification with logprob-based scoring, per-call tracking, and cached re-scoring. Scores trajectory pairs on criteria via Gemini 2.5 Flash logprobs, runs round-robin tournament selection.",
+  "contracts": {
+    "tools": ["score", "select"]
+  },
+  "activation": {
+    "onStartup": false
+  },
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "geminiApiKey": {
+        "type": "string",
+        "description": "Gemini API key (or set GEMINI_API_KEY / VERTEX_API_KEY env var)"
+      },
+      "granularity": {
+        "type": "number",
+        "default": 20,
+        "description": "Scoring granularity (g=20: A-T tokens)"
+      },
+      "nReps": {
+        "type": "number",
+        "default": 4,
+        "description": "Number of repeated verifications per pair"
+      }
+    }
+  }
+}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "openclaw-llm-verifier",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "OpenClaw plugin: call-chain verification with logprob-based scoring, per-call tracking, and cached re-scoring.",
+  "license": "MIT",
+  "author": "ninetyhe",
+  "keywords": ["openclaw", "plugin", "verifier", "llm", "gemini", "logprobs", "scoring"],
+  "files": [
+    "dist",
+    "openclaw.plugin.json"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ninetyhe-90/openclaw-llm-verifier"
+  },
+  "homepage": "https://github.com/ninetyhe-90/openclaw-llm-verifier",
+  "bugs": {
+    "url": "https://github.com/ninetyhe-90/openclaw-llm-verifier/issues"
+  },
+  "dependencies": {
+    "@google/genai": "^1.0.0",
+    "typebox": "^1.1.39"
+  },
+  "peerDependencies": {
+    "openclaw": ">=2026.3.24-beta.2"
+  },
+  "devDependencies": {
+    "typescript": "^5.5.0",
+    "vitest": "^2.0.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run"
+  },
+  "openclaw": {
+    "extensions": ["./dist/index.js"],
+    "compat": {
+      "pluginApi": ">=2026.3.24-beta.2",
+      "minGatewayVersion": "2026.3.24-beta.2"
+    },
+    "build": {
+      "openclawVersion": "2026.3.24-beta.2",
+      "pluginSdkVersion": "2026.3.24-beta.2"
+    }
+  }
+}