@openlines/opl-pi-sht 0.1.14

package/extensions/opl-simplebench/llama-server.ts

@@ -0,0 +1,290 @@
+import { CONFIG } from "./util/config";
+
+// ── llama-server metadata (opt-in via --llama-server) ──────────────────────
+// Queries the direct llama-server management base URL for /props configuration
+// and /metrics counters. All values unavailable from the running build are null.
+// /metrics counters are server-wide cumulative telemetry, not per-request.
+
+export interface ModelConfig {
+  model: string | null;
+  ctx: number | null;
+  ngl: number | null;
+  "flash-attn": boolean | null;
+  threads: number | null;
+  batch: number | null;
+  slots: number | null;
+  "kv-k/v": string | null;
+  temp: number | null;
+  "top-k": number | null;
+  "top-p": number | null;
+  "min-p": number | null;
+  repeat: number | null;
+  "spec-type": string | null;
+  "n-max": number | null;
+  "draft-kv": string | null;
+}
+
+export interface LlamaServerInfo {
+  modelConfig: ModelConfig;
+  reasoning: string | null;
+}
+
+export interface ModelStats {
+  prefill: number | null;
+  gen: number | null;
+  "session-avg": number | null;
+  reasoning: string | null;
+  "draft-accepted-p": number | null;
+  "draft-accepted-tok": number | null;
+}
+
+export interface ServerStats {
+  modelConfig: ModelConfig;
+  modelStats: ModelStats;
+  errors: string[];
+}
+
+const NULL_CONFIG: ModelConfig = {
+  model: null, ctx: null, ngl: null, "flash-attn": null, threads: null, batch: null,
+  slots: null, "kv-k/v": null, temp: null, "top-k": null, "top-p": null,
+  "min-p": null, repeat: null, "spec-type": null, "n-max": null, "draft-kv": null,
+};
+
+function num(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function str(value: unknown): string | null {
+  return typeof value === "string" ? value : null;
+}
+
+function bool(value: unknown): boolean | null {
+  return typeof value === "boolean" ? value : null;
+}
+
+function firstNumber(obj: Record<string, unknown>, keys: string[]): number | null {
+  for (const key of keys) { const v = num(obj[key]); if (v !== null) return v; }
+  return null;
+}
+
+function round2(value: number): number {
+  return Math.round(value * 100) / 100;
+}
+
+// Sampling params arrive as float32 widened to float64 (e.g. 0.6 -> 0.6000000238).
+function round4(value: number | null): number | null {
+  return value === null ? null : Math.round(value * 10000) / 10000;
+}
+
+export function normalizeLlamaServerProps(raw: unknown): LlamaServerInfo {
+  const root = (raw && typeof raw === "object" ? raw : {}) as Record<string, unknown>;
+  const dgs = (root.default_generation_settings && typeof root.default_generation_settings === "object"
+    ? root.default_generation_settings : {}) as Record<string, unknown>;
+  const params = (dgs.params && typeof dgs.params === "object" ? dgs.params : {}) as Record<string, unknown>;
+  return {
+    modelConfig: {
+      model: str(root.model_alias) ?? str(root.model),
+      ctx: num(dgs.n_ctx),
+      // Only populated by builds that expose these directly; never inferred.
+      ngl: firstNumber(root, ["n_gpu_layers", "ngl"]),
+      "flash-attn": bool(root.flash_attn),
+      threads: firstNumber(root, ["n_threads", "threads"]),
+      batch: firstNumber(root, ["n_batch", "batch"]),
+      slots: num(root.total_slots),
+      "kv-k/v": str(root.kv_cache_type) ?? str(root["cache_type_k"]),
+      temp: round4(num(params.temperature)),
+      "top-k": num(params.top_k),
+      "top-p": round4(num(params.top_p)),
+      "min-p": round4(num(params.min_p)),
+      repeat: round4(num(params.repeat_penalty)),
+      "spec-type": str(params["speculative.types"]),
+      "n-max": firstNumber(root, ["n_max", "n_ctx_max"]),
+      "draft-kv": str(root.draft_cache_type),
+    },
+    reasoning: str(params.reasoning_format),
+  };
+}
+
+export function parsePrometheusMetrics(text: string): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    // Skip labelled series (e.g. foo{position="0"}); we only need scalars.
+    const match = /^([a-zA-Z_:][a-zA-Z0-9_:]*)\s+([-+0-9.eE]+)$/.exec(trimmed);
+    if (!match) continue;
+    const value = Number(match[2]);
+    // llama.cpp namespaces every metric as `llamacpp:<name>`; key on the bare name.
+    if (Number.isFinite(value)) out[match[1].replace(/^llamacpp:/, "")] = value;
+  }
+  return out;
+}
+
+export function diffPrometheusMetrics(before: Record<string, number>, after: Record<string, number>): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const key of Object.keys(after)) out[key] = after[key] - (before[key] ?? 0);
+  return out;
+}
+
+function rate(tokens: number | undefined, seconds: number | undefined): number | null {
+  if (typeof tokens !== "number" || typeof seconds !== "number" || seconds <= 0) return null;
+  return round2(tokens / seconds);
+}
+
+export function buildServerStats(
+  props: LlamaServerInfo | null,
+  before: Record<string, number>,
+  after: Record<string, number>,
+  errors: string[],
+): ServerStats {
+  const d = diffPrometheusMetrics(before, after);
+  const promptTok = d.prompt_tokens_total;
+  const promptSec = d.prompt_seconds_total;
+  const genTok = d.tokens_predicted_total;
+  const genSec = d.tokens_predicted_seconds_total;
+  const draftTok = d.spec_decode_num_draft_tokens_total;
+  const acceptedTok = d.spec_decode_num_accepted_tokens_total;
+  const drafts = d.spec_decode_num_drafts_total;
+  const sessionTok = typeof promptTok === "number" && typeof genTok === "number" ? promptTok + genTok : undefined;
+  const sessionSec = typeof promptSec === "number" && typeof genSec === "number" ? promptSec + genSec : undefined;
+  return {
+    modelConfig: props ? props.modelConfig : { ...NULL_CONFIG },
+    modelStats: {
+      prefill: rate(promptTok, promptSec),
+      gen: rate(genTok, genSec),
+      "session-avg": rate(sessionTok, sessionSec),
+      reasoning: props ? props.reasoning : null,
+      "draft-accepted-p": typeof draftTok === "number" && draftTok > 0 && typeof acceptedTok === "number"
+        ? round2((acceptedTok / draftTok) * 100) : null,
+      "draft-accepted-tok": typeof drafts === "number" && drafts > 0 && typeof acceptedTok === "number"
+        ? round2(acceptedTok / drafts) : null,
+    },
+    errors,
+  };
+}
+
+// Strip a trailing /v1 (or /v1/) so /props and /metrics resolve against the root.
+export function managementBaseUrl(baseUrl: string): string {
+  return baseUrl.replace(/\/v1\/?$/, "").replace(/\/$/, "");
+}
+
+async function fetchText(url: string): Promise<string> {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), CONFIG.TOOL_TEST_TIMEOUT_MS);
+  try {
+    const res = await fetch(url, { signal: controller.signal });
+    if (!res.ok) throw new Error(`${url} returned ${res.status}`);
+    return await res.text();
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+export async function fetchLlamaServerProps(baseUrl: string): Promise<{ info: LlamaServerInfo | null; error: string | null }> {
+  try {
+    const text = await fetchText(`${managementBaseUrl(baseUrl)}/props`);
+    return { info: normalizeLlamaServerProps(JSON.parse(text)), error: null };
+  } catch (e: any) {
+    return { info: null, error: `props: ${e?.message || String(e)}` };
+  }
+}
+
+export async function fetchLlamaServerMetrics(baseUrl: string): Promise<{ metrics: Record<string, number>; error: string | null }> {
+  try {
+    const text = await fetchText(`${managementBaseUrl(baseUrl)}/metrics`);
+    return { metrics: parsePrometheusMetrics(text), error: null };
+  } catch (e: any) {
+    return { metrics: {}, error: `metrics: ${e?.message || String(e)}` };
+  }
+}
+
+export interface LlamagputopInfo {
+  modelIds: string[];
+  modelConfig: Partial<ModelConfig>;
+  modelStats: Partial<ModelStats>;
+}
+
+function configValue(key: keyof ModelConfig, value: unknown): ModelConfig[typeof key] | undefined {
+  if (key === "flash-attn") {
+    if (typeof value === "boolean") return value;
+    if (typeof value === "string") {
+      if (["on", "enabled", "true", "1"].includes(value.toLowerCase())) return true;
+      if (["off", "disabled", "false", "0"].includes(value.toLowerCase())) return false;
+    }
+    return undefined;
+  }
+  if (key === "batch" && typeof value === "string") {
+    const n = Number(value.split("/")[0]);
+    return Number.isFinite(n) ? n : undefined;
+  }
+  if (["ctx", "ngl", "threads", "slots", "temp", "top-k", "top-p", "min-p", "repeat", "n-max"].includes(key)) {
+    return num(value) ?? undefined;
+  }
+  return typeof value === "string" ? value : undefined;
+}
+
+const CONFIG_KEYS = Object.keys(NULL_CONFIG) as (keyof ModelConfig)[];
+
+export function normalizeLlamagputopStats(raw: unknown): { info: LlamagputopInfo | null; error: string | null } {
+  const root = (raw && typeof raw === "object" ? raw : {}) as Record<string, unknown>;
+  const llama = (root.llama && typeof root.llama === "object" ? root.llama : {}) as Record<string, unknown>;
+  const modelIds = [root.model, root.model_id, root.modelId, root.model_alias, llama.model, llama.model_id, llama.modelId]
+    .filter((value): value is string => typeof value === "string" && value.trim().length > 0);
+  const modelConfig: Partial<ModelConfig> = modelIds[0] ? { model: modelIds[0] } : {};
+  const rawConfig = (root.modelConfig && typeof root.modelConfig === "object" ? root.modelConfig : {}) as Record<string, unknown>;
+  for (const key of CONFIG_KEYS) {
+    const value = configValue(key, rawConfig[key]);
+    if (value !== undefined) modelConfig[key] = value as never;
+  }
+  const modelStats: Partial<ModelStats> = {};
+  const rawStats = (root.modelStats && typeof root.modelStats === "object" ? root.modelStats : {}) as Record<string, unknown>;
+  for (const key of ["prefill", "gen", "session-avg", "draft-accepted-p", "draft-accepted-tok"] as const) {
+    let value = num(rawStats[key]);
+    if (key === "draft-accepted-p" && value !== null && value <= 1) value *= 100;
+    if (value !== null) modelStats[key] = round2(value);
+  }
+  const reasoning = str(rawStats.reasoning) ?? str(llama.reasoning_format);
+  if (reasoning !== null) modelStats.reasoning = reasoning;
+  return { info: { modelIds, modelConfig, modelStats }, error: null };
+}
+
+export function applyLlamagputop(config: ModelConfig, stats: Partial<ModelConfig>): ModelConfig {
+  const out = { ...config };
+  for (const key of Object.keys(stats) as (keyof ModelConfig)[]) if ((key === "model" || key === "spec-type" || out[key] === null) && stats[key] !== null && stats[key] !== undefined) out[key] = stats[key] as never;
+  return out;
+}
+
+export function applyLlamagputopModelStats(stats: ModelStats, incoming: Partial<ModelStats>): ModelStats {
+  const out = { ...stats };
+  for (const key of Object.keys(incoming) as (keyof ModelStats)[]) if ((key === "reasoning" || out[key] === null) && incoming[key] !== null && incoming[key] !== undefined) out[key] = incoming[key] as never;
+  return out;
+}
+
+export function statsUrl(url: string): string {
+  const value = url.trim().replace(/\/$/, "");
+  return value.endsWith("/stats") ? value : `${value}/stats`;
+}
+
+export function healthUrl(url: string): string {
+  return statsUrl(url).replace(/\/stats$/, "/health");
+}
+
+export async function fetchLlamagputopStats(url: string): Promise<{ info: LlamagputopInfo | null; error: string | null; unavailable: boolean }> {
+  if (!url) return { info: null, error: "stats: no url", unavailable: true };
+  try {
+    const text = await fetchText(statsUrl(url));
+    const result = normalizeLlamagputopStats(JSON.parse(text));
+    return { ...result, unavailable: false };
+  } catch (e: any) {
+    return { info: null, error: `stats: ${e?.message || String(e)}`, unavailable: true };
+  }
+}
+
+export async function fetchLlamagputopHealth(url: string): Promise<string | null> {
+  try {
+    await fetchText(healthUrl(url));
+    return null;
+  } catch (e: any) {
+    return `health: ${e?.message || String(e)}`;
+  }
+}

package/extensions/opl-simplebench/metrics.ts

@@ -0,0 +1,73 @@
+import type { RequestMetrics, TestRecord } from "./types";
+
+export function usageFromRaw(raw: any): Pick<RequestMetrics, "inputTokens" | "outputTokens" | "totalTokens" | "outputTokensPerSecond"> {
+  const usage = raw?.usage;
+  const inputTokens = usage?.prompt_tokens ?? usage?.inputTokens ?? raw?.prompt_eval_count ?? null;
+  const outputTokens = usage?.completion_tokens ?? usage?.outputTokens ?? raw?.eval_count ?? null;
+  const totalTokens = usage?.total_tokens ?? usage?.totalTokens ?? (inputTokens !== null && outputTokens !== null ? inputTokens + outputTokens : null);
+  const evalDurationNs = raw?.eval_duration;
+  const outputTokensPerSecond = outputTokens !== null && typeof evalDurationNs === "number" && evalDurationNs > 0
+    ? outputTokens / (evalDurationNs / 1_000_000_000)
+    : null;
+  return { inputTokens, outputTokens, totalTokens, outputTokensPerSecond };
+}
+
+export function metricsFromChat(result: { elapsedMs: number; raw?: unknown; requestCount?: number; retryCount?: number; startedAt?: string; finishedAt?: string; timeToFirstTokenMs?: number | null; toolCalls?: any[] }): RequestMetrics {
+  return {
+    requestCount: result.requestCount ?? 1,
+    retryCount: result.retryCount ?? 0,
+    wallTimeMs: result.elapsedMs,
+    timeToAnswerMs: result.elapsedMs,
+    timeToFirstTokenMs: result.timeToFirstTokenMs ?? null,
+    startedAt: result.startedAt ?? null,
+    finishedAt: result.finishedAt ?? null,
+    ...usageFromRaw(result.raw),
+    toolCalls: (result.toolCalls ?? []).map((call: any) => ({
+      name: call.function?.name ?? call.name ?? "?",
+      arguments: call.function?.arguments ?? call.arguments,
+    })),
+  };
+}
+
+export function emptyMetrics(): RequestMetrics {
+  return { requestCount: 1, retryCount: 0, wallTimeMs: 0, timeToAnswerMs: 0, timeToFirstTokenMs: null, startedAt: null, finishedAt: null, inputTokens: null, outputTokens: null, totalTokens: null, outputTokensPerSecond: null, toolCalls: [] };
+}
+
+export function mergeRequestMetrics(metrics: RequestMetrics[]): RequestMetrics {
+  const knownTotal = (key: "inputTokens" | "outputTokens" | "totalTokens") => metrics.length && metrics.every(metric => metric[key] !== null)
+    ? metrics.reduce((sum, metric) => sum + (metric[key] as number), 0) : null;
+  const outputSeconds = metrics.reduce((sum, metric) => sum + (metric.outputTokens && metric.outputTokensPerSecond ? metric.outputTokens / metric.outputTokensPerSecond : 0), 0);
+  const outputTokens = knownTotal("outputTokens");
+  return {
+    requestCount: metrics.reduce((sum, metric) => sum + metric.requestCount, 0),
+    retryCount: metrics.reduce((sum, metric) => sum + metric.retryCount, 0),
+    wallTimeMs: metrics.reduce((sum, metric) => sum + metric.wallTimeMs, 0),
+    timeToAnswerMs: metrics.reduce((sum, metric) => sum + metric.timeToAnswerMs, 0),
+    timeToFirstTokenMs: metrics.find(metric => metric.timeToFirstTokenMs !== null)?.timeToFirstTokenMs ?? null,
+    startedAt: metrics.find(metric => metric.startedAt !== null)?.startedAt ?? null,
+    finishedAt: [...metrics].reverse().find(metric => metric.finishedAt !== null)?.finishedAt ?? null,
+    inputTokens: knownTotal("inputTokens"), outputTokens, totalTokens: knownTotal("totalTokens"),
+    outputTokensPerSecond: outputTokens !== null && outputSeconds > 0 ? outputTokens / outputSeconds : null,
+    toolCalls: metrics.flatMap(metric => metric.toolCalls),
+  };
+}
+
+export function aggregateMetrics(tests: TestRecord[]) {
+  const values = tests.map(t => t.metrics.wallTimeMs).sort((a, b) => a - b);
+  const knownTotal = (key: "inputTokens" | "outputTokens" | "totalTokens") => {
+    const known = tests.map(t => t.metrics[key]).filter((value): value is number => value !== null);
+    return known.length === tests.length ? known.reduce((sum, value) => sum + value, 0) : null;
+  };
+  const toolCallsByName: Record<string, number> = {};
+  for (const test of tests) for (const call of test.metrics.toolCalls) toolCallsByName[call.name] = (toolCallsByName[call.name] ?? 0) + 1;
+  const outputRateNumerator = tests.reduce((sum, test) => sum + (test.metrics.outputTokens ?? 0), 0);
+  const outputRateDenominator = tests.reduce((sum, test) => sum + (test.metrics.outputTokensPerSecond && test.metrics.outputTokens ? test.metrics.outputTokens / test.metrics.outputTokensPerSecond : 0), 0);
+  const percentile = values.length ? values[Math.min(values.length - 1, Math.ceil(values.length * .95) - 1)] : 0;
+  return {
+    requests: { total: tests.reduce((sum, test) => sum + test.metrics.requestCount, 0), successful: tests.filter(test => !test.error).length, failed: tests.filter(test => !!test.error).length, retries: tests.reduce((sum, test) => sum + test.metrics.retryCount, 0) },
+    toolCalls: { total: tests.reduce((sum, test) => sum + test.metrics.toolCalls.length, 0), byName: toolCallsByName },
+    inputTokens: knownTotal("inputTokens"), outputTokens: knownTotal("outputTokens"), totalTokens: knownTotal("totalTokens"),
+    outputTokensPerSecond: outputRateDenominator > 0 ? outputRateNumerator / outputRateDenominator : null,
+    latency: { averageMs: values.length ? values.reduce((a, b) => a + b, 0) / values.length : 0, medianMs: values.length ? values[Math.floor(values.length / 2)] : 0, p95Ms: percentile, minMs: values[0] ?? 0, maxMs: values.at(-1) ?? 0 },
+  };
+}

package/extensions/opl-simplebench/report.ts

@@ -0,0 +1,46 @@
+import { fail, info, ok, section, warn, msHuman } from "./util/format";
+
+export const branding = "  ⚡ OPL Simplebench";
+
+export function formatInstructionScore(instructions: { pass: boolean; score: string; schemaValid: boolean }): string {
+  const message = `Instruction following: ${instructions.score} (schema ${instructions.schemaValid ? "valid" : "invalid"})`;
+  return instructions.pass ? ok(message) : fail(message);
+}
+
+export function formatTestSummary(tests: Array<{ name: string; pass: boolean; score: string }>, totalMs: number): string[] {
+  return [section("SUMMARY"), ...tests.map(t => t.pass ? ok(`${t.name}: ${t.score}`) : fail(`${t.name}: ${t.score}`)), info(`Total time: ${msHuman(totalMs)}`), info(`Score: ${tests.filter(t => t.pass).length}/${tests.length} tests passed`)];
+}
+/** Legacy-compatible category recommendation. New runner uses recommendation() above. */
+export function formatRecommendation(model: string, passed: number, total: number): string[] {
+  const label = passed === total ? "STRONG" : passed >= total - 1 ? "GOOD" : passed >= total - 2 ? "USABLE" : "WEAK";
+  return [section("RECOMMENDATION"), label === "WEAK" ? fail(`${model} is ${label}`) : ok(`${model} is ${label}`)];
+}
+
+export function recommendation(reasoning: string, instructionsPass: boolean, toolsPass: boolean, coding?: { passed: number; total: number; efficiency?: { strong: number; moderate: number; weak: number; fail: number } }) {
+  const reasoningPass = reasoning === "STRONG" || reasoning === "MODERATE";
+  // Efficiency-weighted coding pass: STRONG=1.0, MODERATE=0.7, WEAK=0.4, FAIL=0
+  const codingPass = coding ? (() => {
+    if (!coding.efficiency) return coding.passed >= Math.ceil(coding.total / 2);
+    const weighted = coding.efficiency.strong * 1.0 + coding.efficiency.moderate * 0.7 + coding.efficiency.weak * 0.4;
+    return coding.total > 0 && weighted / coding.total >= 0.5;
+  })() : undefined;
+  const passed = Number(reasoningPass) + Number(instructionsPass) + Number(toolsPass) + (codingPass !== undefined ? Number(codingPass) : 0);
+  const total = coding ? 4 : 3;
+  const allCodingStrong = coding?.efficiency ? coding.efficiency.strong === coding.total : coding?.passed === coding?.total;
+  const label = coding
+    ? passed === 4 ? reasoning === "STRONG" && allCodingStrong ? "STRONG" : "GOOD" : passed === 3 ? "USABLE" : passed === 2 ? "LIMITED" : passed === 1 ? "LIMITED" : "WEAK"
+    : passed === 3 ? reasoning === "STRONG" ? "STRONG" : "GOOD" : passed === 2 ? "USABLE" : passed === 1 ? "LIMITED" : "WEAK";
+  return { label, passed, total };
+}
+
+export function codingRecommendation(passed: number, total: number) {
+  const ratio = total > 0 ? passed / total : 0;
+  const label = total > 0 && passed === total ? "STRONG" : ratio >= 0.5 ? "GOOD" : passed > 0 ? "USABLE" : "WEAK";
+  return { label, passed, total };
+}
+
+export function renderSummary(model: string, reasoning: { score: string; passed: number; total: number }, instructions: { score: string; passed: boolean; output: string }, tools: { score: string; passed: boolean; calls: string[]; response: string }, artifact: string | null, metrics: any, coding?: { passed: number; total: number }): string {
+  const overall = recommendation(reasoning.score, instructions.passed, tools.passed, coding);
+  const render = (score: string, text: string) => score === "STRONG" || score === "MODERATE" ? ok(text) : score === "WEAK" ? warn(text) : fail(text);
+  return [section(`SIMPLEBENCH: ${model}`), render(reasoning.score, `Closed-answer contract: ${reasoning.score} (${reasoning.passed}/${reasoning.total})`), render(instructions.score, `Instructions: ${instructions.score}`), info(`Instruction output: ${instructions.output}`), render(tools.score, `Tool usage: ${tools.score} (${tools.calls.join(", ") || "none"})`), info(`Tool response: ${tools.response}`), ...(coding ? [render(coding.passed >= Math.ceil(coding.total / 2) ? "MODERATE" : "FAIL", `Coding Lite: ${coding.passed}/${coding.total}`)] : []), section("METRICS"), info(`Requests: ${metrics.requests}; tool calls: ${metrics.toolCalls}; latency avg/p95: ${Math.round(metrics.latency.averageMs)}ms/${metrics.latency.p95Ms}ms`), section("RECOMMENDATION"), render(overall.label, `${model} is ${overall.label} (${overall.passed}/${overall.total} capability categories passed)`), info(artifact ? `Artifact: ${artifact}` : "Artifact: disabled (--no-artifact)")].join("\n");
+}

package/extensions/opl-simplebench/research.ts

@@ -0,0 +1,200 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+
+import { buildToolResultMessages, type ChatFn, type ChatMessage } from "./util/config";
+import { mergeRequestMetrics, metricsFromChat } from "./metrics";
+import type { RequestMetrics } from "./types";
+
+export interface ResearchSource { id?: string; title: string; url: string; snippet: string; facts?: string[]; }
+
+export interface GroundedResearchFixture {
+  id: string;
+  topic: string;
+  sources: Array<{ id: "S1" | "S2" | "S3"; title: string; url: string; facts: string[] }>;
+  claims: Array<{ text: string; sourceId: "S1" | "S2" | "S3" }>;
+}
+
+export const GROUNDED_URBAN_TREES_FIXTURE: GroundedResearchFixture = {
+  id: "urban-trees-1",
+  topic: "benefits of urban trees",
+  sources: [
+    { id: "S1", title: "Urban shade study", url: "https://research.fixture/S1", facts: ["Tree canopy reduces direct solar exposure on streets.", "Shaded pavement has lower surface temperatures."] },
+    { id: "S2", title: "Urban stormwater study", url: "https://research.fixture/S2", facts: ["Tree canopies intercept some rainfall before it reaches the ground."] },
+    { id: "S3", title: "Urban biodiversity study", url: "https://research.fixture/S3", facts: ["Urban trees provide habitat for birds."] },
+  ],
+  claims: [
+    { text: "Tree canopy reduces direct solar exposure on streets.", sourceId: "S1" },
+    { text: "Shaded pavement has lower surface temperatures.", sourceId: "S1" },
+    { text: "Tree canopies intercept some rainfall before it reaches the ground.", sourceId: "S2" },
+  ],
+};
+
+export async function searchConfiguredResearch(query: string, config: { researchSearchProvider?: "ddgs" | "searxng"; researchSearchUrl?: string; researchMaxResults?: number }): Promise<ResearchSource[]> {
+  const base = config.researchSearchUrl?.replace(/\/$/, "");
+  if (!base) throw new Error("researchSearchUrl is not configured in opl-simplebench.json");
+  const limit = config.researchMaxResults ?? 5;
+  const provider = config.researchSearchProvider ?? "ddgs";
+  const params = provider === "ddgs"
+    ? new URLSearchParams({ query, max_results: String(limit) })
+    : new URLSearchParams({ q: query, format: "json", categories: "general", safesearch: "0" });
+  const endpoint = provider === "ddgs" ? `${base}/search/text?${params}` : `${base}/search?${params}`;
+  const response = await fetch(endpoint);
+  if (!response.ok) throw new Error(`${provider} search returned ${response.status}`);
+  const data = await response.json() as any;
+  const results = Array.isArray(data.results) ? data.results : [];
+  return results.slice(0, limit).map((item: any) => ({ title: String(item.title ?? ""), url: String(item.href ?? item.url ?? ""), snippet: String(item.body ?? item.content ?? "") })).filter((item: ResearchSource) => item.url);
+}
+export interface ResearchArtifactResult {
+  id: "research-artifact" | "research-grounded";
+  passed: boolean;
+  score: "STRONG" | "MODERATE" | "WEAK" | "FAIL" | "ERROR";
+  error: string | null;
+  toolCalls: string[];
+  files: Record<string, string>;
+  metrics: RequestMetrics;
+}
+
+const MINIMALIST_UI_SKILL = "Use semantic HTML, a restrained palette, readable typography, no gradients, and no heavy shadows. Do not use any CSS box-shadow declaration.";
+export const RESEARCH_TASK_PROMPT = `Research benefits of urban trees with the provided tools.
+
+You MUST:
+1. Call web_search for real sources before writing.
+2. Call read_skill and apply its minimalist UI guidance to page.html.
+3. Write research.md with a concise synthesis and a ## Sources section containing Markdown links to returned source URLs.
+4. Write page.html as a minimal, responsive editorial page: semantic HTML with <main>, a viewport meta tag, restrained colors, readable type, no gradients, and no heavy shadows. Do not use any CSS box-shadow declaration.
+Do not answer only in chat: create both files.`;
+export const GROUNDED_RESEARCH_TASK_PROMPT = `Research benefits of urban trees with the provided tools.
+
+You MUST:
+1. Call web_search and use only its source cards.
+2. Call read_skill and apply its minimalist UI guidance to page.html.
+3. Write research.md with a ## Findings section containing every provided required fact as a Markdown bullet followed by its inline source ID, such as [S1]. Under ## Sources, use exactly - [S1](https://research.fixture/S1) for S1 and the equivalent Markdown-link syntax for each cited source.
+4. Write page.html as a minimal, responsive editorial page: semantic HTML with <main>, a viewport meta tag, restrained colors, readable type, no gradients, and no heavy shadows. Do not use any CSS box-shadow declaration.
+Do not answer only in chat: create both files.`;
+const RESEARCH_TOOLS = [
+  { type: "function", function: { name: "web_search", description: "Search the web for sources.", parameters: { type: "object", properties: { query: { type: "string" } }, required: ["query"] } } },
+  { type: "function", function: { name: "read_skill", description: "Read the benchmark-local minimalist UI design guidance.", parameters: { type: "object", properties: {} } } },
+  { type: "function", function: { name: "write_file", description: "Write research.md or page.html in the task directory.", parameters: { type: "object", properties: { path: { type: "string", enum: ["research.md", "page.html"] }, content: { type: "string" } }, required: ["path", "content"] } } },
+];
+
+function parseArgs(value: unknown): Record<string, unknown> {
+  try { return typeof value === "string" ? JSON.parse(value) : (value as Record<string, unknown>) ?? {}; }
+  catch { return {}; }
+}
+
+function normalizeClaim(value: string): string {
+  return value.trim().replace(/[.!?]+$/, "").trim().toLowerCase();
+}
+
+export function verifyGroundedResearch(markdown: string, fixture: GroundedResearchFixture): { passed: boolean; missingClaims: string[]; wrongCitations: string[]; unknownCitations: string[] } {
+  const knownSources = new Map(fixture.sources.map(source => [source.id, source]));
+  const findings = markdown.match(/##\s+Findings\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
+  const sourceSection = markdown.match(/##\s+Sources\s*\n([\s\S]*?)(?=\n##\s|$)/i)?.[1] ?? "";
+  const citedClaims = [...findings.matchAll(/^\s*[-*]\s+(.+?)\s+\[([A-Za-z0-9]+)\]\s*$/gm)];
+  const unknownCitations = citedClaims.map(match => match[2]).filter(id => !knownSources.has(id));
+  const missingClaims: string[] = [];
+  const wrongCitations: string[] = [];
+  for (const claim of fixture.claims) {
+    const match = citedClaims.find(item => normalizeClaim(item[1]) === normalizeClaim(claim.text));
+    if (!match) missingClaims.push(claim.text);
+    else if (match[2] !== claim.sourceId) wrongCitations.push(claim.text);
+  }
+  for (const id of new Set(citedClaims.map(match => match[2]).filter(id => knownSources.has(id)))) {
+    const source = knownSources.get(id)!;
+    if (!new RegExp(`\\[${id}\\]\\(${source.url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\)`).test(sourceSection)) wrongCitations.push(id);
+  }
+  return { passed: !missingClaims.length && !wrongCitations.length && !unknownCitations.length, missingClaims, wrongCitations, unknownCitations };
+}
+
+export function isMinimalistResearchHtml(html: string): boolean {
+  const shadows = html.match(/box-shadow\s*:\s*[^;}"']+/gi) ?? [];
+  return /<html[\s>]/i.test(html)
+    && /<main[\s>]/i.test(html)
+    && /name=["']viewport["']/i.test(html)
+    && !/(linear-gradient|radial-gradient)/i.test(html)
+    && shadows.every(shadow => /:\s*none\s*(?:!important)?\s*$/i.test(shadow));
+}
+
+function verify(files: Record<string, string>, tools: string[], sources: ResearchSource[]): { score: ResearchArtifactResult["score"]; passed: boolean; error: string | null } {
+  const markdown = files["research.md"] ?? "";
+  const html = files["page.html"] ?? "";
+  const searched = tools.includes("web_search");
+  const skillRead = tools.includes("read_skill");
+  const citesSource = /##\s+Sources/i.test(markdown) && sources.some(source => markdown.includes(source.url));
+  const htmlPasses = isMinimalistResearchHtml(html);
+  const complete = searched && skillRead && !!markdown && !!html && citesSource && htmlPasses;
+  if (complete) return { score: "STRONG", passed: true, error: null };
+  if (searched && (!!markdown || !!html)) return { score: "MODERATE", passed: false, error: "missing required research, skill, citation, or minimalist HTML evidence" };
+  return { score: "FAIL", passed: false, error: "research artifact task incomplete" };
+}
+
+export async function runResearchArtifactTask(chatFn: ChatFn, model: string, options: { search: (query: string) => Promise<ResearchSource[]>; maxTurns?: number; onProgress?: (message: string) => void; grounded?: boolean } ): Promise<ResearchArtifactResult> {
+  const grounded = options.grounded === true;
+  const id = grounded ? "research-grounded" as const : "research-artifact" as const;
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "simplebench-research-"));
+  const tools: string[] = [];
+  const requestMetrics: RequestMetrics[] = [];
+  let sources: ResearchSource[] = [];
+  let error: string | null = null;
+  const messages: ChatMessage[] = [
+    { role: "system", content: "Complete a research artifact task using benchmark-local tools. Follow every requirement in the user task." },
+    { role: "user", content: grounded ? GROUNDED_RESEARCH_TASK_PROMPT : RESEARCH_TASK_PROMPT },
+  ];
+  try {
+    for (let turn = 0; turn < (options.maxTurns ?? 8); turn += 1) {
+      options.onProgress?.(`research-artifact: agent turn ${turn + 1}/${options.maxTurns ?? 8}...`);
+      const response = await chatFn(model, messages, { tools: RESEARCH_TOOLS });
+      requestMetrics.push(metricsFromChat(response));
+      const calls = response.toolCalls ?? [];
+      if (!calls.length) break;
+      const results: string[] = [];
+      for (const call of calls) {
+        const fn = call.function ?? call;
+        const name = fn.name;
+        const args = parseArgs(fn.arguments);
+        tools.push(name);
+        options.onProgress?.(`research-artifact: ${name}`);
+        try {
+          if (name === "web_search") {
+            sources = await options.search(String(args.query ?? ""));
+            results.push(JSON.stringify(sources));
+          } else if (name === "read_skill") {
+            results.push(MINIMALIST_UI_SKILL);
+          } else if (name === "write_file") {
+            const relative = String(args.path ?? "");
+            if (relative !== "research.md" && relative !== "page.html") throw new Error("only research.md and page.html are allowed");
+            fs.writeFileSync(path.join(root, relative), String(args.content ?? ""));
+            results.push("written");
+          } else throw new Error(`unknown tool: ${name}`);
+        } catch (e: any) { results.push(JSON.stringify({ error: e?.message || String(e) })); }
+      }
+      messages.push(...buildToolResultMessages(response.content, calls, results));
+    }
+  } catch (e: any) { error = e?.message || String(e); }
+  const files: Record<string, string> = {};
+  for (const name of ["research.md", "page.html"]) {
+    const file = path.join(root, name);
+    if (fs.existsSync(file)) files[name] = fs.readFileSync(file, "utf8");
+  }
+  fs.rmSync(root, { recursive: true, force: true });
+  const verified = error
+    ? { score: "ERROR" as const, passed: false, error }
+    : grounded
+      ? (() => {
+          const citations = verifyGroundedResearch(files["research.md"] ?? "", GROUNDED_URBAN_TREES_FIXTURE);
+          const html = files["page.html"] ?? "";
+          const complete = tools.includes("web_search") && tools.includes("read_skill") && citations.passed && isMinimalistResearchHtml(html);
+          return { score: complete ? "STRONG" as const : "FAIL" as const, passed: complete, error: complete ? null : "missing grounded claim, citation, source, skill, or minimalist HTML evidence" };
+        })()
+      : verify(files, tools, sources);
+  return { id, ...verified, toolCalls: tools, files, metrics: mergeRequestMetrics(requestMetrics) };
+}
+
+export function runGroundedResearchTask(chatFn: ChatFn, model: string, options: { maxTurns?: number; onProgress?: (message: string) => void } = {}): Promise<ResearchArtifactResult> {
+  return runResearchArtifactTask(chatFn, model, {
+    ...options,
+    grounded: true,
+    search: async () => GROUNDED_URBAN_TREES_FIXTURE.sources.map(source => ({ id: source.id, title: source.title, url: source.url, snippet: source.facts.join(" "), facts: source.facts })),
+  });
+}

package/extensions/opl-simplebench/scoring.ts

@@ -0,0 +1,48 @@
+export function normalizeAnswer(value: string): string {
+  return value.trim().replace(/[.!?]+$/, "").trim().toLowerCase();
+}
+
+const NUMERIC_ANSWER = /^\d+$/;
+
+function lastNumericToken(line: string): string | null {
+  const tokens = line.match(/\d+/g);
+  return tokens && tokens.length ? tokens[tokens.length - 1] : null;
+}
+
+export function extractAnswer(response: string, expected?: string): { answer: string; method: string } {
+  const lines = response.trimEnd().split("\n").map(line => line.trim()).filter(Boolean);
+  // Numeric contracts: scan backward for a line whose last numeric token is the
+  // expected value. This accepts naked "8" / "8 days" answers and skips trailing
+  // bare-number remnants that thinking templates leak after the real final line.
+  if (expected && NUMERIC_ANSWER.test(expected)) {
+    for (let i = lines.length - 1; i >= 0; i -= 1) {
+      const token = lastNumericToken(lines[i]);
+      if (token === expected) {
+        return { answer: token, method: i === lines.length - 1 ? "final-line" : "expected-last-numeric" };
+      }
+    }
+  }
+  const line = lines.at(-1);
+  return line ? { answer: normalizeAnswer(line), method: "final-line" } : { answer: "?", method: "none" };
+}
+
+export function scoreReasoning(response: string, expected: string) {
+  const extracted = extractAnswer(response, expected);
+  const correct = extracted.answer === normalizeAnswer(expected);
+  return {
+    answer: extracted.answer,
+    extractionMethod: extracted.method,
+    matchedWords: [],
+    score: correct ? "STRONG" : "FAIL",
+    pass: correct,
+  };
+}
+
+export function averageScore(scores: string[]): string {
+  const weights: Record<string, number> = { STRONG: 3, MODERATE: 2, WEAK: 1, FAIL: 0, ERROR: 0 };
+  const average = scores.reduce((sum, score) => sum + (weights[score] || 0), 0) / scores.length;
+  if (average >= 2.5) return "STRONG";
+  if (average >= 1.5) return "MODERATE";
+  if (average >= 0.5) return "WEAK";
+  return "FAIL";
+}