@remnic/cli 1.0.3 → 1.0.4

package/dist/chunk-U4MQO3IF.js

@@ -0,0 +1,1144 @@
+// ../bench/dist/index.js
+import { mkdir, mkdtemp, rm } from "fs/promises";
+import { tmpdir } from "os";
+import path from "path";
+import { Orchestrator, parseConfig } from "@remnic/core";
+import fs from "fs";
+import path4 from "path";
+import { randomUUID } from "crypto";
+import { readFile as readFile2 } from "fs/promises";
+import path3 from "path";
+import { execSync } from "child_process";
+import { mkdir as mkdir2, readFile, writeFile } from "fs/promises";
+import path2 from "path";
+async function createBenchOrchestrator(mode, overrides) {
+  const tempDir = await mkdtemp(path.join(tmpdir(), `remnic-bench-${mode}-`));
+  await mkdir(path.join(tempDir, "state"), { recursive: true });
+  const orchestrator = new Orchestrator(
+    parseConfig({
+      memoryDir: tempDir,
+      workspaceDir: tempDir,
+      qmdEnabled: false,
+      qmdColdTierEnabled: false,
+      transcriptEnabled: false,
+      hourlySummariesEnabled: false,
+      daySummaryEnabled: false,
+      identityEnabled: false,
+      identityContinuityEnabled: false,
+      namespacesEnabled: false,
+      sharedContextEnabled: false,
+      workTasksEnabled: false,
+      workProjectsEnabled: false,
+      commitmentLedgerEnabled: false,
+      resumeBundlesEnabled: false,
+      nativeKnowledge: { enabled: false },
+      lcmEnabled: true,
+      lcmLeafBatchSize: 4,
+      lcmRollupFanIn: 3,
+      lcmFreshTailTurns: 8,
+      lcmMaxDepth: 4,
+      lcmDeterministicMaxTokens: 512,
+      lcmRecallBudgetShare: 1,
+      extractionDedupeEnabled: mode === "direct",
+      extractionMinChars: mode === "direct" ? 10 : 1e6,
+      extractionMinUserTurns: mode === "direct" ? 0 : 1e6,
+      recallPlannerEnabled: mode === "direct",
+      queryExpansionEnabled: false,
+      rerankEnabled: false,
+      memoryBoxesEnabled: false,
+      traceWeaverEnabled: false,
+      threadingEnabled: false,
+      factDeduplicationEnabled: false,
+      knowledgeIndexEnabled: false,
+      entityRetrievalEnabled: false,
+      verifiedRecallEnabled: false,
+      queryAwareIndexingEnabled: false,
+      contradictionDetectionEnabled: false,
+      memoryLinkingEnabled: false,
+      topicExtractionEnabled: false,
+      chunkingEnabled: true,
+      episodeNoteModeEnabled: false,
+      ...overrides
+    })
+  );
+  await orchestrator.initialize();
+  if (!orchestrator.lcmEngine) {
+    throw new Error("Remnic benchmark adapter requires LCM to be enabled.");
+  }
+  return { tempDir, orchestrator };
+}
+function createAdapterFactory(mode) {
+  return async function createAdapter(options = {}) {
+    let state = await createBenchOrchestrator(mode, options.configOverrides);
+    const getEngine = () => {
+      const engine = state.orchestrator.lcmEngine;
+      if (!engine) {
+        throw new Error("LCM engine unavailable for Remnic benchmark adapter.");
+      }
+      return engine;
+    };
+    const cleanup = async () => {
+      state.orchestrator.lcmEngine?.close();
+      await rm(state.tempDir, { recursive: true, force: true });
+    };
+    const rebuild = async () => {
+      await cleanup();
+      state = await createBenchOrchestrator(mode, options.configOverrides);
+    };
+    return {
+      async store(sessionId, messages) {
+        await getEngine().observeMessages(
+          sessionId,
+          messages.map((message) => ({
+            role: message.role,
+            content: message.content
+          }))
+        );
+      },
+      async recall(sessionId, query, budgetChars) {
+        const engine = getEngine();
+        const budget = budgetChars ?? 32e3;
+        const sections = [];
+        if (query) {
+          const searchResults = await engine.searchContextFull(query, 20, sessionId);
+          if (searchResults.length > 0) {
+            sections.push(
+              `## Search results
+${searchResults.map((result) => `[turn ${result.turn_index}, ${result.role}]: ${result.content}`).join("\n\n")}`
+            );
+          }
+        }
+        const recallText = await engine.assembleRecall(sessionId, budget);
+        if (recallText) {
+          sections.push(recallText);
+        }
+        if (sections.length === 0) {
+          const stats = await engine.getStats(sessionId);
+          if (stats.totalMessages > 0) {
+            const expanded = await engine.expandContext(
+              sessionId,
+              0,
+              stats.totalMessages - 1,
+              Math.floor(budget / 4)
+            );
+            if (expanded.length > 0) {
+              sections.push(
+                `## Raw messages
+${expanded.map((message) => `[${message.role}]: ${message.content}`).join("\n")}`
+              );
+            }
+          }
+        }
+        const joined = sections.join("\n\n");
+        return joined.length > budget ? joined.slice(0, budget) : joined;
+      },
+      async search(query, limit, sessionId) {
+        const results = await getEngine().searchContext(query, limit, sessionId);
+        return results.map((result) => ({
+          turnIndex: result.turn_index,
+          role: result.role,
+          snippet: result.snippet,
+          sessionId: result.session_id
+        }));
+      },
+      async reset(_sessionId) {
+        await rebuild();
+      },
+      async getStats(sessionId) {
+        return getEngine().getStats(sessionId);
+      },
+      async destroy() {
+        await cleanup();
+      }
+    };
+  };
+}
+var createLightweightAdapter = createAdapterFactory("lightweight");
+var createRemnicAdapter = createAdapterFactory("direct");
+var BENCHMARK_RESULT_SCHEMA = {
+  type: "object",
+  required: ["meta", "config", "cost", "results", "environment"],
+  properties: {
+    meta: {
+      type: "object",
+      required: [
+        "id",
+        "benchmark",
+        "benchmarkTier",
+        "version",
+        "remnicVersion",
+        "gitSha",
+        "timestamp",
+        "mode",
+        "runCount",
+        "seeds"
+      ],
+      properties: {
+        id: { type: "string" },
+        benchmark: { type: "string" },
+        benchmarkTier: {
+          type: "string",
+          enum: ["published", "remnic", "custom"]
+        },
+        version: { type: "string" },
+        remnicVersion: { type: "string" },
+        gitSha: { type: "string" },
+        timestamp: { type: "string" },
+        mode: { type: "string", enum: ["full", "quick"] },
+        runCount: { type: "number" },
+        seeds: {
+          type: "array",
+          items: { type: "number" }
+        }
+      }
+    },
+    config: {
+      type: "object",
+      required: [
+        "systemProvider",
+        "judgeProvider",
+        "adapterMode",
+        "remnicConfig"
+      ],
+      properties: {
+        systemProvider: {
+          anyOf: [
+            { type: "null" },
+            {
+              type: "object",
+              required: ["provider", "model"],
+              properties: {
+                provider: { type: "string" },
+                model: { type: "string" },
+                baseUrl: { type: "string" }
+              }
+            }
+          ]
+        },
+        judgeProvider: {
+          anyOf: [
+            { type: "null" },
+            {
+              type: "object",
+              required: ["provider", "model"],
+              properties: {
+                provider: { type: "string" },
+                model: { type: "string" },
+                baseUrl: { type: "string" }
+              }
+            }
+          ]
+        },
+        adapterMode: { type: "string" },
+        remnicConfig: { type: "object" }
+      }
+    },
+    cost: {
+      type: "object",
+      required: [
+        "totalTokens",
+        "inputTokens",
+        "outputTokens",
+        "estimatedCostUsd",
+        "totalLatencyMs",
+        "meanQueryLatencyMs"
+      ],
+      properties: {
+        totalTokens: { type: "number" },
+        inputTokens: { type: "number" },
+        outputTokens: { type: "number" },
+        estimatedCostUsd: { type: "number" },
+        totalLatencyMs: { type: "number" },
+        meanQueryLatencyMs: { type: "number" }
+      }
+    },
+    results: {
+      type: "object",
+      required: ["tasks", "aggregates"],
+      properties: {
+        tasks: {
+          type: "array",
+          items: {
+            type: "object",
+            required: [
+              "taskId",
+              "question",
+              "expected",
+              "actual",
+              "scores",
+              "latencyMs",
+              "tokens"
+            ],
+            properties: {
+              taskId: { type: "string" },
+              question: { type: "string" },
+              expected: { type: "string" },
+              actual: { type: "string" },
+              scores: { type: "object" },
+              latencyMs: { type: "number" },
+              tokens: {
+                type: "object",
+                required: ["input", "output"],
+                properties: {
+                  input: { type: "number" },
+                  output: { type: "number" }
+                }
+              }
+            }
+          }
+        },
+        aggregates: { type: "object" },
+        statistics: { type: "object" }
+      }
+    },
+    environment: {
+      type: "object",
+      required: ["os", "nodeVersion"],
+      properties: {
+        os: { type: "string" },
+        nodeVersion: { type: "string" },
+        hardware: { type: "string" }
+      }
+    }
+  }
+};
+var OpenAiCompatibleProvider = class {
+  provider = "openai";
+  id;
+  name;
+  config;
+  usage = {
+    inputTokens: 0,
+    outputTokens: 0,
+    totalTokens: 0
+  };
+  constructor(config) {
+    this.config = config;
+    this.id = `openai:${config.model}`;
+    this.name = config.model;
+  }
+  async complete(prompt, opts = {}) {
+    const startedAt = performance.now();
+    const response = await fetch(this.urlFor("chat/completions"), {
+      method: "POST",
+      headers: this.headers(opts.headers),
+      body: JSON.stringify({
+        model: this.config.model,
+        messages: [
+          ...opts.systemPrompt ? [{ role: "system", content: opts.systemPrompt }] : [],
+          { role: "user", content: prompt }
+        ],
+        temperature: opts.temperature,
+        max_tokens: opts.maxTokens
+      })
+    });
+    if (!response.ok) {
+      throw new Error(
+        `OpenAI-compatible completion failed: ${response.status} ${response.statusText}`
+      );
+    }
+    const payload = await response.json();
+    const promptTokens = payload.usage?.prompt_tokens ?? 0;
+    const completionTokens = payload.usage?.completion_tokens ?? 0;
+    this.recordUsage(promptTokens, completionTokens);
+    return {
+      text: readMessageText(payload),
+      tokens: {
+        input: promptTokens,
+        output: completionTokens
+      },
+      latencyMs: Math.round(performance.now() - startedAt),
+      model: payload.model ?? this.config.model
+    };
+  }
+  async discover() {
+    const response = await fetch(this.urlFor("models"), {
+      method: "GET",
+      headers: this.headers()
+    });
+    if (!response.ok) {
+      throw new Error(
+        `OpenAI-compatible model discovery failed: ${response.status} ${response.statusText}`
+      );
+    }
+    const payload = await response.json();
+    return (payload.data ?? []).map((model) => ({
+      id: model.id,
+      name: model.name ?? model.id,
+      contextLength: model.context_length ?? 0,
+      capabilities: model.capabilities ?? ["completion"],
+      ...model.quantization ? { quantization: model.quantization } : {},
+      ...model.parameter_count ? { parameterCount: model.parameter_count } : {}
+    }));
+  }
+  getUsage() {
+    return { ...this.usage };
+  }
+  resetUsage() {
+    this.usage = {
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0
+    };
+  }
+  headers(extraHeaders = {}) {
+    return {
+      "content-type": "application/json",
+      ...this.config.apiKey ? { authorization: `Bearer ${this.config.apiKey}` } : {},
+      ...this.config.headers ?? {},
+      ...extraHeaders
+    };
+  }
+  recordUsage(inputTokens, outputTokens) {
+    this.usage = {
+      inputTokens: this.usage.inputTokens + inputTokens,
+      outputTokens: this.usage.outputTokens + outputTokens,
+      totalTokens: this.usage.totalTokens + inputTokens + outputTokens
+    };
+  }
+  urlFor(pathname) {
+    const baseUrl = this.config.baseUrl ?? "https://api.openai.com/v1";
+    const normalizedBase = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
+    const normalizedPath = pathname.startsWith("/") ? pathname.slice(1) : pathname;
+    return `${normalizedBase}/${normalizedPath}`;
+  }
+};
+function readMessageText(payload) {
+  const content = payload.choices?.[0]?.message?.content;
+  if (typeof content === "string") {
+    return content;
+  }
+  if (Array.isArray(content)) {
+    return content.map((part) => part.text ?? "").join("").trim();
+  }
+  return "";
+}
+function createOpenAiCompatibleProvider(config) {
+  return new OpenAiCompatibleProvider(config);
+}
+var LONG_MEM_EVAL_SMOKE_FIXTURE = [
+  {
+    question_id: 1,
+    question_type: "single-session-user",
+    question: "What city does the user live in?",
+    answer: "Paris",
+    question_date: "2025-01-01",
+    haystack_dates: ["2024-12-01"],
+    haystack_session_ids: ["session-1"],
+    haystack_sessions: [
+      [
+        { role: "user", content: "I moved to Paris last year." },
+        { role: "assistant", content: "Paris sounds great." }
+      ]
+    ],
+    answer_session_ids: ["session-1"]
+  }
+];
+function exactMatch(predicted, expected) {
+  return normalizeText(predicted) === normalizeText(expected) ? 1 : 0;
+}
+function f1Score(predicted, expected) {
+  const predictedTokens = tokenize(predicted);
+  const expectedTokens = tokenize(expected);
+  if (predictedTokens.length === 0 && expectedTokens.length === 0) return 1;
+  if (predictedTokens.length === 0 || expectedTokens.length === 0) return 0;
+  const predictedCounts = frequencyMap(predictedTokens);
+  const expectedCounts = frequencyMap(expectedTokens);
+  let overlap = 0;
+  for (const [token, count] of expectedCounts.entries()) {
+    overlap += Math.min(count, predictedCounts.get(token) ?? 0);
+  }
+  if (overlap === 0) return 0;
+  const precision = overlap / predictedTokens.length;
+  const recall = overlap / expectedTokens.length;
+  return 2 * precision * recall / (precision + recall);
+}
+function rougeL(predicted, expected) {
+  const predictedTokens = tokenize(predicted);
+  const expectedTokens = tokenize(expected);
+  if (predictedTokens.length === 0 && expectedTokens.length === 0) return 1;
+  if (predictedTokens.length === 0 || expectedTokens.length === 0) return 0;
+  const lcsLength = longestCommonSubsequence(predictedTokens, expectedTokens);
+  const precision = lcsLength / predictedTokens.length;
+  const recall = lcsLength / expectedTokens.length;
+  if (precision + recall === 0) return 0;
+  return 2 * precision * recall / (precision + recall);
+}
+function recallAtK(retrieved, relevant, k) {
+  if (relevant.length === 0) return 1;
+  const topK = retrieved.slice(0, k).map(normalizeText);
+  const relevantSet = new Set(relevant.map(normalizeText));
+  const hits = new Set(
+    topK.filter((candidate) => relevantSet.has(candidate))
+  ).size;
+  return hits / relevantSet.size;
+}
+function containsAnswer(predicted, expected) {
+  const normalizedExpected = normalizeText(expected);
+  if (normalizedExpected.length === 0) return 0;
+  return normalizeText(predicted).includes(normalizedExpected) ? 1 : 0;
+}
+async function llmJudgeScore(judge, question, predicted, expected) {
+  if (!judge) return -1;
+  try {
+    return await judge.score(question, predicted, expected);
+  } catch {
+    return -1;
+  }
+}
+async function timed(fn) {
+  const startedAt = performance.now();
+  const result = await fn();
+  return {
+    result,
+    durationMs: Math.round(performance.now() - startedAt)
+  };
+}
+function aggregateTaskScores(metricsList) {
+  const metricValues = collectMetricValues(metricsList);
+  const aggregates = {};
+  for (const [metricName, values] of Object.entries(metricValues)) {
+    aggregates[metricName] = summarizeMetricValues(values);
+  }
+  return aggregates;
+}
+function collectMetricValues(metricsList) {
+  if (metricsList.length === 0) return {};
+  const metricNames = /* @__PURE__ */ new Set();
+  for (const metrics of metricsList) {
+    for (const metricName of Object.keys(metrics)) {
+      metricNames.add(metricName);
+    }
+  }
+  const metricValues = {};
+  for (const metricName of metricNames) {
+    const values = metricsList.map((metrics) => metrics[metricName]).filter((value) => typeof value === "number" && !Number.isNaN(value)).sort((left, right) => left - right);
+    if (values.length > 0) {
+      metricValues[metricName] = values;
+    }
+  }
+  return metricValues;
+}
+function summarizeMetricValues(values) {
+  const mean = values.reduce((sum, value) => sum + value, 0) / values.length;
+  const median = values.length % 2 === 0 ? (values[values.length / 2 - 1] + values[values.length / 2]) / 2 : values[Math.floor(values.length / 2)];
+  const variance = values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / values.length;
+  return {
+    mean,
+    median,
+    stdDev: Math.sqrt(variance),
+    min: values[0],
+    max: values[values.length - 1]
+  };
+}
+function normalizeText(value) {
+  return String(value ?? "").trim().toLowerCase();
+}
+function tokenize(value) {
+  return normalizeText(value).replace(/[^\w\s]/g, " ").split(/\s+/).filter((token) => token.length > 0);
+}
+function frequencyMap(tokens) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const token of tokens) {
+    counts.set(token, (counts.get(token) ?? 0) + 1);
+  }
+  return counts;
+}
+function longestCommonSubsequence(left, right) {
+  let previous = new Array(right.length + 1).fill(0);
+  let current = new Array(right.length + 1).fill(0);
+  for (let leftIndex = 1; leftIndex <= left.length; leftIndex += 1) {
+    for (let rightIndex = 1; rightIndex <= right.length; rightIndex += 1) {
+      if (left[leftIndex - 1] === right[rightIndex - 1]) {
+        current[rightIndex] = previous[rightIndex - 1] + 1;
+      } else {
+        current[rightIndex] = Math.max(
+          previous[rightIndex],
+          current[rightIndex - 1]
+        );
+      }
+    }
+    [previous, current] = [current, previous];
+    current.fill(0);
+  }
+  return previous[right.length];
+}
+function sanitizeFilenameSegment(value) {
+  const sanitized = value.trim().replace(/[^a-zA-Z0-9._-]/g, "_");
+  return sanitized.length > 0 ? sanitized : "unknown";
+}
+async function writeBenchmarkResult(result, outputDir) {
+  await mkdir2(outputDir, { recursive: true });
+  const safeRemnicVersion = sanitizeFilenameSegment(result.meta.remnicVersion);
+  const timestamp = result.meta.timestamp.replace(/[:.]/g, "-");
+  const filePath = path2.join(
+    outputDir,
+    `${result.meta.benchmark}-v${safeRemnicVersion}-${timestamp}.json`
+  );
+  await writeFile(filePath, JSON.stringify(result, null, 2) + "\n");
+  return filePath;
+}
+async function getRemnicVersion() {
+  try {
+    const packageJson = JSON.parse(
+      await readFile(
+        path2.resolve(import.meta.dirname, "../../../package.json"),
+        "utf8"
+      )
+    );
+    return typeof packageJson.version === "string" ? packageJson.version : "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+function getGitSha() {
+  try {
+    return execSync("git rev-parse --short HEAD", { encoding: "utf8" }).trim();
+  } catch {
+    return "unknown";
+  }
+}
+var longMemEvalDefinition = {
+  id: "longmemeval",
+  title: "LongMemEval",
+  tier: "published",
+  status: "ready",
+  runnerAvailable: true,
+  meta: {
+    name: "longmemeval",
+    version: "2.0.0",
+    description: "Long-term memory evaluation across information extraction, multi-session reasoning, temporal reasoning, and knowledge updates.",
+    category: "retrieval",
+    citation: "Wu et al. LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory. ICLR 2025."
+  }
+};
+async function runLongMemEvalBenchmark(options) {
+  const dataset = await loadDataset(options.mode, options.datasetDir, options.limit);
+  const tasks = [];
+  for (const item of dataset) {
+    await options.system.reset();
+    const sessionIds = [];
+    for (let sessionIndex = 0; sessionIndex < item.haystack_sessions.length; sessionIndex += 1) {
+      const sessionId = item.haystack_session_ids[sessionIndex] ?? `session-${sessionIndex}`;
+      const messages = item.haystack_sessions[sessionIndex].map(
+        (turn) => ({
+          role: turn.role,
+          content: turn.content
+        })
+      );
+      sessionIds.push(sessionId);
+      if (messages.length > 0) {
+        await options.system.store(sessionId, messages);
+      }
+    }
+    const { result: recalledText, durationMs } = await timed(async () => {
+      const recalledSessions = await Promise.all(
+        sessionIds.map(
+          (sessionId) => options.system.recall(sessionId, item.question)
+        )
+      );
+      return recalledSessions.filter(Boolean).join("\n\n");
+    });
+    const searchResults = await options.system.search(item.question, 10);
+    const judgeScore = await llmJudgeScore(
+      options.system.judge,
+      item.question,
+      recalledText,
+      item.answer
+    );
+    const scores = {
+      f1: f1Score(recalledText, item.answer),
+      contains_answer: containsAnswer(recalledText, item.answer),
+      search_hits: searchResults.length
+    };
+    if (judgeScore >= 0) {
+      scores.llm_judge = judgeScore;
+    }
+    tasks.push({
+      taskId: `q${item.question_id}`,
+      question: item.question,
+      expected: item.answer,
+      actual: recalledText,
+      scores,
+      latencyMs: durationMs,
+      tokens: { input: 0, output: 0 },
+      details: {
+        questionType: item.question_type,
+        questionDate: item.question_date,
+        haystackDates: item.haystack_dates,
+        haystackSessionIds: item.haystack_session_ids,
+        answerSessionIds: item.answer_session_ids
+      }
+    });
+  }
+  const remnicVersion = await getRemnicVersion();
+  const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
+  const totalInputTokens = tasks.reduce(
+    (sum, task) => sum + task.tokens.input,
+    0
+  );
+  const totalOutputTokens = tasks.reduce(
+    (sum, task) => sum + task.tokens.output,
+    0
+  );
+  return {
+    meta: {
+      id: randomUUID(),
+      benchmark: options.benchmark.id,
+      benchmarkTier: options.benchmark.tier,
+      version: options.benchmark.meta.version,
+      remnicVersion,
+      gitSha: getGitSha(),
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      mode: options.mode,
+      runCount: 1,
+      seeds: [options.seed ?? 0]
+    },
+    config: {
+      systemProvider: options.systemProvider ?? null,
+      judgeProvider: options.judgeProvider ?? null,
+      adapterMode: options.adapterMode ?? "direct",
+      remnicConfig: options.remnicConfig ?? {}
+    },
+    cost: {
+      totalTokens: totalInputTokens + totalOutputTokens,
+      inputTokens: totalInputTokens,
+      outputTokens: totalOutputTokens,
+      estimatedCostUsd: 0,
+      totalLatencyMs,
+      meanQueryLatencyMs: tasks.length > 0 ? totalLatencyMs / tasks.length : 0
+    },
+    results: {
+      tasks,
+      aggregates: aggregateTaskScores(tasks.map((task) => task.scores))
+    },
+    environment: {
+      os: process.platform,
+      nodeVersion: process.version,
+      hardware: process.arch
+    }
+  };
+}
+async function loadDataset(mode, datasetDir, limit) {
+  const ensureDatasetItems = (items) => {
+    if (items.length === 0) {
+      throw new Error(
+        "LongMemEval dataset is empty after applying the requested limit."
+      );
+    }
+    return items;
+  };
+  if (datasetDir) {
+    const datasetErrors = [];
+    for (const filename of [
+      "longmemeval_oracle.json",
+      "longmemeval_s_cleaned.json",
+      "longmemeval.json"
+    ]) {
+      try {
+        const raw = await readFile2(path3.join(datasetDir, filename), "utf8");
+        const parsed = JSON.parse(raw);
+        return ensureDatasetItems(limit ? parsed.slice(0, limit) : parsed);
+      } catch (error) {
+        datasetErrors.push(
+          `${filename}: ${error instanceof Error ? error.message : String(error)}`
+        );
+        continue;
+      }
+    }
+    throw new Error(
+      `LongMemEval dataset not found under ${datasetDir}. Tried longmemeval_oracle.json, longmemeval_s_cleaned.json, and longmemeval.json. Errors: ${datasetErrors.join(" | ")}`
+    );
+  }
+  if (mode === "full") {
+    throw new Error(
+      "LongMemEval full mode requires datasetDir. Pass a dataset path or use quick mode to run the bundled smoke fixture."
+    );
+  }
+  const bundledFixture = limit ? LONG_MEM_EVAL_SMOKE_FIXTURE.slice(0, limit) : LONG_MEM_EVAL_SMOKE_FIXTURE;
+  if (bundledFixture.length > 0) {
+    return ensureDatasetItems(bundledFixture);
+  }
+  throw new Error("LongMemEval dataset not found and bundled smoke fixture is empty.");
+}
+var REGISTERED_BENCHMARKS = [
+  {
+    id: "ama-bench",
+    title: "AMA-Bench",
+    tier: "published",
+    status: "planned",
+    runnerAvailable: false,
+    meta: {
+      name: "ama-bench",
+      version: "1.0.0",
+      description: "Long-horizon agentic memory benchmark.",
+      category: "agentic"
+    }
+  },
+  {
+    id: "memory-arena",
+    title: "MemoryArena",
+    tier: "published",
+    status: "planned",
+    runnerAvailable: false,
+    meta: {
+      name: "memory-arena",
+      version: "1.0.0",
+      description: "Interdependent multi-session task benchmark.",
+      category: "agentic"
+    }
+  },
+  {
+    id: "amemgym",
+    title: "AMemGym",
+    tier: "published",
+    status: "planned",
+    runnerAvailable: false,
+    meta: {
+      name: "amemgym",
+      version: "1.0.0",
+      description: "Interactive personalization benchmark.",
+      category: "agentic"
+    }
+  },
+  {
+    ...longMemEvalDefinition,
+    run: runLongMemEvalBenchmark
+  },
+  {
+    id: "locomo",
+    title: "LoCoMo",
+    tier: "published",
+    status: "planned",
+    runnerAvailable: false,
+    meta: {
+      name: "locomo",
+      version: "1.0.0",
+      description: "Long conversation memory benchmark.",
+      category: "conversational"
+    }
+  }
+];
+function listBenchmarks() {
+  return REGISTERED_BENCHMARKS.map(stripRuntimeFields);
+}
+function getBenchmark(id) {
+  const benchmark = REGISTERED_BENCHMARKS.find((candidate) => candidate.id === id);
+  return benchmark ? stripRuntimeFields(benchmark) : void 0;
+}
+function getRegisteredBenchmark(id) {
+  return REGISTERED_BENCHMARKS.find((candidate) => candidate.id === id);
+}
+function stripRuntimeFields(benchmark) {
+  return {
+    id: benchmark.id,
+    title: benchmark.title,
+    tier: benchmark.tier,
+    status: benchmark.status,
+    runnerAvailable: benchmark.runnerAvailable,
+    meta: benchmark.meta
+  };
+}
+var DEFAULT_BASELINE_PATH = path4.join(process.cwd(), "benchmarks", "baseline.json");
+var DEFAULT_REPORT_PATH = path4.join(process.cwd(), "benchmarks", "report.json");
+var BASELINE_VERSION = 1;
+var DEFAULT_TOLERANCE = 10;
+var DEFAULT_QUERIES = [
+  "What is the storage?",
+  "How do I access storage?",
+  "What categories exist?",
+  "How is memory organized?",
+  "What is the recall budget?",
+  "What is the extraction pipeline?",
+  "What facts are stored about the project?",
+  "What is the architecture?"
+];
+function hrTimeMs() {
+  const [seconds, nanos] = process.hrtime();
+  return seconds * 1e3 + Math.round(nanos / 1e6);
+}
+async function runBenchmark(benchmarkId, options) {
+  const registeredBenchmark = getRegisteredBenchmark(benchmarkId);
+  if (!registeredBenchmark) {
+    throw new Error(
+      `Unknown benchmark "${benchmarkId}". Available benchmarks: ${listBenchmarks().map((benchmark) => benchmark.id).join(", ")}`
+    );
+  }
+  if (!registeredBenchmark.run) {
+    throw new Error(
+      `Benchmark "${benchmarkId}" is listed but has not been migrated into @remnic/bench yet.`
+    );
+  }
+  return registeredBenchmark.run({
+    ...options,
+    mode: options.mode ?? "quick",
+    benchmark: benchmarkDefinition(registeredBenchmark.id)
+  });
+}
+function benchmarkDefinition(id) {
+  const definition = getBenchmark(id);
+  if (!definition) {
+    throw new Error(`Benchmark definition disappeared for "${id}".`);
+  }
+  return definition;
+}
+function loadBaseline(baselinePath) {
+  const resolvedPath = baselinePath ?? DEFAULT_BASELINE_PATH;
+  if (!fs.existsSync(resolvedPath)) {
+    return void 0;
+  }
+  try {
+    const raw = JSON.parse(fs.readFileSync(resolvedPath, "utf8"));
+    if (raw.version !== BASELINE_VERSION) {
+      console.warn(
+        `Baseline version mismatch: expected ${BASELINE_VERSION}, got ${raw.version}`
+      );
+    }
+    return raw;
+  } catch {
+    return void 0;
+  }
+}
+function saveBaseline(baselinePath, baseline) {
+  fs.mkdirSync(path4.dirname(baselinePath), { recursive: true });
+  fs.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}
+`);
+}
+async function recallWithTiers(service, query) {
+  const tiers = [];
+  const tierDetails = [];
+  const exactStart = hrTimeMs();
+  const exactResponse = await service.recall({
+    query,
+    mode: "auto"
+  });
+  const exactLatency = hrTimeMs() - exactStart;
+  if (exactResponse.results?.some(
+    (memory) => memory.preview.toLowerCase().includes(query.toLowerCase())
+  )) {
+    tiers.push("exact_match");
+    tierDetails.push({
+      tier: "exact_match",
+      latencyMs: exactLatency,
+      resultsCount: exactResponse.results.length
+    });
+    return { tiers, tierDetails };
+  }
+  const keywordStart = hrTimeMs();
+  const keywordResponse = await service.recall({
+    query,
+    mode: "auto"
+  });
+  const keywordLatency = hrTimeMs() - keywordStart;
+  const queryWords = query.toLowerCase().split(/\s+/).filter((word) => word.length > 2);
+  if (keywordResponse.results?.some(
+    (memory) => queryWords.some((word) => memory.preview.toLowerCase().includes(word))
+  )) {
+    tiers.push("category_match");
+    tierDetails.push({
+      tier: "category_match",
+      latencyMs: keywordLatency,
+      resultsCount: keywordResponse.results.length
+    });
+    return { tiers, tierDetails };
+  }
+  const confidenceStart = hrTimeMs();
+  const confidenceResponse = await service.recall({
+    query,
+    mode: "auto"
+  });
+  const confidenceLatency = hrTimeMs() - confidenceStart;
+  const taggedResults = (confidenceResponse.results ?? []).filter(
+    (memory) => memory.tags?.length > 0
+  );
+  if (taggedResults.length > 0) {
+    tiers.push("high_confidence");
+    tierDetails.push({
+      tier: "high_confidence",
+      latencyMs: confidenceLatency,
+      resultsCount: taggedResults.length
+    });
+    return { tiers, tierDetails };
+  }
+  const semanticStart = hrTimeMs();
+  const semanticResponse = await service.recall({
+    query,
+    mode: "auto"
+  });
+  const semanticLatency = hrTimeMs() - semanticStart;
+  if ((semanticResponse.results ?? []).length > 0) {
+    tiers.push("semantic_search");
+    tierDetails.push({
+      tier: "semantic_search",
+      latencyMs: semanticLatency,
+      resultsCount: semanticResponse.results.length
+    });
+    return { tiers, tierDetails };
+  }
+  const fullStart = hrTimeMs();
+  const fullResponse = await service.recall({
+    query,
+    mode: "full"
+  });
+  const fullLatency = hrTimeMs() - fullStart;
+  if ((fullResponse.results ?? []).length > 0) {
+    tiers.push("full_search");
+    tierDetails.push({
+      tier: "full_search",
+      latencyMs: fullLatency,
+      resultsCount: fullResponse.results.length
+    });
+    return { tiers, tierDetails };
+  }
+  tiers.push("no_results");
+  tierDetails.push({
+    tier: "no_results",
+    latencyMs: exactLatency + keywordLatency + confidenceLatency + semanticLatency + fullLatency,
+    resultsCount: 0
+  });
+  return { tiers, tierDetails };
+}
+async function runExplain(service, query) {
+  const start = hrTimeMs();
+  const { tiers, tierDetails } = await recallWithTiers(service, query);
+  const totalDurationMs = hrTimeMs() - start;
+  return {
+    query,
+    tiersUsed: tiers,
+    tierResults: tierDetails,
+    durationMs: tierDetails[0]?.latencyMs ?? totalDurationMs,
+    totalDurationMs
+  };
+}
+async function runSingle(service, queryText) {
+  const start = hrTimeMs();
+  const { tiers, tierDetails } = await recallWithTiers(service, queryText);
+  const totalDurationMs = hrTimeMs() - start;
+  return {
+    query: queryText,
+    latencyMs: totalDurationMs,
+    tiersUsed: tiers,
+    throughput: totalDurationMs > 0 ? 1 / (totalDurationMs / 1e3) : 0,
+    resultsCount: tierDetails.reduce((sum, tier) => sum + tier.resultsCount, 0),
+    totalDurationMs,
+    tierDetails
+  };
+}
+async function runBenchSuite(service, config = {}) {
+  const queries = config.queries ?? DEFAULT_QUERIES;
+  const regressionTolerance = config.regressionTolerance ?? DEFAULT_TOLERANCE;
+  const baselinePath = config.baselinePath ?? DEFAULT_BASELINE_PATH;
+  const reportPath = config.reportPath ?? DEFAULT_REPORT_PATH;
+  const explain = config.explain ?? false;
+  const results = [];
+  const suiteStart = hrTimeMs();
+  for (const query of queries) {
+    if (explain) {
+      const explained = await runExplain(service, query);
+      results.push({
+        query: explained.query,
+        latencyMs: explained.durationMs,
+        tiersUsed: explained.tiersUsed,
+        throughput: explained.totalDurationMs > 0 ? 1 / (explained.totalDurationMs / 1e3) : 0,
+        resultsCount: explained.tierResults.reduce(
+          (sum, tier) => sum + tier.resultsCount,
+          0
+        ),
+        totalDurationMs: explained.totalDurationMs,
+        tierDetails: explained.tierResults
+      });
+    } else {
+      results.push(await runSingle(service, query));
+    }
+  }
+  const totalDurationMs = hrTimeMs() - suiteStart;
+  const metrics = {};
+  for (const result of results) {
+    metrics[result.query] = result.latencyMs;
+  }
+  const report = generateReport(results, reportPath);
+  const baseline = loadBaseline(baselinePath);
+  const regressionResult = checkRegression(metrics, baseline, regressionTolerance);
+  if (!baseline) {
+    saveBaseline(baselinePath, {
+      version: BASELINE_VERSION,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      metrics
+    });
+  }
+  return {
+    results,
+    report,
+    totalDurationMs,
+    regressions: regressionResult.regressions
+  };
+}
+function checkRegression(metrics, baseline, tolerance) {
+  if (!baseline) {
+    return { passed: true, regressions: [] };
+  }
+  const regressions = [];
+  for (const [metric, currentValue] of Object.entries(metrics)) {
+    const baselineValue = baseline.metrics[metric];
+    if (baselineValue === void 0) {
+      continue;
+    }
+    const changePercent = baselineValue > 0 ? (currentValue - baselineValue) / baselineValue * 100 : 0;
+    regressions.push({
+      metric,
+      currentValue,
+      baselineValue,
+      tolerance,
+      passed: changePercent <= tolerance
+    });
+  }
+  return {
+    passed: regressions.every((regression) => regression.passed),
+    regressions
+  };
+}
+function generateReport(results, reportPath) {
+  const report = {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    queries: results.map((result) => ({
+      query: result.query,
+      tiersUsed: result.tiersUsed,
+      durationMs: result.latencyMs,
+      resultsCount: result.resultsCount,
+      throughput: result.throughput,
+      tierDetails: result.tierDetails
+    })),
+    totalDurationMs: results.reduce((sum, result) => sum + result.totalDurationMs, 0)
+  };
+  if (reportPath) {
+    fs.mkdirSync(path4.dirname(reportPath), { recursive: true });
+    fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}
+`);
+  }
+  return report;
+}
+
+export {
+  createLightweightAdapter,
+  createRemnicAdapter,
+  BENCHMARK_RESULT_SCHEMA,
+  createOpenAiCompatibleProvider,
+  exactMatch,
+  f1Score,
+  rougeL,
+  recallAtK,
+  containsAnswer,
+  llmJudgeScore,
+  timed,
+  aggregateTaskScores,
+  writeBenchmarkResult,
+  listBenchmarks,
+  getBenchmark,
+  runBenchmark,
+  loadBaseline,
+  saveBaseline,
+  runExplain,
+  runBenchSuite,
+  checkRegression,
+  generateReport
+};