@eidentic/bench 0.1.1 → 0.1.3

package/dist/chunk-KOMVTEKE.js

@@ -0,0 +1,98 @@
+// src/lme-loader.ts
+import { readFile, stat } from "node:fs/promises";
+var LONGMEMEVAL_SOURCE = {
+  url: "https://huggingface.co/datasets/xiaowu0162/longmemeval",
+  snapshotSha: "2ec2a557f339b6c0369619b1ed5793734cc87533",
+  file: "longmemeval_s",
+  license: "MIT"
+};
+var DEFAULT_MAX_BYTES = 512 * 1024 * 1024;
+async function assertFileSize(filePath, maxBytes) {
+  let fileSize;
+  try {
+    const s = await stat(filePath);
+    fileSize = s.size;
+  } catch (err) {
+    throw new Error(
+      `bench loader: cannot stat file "${filePath}": ${err.message}`
+    );
+  }
+  if (fileSize > maxBytes) {
+    const mb = (fileSize / (1024 * 1024)).toFixed(1);
+    const capMb = (maxBytes / (1024 * 1024)).toFixed(0);
+    throw new Error(
+      `bench loader: file "${filePath}" is ${mb} MiB, which exceeds the ${capMb} MiB cap. Pass a larger maxBytes option if this is intentional.`
+    );
+  }
+}
+function parseLmeDateTimeString(raw) {
+  if (!raw) return 0;
+  const cleaned = raw.replace(/\s*\([A-Za-z]+\)\s*/, " ").trim();
+  const iso = cleaned.replace(/^(\d{4})\/(\d{2})\/(\d{2})/, "$1-$2-$3");
+  const ms = Date.parse(iso);
+  if (Number.isNaN(ms)) return 0;
+  return ms;
+}
+function extractBaseType(rawType) {
+  if (rawType.endsWith("_abs")) {
+    return rawType.slice(0, -4);
+  }
+  return rawType;
+}
+function parseSession(id, dateTime, rawTurns) {
+  const turns = rawTurns.map((t) => ({
+    role: t.role === "assistant" ? "assistant" : "user",
+    content: t.content ?? "",
+    hasAnswer: t.has_answer === true
+  }));
+  return {
+    id,
+    dateTime,
+    dateTimeMs: parseLmeDateTimeString(dateTime),
+    turns
+  };
+}
+async function loadLongMemEval(jsonPath, opts) {
+  await assertFileSize(jsonPath, opts?.maxBytes ?? DEFAULT_MAX_BYTES);
+  const raw = JSON.parse(await readFile(jsonPath, "utf-8"));
+  if (!Array.isArray(raw)) {
+    throw new Error(
+      `bench loader: expected the LongMemEval JSON root to be an array, but got ${typeof raw}. Did you pass the correct file?`
+    );
+  }
+  const rawQuestions = raw;
+  const questions = rawQuestions.map((q, i) => {
+    const id = q.question_id ?? String(i);
+    const rawType = q.question_type ?? "single-session-user";
+    const baseType = extractBaseType(rawType);
+    const isAbstention = rawType.endsWith("_abs");
+    const rawSessions = Array.isArray(q.haystack_sessions) ? q.haystack_sessions : [];
+    const dates = Array.isArray(q.haystack_dates) ? q.haystack_dates : [];
+    const sessionIds = Array.isArray(q.haystack_session_ids) ? q.haystack_session_ids : [];
+    const sessions = rawSessions.map((turns, idx) => {
+      const sessId = sessionIds[idx] ?? `sess-${idx}`;
+      const dateTime = dates[idx] ?? "";
+      return parseSession(sessId, dateTime, Array.isArray(turns) ? turns : []);
+    });
+    sessions.sort((a, b) => a.dateTimeMs - b.dateTimeMs);
+    return {
+      id,
+      type: rawType,
+      baseType,
+      isAbstention,
+      question: q.question ?? "",
+      answer: q.answer ?? "",
+      questionDate: q.question_date ?? "",
+      questionDateMs: parseLmeDateTimeString(q.question_date ?? ""),
+      sessions,
+      answerSessionIds: Array.isArray(q.answer_session_ids) ? q.answer_session_ids : []
+    };
+  });
+  return { questions };
+}
+
+export {
+  LONGMEMEVAL_SOURCE,
+  parseLmeDateTimeString,
+  loadLongMemEval
+};