localarena 0.1.0 → 0.2.0

package/javascript/src/tasks.js

@@ -0,0 +1,828 @@
+import { ChatMessage } from "./providers.js";
+
+function assertRecord(value, field) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    throw new TypeError(`${field} must be an object`);
+  }
+}
+
+function assertNonEmptyString(value, field) {
+  if (typeof value !== "string") {
+    throw new TypeError(`${field} must be a string`);
+  }
+  if (value.trim().length === 0) {
+    throw new RangeError(`${field} must not be empty`);
+  }
+}
+
+function cloneJSON(value, field, seen = new Set(), depth = 0) {
+  if (depth > 100) {
+    throw new RangeError(`${field} exceeds the maximum JSON depth`);
+  }
+  if (
+    value === null ||
+    typeof value === "string" ||
+    typeof value === "boolean"
+  ) {
+    return value;
+  }
+  if (typeof value === "number") {
+    if (!Number.isFinite(value)) {
+      throw new TypeError(`${field} must contain only finite numbers`);
+    }
+    if (Number.isInteger(value) && !Number.isSafeInteger(value)) {
+      throw new TypeError(`${field} integers must be within the safe range`);
+    }
+    return value;
+  }
+  if (typeof value !== "object") {
+    throw new TypeError(`${field} must contain only JSON values`);
+  }
+  if (seen.has(value)) {
+    throw new TypeError(`${field} must not contain cycles`);
+  }
+
+  seen.add(value);
+  let result;
+  if (Array.isArray(value)) {
+    result = value.map((item, index) =>
+      cloneJSON(item, `${field}[${index}]`, seen, depth + 1),
+    );
+  } else {
+    const prototype = Object.getPrototypeOf(value);
+    if (prototype !== Object.prototype && prototype !== null) {
+      throw new TypeError(`${field} must contain only plain objects`);
+    }
+    result = {};
+    for (const [key, item] of Object.entries(value)) {
+      Object.defineProperty(result, key, {
+        configurable: true,
+        enumerable: true,
+        value: cloneJSON(item, `${field}.${key}`, seen, depth + 1),
+        writable: true,
+      });
+    }
+  }
+  seen.delete(value);
+  return result;
+}
+
+function freezeJSON(value) {
+  if (Array.isArray(value)) {
+    return Object.freeze(value.map(freezeJSON));
+  }
+  if (value !== null && typeof value === "object") {
+    return Object.freeze(
+      Object.fromEntries(
+        Object.entries(value).map(([key, item]) => [key, freezeJSON(item)]),
+      ),
+    );
+  }
+  return value;
+}
+
+function frozenJSONObject(value, field) {
+  assertRecord(value, field);
+  return freezeJSON(cloneJSON(value, field));
+}
+
+// Unicode 15 full case-fold mappings that differ from default lowercase.
+// Cherokee ranges are handled arithmetically below to keep this table compact.
+const CASE_FOLD_SPECIALS = Object.freeze({
+  µ: "μ",
+  ß: "ss",
+  ʼn: "ʼn",
+  ſ: "s",
+  ǰ: "ǰ",
+  "ͅ": "ι",
+  ΐ: "ΐ",
+  ΰ: "ΰ",
+  ς: "σ",
+  ϐ: "β",
+  ϑ: "θ",
+  ϕ: "φ",
+  ϖ: "π",
+  ϰ: "κ",
+  ϱ: "ρ",
+  ϵ: "ε",
+  և: "եւ",
+  ᲀ: "в",
+  ᲁ: "д",
+  ᲂ: "о",
+  ᲃ: "с",
+  ᲄ: "т",
+  ᲅ: "т",
+  ᲆ: "ъ",
+  ᲇ: "ѣ",
+  ᲈ: "ꙋ",
+  ẖ: "ẖ",
+  ẗ: "ẗ",
+  ẘ: "ẘ",
+  ẙ: "ẙ",
+  ẚ: "aʾ",
+  ẛ: "ṡ",
+  ὐ: "ὐ",
+  ὒ: "ὒ",
+  ὔ: "ὔ",
+  ὖ: "ὖ",
+  ᾀ: "ἀι",
+  ᾁ: "ἁι",
+  ᾂ: "ἂι",
+  ᾃ: "ἃι",
+  ᾄ: "ἄι",
+  ᾅ: "ἅι",
+  ᾆ: "ἆι",
+  ᾇ: "ἇι",
+  ᾐ: "ἠι",
+  ᾑ: "ἡι",
+  ᾒ: "ἢι",
+  ᾓ: "ἣι",
+  ᾔ: "ἤι",
+  ᾕ: "ἥι",
+  ᾖ: "ἦι",
+  ᾗ: "ἧι",
+  ᾠ: "ὠι",
+  ᾡ: "ὡι",
+  ᾢ: "ὢι",
+  ᾣ: "ὣι",
+  ᾤ: "ὤι",
+  ᾥ: "ὥι",
+  ᾦ: "ὦι",
+  ᾧ: "ὧι",
+  ᾲ: "ὰι",
+  ᾳ: "αι",
+  ᾴ: "άι",
+  ᾶ: "ᾶ",
+  ᾷ: "ᾶι",
+  ι: "ι",
+  ῂ: "ὴι",
+  ῃ: "ηι",
+  ῄ: "ήι",
+  ῆ: "ῆ",
+  ῇ: "ῆι",
+  ῒ: "ῒ",
+  ΐ: "ΐ",
+  ῖ: "ῖ",
+  ῗ: "ῗ",
+  ῢ: "ῢ",
+  ΰ: "ΰ",
+  ῤ: "ῤ",
+  ῦ: "ῦ",
+  ῧ: "ῧ",
+  ῲ: "ὼι",
+  ῳ: "ωι",
+  ῴ: "ώι",
+  ῶ: "ῶ",
+  ῷ: "ῶι",
+  ff: "ff",
+  fi: "fi",
+  fl: "fl",
+  ffi: "ffi",
+  ffl: "ffl",
+  ſt: "st",
+  st: "st",
+  ﬓ: "մն",
+  ﬔ: "մե",
+  ﬕ: "մի",
+  ﬖ: "վն",
+  ﬗ: "մխ",
+});
+
+function unicodeCaseFold(value) {
+  return Array.from(value.toLowerCase(), (character) => {
+    const codePoint = character.codePointAt(0);
+    if (codePoint >= 0xab70 && codePoint <= 0xabbf) {
+      return String.fromCodePoint(codePoint - 0x97d0);
+    }
+    if (codePoint >= 0x13f8 && codePoint <= 0x13fd) {
+      return String.fromCodePoint(codePoint - 8);
+    }
+    return CASE_FOLD_SPECIALS[character] ?? character;
+  }).join("");
+}
+
+function normalizedText(value, { strip, ignoreCase }) {
+  let result = strip ? value.trim() : value;
+  if (ignoreCase) {
+    result = unicodeCaseFold(result);
+  }
+  return result;
+}
+
+function assertEvaluator(value) {
+  if (
+    value === null ||
+    typeof value !== "object" ||
+    typeof value.evaluate !== "function" ||
+    typeof value.toConfig !== "function"
+  ) {
+    throw new TypeError("evaluator must implement evaluate() and toConfig()");
+  }
+}
+
+function assertGeneration(value) {
+  if (
+    value === null ||
+    typeof value !== "object" ||
+    typeof value.text !== "string"
+  ) {
+    throw new TypeError("generation must contain text");
+  }
+}
+
+function deepJSONEqual(left, right) {
+  if (Object.is(left, right)) {
+    return true;
+  }
+  if (Array.isArray(left) || Array.isArray(right)) {
+    if (!Array.isArray(left) || !Array.isArray(right)) {
+      return false;
+    }
+    return (
+      left.length === right.length &&
+      left.every((value, index) => deepJSONEqual(value, right[index]))
+    );
+  }
+  if (
+    left !== null &&
+    right !== null &&
+    typeof left === "object" &&
+    typeof right === "object"
+  ) {
+    const leftKeys = Object.keys(left).sort();
+    const rightKeys = Object.keys(right).sort();
+    return (
+      leftKeys.length === rightKeys.length &&
+      leftKeys.every(
+        (key, index) =>
+          key === rightKeys[index] && deepJSONEqual(left[key], right[key]),
+      )
+    );
+  }
+  return false;
+}
+
+function jsonErrorLocation(error, source) {
+  const match = /position\s+(\d+)/iu.exec(error.message);
+  if (match === null) {
+    return "invalid JSON";
+  }
+  const position = Math.min(source.length, Number(match[1]));
+  const prefix = source.slice(0, position);
+  const lines = prefix.split("\n");
+  return `invalid JSON at line ${lines.length}, column ${lines.at(-1).length + 1}`;
+}
+
+function assertUniqueJSONKeys(source) {
+  let cursor = 0;
+
+  const skipWhitespace = () => {
+    while (/[\t\n\r ]/u.test(source[cursor] ?? "")) {
+      cursor += 1;
+    }
+  };
+
+  const parseString = () => {
+    const start = cursor;
+    cursor += 1;
+    while (cursor < source.length) {
+      if (source[cursor] === "\\") {
+        cursor += 2;
+        continue;
+      }
+      if (source[cursor] === '"') {
+        cursor += 1;
+        return JSON.parse(source.slice(start, cursor));
+      }
+      cursor += 1;
+    }
+    throw new SyntaxError("invalid JSON string");
+  };
+
+  const parseValue = () => {
+    skipWhitespace();
+    const token = source[cursor];
+    if (token === "{") {
+      cursor += 1;
+      skipWhitespace();
+      const keys = new Set();
+      if (source[cursor] === "}") {
+        cursor += 1;
+        return;
+      }
+      while (cursor < source.length) {
+        const key = parseString();
+        if (keys.has(key)) {
+          throw new SyntaxError("duplicate JSON key");
+        }
+        keys.add(key);
+        skipWhitespace();
+        cursor += 1;
+        parseValue();
+        skipWhitespace();
+        if (source[cursor] === "}") {
+          cursor += 1;
+          return;
+        }
+        cursor += 1;
+        skipWhitespace();
+      }
+      return;
+    }
+    if (token === "[") {
+      cursor += 1;
+      skipWhitespace();
+      if (source[cursor] === "]") {
+        cursor += 1;
+        return;
+      }
+      while (cursor < source.length) {
+        parseValue();
+        skipWhitespace();
+        if (source[cursor] === "]") {
+          cursor += 1;
+          return;
+        }
+        cursor += 1;
+      }
+      return;
+    }
+    if (token === '"') {
+      parseString();
+      return;
+    }
+    while (
+      cursor < source.length &&
+      !/[\t\n\r ,\]}]/u.test(source[cursor])
+    ) {
+      cursor += 1;
+    }
+  };
+
+  parseValue();
+  skipWhitespace();
+  if (cursor !== source.length) {
+    throw new SyntaxError("invalid JSON");
+  }
+}
+
+export function parseStrictJSON(source) {
+  const parsed = JSON.parse(source);
+  assertUniqueJSONKeys(source);
+  return parsed;
+}
+
+function evaluatorOptions(first, second, requiredKey) {
+  if (
+    first !== null &&
+    typeof first === "object" &&
+    !Array.isArray(first) &&
+    Object.hasOwn(first, requiredKey) &&
+    second === undefined
+  ) {
+    return first;
+  }
+  return { [requiredKey]: first, ...(second ?? {}) };
+}
+
+export class Score {
+  constructor(valueOrOptions, options = {}) {
+    const values =
+      valueOrOptions !== null &&
+      typeof valueOrOptions === "object" &&
+      !Array.isArray(valueOrOptions)
+        ? valueOrOptions
+        : { value: valueOrOptions, ...options };
+    assertRecord(values, "score");
+    if (typeof values.value !== "number" || !Number.isFinite(values.value)) {
+      throw new TypeError("score.value must be a finite number");
+    }
+    if (values.value < 0 || values.value > 1) {
+      throw new RangeError("score.value must be between zero and one");
+    }
+    const passed = values.passed ?? null;
+    if (passed !== null && typeof passed !== "boolean") {
+      throw new TypeError("score.passed must be a boolean or null");
+    }
+    const reason = values.reason ?? "";
+    if (typeof reason !== "string") {
+      throw new TypeError("score.reason must be a string");
+    }
+
+    this.value = values.value;
+    this.passed = passed;
+    this.reason = reason;
+    this.metadata = frozenJSONObject(values.metadata ?? {}, "score.metadata");
+    Object.freeze(this);
+  }
+
+  snapshot() {
+    return {
+      value: this.value,
+      passed: this.passed,
+      reason: this.reason,
+      metadata: cloneJSON(this.metadata, "score.metadata"),
+    };
+  }
+
+  toJSON() {
+    return this.snapshot();
+  }
+
+  static fromSnapshot(value) {
+    return new Score(value);
+  }
+}
+
+export class PromptTask {
+  constructor(idOrOptions, messages, options = {}) {
+    const values =
+      idOrOptions !== null &&
+      typeof idOrOptions === "object" &&
+      !Array.isArray(idOrOptions)
+        ? idOrOptions
+        : { id: idOrOptions, messages, ...options };
+    assertRecord(values, "task");
+    assertNonEmptyString(values.id, "task.id");
+    if (!Array.isArray(values.messages) || values.messages.length === 0) {
+      throw new TypeError("task.messages must be a non-empty array");
+    }
+    const normalizedMessages = values.messages.map((message, index) => {
+      if (!(message instanceof ChatMessage)) {
+        throw new TypeError(
+          `task.messages[${index}] must be a ChatMessage`,
+        );
+      }
+      if (!["system", "user", "assistant"].includes(message.role)) {
+        throw new RangeError(
+          `task.messages[${index}].role must be system, user, or assistant`,
+        );
+      }
+      return message;
+    });
+    if (values.evaluator !== undefined && values.evaluator !== null) {
+      assertEvaluator(values.evaluator);
+    }
+
+    this.id = values.id;
+    this.messages = Object.freeze(normalizedMessages);
+    this.evaluator = values.evaluator ?? null;
+    this.metadata = frozenJSONObject(values.metadata ?? {}, "task.metadata");
+    Object.freeze(this);
+  }
+
+  static fromText(id, prompt, options = {}) {
+    assertRecord(options, "options");
+    if (typeof prompt !== "string") {
+      throw new TypeError("prompt must be a string");
+    }
+    const messages = [];
+    if (options.system !== undefined && options.system !== null) {
+      if (typeof options.system !== "string") {
+        throw new TypeError("system must be a string or null");
+      }
+      messages.push(new ChatMessage("system", options.system));
+    }
+    messages.push(new ChatMessage("user", prompt));
+    return new PromptTask({
+      id,
+      messages,
+      evaluator: options.evaluator,
+      metadata: options.metadata,
+    });
+  }
+
+  snapshot() {
+    return {
+      id: this.id,
+      messages: this.messages.map(({ role, content }) => ({ role, content })),
+      evaluator: this.evaluator?.toConfig() ?? null,
+      metadata: cloneJSON(this.metadata, "task.metadata"),
+    };
+  }
+
+  toJSON() {
+    return this.snapshot();
+  }
+}
+
+export class ExactMatch {
+  constructor(expectedOrOptions, options) {
+    const values = evaluatorOptions(expectedOrOptions, options, "expected");
+    if (typeof values.expected !== "string") {
+      throw new TypeError("expected must be a string");
+    }
+    const strip = values.strip ?? true;
+    const ignoreCase = values.ignoreCase ?? values.ignore_case ?? false;
+    if (typeof strip !== "boolean" || typeof ignoreCase !== "boolean") {
+      throw new TypeError("strip and ignoreCase must be booleans");
+    }
+
+    this.expected = values.expected;
+    this.strip = strip;
+    this.ignoreCase = ignoreCase;
+    Object.freeze(this);
+  }
+
+  get name() {
+    return "exact";
+  }
+
+  evaluate(_task, generation) {
+    assertGeneration(generation);
+    const options = { strip: this.strip, ignoreCase: this.ignoreCase };
+    const passed =
+      normalizedText(generation.text, options) ===
+      normalizedText(this.expected, options);
+    return new Score(passed ? 1 : 0, {
+      passed,
+      reason: passed
+        ? "exact match"
+        : "answer did not exactly match",
+    });
+  }
+
+  toConfig() {
+    return {
+      type: this.name,
+      expected: this.expected,
+      strip: this.strip,
+      ignore_case: this.ignoreCase,
+    };
+  }
+}
+
+export class Contains {
+  constructor(expectedOrOptions, options) {
+    const values = evaluatorOptions(expectedOrOptions, options, "expected");
+    const expected =
+      typeof values.expected === "string"
+        ? [values.expected]
+        : values.expected;
+    if (
+      !Array.isArray(expected) ||
+      expected.length === 0 ||
+      expected.some(
+        (value) => typeof value !== "string" || value.length === 0,
+      )
+    ) {
+      throw new RangeError(
+        "expected must contain one or more non-empty strings",
+      );
+    }
+    const mode = values.mode ?? "all";
+    if (mode !== "all" && mode !== "any") {
+      throw new RangeError('mode must be "all" or "any"');
+    }
+    const ignoreCase = values.ignoreCase ?? values.ignore_case ?? false;
+    if (typeof ignoreCase !== "boolean") {
+      throw new TypeError("ignoreCase must be a boolean");
+    }
+
+    this.expected = Object.freeze([...expected]);
+    this.mode = mode;
+    this.ignoreCase = ignoreCase;
+    Object.freeze(this);
+  }
+
+  get name() {
+    return "contains";
+  }
+
+  evaluate(_task, generation) {
+    assertGeneration(generation);
+    const normalize = (value) =>
+      normalizedText(value, { strip: false, ignoreCase: this.ignoreCase });
+    const haystack = normalize(generation.text);
+    const matches = this.expected.map((value) =>
+      haystack.includes(normalize(value)),
+    );
+    const passed =
+      this.mode === "all" ? matches.every(Boolean) : matches.some(Boolean);
+    const matched = matches.filter(Boolean).length;
+    return new Score(passed ? 1 : 0, {
+      passed,
+      reason: `matched ${matched} of ${matches.length} required strings`,
+      metadata: { matched, total: matches.length },
+    });
+  }
+
+  toConfig() {
+    return {
+      type: this.name,
+      expected: [...this.expected],
+      mode: this.mode,
+      ignore_case: this.ignoreCase,
+    };
+  }
+}
+
+export class RegexMatch {
+  constructor(patternOrOptions, options) {
+    const values = evaluatorOptions(patternOrOptions, options, "pattern");
+    assertNonEmptyString(values.pattern, "pattern");
+    const ignoreCase = values.ignoreCase ?? values.ignore_case ?? false;
+    const fullMatch = values.fullMatch ?? values.full_match ?? false;
+    if (typeof ignoreCase !== "boolean" || typeof fullMatch !== "boolean") {
+      throw new TypeError("ignoreCase and fullMatch must be booleans");
+    }
+
+    new RegExp(values.pattern, ignoreCase ? "iu" : "u");
+    this.pattern = values.pattern;
+    this.ignoreCase = ignoreCase;
+    this.fullMatch = fullMatch;
+    Object.freeze(this);
+  }
+
+  get name() {
+    return "regex";
+  }
+
+  evaluate(_task, generation) {
+    assertGeneration(generation);
+    const expression = new RegExp(
+      this.pattern,
+      this.ignoreCase ? "iu" : "u",
+    );
+    const match = expression.exec(generation.text);
+    const passed =
+      match !== null &&
+      (!this.fullMatch ||
+        (match.index === 0 && match[0].length === generation.text.length));
+    return new Score(passed ? 1 : 0, {
+      passed,
+      reason: passed
+        ? "regular expression matched"
+        : "regular expression did not match",
+    });
+  }
+
+  toConfig() {
+    return {
+      type: this.name,
+      pattern: this.pattern,
+      ignore_case: this.ignoreCase,
+      full_match: this.fullMatch,
+    };
+  }
+}
+
+export class JSONMatch {
+  constructor(expected = null, options) {
+    let values;
+    if (
+      options === undefined &&
+      expected !== null &&
+      typeof expected === "object" &&
+      !Array.isArray(expected) &&
+      (Object.hasOwn(expected, "compare") ||
+        (Object.hasOwn(expected, "expected") &&
+          Object.keys(expected).every((key) =>
+            ["expected", "compare"].includes(key),
+          )))
+    ) {
+      values = expected;
+    } else {
+      values = { expected, ...(options ?? {}) };
+    }
+    const compare = values.compare ?? false;
+    if (typeof compare !== "boolean") {
+      throw new TypeError("compare must be a boolean");
+    }
+
+    this.compare = compare;
+    this.expected = compare
+      ? freezeJSON(cloneJSON(values.expected, "expected"))
+      : null;
+    Object.freeze(this);
+  }
+
+  get name() {
+    return "json";
+  }
+
+  evaluate(_task, generation) {
+    assertGeneration(generation);
+    let actual;
+    try {
+      actual = parseStrictJSON(generation.text);
+    } catch (error) {
+      return new Score(0, {
+        passed: false,
+        reason: jsonErrorLocation(error, generation.text),
+      });
+    }
+    const passed = !this.compare || deepJSONEqual(actual, this.expected);
+    return new Score(passed ? 1 : 0, {
+      passed,
+      reason: !this.compare
+        ? "valid JSON"
+        : passed
+          ? "JSON matched"
+          : "JSON did not match",
+    });
+  }
+
+  toConfig() {
+    const config = {
+      type: this.name,
+      compare: this.compare,
+    };
+    if (this.compare) {
+      config.expected = cloneJSON(this.expected, "expected");
+    }
+    return config;
+  }
+}
+
+export const JsonMatch = JSONMatch;
+
+export class NumericMatch {
+  constructor(expectedOrOptions, options) {
+    const values = evaluatorOptions(expectedOrOptions, options, "expected");
+    const tolerance = values.tolerance ?? 0;
+    if (
+      typeof values.expected !== "number" ||
+      !Number.isFinite(values.expected)
+    ) {
+      throw new TypeError("expected must be a finite number");
+    }
+    if (typeof tolerance !== "number" || !Number.isFinite(tolerance)) {
+      throw new TypeError("tolerance must be a finite number");
+    }
+    if (tolerance < 0) {
+      throw new RangeError("tolerance must not be negative");
+    }
+
+    this.expected = values.expected;
+    this.tolerance = tolerance;
+    Object.freeze(this);
+  }
+
+  get name() {
+    return "numeric";
+  }
+
+  evaluate(_task, generation) {
+    assertGeneration(generation);
+    const source = generation.text.trim();
+    const numeric =
+      /^[+-]?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|inf(?:inity)?|nan)$/iu;
+    if (source.length === 0 || !numeric.test(source)) {
+      return new Score(0, {
+        passed: false,
+        reason: "answer was not a number",
+      });
+    }
+    const actual = Number(source);
+    if (!Number.isFinite(actual)) {
+      return new Score(0, {
+        passed: false,
+        reason: "answer was not finite",
+      });
+    }
+    const difference = Math.abs(actual - this.expected);
+    const passed = difference <= this.tolerance;
+    return new Score(passed ? 1 : 0, {
+      passed,
+      reason: passed
+        ? "numeric answer matched"
+        : `absolute error ${difference} exceeded tolerance`,
+      metadata: {
+        actual,
+        absolute_error: difference,
+      },
+    });
+  }
+
+  toConfig() {
+    return {
+      type: this.name,
+      expected: this.expected,
+      tolerance: this.tolerance,
+    };
+  }
+}
+
+export function evaluatorFromConfig(config) {
+  if (config === null || config === undefined || config.type === "none") {
+    return null;
+  }
+  assertRecord(config, "evaluator");
+  switch (config.type) {
+    case "exact":
+      return new ExactMatch(config);
+    case "contains":
+      return new Contains(config);
+    case "regex":
+      return new RegexMatch(config);
+    case "json":
+      return new JSONMatch(config);
+    case "numeric":
+      return new NumericMatch(config);
+    default:
+      throw new RangeError(
+        `unsupported evaluator type: ${String(config.type)}`,
+      );
+  }
+}