@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/src/internal/CanonicalJson.ts

@@ -0,0 +1,234 @@
+/**
+ * The one canonical JSON encoder both serializers use.
+ *
+ * A report embeds the raw output of an arbitrary target flow, so the value
+ * being serialized is not JSON and cannot be assumed to be finite, acyclic, or
+ * even readable. A recursive walker over `JSON.stringify` answered that with a
+ * `RangeError` thrown out of a function typed `string`. This encoder is total
+ * instead: every shape JSON cannot express is replaced by a marker that names
+ * what was there, so a report of a broken run is still a report.
+ *
+ * @since 0.1.0
+ */
+
+/**
+ * The nesting depth beyond which a value is replaced by a marker.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const maxDepth = 64
+
+/**
+ * Default cap on an embedded string, in UTF-16 code units.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const maxStringLength = 8192
+
+/**
+ * Total number of values one encode may visit.
+ *
+ * The cycle detector forgets a value once it leaves it, which is what keeps a
+ * value referenced twice from being reported as a cycle. The cost is that a
+ * shared acyclic graph is re-expanded at every reference, so `n` nested
+ * two-child wrappers over one shared child expand to `2^n` values while
+ * staying far below {@link maxDepth}. The ceiling is roughly three times what
+ * a suite at its declared case limit produces.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const maxNodes = 2_000_000
+
+/**
+ * Approximate cap on the encoded output, in UTF-16 code units.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const maxBytes = 33_554_432
+
+/**
+ * Marker left where a total budget ran out.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const budgetExceeded = "[budget exceeded]"
+
+/**
+ * Options accepted by {@link encode}.
+ *
+ * @since 0.1.0
+ * @private
+ */
+interface Options {
+  /** Cap on embedded strings; `undefined` keeps them whole. */
+  readonly maxStringLength?: number | undefined
+  /** Cap on values visited; defaults to {@link maxNodes}. */
+  readonly maxNodes?: number | undefined
+  /** Cap on encoded output length; defaults to {@link maxBytes}. */
+  readonly maxBytes?: number | undefined
+}
+
+/** Remaining total work, decremented in traversal order so truncation is stable. */
+interface Budget {
+  nodes: number
+  bytes: number
+}
+
+/**
+ * Orders keys by UTF-16 code unit, never by locale.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const compareText = (left: string, right: string): number => left < right ? -1 : left > right ? 1 : 0
+
+const truncate = (value: string, limit: number | undefined): string =>
+  limit === undefined || value.length <= limit
+    ? value
+    : `${value.slice(0, limit)}[truncated ${value.length - limit} chars]`
+
+const primitive = (value: unknown, options: Options): unknown => {
+  if (typeof value === "string") return truncate(value, options.maxStringLength)
+  if (typeof value === "number") {
+    if (Number.isNaN(value)) return "[NaN]"
+    if (value === Number.POSITIVE_INFINITY) return "[Infinity]"
+    if (value === Number.NEGATIVE_INFINITY) return "[-Infinity]"
+    return Object.is(value, -0) ? 0 : value
+  }
+  if (typeof value === "bigint") return `[bigint ${value}]`
+  if (typeof value === "function") return "[function]"
+  if (typeof value === "symbol") return "[symbol]"
+  return value
+}
+
+const unreadable = (cause: unknown): string => {
+  try {
+    return `[unreadable: ${String(cause)}]`
+  } catch {
+    return "[unreadable]"
+  }
+}
+
+const entriesOf = (value: object): ReadonlyArray<readonly [string, unknown]> =>
+  Object.keys(value).map((key) => {
+    try {
+      return [key, (value as { readonly [key: string]: unknown })[key]] as const
+    } catch (cause) {
+      return [key, unreadable(cause)] as const
+    }
+  })
+
+const objectOf = (
+  entries: ReadonlyArray<readonly [string, unknown]>,
+  depth: number,
+  seen: Set<object>,
+  options: Options,
+  budget: Budget
+): object =>
+  Object.fromEntries(
+    entries
+      .filter(([, entry]) => entry !== undefined)
+      .sort(([left], [right]) => compareText(left, right))
+      .map(([key, entry]) => {
+        // The key is rendered too, so a wide object of long keys spends the
+        // byte budget even when every value is a marker.
+        budget.bytes -= key.length + 3
+        return [key, walk(entry, depth + 1, seen, options, budget)]
+      })
+  )
+
+const walk = (value: unknown, depth: number, seen: Set<object>, options: Options, budget: Budget): unknown => {
+  if (budget.nodes <= 0 || budget.bytes <= 0) return budgetExceeded
+  budget.nodes -= 1
+  if (value === null || typeof value !== "object") {
+    const encoded = primitive(value, options)
+    budget.bytes -= typeof encoded === "string" ? encoded.length + 2 : 8
+    return encoded
+  }
+  if (seen.has(value)) return "[circular]"
+  if (depth > maxDepth) return "[depth exceeded]"
+  budget.bytes -= 2
+  try {
+    if (value instanceof Date) return Number.isNaN(value.getTime()) ? "[invalid Date]" : value.toISOString()
+    seen.add(value)
+    try {
+      if (value instanceof Error) {
+        const properties = new Map(entriesOf(value))
+        properties.set("name", String(value.name))
+        properties.set("message", String(value.message))
+        return objectOf([...properties], depth, seen, options, budget)
+      }
+      if (Array.isArray(value)) return value.map((entry) => walk(entry, depth + 1, seen, options, budget))
+      if (value instanceof Set) return [...value].map((entry) => walk(entry, depth + 1, seen, options, budget))
+      if (value instanceof Map) {
+        return [...value.entries()]
+          .map((
+            [key, entry]
+          ) => [walk(key, depth + 1, seen, options, budget), walk(entry, depth + 1, seen, options, budget)])
+      }
+      return objectOf(entriesOf(value), depth, seen, options, budget)
+    } finally {
+      seen.delete(value)
+    }
+  } catch (cause) {
+    return unreadable(cause)
+  }
+}
+
+/**
+ * Rewrites a value into a shape `JSON.stringify` renders deterministically.
+ *
+ * Object keys are sorted by code unit, `-0` becomes `0`, and `undefined`
+ * members are dropped. Everything JSON cannot express becomes a bracketed
+ * marker rather than a throw or a silent `null`: `[circular]`,
+ * `[depth exceeded]`, `[NaN]`, `[Infinity]`, `[-Infinity]`, `[bigint n]`,
+ * `[function]`, `[symbol]`, and `[unreadable: …]` when a foreign operation
+ * throws. An `Error` becomes an object containing its own enumerable fields
+ * plus `name` and `message`, so typed error fields survive serialization. A
+ * `Date` becomes its ISO string, a `Set` an array, and a `Map` an array of
+ * key/value pairs.
+ *
+ * Traversal is bounded in total, not only per branch: once {@link maxNodes}
+ * values or {@link maxBytes} output code units are spent, every remaining
+ * value becomes `[budget exceeded]`. The budget is spent in traversal order,
+ * and traversal order is fixed by the key sort, so one value always truncates
+ * at the same place.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const encode = (value: unknown, options: Options = {}): unknown =>
+  walk(value, 0, new Set<object>(), options, {
+    nodes: options.maxNodes ?? maxNodes,
+    bytes: options.maxBytes ?? maxBytes
+  })
+
+/**
+ * Serializes a value as canonical JSON with a trailing newline.
+ *
+ * @since 0.1.0
+ * @private
+ */
+const stringify = (value: unknown, options: Options = {}): string => `${JSON.stringify(encode(value, options))}\n`
+
+/**
+ * The canonical JSON encoder: its limits, key order, and serializers.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export const CanonicalJson = {
+  maxDepth,
+  maxStringLength,
+  maxNodes,
+  maxBytes,
+  budgetExceeded,
+  compareText,
+  encode,
+  stringify
+} as const

package/src/internal/controlCharacters.ts

@@ -0,0 +1,66 @@
+/**
+ * The one control-character rule every trust boundary in this package shares.
+ *
+ * A control character in a name corrupts a Markdown report and a CI log line:
+ * a newline lets the value print its own line, which on GitHub Actions is a
+ * workflow command the runner executes, an ESC or a C1 CSI (U+009B) sequence
+ * rewrites the visible log, and a Unicode bidirectional control reorders what
+ * a reader sees. Values an author declares (suite and case names, baseline
+ * suites and records) are rejected where they enter the system; values a
+ * target returns at runtime (step keys, failure messages) are flattened
+ * instead, so a broken target stays a readable failure rather than a rejected
+ * run.
+ *
+ * The rule covers C0 controls (U+0000 through U+001F), DEL (U+007F), C1
+ * controls (U+0080 through U+009F), and the bidirectional controls U+061C,
+ * U+200E, U+200F, U+202A through U+202E, and U+2066 through U+2069.
+ *
+ * @since 0.1.0
+ */
+
+/**
+ * Whether one code point falls under the control-character rule.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export const isControlCodePoint = (code: number): boolean =>
+  code < 0x20 ||
+  (code >= 0x7f && code <= 0x9f) ||
+  code === 0x061c ||
+  code === 0x200e ||
+  code === 0x200f ||
+  (code >= 0x202a && code <= 0x202e) ||
+  (code >= 0x2066 && code <= 0x2069)
+
+/**
+ * Names the first control character in `value`, or `undefined` when there is
+ * none.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export const controlCharacter = (value: string): string | undefined => {
+  for (const character of value) {
+    const code = character.codePointAt(0)!
+    if (isControlCodePoint(code)) return `U+${code.toString(16).toUpperCase().padStart(4, "0")}`
+  }
+  return undefined
+}
+
+/**
+ * Replaces every control character with a space, the way a report cell does,
+ * so a hostile string cannot emit its own log line, terminal escape, or
+ * reordered text.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export const flattenControlCharacters = (value: string): string => {
+  if (controlCharacter(value) === undefined) return value
+  let flattened = ""
+  for (const character of value) {
+    flattened += isControlCodePoint(character.codePointAt(0)!) ? " " : character
+  }
+  return flattened
+}

package/src/internal/scorerLabel.ts

@@ -0,0 +1,35 @@
+/**
+ * The one scorer label rule every rendering in this package shares.
+ *
+ * A scorer has two identities: `scorer` is the key, a digest of the scorer's
+ * own declaration and the only thing a baseline matches on, and `scorerName`
+ * is the readable name carried beside it. A digest is unreadable in a report
+ * cell, a CI log line, or a diagnostic, so each of them prints the name with
+ * the first eight key characters instead. Without a name there is nothing to
+ * abbreviate the key down to, so the label is the key itself: a truncated
+ * digest and no name would leave a reader nothing to match a baseline record
+ * on. One rule is what keeps a report from naming one scorer two ways, an
+ * inconclusive reason built by the runner and a Missing row built by the
+ * reporter agree.
+ *
+ * @since 0.1.0
+ */
+
+/**
+ * Renders `name (first 8 of the key)`, or the bare key when the scorer carries
+ * no name.
+ *
+ * Every flow carries a name, so a scorer a run holds always has one. A stored
+ * observation may not: a baseline recorded before a scorer was named, or one
+ * written by another tool, carries the key alone, and it labels as the key
+ * rather than as a blank followed by a digest.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export const scorerLabel = (
+  identity: { readonly scorer: string; readonly scorerName?: string | undefined }
+): string => {
+  const name = identity.scorerName
+  return name === undefined || name.length === 0 ? identity.scorer : `${name} (${identity.scorer.slice(0, 8)})`
+}

package/src/internal/tupleKey.ts

@@ -0,0 +1,20 @@
+/**
+ * The one injective tuple encoder every key in this package is built with.
+ *
+ * Joining caller-supplied strings on a delimiter is not injective: under a NUL
+ * separator the tuples `["a", "b" + NUL + "c"]` and `["a" + NUL + "b", "c"]`
+ * produce one key, so two distinct score jobs could share an identity and two
+ * distinct baseline records could sort as one. Encoding the tuple as a JSON
+ * array keeps the components separated by the encoding itself, whatever they
+ * contain.
+ *
+ * @since 0.1.0
+ */
+
+/**
+ * Encodes a tuple of strings as one key.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export const tupleKey = (...parts: ReadonlyArray<string>): string => JSON.stringify(parts)