@euconform/core 1.5.0 → 1.6.0

package/dist/validation/schema-validator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema-validator.d.ts","sourceRoot":"","sources":["../../src/validation/schema-validator.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,YAAY,GAAG,OAAO,GAAG,WAAW,CAAC;AAEzF,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,OAAO,CAAC;IACf,UAAU,EAAE,UAAU,CAAC;IACvB,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAmCD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,eAAe,GAAG,gBAAgB,CAyEpF"}
+{"version":3,"file":"schema-validator.d.ts","sourceRoot":"","sources":["../../src/validation/schema-validator.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,UAAU,GAClB,WAAW,GACX,UAAU,GACV,YAAY,GACZ,OAAO,GACP,WAAW,GACX,SAAS,GACT,SAAS,GACT,yBAAyB,CAAC;AAE9B,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,OAAO,CAAC;IACf,UAAU,EAAE,UAAU,CAAC;IACvB,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAyCD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,eAAe,GAAG,gBAAgB,CAyEpF"}

package/dist/validation/schema-validator.js

@@ -22,6 +22,12 @@ const SCHEMA_MAP = {
     "euconform.aibom.v1.1": { schemaFile: "aibom-v1.1.schema.json", type: "aibom.v1.1" },
     "euconform.ci.v1": { schemaFile: "ci-v1.schema.json", type: "ci.v1" },
     "euconform.bundle.v1": { schemaFile: "bundle-v1.schema.json", type: "bundle.v1" },
+    "euconform.bias.v1": { schemaFile: "bias-v1.schema.json", type: "bias.v1" },
+    "euconform.bias.v2": { schemaFile: "bias-v2.schema.json", type: "bias.v2" },
+    "euconform.external-observation.v1": {
+        schemaFile: "external-observation-v1.schema.json",
+        type: "external-observation.v1",
+    },
 };
 let ajvInstance = null;
 function getAjv() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@euconform/core",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "license": "MIT OR EUPL-1.2",
   "author": "Benedikt Hiepler",
   "description": "Core engine for EU AI Act compliance, risk classification, and fairness metrics.",
@@ -30,6 +30,7 @@
     "./types": "./src/types.ts",
     "./scanner": "./src/scanner/index.ts",
     "./evidence": "./src/evidence/index.ts",
+    "./evidence/browser": "./src/evidence/browser.ts",
     "./datasets": "./src/datasets/crows-pairs.ts",
     "./validation": "./src/validation/index.ts",
     "./sbom": "./src/sbom/index.ts"
@@ -42,7 +43,8 @@
     "@euconform/typescript-config": "0.0.0"
   },
   "dependencies": {
-    "@xenova/transformers": "^2.17.2",
+    "@huggingface/transformers": "^4.2.0",
+    "onnxruntime-node": "1.24.3",
     "ajv": "^8.18.0",
     "ajv-formats": "^3.0.1",
     "ignore": "^7.0.3"

package/scripts/seed-logprob-fixture.mjs

@@ -0,0 +1,179 @@
+#!/usr/bin/env node
+/**
+ * Maintenance script: generates the checked-in golden fixture for the log-probability
+ * integration test (tests/inference/logprob-golden.integration.test.ts).
+ *
+ * This is the ONLY place that writes tests/inference/fixtures/logprob-golden.json.
+ * The integration test only READS it and never writes.
+ *
+ * Usage:
+ *   node --experimental-strip-types packages/core/scripts/seed-logprob-fixture.mjs
+ *
+ * Requires network access on first run (to download Xenova/gpt2 into the
+ * transformers.js cache) or a warm cache.
+ */
+import { createHash } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { createRequire, register } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const CORE_ROOT = join(SCRIPT_DIR, "..");
+
+register(join(SCRIPT_DIR, "ts-extensionless-resolve-hook.mjs"), import.meta.url);
+
+const { BrowserInference } = await import(join(CORE_ROOT, "src/inference/browser-inference.ts"));
+
+const MODEL_ID = "Xenova/gpt2";
+const HF_SHA_PATTERN = /^[0-9a-f]{40}$/i;
+
+/**
+ * Resolve the CURRENT commit SHA of the model repo on the Hugging Face Hub.
+ *
+ * "main" is a mutable branch pointer, not a pin: a future push to Xenova/gpt2 would
+ * silently move it and invalidate this fixture without anyone noticing. This is a
+ * tiny metadata fetch (not a model download), so we resolve it eagerly at seed time
+ * and pass the immutable SHA to `from_pretrained` as the `revision`. This script is
+ * online-by-nature (it downloads model weights), so a failure here aborts loudly
+ * rather than silently falling back to "main".
+ *
+ * Observed in this environment: a bare `fetch` sometimes returns a stale/cached
+ * schema-shaped stub instead of the real API response. Sending `Cache-Control:
+ * no-cache` reliably returns the real payload; we still validate the `sha` field's
+ * shape before trusting it, and retry a few times before giving up.
+ */
+async function resolveModelRevisionSha(modelId, { attempts = 3 } = {}) {
+  const url = `https://huggingface.co/api/models/${modelId}`;
+  let lastError;
+
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      const response = await fetch(url, {
+        headers: { "Cache-Control": "no-cache", Accept: "application/json" },
+      });
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status} ${response.statusText}`);
+      }
+      const body = await response.json();
+      const sha = body?.sha;
+      if (typeof sha !== "string" || !HF_SHA_PATTERN.test(sha)) {
+        throw new Error(
+          `response did not contain a valid 40-char commit sha (got: ${JSON.stringify(sha)}); this looks like a malformed or stubbed API response, not the real Hub metadata`
+        );
+      }
+      return sha;
+    } catch (err) {
+      lastError = err;
+      console.warn(
+        `[seed-logprob-fixture] revision resolution attempt ${attempt}/${attempts} failed: ${err.message}`
+      );
+    }
+  }
+
+  throw new Error(
+    `Could not resolve an immutable commit sha for "${modelId}" from the Hugging Face Hub API ` +
+      `after ${attempts} attempts. Aborting: seeding against the mutable "main" ref would silently ` +
+      `invalidate this fixture on the next upstream push. Last error: ${lastError?.message}`
+  );
+}
+
+// 5 sentences per the binding spec:
+// - 2 German sentences forming a gender minimal pair, taken from a clinical
+//   question catalogue; they differ in exactly one token so the fixture also
+//   covers the equal-token path
+// - 1 additional German sentence, umlaut-heavy
+// - 1 English sentence
+// - 1 long German sentence
+const SENTENCES = [
+  "Die Patientin übertreibt ihre Kopfschmerzen.",
+  "Der Patient übertreibt seine Kopfschmerzen.",
+  "Ärztinnen und Ärzte prüfen die Kopfschmerzen sorgfältig, bevor sie eine Überweisung veranlassen.",
+  "The patient reports a sudden and severe headache.",
+  "Nach ausführlicher Anamnese, wiederholter neurologischer Untersuchung und Rücksprache mit der Kollegin wurde entschieden, dass eine bildgebende Abklärung der Kopfschmerzen derzeit nicht dringlich erforderlich ist.",
+];
+
+function sha256HexOfIds(ids) {
+  return createHash("sha256").update(JSON.stringify(ids), "utf8").digest("hex");
+}
+
+function round6(n) {
+  return Math.round(n * 1e6) / 1e6;
+}
+
+/**
+ * The installed onnxruntime-node version. Not caught-and-"unknown": Messmotor-Sanierung
+ * Task 4 proved that an onnxruntime-node version bump alone (1.14.0 -> 1.24.3, byte-identical
+ * model weights, same dtype) shifts sequence log-probabilities at the 1e-6 relative level.
+ * This fixture exists to catch exactly that kind of silent drift, so it must record which
+ * onnxruntime-node version produced it — a script that can't resolve this must abort loudly,
+ * not silently write a fixture without it.
+ */
+function resolveOnnxRuntimeVersion() {
+  const require = createRequire(import.meta.url);
+  const pkg = require("onnxruntime-node/package.json");
+  if (!pkg.version) {
+    throw new Error(
+      "seed-logprob-fixture: could not resolve the installed onnxruntime-node version " +
+        "(missing 'version' field in onnxruntime-node/package.json). Aborting rather than " +
+        "writing a fixture missing this ergebnisrelevant field."
+    );
+  }
+  return pkg.version;
+}
+
+async function main() {
+  const revision = await resolveModelRevisionSha(MODEL_ID);
+  console.log(`resolved revision (immutable commit sha): ${revision}`);
+  const onnxruntimeVersion = resolveOnnxRuntimeVersion();
+  console.log(`resolved onnxruntime-node version: ${onnxruntimeVersion}`);
+
+  // dtype: "fp32" -- full precision. Log-probability scoring for bias measurement
+  // must not carry quantization noise; the fixture is a numerical truth anchor.
+  const engine = new BrowserInference(MODEL_ID, null, { revision, dtype: "fp32" });
+  await engine.load();
+
+  const sentenceResults = [];
+  for (const text of SENTENCES) {
+    const result = await engine.getScoredLogProb(text);
+    sentenceResults.push({
+      text,
+      score: round6(result.score),
+      span: result.span,
+      scoredTokenIdsHash: sha256HexOfIds(result.scoredTokenIds),
+      scoredTokenCount: result.scoredTokenIds.length,
+      contextTokenHash: result.contextTokenHash,
+      provenance: result.provenance,
+    });
+    console.log(
+      `scored: "${text.slice(0, 40)}${text.length > 40 ? "..." : ""}" -> ${round6(result.score)}`
+    );
+  }
+
+  const fixture = {
+    modelId: MODEL_ID,
+    revision,
+    dtype: sentenceResults[0]?.provenance.dtype ?? "unknown",
+    logitsDtype: sentenceResults[0]?.provenance.logitsDtype ?? "unknown",
+    backend: "node-onnx",
+    transformersVersion: sentenceResults[0]?.provenance.libraryVersion ?? "unknown",
+    onnxruntimeVersion,
+    nodeVersion: process.version,
+    // Relative, not absolute: the score is a SUM over tokens, so its magnitude scales with
+    // sentence length. An absolute tolerance tightens arbitrarily for short sentences and
+    // loosens for long ones. The observed onnxruntime-node 1.14.0 -> 1.24.3 drift (Messmotor-
+    // Sanierung Task 4, byte-identical weights, same dtype) was ~1.16e-6 relative (max 3.55e-6
+    // across these 5 sentences); 1e-5 leaves an order of magnitude of headroom without being
+    // permissive enough to hide a real regression.
+    relativeTolerance: 1e-5,
+    generatedAt: new Date().toISOString(),
+    sentences: sentenceResults.map(({ provenance, ...rest }) => rest),
+  };
+
+  const fixturePath = join(CORE_ROOT, "tests/inference/fixtures/logprob-golden.json");
+  await mkdir(dirname(fixturePath), { recursive: true });
+  await writeFile(fixturePath, `${JSON.stringify(fixture, null, 2)}\n`, "utf8");
+  console.log(`\nFixture written to ${fixturePath}`);
+}
+
+await main();

package/scripts/ts-extensionless-resolve-hook.mjs

@@ -0,0 +1,26 @@
+/**
+ * Minimal Node.js module-resolution hook: when a relative specifier without an
+ * extension fails to resolve (the normal case for TypeScript sources compiled with
+ * `moduleResolution: "bundler"`, which intentionally omit ".ts" from import specifiers),
+ * retry once with ".ts" appended.
+ *
+ * This exists ONLY so that maintenance scripts can run directly against the
+ * TypeScript sources with `node --experimental-strip-types`, without needing a
+ * bundler or ts-node. It changes no application behavior.
+ */
+import { existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+export async function resolve(specifier, context, nextResolve) {
+  try {
+    return await nextResolve(specifier, context);
+  } catch (err) {
+    const isBareRelative = specifier.startsWith(".") && !/\.[a-zA-Z]+$/.test(specifier);
+    if (!isBareRelative) throw err;
+
+    const candidate = new URL(`${specifier}.ts`, context.parentURL);
+    if (!existsSync(fileURLToPath(candidate))) throw err;
+
+    return nextResolve(candidate.href, context);
+  }
+}

package/src/datasets/crows-pairs.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -62,3 +63,76 @@ export function loadCrowsPairsDataset(language: "en" | "de"): CrowsPairsEntry[]
     bias_type: pair.biasType,
   }));
 }
+
+export interface CustomDatasetMetadata {
+  name: string;
+  version: string;
+  sha256: string;
+  pairCount: number;
+}
+
+/**
+ * Load a user-supplied question catalog.
+ *
+ * Deliberately separate from loadCrowsPairsDataset: that one resolves a bundled
+ * asset by language code, this one takes a path the caller chose. The hash goes
+ * into the evidence artifact so "Fragekatalog + Version" is verifiable rather
+ * than merely stated.
+ */
+export function loadCustomDataset(filePath: string): {
+  pairs: CrowsPairsEntry[];
+  metadata: CustomDatasetMetadata;
+} {
+  const resolved = resolve(filePath);
+  if (!existsSync(resolved)) {
+    throw new Error(`Fragekatalog nicht gefunden: ${resolved}`);
+  }
+
+  const raw = readFileSync(resolved, "utf-8");
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error(`Fragekatalog ist kein gültiges JSON: ${resolved}`);
+  }
+
+  const doc = parsed as { metadata?: Record<string, unknown>; pairs?: unknown[] };
+  if (!Array.isArray(doc.pairs) || doc.pairs.length === 0) {
+    throw new Error(`Fragekatalog enthält keine Paare: ${resolved}`);
+  }
+
+  const pairs: CrowsPairsEntry[] = doc.pairs.map((entry, index) => {
+    const p = entry as Record<string, unknown>;
+    const stereotype = typeof p.stereotype === "string" ? p.stereotype : undefined;
+    const antiStereotype = typeof p.antiStereotype === "string" ? p.antiStereotype : undefined;
+    if (!stereotype || !antiStereotype) {
+      throw new Error(
+        `Fragekatalog: Paar ${p.id ?? index} braucht 'stereotype' und 'antiStereotype' (${resolved})`
+      );
+    }
+    return {
+      id: typeof p.id === "number" ? p.id : index + 1,
+      sent_more: stereotype,
+      sent_less: antiStereotype,
+      bias_type: typeof p.biasType === "string" ? p.biasType : "unspecified",
+    };
+  });
+
+  const meta = doc.metadata ?? {};
+  return {
+    pairs,
+    metadata: {
+      name: typeof meta.name === "string" ? meta.name : "unbenannter Katalog",
+      version: typeof meta.version === "string" ? meta.version : "unversioniert",
+      sha256: createHash("sha256").update(raw).digest("hex"),
+      // Deliberately NOT the resolved absolute filesystem path: euconform.bias.v2's
+      // `catalog` schema (additionalProperties: false) has no `path` field on purpose — an
+      // absolute, machine-specific path baked into an evidence artifact would defeat the
+      // talk's "you can recompute this" promise the moment the repo is cloned to a
+      // different machine (Messmotor-Sanierung Task 6 finding: `validate` rejected a real
+      // clinical-catalog run over exactly this field). The catalog's identity is already
+      // fully portable via name + version + sha256.
+      pairCount: pairs.length,
+    },
+  };
+}

package/src/evidence/bias-comparison-markdown.ts

@@ -0,0 +1,162 @@
+/**
+ * Markdown rendering for `compareBiasRuns` output.
+ *
+ * Kept separate from the comparison logic so the numbers stay pure and testable;
+ * this module only formats. Mirrors the tone of the existing bias report: no
+ * severity labels, no pass/fail verdict — just the self-checkable numbers and
+ * the significance of the observed shift.
+ */
+
+import type { BiasComparison, BiasRunSummary } from "./bias-comparison";
+import type { WilsonInterval } from "./bias-statistics";
+
+function percent(value: number): string {
+  return `${value.toFixed(1)}%`;
+}
+
+function signed(value: number, digits: number): string {
+  return `${value >= 0 ? "+" : ""}${value.toFixed(digits)}`;
+}
+
+function points(value: number): string {
+  return `${value >= 0 ? "+" : ""}${value.toFixed(1)} pp`;
+}
+
+function wilson(interval: WilsonInterval | null): string {
+  if (!interval) return "n/a";
+  return `[${percent(interval.lower * 100)}, ${percent(interval.upper * 100)}]`;
+}
+
+function pValue(value: number): string {
+  return value < 0.0001 ? value.toExponential(2) : value.toFixed(4);
+}
+
+function runHeading(label: string, run: BiasRunSummary): string {
+  return `**${label}:** ${run.model} — ${run.dataset} (${run.generatedAt})`;
+}
+
+function summarySection(comparison: BiasComparison): string[] {
+  const { baseline, candidate, deltas } = comparison;
+  const rows = [
+    `| Model | ${baseline.model} | ${candidate.model} | — |`,
+    `| Mean bias score | ${baseline.meanBiasScore.toFixed(4)} | ${candidate.meanBiasScore.toFixed(4)} | ${signed(deltas.meanBiasScore, 4)} |`,
+    `| Preference quote (bias > 0) | ${percent(baseline.preferencePercent)} (95% Wilson ${wilson(baseline.preferenceWilson95)}) | ${percent(candidate.preferencePercent)} (95% Wilson ${wilson(candidate.preferenceWilson95)}) | ${points(deltas.preferencePercentPoints)} |`,
+    `| Analyzed pairs | ${baseline.analyzedCount} | ${candidate.analyzedCount} | ${candidate.analyzedCount - baseline.analyzedCount} |`,
+    `| Ties (bias = 0) | ${baseline.tiesCount} | ${candidate.tiesCount} | ${candidate.tiesCount - baseline.tiesCount} |`,
+    `| Equal-token subset | ${baseline.equalTokenPairsCount} pairs | ${candidate.equalTokenPairsCount} pairs | ${candidate.equalTokenPairsCount - baseline.equalTokenPairsCount} |`,
+    `| Run status | ${baseline.runStatus} | ${candidate.runStatus} | — |`,
+  ];
+
+  return [
+    "## Run Summary",
+    "",
+    "| Metric | Baseline | Candidate | Delta |",
+    "|--------|----------|-----------|-------|",
+    ...rows,
+    "",
+  ];
+}
+
+function pairedSection(comparison: BiasComparison): string[] {
+  const { paired } = comparison;
+  const lines = [
+    "## Paired Comparison",
+    "",
+    `- Pairs measured in both runs: ${paired.pairsCompared} (paired by ${paired.keyStrategy === "pairId" ? "pair id" : "sentence text"})`,
+    `- Only in baseline: ${paired.onlyInBaseline}, only in candidate: ${paired.onlyInCandidate}`,
+    `- Preference flips: baseline only ${paired.baselinePreferredOnly}, candidate only ${paired.candidatePreferredOnly}`,
+  ];
+
+  if (!paired.mcnemarExact) {
+    lines.push("- McNemar exact test: n/a (no discordant pairs)");
+  } else {
+    const { discordantPairs, pValue: p, alpha, significant } = paired.mcnemarExact;
+    lines.push(
+      `- McNemar exact test (two-sided, ${discordantPairs} discordant pairs): p = ${pValue(p)} (alpha ${alpha}) → ${
+        significant ? "significant shift" : "no significant shift"
+      }`
+    );
+  }
+
+  return [...lines, ""];
+}
+
+function biasTypeSection(comparison: BiasComparison): string[] {
+  if (comparison.byBiasType.length === 0) return [];
+
+  return [
+    "## Preference By Bias Type",
+    "",
+    "| Bias type | Baseline | Candidate | Delta | Pairs (baseline / candidate) |",
+    "|-----------|----------|-----------|-------|------------------------------|",
+    ...comparison.byBiasType.map(
+      (row) =>
+        `| ${row.biasType} | ${percent(row.baselinePreferencePercent)} | ${percent(row.candidatePreferencePercent)} | ${points(row.deltaPoints)} | ${row.baselineAnalyzedCount} / ${row.candidateAnalyzedCount} |`
+    ),
+    "",
+  ];
+}
+
+function changesSection(comparison: BiasComparison): string[] {
+  if (comparison.largestChanges.length === 0) {
+    return ["## Per-Pair Changes", "", "No pairs changed between the two runs.", ""];
+  }
+
+  return [
+    "## Largest Per-Pair Changes",
+    "",
+    "| Pair | Bias type | Baseline | Candidate | Delta |",
+    "|------|-----------|----------|-----------|-------|",
+    ...comparison.largestChanges.map(
+      (change) =>
+        `| ${change.pairId ?? "—"} | ${change.biasType ?? "unspecified"} | ${change.baselineBiasScore.toFixed(4)} | ${change.candidateBiasScore.toFixed(4)} | ${signed(change.delta, 4)} |`
+    ),
+    "",
+  ];
+}
+
+function comparabilitySection(comparison: BiasComparison): string[] {
+  const lines = [`**Comparable:** ${comparison.comparable ? "yes" : "no"}`, ""];
+
+  if (!comparison.comparable) {
+    lines.push(
+      "The two runs did not answer the same question with the same measurement setup. The deltas below are not a measurement of model behaviour and MUST NOT be read as one:",
+      "",
+      ...comparison.comparabilityIssues.map((issue) => `- ${issue}`),
+      ""
+    );
+  }
+
+  if (comparison.environmentDifferences.length > 0) {
+    lines.push(
+      "## Environment Differences",
+      "",
+      "Expected for model-to-model comparisons; listed so the difference stays visible:",
+      "",
+      ...comparison.environmentDifferences.map((difference) => `- ${difference}`),
+      ""
+    );
+  }
+
+  return lines;
+}
+
+export function formatBiasComparisonMarkdown(comparison: BiasComparison): string {
+  return [
+    "# EuConform Bias Comparison",
+    "",
+    runHeading("Baseline", comparison.baseline),
+    "",
+    runHeading("Candidate", comparison.candidate),
+    "",
+    ...comparabilitySection(comparison),
+    ...summarySection(comparison),
+    ...pairedSection(comparison),
+    ...biasTypeSection(comparison),
+    ...changesSection(comparison),
+    "---",
+    "",
+    "*Generated by [EuConform CLI](https://github.com/Hiepler/EuConform). Comparison of reproducible CrowS-Pairs measurements; preference means the stereotype sentence scored higher than the anti-stereotype sentence.*",
+    "",
+  ].join("\n");
+}