@intellectif/lk-core 0.8.1 → 0.9.0

package/vectors/README.md

@@ -0,0 +1,97 @@
+# Grade-stability vectors
+
+`scoring.json` is lk-core's binding rule written down as data:
+
+> Anything that can change a historical grade is a package **major**, always.
+
+Each vector is a call to a scoring function and the exact value that call
+returned when the vector was frozen. If a release of `@intellectif/lk-core`
+returns anything different for any vector, a grade you stored would come out
+differently today than on the day you recorded it — and that release is not
+allowed to be a minor or a patch.
+
+The SDK replays this corpus against its own source and against its built
+package, CommonJS and ES module, on every pull request, on the Node version
+consumers run and on the one the release is built with. It ships in the tarball
+so you can replay it against the build **you** install.
+
+## Replaying it in your own test suite
+
+```js
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import * as core from '@intellectif/lk-core';
+
+const root = dirname(createRequire(import.meta.url).resolve('@intellectif/lk-core/package.json'));
+const { replay } = await import(pathToFileURL(join(root, 'vectors/replay.mjs')).href);
+const corpus = JSON.parse(readFileSync(join(root, 'vectors/scoring.json'), 'utf8'));
+
+test('lk-core still grades exactly as it did', () => {
+  const changed = replay(core, corpus).filter((result) => !result.ok);
+  expect(changed).toEqual([]);
+});
+```
+
+These files are deliberately **not** package exports. An export would be new
+public API — a lk-core minor, which forces a lk-react major — for what is test
+data. They are reached by resolving `package.json`, which lk-core does export.
+
+## If a vector fails
+
+Do not change the expectation to make it pass. A failing vector means an
+upgrade changed how a learner's answer scores. Decide whether that change is
+wanted, and if it is, work out what it does to grades you already hold before
+you accept the new number. Every failure names its vector, and the ones that pin
+a known hazard carry a `note` saying why they exist.
+
+## What it covers
+
+- **`score` and `evaluate`** for every built-in activity type, including each
+  default a future release could plausibly "improve" — diacritic folding, Unicode
+  normalisation, empty-input handling — the legacy per-blank `caseSensitive` and
+  `trimWhitespace` flags and the way `match` overrides them, and the refusal to
+  score redacted data or an answer key that cannot produce a finite score.
+- **Written-response word counts**, recomputed from the text rather than taken
+  from the client's `wordCount`, with inclusive `minWords` / `maxWords` bounds.
+- **`matchText` and `levenshteinDistance`** across every authored policy.
+- **`computePassThreshold` and `DEFAULT_PASS_THRESHOLD`**, including an authored
+  threshold of `0`.
+- **`roundGrade`, `gte` and `classifyBand`** in every mode, including float
+  edges, negative zero, half-even ties and near-ties, and bands that share a
+  minimum.
+- **`countWords`**, **`seededShuffle`** in both versions, **`gradeFromRubric`**
+  including its unscorable paths, and **`outcomeFromGrade`**.
+- **`scoredItemsFromPlan`**, which decides what a stored paper's denominator is
+  when an outcome is missing, and **`composeAssessmentScore`** across weights,
+  points, provisional and unscorable items, and section thresholds.
+
+## What it does not cover
+
+- **Planning an attempt** (`planAttempt`, `planHash`, `verifyAttemptPlan`) and
+  **redaction** (`redact`). A plan's hashes are identity rather than a grade, and
+  both are verified by their own tests. How a stored plan becomes scored items is
+  covered, above.
+- **Custom missing-outcome policies.** `scoredItemsFromPlan` accepts a function
+  for `missing`; a function cannot be stored as data, so only the built-in
+  policies are pinned.
+- **xAPI statements**, and **anything `@intellectif/lk-react` renders**.
+- **Your own arithmetic.** If you round, aggregate or canonicalise answers
+  yourself, this corpus says nothing about that code.
+- **Text of developer-facing diagnostics.** A vector can mark a field such as an
+  `unscorable` reason as ignored, because rewording a diagnostic is not a grade
+  change. Codes that are contract, like `'requires_async_grading'`, are never
+  ignored.
+
+## Encoding
+
+JSON cannot express four values these functions produce or accept, so they are
+tagged: `{ "$number": "NaN" | "Infinity" | "-Infinity" | "-0" }` and
+`{ "$undefined": true }`. A call that throws is recorded as
+`{ "$throws": "<error.name>" }`. Every non-ASCII character is stored as a
+`\uXXXX` escape, so a no-break space can never pass for an ordinary one.
+Comparison ignores object key order.
+
+`corpusVersion` changes only if this encoding does. Adding vectors does not
+change it, so always load the `replay.mjs` that sits beside the corpus.

package/vectors/replay.d.mts

@@ -0,0 +1,44 @@
+/** Types for `replay.mjs`. See that file for the contract and the encoding. */
+
+/** The encoding version this replayer reads. */
+export declare const CORPUS_VERSION: 1;
+
+/** One frozen call. Exactly one of `fn` / `const` is present. */
+export interface ScoringVector {
+  id: string;
+  fn?: string;
+  const?: string;
+  /** Encoded arguments. */
+  args?: unknown[];
+  /** Encoded return value, or `{ $throws: "<error.name>" }`. */
+  expect: unknown;
+  /** Keys excluded from comparison: developer-facing diagnostics only. */
+  ignore?: string[];
+  /** Why this vector exists, when that is not obvious from its id. */
+  note?: string;
+}
+
+export interface ScoringCorpus {
+  $comment?: string;
+  corpusVersion: number;
+  frozenFrom: string;
+  vectors: ScoringVector[];
+}
+
+export interface VectorResult {
+  id: string;
+  ok: boolean;
+  /** Canonical JSON of the frozen expectation. */
+  expected: string;
+  /** Canonical JSON of what this build returned. */
+  actual: string;
+  note?: string;
+}
+
+export declare function encode(value: unknown): unknown;
+export declare function decode(value: unknown): unknown;
+export declare function canonical(value: unknown): string;
+/** `core` is the lk-core module namespace: `import * as core from '@intellectif/lk-core'`. */
+export declare function callVector(core: object, vector: ScoringVector): unknown;
+export declare function runVector(core: object, vector: ScoringVector): VectorResult;
+export declare function replay(core: object, corpus: ScoringCorpus): VectorResult[];

package/vectors/replay.mjs

@@ -0,0 +1,159 @@
+/**
+ * Replays the grade-stability corpus (`scoring.json`) against an lk-core build.
+ *
+ * The corpus is the SDK's binding rule — "anything that can change a historical
+ * grade is a package major" — written down as data. Each vector is a call and
+ * the exact value that call returned when the vector was frozen. If a release
+ * returns anything else, a stored grade would score differently today than on
+ * the day it was recorded, and that release is not allowed to be a minor.
+ *
+ * This file ships in the package so a consumer can run the same corpus against
+ * the build they actually install, without re-implementing the comparison:
+ *
+ * ```js
+ * import { createRequire } from 'node:module';
+ * import { dirname, join } from 'node:path';
+ * import { pathToFileURL } from 'node:url';
+ * import { readFileSync } from 'node:fs';
+ * import * as core from '@intellectif/lk-core';
+ *
+ * const root = dirname(createRequire(import.meta.url).resolve('@intellectif/lk-core/package.json'));
+ * const { replay } = await import(pathToFileURL(join(root, 'vectors/replay.mjs')).href);
+ * const corpus = JSON.parse(readFileSync(join(root, 'vectors/scoring.json'), 'utf8'));
+ * const failures = replay(core, corpus).filter((result) => !result.ok);
+ * ```
+ *
+ * It is plain ESM with no imports, so it runs unchanged under Node, Vitest and
+ * Jest, and it is deliberately NOT a package export: an export would be new API
+ * and a lk-core minor, which forces a lk-react major for a file of test data.
+ *
+ * ## Encoding
+ *
+ * JSON cannot express four values the scoring functions genuinely produce or
+ * accept, so they are tagged: `{ "$number": "NaN" | "Infinity" | "-Infinity" |
+ * "-0" }` and `{ "$undefined": true }`. `-0` is tagged because `roundGrade`
+ * deliberately returns `0`, not `-0`, for a negative value that rounds to
+ * zero, and JSON would erase the difference. A call that throws is recorded as
+ * `{ "$throws": "<error.name>" }`.
+ */
+
+/** Bump only when the ENCODING changes. Adding vectors does not change it. */
+export const CORPUS_VERSION = 1;
+
+/** Turns a runtime value into the corpus's JSON-safe form. */
+export function encode(value) {
+  if (typeof value === 'number') {
+    if (Number.isNaN(value)) return { $number: 'NaN' };
+    if (value === Number.POSITIVE_INFINITY) return { $number: 'Infinity' };
+    if (value === Number.NEGATIVE_INFINITY) return { $number: '-Infinity' };
+    if (Object.is(value, -0)) return { $number: '-0' };
+    return value;
+  }
+  if (value === undefined) return { $undefined: true };
+  if (Array.isArray(value)) return value.map(encode);
+  if (value !== null && typeof value === 'object') {
+    const out = {};
+    for (const [key, entry] of Object.entries(value)) {
+      // An absent key and a key set to `undefined` serialise identically in
+      // JSON; recording both as absent keeps the corpus honest about that.
+      if (entry !== undefined) out[key] = encode(entry);
+    }
+    return out;
+  }
+  return value;
+}
+
+/** Inverse of {@link encode}. Always returns fresh objects. */
+export function decode(value) {
+  if (Array.isArray(value)) return value.map(decode);
+  if (value !== null && typeof value === 'object') {
+    const keys = Object.keys(value);
+    if (keys.length === 1 && keys[0] === '$number') {
+      return {
+        NaN: Number.NaN,
+        Infinity: Number.POSITIVE_INFINITY,
+        '-Infinity': Number.NEGATIVE_INFINITY,
+        '-0': -0,
+      }[value.$number];
+    }
+    if (keys.length === 1 && keys[0] === '$undefined') return undefined;
+    const out = {};
+    for (const [key, entry] of Object.entries(value)) out[key] = decode(entry);
+    return out;
+  }
+  return value;
+}
+
+/**
+ * JSON with keys sorted at every depth. Object key ORDER is not a grade: a
+ * refactor that reorders an object literal must not fail the gate, or people
+ * learn to regenerate the corpus to make it pass — which is the one habit that
+ * would turn this tripwire into a formality.
+ */
+export function canonical(value) {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  if (value !== null && typeof value === 'object') {
+    return `{${Object.keys(value)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`)
+      .join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+/** Removes the named keys at every depth. Used only where a vector opts in. */
+function without(value, ignore) {
+  if (ignore.size === 0) return value;
+  if (Array.isArray(value)) return value.map((entry) => without(entry, ignore));
+  if (value !== null && typeof value === 'object') {
+    const out = {};
+    for (const [key, entry] of Object.entries(value)) {
+      if (!ignore.has(key)) out[key] = without(entry, ignore);
+    }
+    return out;
+  }
+  return value;
+}
+
+/** Executes one vector against `core` and returns the encoded result. */
+export function callVector(core, vector) {
+  if (vector.const !== undefined) {
+    return core[vector.const] === undefined
+      ? { $missing: vector.const }
+      : encode(core[vector.const]);
+  }
+  const fn = core[vector.fn];
+  if (typeof fn !== 'function') return { $missing: vector.fn };
+  try {
+    return encode(fn(...vector.args.map(decode)));
+  } catch (error) {
+    return { $throws: error?.name ?? 'Error' };
+  }
+}
+
+/**
+ * Executes one vector and compares it with its frozen expectation.
+ *
+ * `ignore` exists for exactly one kind of field: a developer-facing diagnostic
+ * sentence such as an `unscorable` reason. Rewording one of those does not
+ * change a grade, and pinning its text would make copy edits fail a gate that
+ * exists for grades. Codes that ARE contract — a deferred outcome's
+ * `'requires_async_grading'` — are never ignored.
+ */
+export function runVector(core, vector) {
+  const ignore = new Set(vector.ignore ?? []);
+  const actual = canonical(without(callVector(core, vector), ignore));
+  const expected = canonical(without(vector.expect, ignore));
+  return { id: vector.id, ok: actual === expected, expected, actual, note: vector.note };
+}
+
+/** Replays a whole corpus. Throws if the corpus uses an encoding this file does not speak. */
+export function replay(core, corpus) {
+  if (corpus?.corpusVersion !== CORPUS_VERSION) {
+    throw new Error(
+      `scoring.json declares corpusVersion ${String(corpus?.corpusVersion)}, but this replay.mjs reads ` +
+        `version ${CORPUS_VERSION}. Load the replay.mjs shipped beside the corpus.`,
+    );
+  }
+  return corpus.vectors.map((vector) => runVector(core, vector));
+}