eklavya 1.14.0 → 1.16.0

package/dist/eval/history-stats.js

@@ -0,0 +1,230 @@
+/**
+ * What a real answer history says about the promises the product makes.
+ *
+ * The question-quality eval next door measures questions before anyone answers
+ * them. This measures what happened after: it reads an actual `knowledge.db`
+ * and asks whether the claims hold on it.
+ *
+ * Three, in the order they matter:
+ *
+ * 1. **The repeat rate.** *Never the same question twice* is the promise the
+ *    whole tool rests on, and it is the one claim measurable today with no new
+ *    harness. If it is not kept, nothing else here is worth reading.
+ * 2. **Tier calibration.** Tiers 1-5 are supposed to be a difficulty ladder. If
+ *    grades do not fall as tiers rise, the ladder is decoration.
+ * 3. **Whether anything survives a gap.** Not retention -- that needs a study
+ *    with people in it -- but the weakest honest version: when a concept came
+ *    back days later, did the answer hold.
+ *
+ * Pure, like `srs.ts`: rows in, numbers out, no database handle and no clock
+ * beyond what the rows carry. Aggregates only, and deliberately so -- nothing
+ * here returns question or answer text, because the output is written to a file
+ * that gets committed and the repo rule is that a learner's data never is.
+ */
+import { questionFingerprint } from '../store.js';
+/**
+ * How far back the product itself looks for a repeat.
+ *
+ * `hasAskedQuestion` in `store.ts` fingerprints the last 20 attempts on a
+ * concept and no further. That is a deliberate bound, not an oversight -- but it
+ * means a repeat older than the window is a *design limit* rather than a broken
+ * promise, and reporting the two as one number would be dishonest in whichever
+ * direction happened to flatter.
+ */
+export const REPEAT_WINDOW = 20;
+/**
+ * How many previous questions the *planner* puts in front of the tutor.
+ *
+ * `get_session_quiz_plan` fills each item's `asked_before` from
+ * `recentQuestions(db, concept.id, ASKED_HISTORY)` with ASKED_HISTORY = 3. That
+ * is the mechanism that actually prevents a repeat, because it acts while the
+ * question is being written.
+ *
+ * `REPEAT_WINDOW` above is a different thing and a weaker one: `hasAskedQuestion`
+ * is consulted by `record_attempt` *after* the question was asked, and it does
+ * not reject anything -- it records the attempt and returns `repeat_question:
+ * true`. Calling a repeat at distance 4-20 "one the product should have caught"
+ * credits a check that never saw it. Both numbers are reported, separately.
+ */
+export const PLANNER_HISTORY = 3;
+export function repeatStats(rows) {
+    const byConcept = new Map();
+    for (const row of rows) {
+        const list = byConcept.get(row.concept_id);
+        if (list)
+            list.push(row);
+        else
+            byConcept.set(row.concept_id, [row]);
+    }
+    let repeats = 0;
+    let repeatable = 0;
+    let plannerSaw = 0;
+    let recorderOnly = 0;
+    let worstRun = 0;
+    for (const list of byConcept.values()) {
+        const ordered = [...list].sort((a, b) => a.id - b.id);
+        const seen = [];
+        const counts = new Map();
+        for (const [index, row] of ordered.entries()) {
+            const print = questionFingerprint(row.question);
+            // An empty fingerprint is not evidence of anything -- it means the stem
+            // was blank or punctuation only, which the product also refuses to match.
+            // It cannot count as a repeat, so it must not swell the denominator
+            // either; that error would run in the flattering direction.
+            if (print) {
+                if (index > 0)
+                    repeatable++;
+                // lastIndexOf, not indexOf: what matters is how far back the NEAREST
+                // earlier copy was, since that is the one both windows would have held.
+                // Taking the oldest copy under-reports repeats the product could see.
+                const previous = seen.lastIndexOf(print);
+                if (previous >= 0) {
+                    repeats++;
+                    const distance = seen.length - previous;
+                    if (distance <= PLANNER_HISTORY)
+                        plannerSaw++;
+                    else if (distance <= REPEAT_WINDOW)
+                        recorderOnly++;
+                }
+                const n = (counts.get(print) ?? 0) + 1;
+                counts.set(print, n);
+                if (n > worstRun)
+                    worstRun = n;
+            }
+            seen.push(print);
+        }
+    }
+    return {
+        attempts: rows.length,
+        concepts: byConcept.size,
+        repeatable,
+        repeats,
+        plannerSaw,
+        recorderOnly,
+        outsideBoth: repeats - plannerSaw - recorderOnly,
+        worstRun,
+    };
+}
+/**
+ * Both readings, because the data cannot choose between them.
+ *
+ * `migrations/004_attempt_outcome.sql` states the contract: NULL means
+ * "recorded before this column existed", and readers "must treat it as unknown,
+ * not as any particular outcome". Keeping those rows counts a grade 0 of unknown
+ * provenance as evidence the learner did not know; dropping them throws away
+ * real answers. On the first published history the choice moved tier 2 from
+ * mean 2.25 to 3.10 -- from below the pass threshold to above it -- and changed
+ * which tier scored worst.
+ *
+ * A single number there would have been a choice presented as a measurement, so
+ * both are reported and the result file has to show both.
+ */
+export function tierReadings(rows) {
+    return {
+        allGraded: tierStats(rows, true),
+        knownOutcome: tierStats(rows, false),
+    };
+}
+export function tierStats(rows, includeUnknownOutcome = true) {
+    // A decline is recorded as grade 0 and means "not now", not "did not know".
+    // Averaging it in would make any tier the developer happened to skip look
+    // harder than it is.
+    const notDeclined = rows.filter((r) => r.outcome !== 'declined');
+    const graded = includeUnknownOutcome ? notDeclined : notDeclined.filter((r) => r.outcome != null);
+    const byTier = new Map();
+    for (const row of graded) {
+        const list = byTier.get(row.difficulty);
+        if (list)
+            list.push(row);
+        else
+            byTier.set(row.difficulty, [row]);
+    }
+    const out = [...byTier.entries()]
+        .sort(([a], [b]) => a - b)
+        .map(([tier, list]) => ({
+        tier,
+        attempts: list.length,
+        meanGrade: list.reduce((n, r) => n + r.grade, 0) / list.length,
+        passRate: list.filter((r) => r.grade >= 3).length / list.length,
+    }));
+    let monotonic = true;
+    for (let i = 1; i < out.length; i++) {
+        if ((out[i]?.meanGrade ?? 0) > (out[i - 1]?.meanGrade ?? 0))
+            monotonic = false;
+    }
+    return {
+        basis: includeUnknownOutcome ? 'all-graded' : 'known-outcome',
+        rows: out,
+        monotonic,
+        excludedDeclines: rows.length - notDeclined.length,
+        excludedUnknown: notDeclined.length - graded.length,
+    };
+}
+const DAY_MS = 24 * 60 * 60 * 1000;
+/**
+ * `ts` is written by SQLite's `datetime('now')`, which is UTC with no zone
+ * marker. `Date.parse('2026-09-01 10:00:00')` reads that as local time, so on a
+ * machine at UTC+5:30 every timestamp shifts and a gap near the threshold lands
+ * on the wrong side of it.
+ */
+function parseTs(ts) {
+    return Date.parse(`${ts.replace(' ', 'T')}Z`);
+}
+/**
+ * Consecutive attempts on the same concept separated by at least `minDays`.
+ *
+ * Consecutive, not all-pairs: counting every combination would weight a concept
+ * asked five times far above one asked twice, and the question being asked is
+ * "did it hold across a gap", once per gap.
+ */
+export function gapStats(rows, minDays = 1) {
+    const byConcept = new Map();
+    for (const row of rows) {
+        const list = byConcept.get(row.concept_id);
+        if (list)
+            list.push(row);
+        else
+            byConcept.set(row.concept_id, [row]);
+    }
+    let pairs = 0;
+    let held = 0;
+    let skippedNoPriorPass = 0;
+    for (const list of byConcept.values()) {
+        // Ordered over EVERY attempt, declines included. Filtering them out first
+        // makes two attempts either side of a skipped one look consecutive, so a
+        // concept the learner saw three times reports one long quiet gap it never
+        // had. Declines are skipped as the *later* member below instead.
+        const ordered = [...list].sort((a, b) => a.id - b.id);
+        for (let i = 1; i < ordered.length; i++) {
+            const before = ordered[i - 1];
+            const after = ordered[i];
+            if (!before || !after)
+                continue;
+            if (after.outcome === 'declined')
+                continue;
+            // `datetime('now')` writes UTC without a zone; treat it as UTC rather than
+            // letting Date.parse guess local, which shifts every gap by the offset.
+            const gap = parseTs(after.ts) - parseTs(before.ts);
+            if (!Number.isFinite(gap) || gap < minDays * DAY_MS)
+                continue;
+            if (before.grade < 3) {
+                skippedNoPriorPass++;
+                continue;
+            }
+            pairs++;
+            if (after.grade >= 3)
+                held++;
+        }
+    }
+    return { minDays, pairs, held, skippedNoPriorPass };
+}
+export function outcomeStats(rows) {
+    const count = (o) => rows.filter((r) => r.outcome === o).length;
+    return {
+        answered: count('answered'),
+        dontKnow: count('dont_know'),
+        declined: count('declined'),
+        unrecorded: rows.filter((r) => r.outcome == null).length,
+    };
+}
+//# sourceMappingURL=history-stats.js.map

package/dist/eval/history-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"history-stats.js","sourceRoot":"","sources":["../../src/eval/history-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAalD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEhC;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AAsCjC,MAAM,UAAU,WAAW,CAAC,IAAkB;IAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;IAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACpB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEzC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChD,wEAAwE;YACxE,0EAA0E;YAC1E,oEAAoE;YACpE,4DAA4D;YAC5D,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,KAAK,GAAG,CAAC;oBAAE,UAAU,EAAE,CAAC;gBAE5B,qEAAqE;gBACrE,wEAAwE;gBACxE,sEAAsE;gBACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACzC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;oBAClB,OAAO,EAAE,CAAC;oBACV,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;oBACxC,IAAI,QAAQ,IAAI,eAAe;wBAAE,UAAU,EAAE,CAAC;yBACzC,IAAI,QAAQ,IAAI,aAAa;wBAAE,YAAY,EAAE,CAAC;gBACrD,CAAC;gBACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBACrB,IAAI,CAAC,GAAG,QAAQ;oBAAE,QAAQ,GAAG,CAAC,CAAC;YACjC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,QAAQ,EAAE,SAAS,CAAC,IAAI;QACxB,UAAU;QACV,OAAO;QACP,UAAU;QACV,YAAY;QACZ,WAAW,EAAE,OAAO,GAAG,UAAU,GAAG,YAAY;QAChD,QAAQ;KACT,CAAC;AACJ,CAAC;AA+BD;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,IAAkB;IAC7C,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC;QAChC,YAAY,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAkB,EAAE,qBAAqB,GAAG,IAAI;IACxE,4EAA4E;IAC5E,0EAA0E;IAC1E,qBAAqB;IACrB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACpB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,GAAG,GAAc,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,IAAI;QACJ,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;QAC9D,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;KAChE,CAAC,CAAC,CAAC;IAEN,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC;YAAE,SAAS,GAAG,KAAK,CAAC;IACjF,CAAC;IAED,OAAO;QACL,KAAK,EAAE,qBAAqB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe;QAC7D,IAAI,EAAE,GAAG;QACT,SAAS;QACT,gBAAgB,EAAE,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;QAClD,eAAe,EAAE,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;KACpD,CAAC;AACJ,CAAC;AAoBD,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEnC;;;;;GAKG;AACH,SAAS,OAAO,CAAC,EAAU;IACzB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAkB,EAAE,OAAO,GAAG,CAAC;IACtD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;IAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACpB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,0EAA0E;QAC1E,yEAAyE;QACzE,0EAA0E;QAC1E,iEAAiE;QACjE,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAEtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK;gBAAE,SAAS;YAChC,IAAI,KAAK,CAAC,OAAO,KAAK,UAAU;gBAAE,SAAS;YAE3C,2EAA2E;YAC3E,wEAAwE;YACxE,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,OAAO,GAAG,MAAM;gBAAE,SAAS;YAE9D,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACrB,kBAAkB,EAAE,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC;gBAAE,IAAI,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;AACtD,CAAC;AAUD,MAAM,UAAU,YAAY,CAAC,IAAkB;IAC7C,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;QAC3B,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC;QAC5B,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;QAC3B,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM;KACzD,CAAC;AACJ,CAAC"}

package/dist/eval/question-checks.js

@@ -0,0 +1,220 @@
+/**
+ * The deterministic half of the question-quality eval.
+ *
+ * Eklavya's claim is that the developer learns, and until now nothing measured
+ * it: the 426 tests next door cover SM-2 arithmetic, plan sizing, gate maths and
+ * migrations -- the machinery -- while the product is a question, and whether a
+ * question is any good was checked by nobody.
+ *
+ * These are the checks that need no model. That matters more than it sounds:
+ * an LLM judge is soft, costs money, disagrees with itself between runs, and is
+ * biased toward output from its own family. Everything that can be settled by
+ * counting is settled here instead, for free and reproducibly, and the judge is
+ * left with only the three questions that genuinely need reading comprehension
+ * (see `eval/README.md`).
+ *
+ * Pure, like `srs.ts`: no database, no clock, no file reads. A check returns
+ * what it found, never a fix.
+ */
+/** Words, for length checks. Punctuation-only tokens do not count. */
+function words(text) {
+    return text.split(/\s+/).filter((w) => /[\p{L}\p{N}]/u.test(w));
+}
+/**
+ * Lowercased words, with identifiers kept whole.
+ *
+ * `_`, `-`, `.` and `/` stay inside a token on purpose. Splitting on them turns
+ * `max_questions_per_task` into four words and `mcp/src/eval/question-checks.ts`
+ * into six, so a stem and an answer that merely name the same identifier -- the
+ * ordinary way to write these questions -- would share a four-word run and trip
+ * `answer_not_in_stem` on its own. The fixtures here are this repo's own code,
+ * so that is the common case, not the corner one.
+ */
+function normalized(text) {
+    return words(text.toLowerCase().replace(/[^\p{L}\p{N}_\-./]+/gu, ' '));
+}
+/**
+ * The longest run of consecutive words the two strings share.
+ *
+ * Used for "the answer is not restated in the stem". A bag-of-words overlap
+ * would fire on every question, because a stem is *supposed* to share its
+ * subject with the answer -- "httpOnly" appears in both by design. A shared
+ * four-word run is a different thing: that is the sentence, not the subject.
+ */
+export function longestSharedRun(a, b) {
+    const x = normalized(a);
+    const y = normalized(b);
+    if (x.length === 0 || y.length === 0)
+        return 0;
+    // Classic LCS-of-substrings table. Both inputs are one sentence, so the
+    // quadratic cost is a few thousand operations.
+    let best = 0;
+    const prev = new Array(y.length + 1).fill(0);
+    for (let i = 1; i <= x.length; i++) {
+        let diagonal = 0;
+        for (let j = 1; j <= y.length; j++) {
+            const above = prev[j] ?? 0;
+            const run = x[i - 1] === y[j - 1] ? diagonal + 1 : 0;
+            diagonal = above;
+            prev[j] = run;
+            if (run > best)
+                best = run;
+        }
+    }
+    return best;
+}
+/** Stems this long stop being read and start being pattern-matched. */
+export const STEM_WORD_LIMIT = 25;
+/** A shared run this long means the stem contains the answer's sentence. */
+export const ANSWER_RUN_LIMIT = 4;
+/**
+ * Ratio between the longest and shortest option. Four options of visibly
+ * different length leak the answer through shape alone, whichever one is
+ * correct.
+ */
+export const OPTION_RATIO_LIMIT = 2.5;
+/**
+ * Below this many words in the longest option, the ratio says nothing: four
+ * answers like `O(1)` / `O(n log n)` sit at 3.0 and are perfectly matched to a
+ * reader. Shape is a tell in sentences, not in tokens.
+ */
+export const PARITY_MIN_WORDS = 6;
+/**
+ * How many words longer the correct option may be than the next-longest before
+ * the length itself is a hint. Below this it is noise; a reader does not count
+ * words, they see a shape.
+ */
+export const LENGTH_MARGIN_WORDS = 3;
+/**
+ * Negation that makes a stem a reading test.
+ *
+ * Deliberately narrow. A plain "not" is often legitimate and contrastive --
+ * "why is this set on the refresh cookie but not on the access token" is a good
+ * stem -- so flagging every "not" would flag good questions and train whoever
+ * reads the report to ignore it. These three forms are the ones that turn the
+ * question itself inside out.
+ */
+const NEGATED_STEM = [
+    // "Which of these is not / will not / can't / is never ..."
+    /\bwhich\b(?:\s+of\s+(?:these|the\s+following))?\s+\w*\s*\b(?:is|are|does|do|will|would|can|could)(?:n't|\s+not|\s+never)\b/i,
+    // "... except which of these", but not the ordinary prose "except a corrupt file"
+    /\bexcept\b\s+(?:for\s+)?(?:which|one|these|the\s+following)\b/i,
+    /\b(?:is|are)\s+(?:not|never)\s+true\b/i,
+    /\b(?:incorrect|least\s+likely)\b/i,
+    /\bnone\s+of\s+(?:these|the\s+above)\b/i,
+];
+/**
+ * The tool renders the labels; a stem that numbers them is showing them twice.
+ *
+ * Two markers, not one. A single `b)` is `retry(db, b)` and a single `3.` is
+ * "the plan asked for tier 3. Why..." -- both ordinary, and both were flagged by
+ * the first version of this. An actual enumeration always has at least two.
+ */
+const OPTION_MARKER = /(^|\s)(?:[a-d]\)|\(?[1-4][).]\s)/gim;
+function numberedOptions(stem) {
+    return (stem.match(OPTION_MARKER) ?? []).length >= 2;
+}
+export function checkQuestion(q) {
+    const checks = [];
+    const push = (id, ok, detail) => checks.push({ id, ok, detail });
+    const labels = q.options.map((o) => o.trim()).filter((o) => o.length > 0);
+    const unique = new Set(labels.map((o) => o.toLowerCase()));
+    push('four_options', labels.length === 4 && unique.size === 4, `${labels.length} non-empty option(s), ${unique.size} distinct`);
+    // The obedience check, and the reason the whole harness drives the real
+    // planner rather than inventing plan items: `answer_position` is assigned
+    // server-side by a hash so the answer cannot cluster, and that only works if
+    // the model actually puts it there.
+    const obeyed = q.correct === q.answer_position;
+    push('answer_at_position', obeyed, obeyed
+        ? `correct option at ${q.answer_position}, as asked`
+        : `plan said ${q.answer_position}, model used ${q.correct}`);
+    const stemWords = words(q.stem).length;
+    push('stem_length', stemWords <= STEM_WORD_LIMIT, `${stemWords} words (limit ${STEM_WORD_LIMIT})`);
+    // Indexed into `q.options`, never `labels`. `labels` drops empty options, so
+    // one blank before the answer shifts every later index by one -- and the
+    // conspicuousness check below then compared the correct option against
+    // itself and always passed, on exactly the malformed question it should have
+    // been loudest about.
+    const inRange = Number.isInteger(q.correct) && q.correct >= 1 && q.correct <= q.options.length;
+    push('correct_in_range', inRange, inRange ? `correct = ${q.correct}` : `correct = ${JSON.stringify(q.correct)}, options = ${q.options.length}`);
+    const correct = inRange ? (q.options[q.correct - 1] ?? '') : '';
+    const run = longestSharedRun(q.stem, correct);
+    push('answer_not_in_stem', run < ANSWER_RUN_LIMIT, `longest run shared with the correct option: ${run} word(s)`);
+    const lengths = q.options.map((o) => words(o).length);
+    const longest = Math.max(...lengths, 0);
+    const shortest = Math.min(...lengths, Infinity);
+    const ratio = shortest > 0 ? longest / shortest : Infinity;
+    const shortEnoughToNotMatter = longest <= PARITY_MIN_WORDS;
+    push('option_parity', shortEnoughToNotMatter || ratio <= OPTION_RATIO_LIMIT, shortEnoughToNotMatter
+        ? `all four options are ${longest} words or fewer, so shape carries nothing`
+        : `longest/shortest = ${ratio.toFixed(2)} (limit ${OPTION_RATIO_LIMIT})`);
+    // Separate from parity on purpose. Four options can sit inside the ratio and
+    // the correct one still stand out, which is the tell a learner notices first:
+    // pick the most careful-sounding answer without reading it.
+    //
+    // The margin is why this is not simply "is it the longest". The first live
+    // run flagged all three of its questions, and one of the three was the
+    // correct option beating the next by two words -- 17 against 15, which nobody
+    // can see. The other two were four-word margins and still flag today. A check
+    // that fires on noise is a check whose report gets skipped. Being the longest
+    // at all is still worth counting, but as a rate across the run
+    // (`correctLongest` below), not as a verdict on one question.
+    const correctLength = words(correct).length;
+    const others = q.options.filter((_, i) => i !== q.correct - 1).map((o) => words(o).length);
+    const runnerUp = Math.max(...others, 0);
+    const margin = correctLength - runnerUp;
+    push('correct_not_conspicuous', margin < LENGTH_MARGIN_WORDS, `correct option is ${correctLength} words against a next-longest ${runnerUp} (margin ${margin}, limit ${LENGTH_MARGIN_WORDS})`);
+    const negated = NEGATED_STEM.find((re) => re.test(q.stem));
+    push('positive_form', !negated, negated ? `stem is negated: ${negated}` : 'stem asks the positive form');
+    const numbered = numberedOptions(q.stem);
+    push('options_not_numbered', !numbered, numbered ? 'stem enumerates the options itself' : 'stem does not letter or number the options');
+    // 1.14 removed the settings line from the stem. A stem that opens with a
+    // bracketed readout is the model composing one anyway, and it would change
+    // the fingerprint the repeat check depends on.
+    push('no_settings_line', !q.stem.trimStart().startsWith('['), 'stem opens with the question');
+    return checks;
+}
+export function scoreAll(questions) {
+    const scored = questions.map((question) => {
+        const checks = checkQuestion(question);
+        return { question, checks, passed: checks.every((c) => c.ok) };
+    });
+    const byCheck = {};
+    for (const { checks } of scored) {
+        for (const c of checks) {
+            const row = (byCheck[c.id] ??= { passed: 0, failed: 0 });
+            if (c.ok)
+                row.passed++;
+            else
+                row.failed++;
+        }
+    }
+    const slots = [0, 0, 0, 0];
+    let correctLongest = 0;
+    for (const { question } of scored) {
+        const i = question.correct - 1;
+        if (i === 0 || i === 1 || i === 2 || i === 3)
+            slots[i] += 1;
+        // Inside the same guard as `slots`: an out-of-range `correct` is a
+        // malformed question, and counting it as "not the longest" reports a
+        // property of a question that does not exist.
+        if (i === 0 || i === 1 || i === 2 || i === 3) {
+            const lengths = question.options.map((o) => words(o).length);
+            const mine = lengths[i] ?? 0;
+            if (mine === Math.max(...lengths, 0) && lengths.filter((l) => l === mine).length === 1) {
+                correctLongest++;
+            }
+        }
+    }
+    return {
+        scored,
+        summary: {
+            questions: scored.length,
+            clean: scored.filter((s) => s.passed).length,
+            byCheck,
+            slots,
+            correctLongest,
+        },
+    };
+}
+//# sourceMappingURL=question-checks.js.map

package/dist/eval/question-checks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"question-checks.js","sourceRoot":"","sources":["../../src/eval/question-checks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AA0BH,sEAAsE;AACtE,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAAS,EAAE,CAAS;IACnD,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IACxB,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAE/C,wEAAwE;IACxE,+CAA+C;IAC/C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,QAAQ,GAAG,KAAK,CAAC;YACjB,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YACd,IAAI,GAAG,GAAG,IAAI;gBAAE,IAAI,GAAG,GAAG,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uEAAuE;AACvE,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAElC,4EAA4E;AAC5E,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAErC;;;;;;;;GAQG;AACH,MAAM,YAAY,GAAG;IACnB,4DAA4D;IAC5D,6HAA6H;IAC7H,kFAAkF;IAClF,gEAAgE;IAChE,wCAAwC;IACxC,mCAAmC;IACnC,wCAAwC;CACzC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,aAAa,GAAG,qCAAqC,CAAC;AAE5D,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,CAAoB;IAChD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,EAAW,EAAE,MAAc,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAE1F,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC3D,IAAI,CACF,cAAc,EACd,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EACxC,GAAG,MAAM,CAAC,MAAM,yBAAyB,MAAM,CAAC,IAAI,WAAW,CAChE,CAAC;IAEF,wEAAwE;IACxE,0EAA0E;IAC1E,6EAA6E;IAC7E,oCAAoC;IACpC,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,eAAe,CAAC;IAC/C,IAAI,CACF,oBAAoB,EACpB,MAAM,EACN,MAAM;QACJ,CAAC,CAAC,qBAAqB,CAAC,CAAC,eAAe,YAAY;QACpD,CAAC,CAAC,aAAa,CAAC,CAAC,eAAe,gBAAgB,CAAC,CAAC,OAAO,EAAE,CAC9D,CAAC;IAEF,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACvC,IAAI,CAAC,aAAa,EAAE,SAAS,IAAI,eAAe,EAAE,GAAG,SAAS,iBAAiB,eAAe,GAAG,CAAC,CAAC;IAEnG,6EAA6E;IAC7E,yEAAyE;IACzE,uEAAuE;IACvE,6EAA6E;IAC7E,sBAAsB;IACtB,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAC/F,IAAI,CACF,kBAAkB,EAClB,OAAO,EACP,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAC7G,CAAC;IACF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,IAAI,CACF,oBAAoB,EACpB,GAAG,GAAG,gBAAgB,EACtB,+CAA+C,GAAG,UAAU,CAC7D,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC3D,MAAM,sBAAsB,GAAG,OAAO,IAAI,gBAAgB,CAAC;IAC3D,IAAI,CACF,eAAe,EACf,sBAAsB,IAAI,KAAK,IAAI,kBAAkB,EACrD,sBAAsB;QACpB,CAAC,CAAC,wBAAwB,OAAO,2CAA2C;QAC5E,CAAC,CAAC,sBAAsB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,kBAAkB,GAAG,CAC3E,CAAC;IAEF,6EAA6E;IAC7E,8EAA8E;IAC9E,4DAA4D;IAC5D,EAAE;IACF,2EAA2E;IAC3E,uEAAuE;IACvE,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,+DAA+D;IAC/D,8DAA8D;IAC9D,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC;IACxC,IAAI,CACF,yBAAyB,EACzB,MAAM,GAAG,mBAAmB,EAC5B,qBAAqB,aAAa,iCAAiC,QAAQ,YAAY,MAAM,WAAW,mBAAmB,GAAG,CAC/H,CAAC;IAEF,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,IAAI,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC;IAEzG,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,CACF,sBAAsB,EACtB,CAAC,QAAQ,EACT,QAAQ,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC,4CAA4C,CAC/F,CAAC;IAEF,yEAAyE;IACzE,2EAA2E;IAC3E,+CAA+C;IAC/C,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,8BAA8B,CAAC,CAAC;IAE9F,OAAO,MAAM,CAAC;AAChB,CAAC;AAkCD,MAAM,UAAU,QAAQ,CAAC,SAA8B;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QACxC,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,CAAC,EAAE;gBAAE,GAAG,CAAC,MAAM,EAAE,CAAC;;gBAClB,GAAG,CAAC,MAAM,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAqC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5D,mEAAmE;QACnE,qEAAqE;QACrE,8CAA8C;QAC9C,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC7D,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,IAAI,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvF,cAAc,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM;QACN,OAAO,EAAE;YACP,SAAS,EAAE,MAAM,CAAC,MAAM;YACxB,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;YAC5C,OAAO;YACP,KAAK;YACL,cAAc;SACf;KACF,CAAC;AACJ,CAAC"}

package/dist/hooks/checkpoint-quiz.js

@@ -129,7 +129,7 @@ Concept: ${row.concept}
 Do exactly this, then get straight back to the task:
   1. get_session_quiz_plan with max: 1 and ignore_cooldown: true (the pacing is already decided -- this hook is the cooldown).
   2. Ask that ONE question with AskUserQuestion: four options, one correct, three plausible, and put the correct one in the slot answer_position names. Header "Eklavya", so it is clear who is asking.
-     Ask the stem on its own -- the dials that chose this question live in the status bar, so do not prepend a settings line to it.
+     Ask the stem on its own -- the dials that chose this question live in the status bar, so what you ask is the question and nothing else.
   3. Grade it with record_attempt: format "mcq", the labels in "options", the stem alone in "question".
   4. Resume the task exactly where you left off. Do not summarise, do not re-plan, do not ask a second question.
 

package/dist/hooks/lib.js

@@ -19,22 +19,23 @@ import fs from 'node:fs';
 import Database from 'better-sqlite3';
 import { dbPath } from '../paths.js';
 import { loadConfig } from '../config.js';
+import { readStdinBounded, stripBom, HOOK_STDIN } from '../stdin.js';
 /**
- * Reads stdin to EOF. Hooks are always given JSON, but "always" is doing a lot
- * of work on a critical path — a hook invoked by hand, or by a harness version
- * that changes its mind, gets an empty object rather than an exception.
+ * The JSON the host piped in, or `{}`.
+ *
+ * Hooks are always given JSON, but "always" is doing a lot of work on a
+ * critical path -- a hook invoked by hand, or by a harness version that changes
+ * its mind, gets an empty object rather than an exception.
+ *
+ * The read is bounded (`readStdinBounded`). It used to be `for await (const
+ * chunk of process.stdin)`, which waits for EOF and has no other exit: on
+ * Windows the host may run a hook through a PowerShell block that swallows the
+ * pipe, so `end` never fires and the hook blocks the session on every tool call
+ * that triggers it. A hook that throws is survivable; a hook that waits is not.
  */
-async function readStdin() {
-    if (process.stdin.isTTY)
-        return '';
-    const chunks = [];
-    for await (const chunk of process.stdin)
-        chunks.push(chunk);
-    return Buffer.concat(chunks).toString('utf8');
-}
 export async function readInput() {
     try {
-        const raw = await readStdin();
+        const raw = stripBom(await readStdinBounded(HOOK_STDIN));
         if (!raw.trim())
             return {};
         const parsed = JSON.parse(raw);
@@ -49,7 +50,7 @@ export async function readInput() {
  *
  * `openDb()` is the server's entry point and does both, which is right for a
  * server and wrong here: a hook is not the thing that should be migrating a
- * schema, and four hooks racing a migration on session start is a corruption
+ * schema, and several hooks racing a migration on session start is a corruption
  * story rather than a feature. A hook that finds no database has nothing to say,
  * which is the same answer `eklavya_have_deps` gave.
  */
@@ -67,6 +68,30 @@ export function openExisting() {
         return null;
     }
 }
+/**
+ * `meta` key holding the UserPromptSubmit nudge's per-session bookkeeping.
+ * Shared here rather than exported from the hook, because `session-start`
+ * clears it and two hooks spelling the same key differently would be a bug that
+ * only shows up as a nudge that never stops.
+ */
+export const NUDGE_KEY_PREFIX = 'prompt_nudge:';
+/**
+ * Forget what the nudge knows about this session.
+ *
+ * Called by `session-start`, which fires on resume and after a compaction as
+ * well as at startup, and reprints the standing directive every time. Without
+ * this the nudge's row survives with its original first-seen stamp, the grace
+ * window is already spent, and the first prompt of a resumed session restates a
+ * directive printed seconds earlier.
+ */
+export function clearNudgeState(db, sessionId) {
+    try {
+        db.prepare('DELETE FROM meta WHERE key = ?').run(`${NUDGE_KEY_PREFIX}${sessionId}`);
+    }
+    catch {
+        /* Bookkeeping. A session that keeps its old row gets one extra nudge. */
+    }
+}
 export function config(cwd) {
     return loadConfig(cwd);
 }

package/dist/hooks/lib.js.map

@@ -1 +1 @@
-{"version":3,"file":"lib.js","sourceRoot":"","sources":["../../src/hooks/lib.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,UAAU,EAAuB,MAAM,cAAc,CAAC;AAa/D;;;;GAIG;AACH,KAAK,UAAU,SAAS;IACtB,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,KAAe,CAAC,CAAC;IACtE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,SAAS,EAAE,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC,CAAC,CAAE,MAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC;IACtB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACjC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAC/B,OAAO,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IAChC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,KAAK,CAAC,KAAgB;IACpC,OAAO,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;AACvE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,KAAgB,EAAE,EAAa;IACvD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC/C,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,IAAI,KAAK,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC,UAAU,CAAC;IAC9C,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,EAErE,CAAC;QACd,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,MAAM,KAAK,GAAG,OAAO,CAAC;AAE7B;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,EAA6B;IACxD,IAAI,CAAC,EAAE;QAAE,OAAO,KAAK,CAAC;IAEtB,MAAM,OAAO,GAAG,yBAAyB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC;IAE7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,MAAM;IACpB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,KAAgC,EAChC,KAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,KAAK,KAAK,YAAY;YAC3B,CAAC,CAAC,4IAA4I;YAC9I,CAAC,CAAC,+LAA+L,CAAC;IACtM,CAAC;IACD,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,KAAK,YAAY;gBAC3B,CAAC,CAAC,2EAA2E;gBAC7E,CAAC,CAAC,qGAAqG,CAAC;QAC5G,CAAC;QACD,OAAO,KAAK,KAAK,YAAY;YAC3B,CAAC,CAAC,wBAAwB,KAAK,0HAA0H;YACzJ,CAAC,CAAC,wBAAwB,KAAK,kKAAkK,CAAC;IACtM,CAAC;IACD,OAAO,KAAK,KAAK,YAAY;QAC3B,CAAC,CAAC,yGAAyG;QAC3G,CAAC,CAAC,wGAAwG,CAAC;AAC/G,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAkD;IAC1E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;QAChC,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC"}
+{"version":3,"file":"lib.js","sourceRoot":"","sources":["../../src/hooks/lib.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,UAAU,EAAuB,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAcrE;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC,CAAC,CAAE,MAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC;IACtB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACjC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAC/B,OAAO,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,eAAe,CAAC;AAEhD;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,EAAM,EAAE,SAAiB;IACvD,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC,GAAG,CAAC,GAAG,gBAAgB,GAAG,SAAS,EAAE,CAAC,CAAC;IACtF,CAAC;IAAC,MAAM,CAAC;QACP,yEAAyE;IAC3E,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IAChC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,KAAK,CAAC,KAAgB;IACpC,OAAO,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;AACvE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,KAAgB,EAAE,EAAa;IACvD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC/C,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,IAAI,KAAK,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC,UAAU,CAAC;IAC9C,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,EAErE,CAAC;QACd,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,MAAM,KAAK,GAAG,OAAO,CAAC;AAE7B;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAAC,EAA6B;IACxD,IAAI,CAAC,EAAE;QAAE,OAAO,KAAK,CAAC;IAEtB,MAAM,OAAO,GAAG,yBAAyB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC;IAE7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,MAAM;IACpB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,KAAgC,EAChC,KAA4B;IAE5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,KAAK,KAAK,YAAY;YAC3B,CAAC,CAAC,4IAA4I;YAC9I,CAAC,CAAC,+LAA+L,CAAC;IACtM,CAAC;IACD,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,KAAK,YAAY;gBAC3B,CAAC,CAAC,2EAA2E;gBAC7E,CAAC,CAAC,qGAAqG,CAAC;QAC5G,CAAC;QACD,OAAO,KAAK,KAAK,YAAY;YAC3B,CAAC,CAAC,wBAAwB,KAAK,0HAA0H;YACzJ,CAAC,CAAC,wBAAwB,KAAK,kKAAkK,CAAC;IACtM,CAAC;IACD,OAAO,KAAK,KAAK,YAAY;QAC3B,CAAC,CAAC,yGAAyG;QAC3G,CAAC,CAAC,wGAAwG,CAAC;AAC/G,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAkD;IAC1E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;QAChC,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC"}