@lernapps/app-templates 0.0.0-stage → 0.1.0

package/runtime/explainer/generated/question.ts

@@ -0,0 +1,165 @@
+// The question of a task (a question of the shared questions), the answer in the shape of the learner's, the feedback
+// on a wrong answer, and what a learner sees of a task.
+import { ExpressionError, type Rec, same, type Value } from "../expression.ts";
+import { Fraction } from "../fraction.ts";
+import { type Answer, isCorrect, parseNumber, type Question } from "../../shared/questions/question.ts";
+import { inLanguage } from "../../shared/languages.ts";
+import { type Generated, type Task, type Templates } from "./types.ts";
+import { kindOf } from "./comparator.ts";
+import { fill, formatNumber, label, plainText, taskText, templatesIn } from "./texts.ts";
+
+const listOf = (value: Value, what: string): readonly Value[] => {
+  if (!Array.isArray(value)) throw new ExpressionError(`the answer of ${what} is a list`);
+  return value as readonly Value[];
+};
+
+/** The task as the question of the comparator's kind, in `lang`. */
+export function questionFor(spec: Generated, task: Task, lang: string, templates: Templates): Question {
+  const kind = kindOf(spec.answer.compare);
+  const base = {
+    id: `${spec.id}-${task.seed}`,
+    text: plainText(taskText(spec, task, lang, templates)),
+    explanation: "",
+    links: [],
+  };
+  switch (kind.kind) {
+    case "number": {
+      if (!(task.answer instanceof Fraction)) throw new ExpressionError("the answer of number is a number");
+      return {
+        ...base,
+        type: "number",
+        answer: task.answer.toNumber(),
+        tolerance: kind.tolerance.toNumber(),
+        ...(kind.unit === undefined ? {} : { unit: inLanguage(kind.unit, lang) }),
+      };
+    }
+    case "choice":
+      return {
+        ...base,
+        type: "single-choice",
+        options: (task.options ?? []).map((option) => ({
+          text: label(option, lang),
+          correct: same(task.answer, option),
+          info: "",
+        })),
+      };
+    case "order":
+      return { ...base, type: "ordering", items: listOf(task.answer, "order").map((item) => label(item, lang)) };
+    case "pairs":
+      return {
+        ...base,
+        type: "matching",
+        pairs: pairValues(spec, task.answer).map(([left, right]) => ({
+          left: label(left, lang),
+          right: label(right, lang),
+        })),
+      };
+  }
+}
+
+/** The pairs of a pairs answer, as values: [left, right]. */
+export function pairValues(spec: Generated, value: Value): [Value, Value][] {
+  const kind = kindOf(spec.answer.compare);
+  return listOf(value, "pairs").map((item) => {
+    if (kind.kind === "pairs" && kind.left !== undefined && kind.right !== undefined) {
+      const fields = item as Rec;
+      return [fields[kind.left] ?? "", fields[kind.right] ?? ""];
+    }
+    if (!Array.isArray(item) || item.length !== 2)
+      throw new ExpressionError("each pair is a list of two: [left, right]");
+    return [item[0] as Value, item[1] as Value];
+  });
+}
+
+/** The learner's answer that gives `value` (e.g. a mistake) through the controls of the task's question. */
+function answerGiving(spec: Generated, task: Task, value: Value): Answer {
+  const kind = kindOf(spec.answer.compare);
+  const index = (list: readonly Value[], item: Value) => list.findIndex((other) => same(other, item));
+  switch (kind.kind) {
+    case "number": {
+      if (!(value instanceof Fraction)) throw new ExpressionError("a mistake of number is a number");
+      const places = Math.min(value.decimals(), Math.max(kind.decimals, 2));
+      return formatNumber(value.round(places), "de");
+    }
+    case "choice": {
+      const at = index(task.options ?? [], value);
+      if (at < 0) throw new ExpressionError(`${label(value, "de")} is not one of the choices`);
+      return at;
+    }
+    case "order": {
+      const order = listOf(value, "order");
+      return listOf(task.answer, "order").map((item) => index(order, item));
+    }
+    case "pairs": {
+      const right = pairValues(spec, task.answer);
+      const given = pairValues(spec, value);
+      return right.map(([left]) => {
+        const chosen = given.find(([other]) => same(other, left))?.[1];
+        return chosen === undefined ? -1 : right.findIndex(([, other]) => same(other, chosen));
+      });
+    }
+  }
+}
+
+/**
+ * The learner's answer that gives `value`, an example, through the controls of the task's question: as answerGiving,
+ * but a number as it is, with up to six decimals, so an example just outside the tolerance stays outside.
+ */
+export function exampleGiving(spec: Generated, task: Task, value: Value): Answer {
+  if (kindOf(spec.answer.compare).kind !== "number") return answerGiving(spec, task, value);
+  if (!(value instanceof Fraction)) throw new ExpressionError("an example of number is a number");
+  return formatNumber(value, "de");
+}
+
+/** The learner's answer that gives the mistake `index` of a task, through the controls of its question. */
+export function answerFor(spec: Generated, task: Task, _question: Question, index: number): Answer {
+  return answerGiving(spec, task, task.mistakes[index] as Value);
+}
+
+/** Whether two answers to `question` are the same, a number within `slack`. */
+function sameAnswer(question: Question, a: Answer, b: Answer, slack: number): boolean {
+  if (question.type === "number") {
+    const left = typeof a === "string" ? parseNumber(a, question.unit) : undefined;
+    const right = typeof b === "string" ? parseNumber(b, question.unit) : undefined;
+    return left !== undefined && right !== undefined && Math.abs(left - right) <= slack + 1e-9;
+  }
+  if (Array.isArray(a) && Array.isArray(b)) return a.join() === b.join();
+  return a === b;
+}
+
+/** The index of the first typical mistake the learner's wrong answer makes; undefined when it makes none. */
+export function mistakeOf(spec: Generated, task: Task, question: Question, answer: Answer): number | undefined {
+  if (isCorrect(question, answer)) return undefined;
+  const kind = kindOf(spec.answer.compare);
+  // a mistake typed rounded to the answer's decimals still counts as that mistake
+  const slack =
+    kind.kind === "number" ? Math.max(kind.tolerance.toNumber(), 0.5 * 10 ** -Math.max(kind.decimals, 2)) : 0;
+  for (const [index] of task.mistakes.entries()) {
+    try {
+      if (sameAnswer(question, answer, answerFor(spec, task, question, index), slack)) return index;
+    } catch {
+      // a mistake the controls cannot give
+    }
+  }
+  return undefined;
+}
+
+/** The message of the first mistake the learner's wrong answer makes, in `lang`; undefined when it makes none. */
+export function diagnose(
+  spec: Generated,
+  task: Task,
+  question: Question,
+  answer: Answer,
+  lang: string,
+  templates: Templates,
+): string | undefined {
+  const index = mistakeOf(spec, task, question, answer);
+  if (index === undefined) return undefined;
+  return fill(templatesIn(templates, lang).messages[index] ?? "", spec, task, lang, "text");
+}
+
+/** What a learner sees of a task: its text and its question's options, items or pairs; "Neue Aufgabe" changes it. */
+export function taskKey(spec: Generated, task: Task, lang: string, templates: Templates): string {
+  const { id: _id, ...question } = questionFor(spec, task, lang, templates);
+  return JSON.stringify(question);
+}

package/runtime/explainer/generated/soundness.ts

@@ -0,0 +1,280 @@
+// The property test of a generated exercise at build time: the tasks of a fixed set of seeds, each checked for
+// its formats, its answer, its mistakes and its options, and its examples run through the check.
+import { evaluate, parse, same, type Value } from "../expression.ts";
+import { Fraction } from "../fraction.ts";
+import { type Answer, isCorrect, type Question, rightAnswer } from "../../shared/questions/question.ts";
+import { verdict } from "../../shared/examples.ts";
+import { type Data, type Generated, type Task, type Templates } from "./types.ts";
+import { kindOf } from "./comparator.ts";
+import { ATTEMPTS, compiled, generate, holds, StarvedError, values } from "./values.ts";
+import { answerLabel, fill, hintOf, label, stepsOf, taskText, templatesIn } from "./texts.ts";
+import { answerFor, exampleGiving, pairValues, questionFor } from "./question.ts";
+
+/** A problem of an exercise: where in it, in which language, and what. */
+export interface GeneratedProblem {
+  path: (string | number)[];
+  lang?: string;
+  message: string;
+}
+
+export const message = (error: unknown) => (error instanceof Error ? error.message : String(error));
+
+/** Whether the answer is in the comparator's format; a problem otherwise. */
+function formatProblem(spec: Generated, task: Task, lang: string): string | undefined {
+  const kind = kindOf(spec.answer.compare);
+  const shown = () => label(task.answer, lang);
+  switch (kind.kind) {
+    case "number":
+      if (!(task.answer instanceof Fraction)) return `the answer is not a number, but compare is number`;
+      if (task.answer.decimals() > kind.decimals) {
+        return `the answer ${shown()} has more than ${kind.decimals} decimals; round it in the answer, or narrow given or where`;
+      }
+      return undefined;
+    case "choice":
+      if (!(task.options ?? []).some((option) => same(option, task.answer)))
+        return `the answer ${shown()} is not one of the choices`;
+      return undefined;
+    case "order": {
+      if (!Array.isArray(task.answer) || task.answer.length < 2) return "the answer of order is a list of at least two";
+      const texts = (task.answer as readonly Value[]).map((item) => label(item, lang));
+      return new Set(texts).size === texts.length ? undefined : "the answer of order has an item twice";
+    }
+    case "pairs": {
+      const pairs = pairValues(spec, task.answer).map(([left, right]) => [label(left, lang), label(right, lang)]);
+      if (pairs.length < 2) return "the answer of pairs is a list of at least two";
+      const lefts = new Set(pairs.map(([left]) => left));
+      const rights = new Set(pairs.map(([, right]) => right));
+      return lefts.size === pairs.length && rights.size === pairs.length
+        ? undefined
+        : "the answer of pairs has an entry twice";
+    }
+  }
+}
+
+/** The problems of one seed's task. */
+function seedProblems(
+  spec: Generated,
+  seed: number,
+  data: Data,
+  templates: Templates,
+  languages: readonly string[],
+): GeneratedProblem[] {
+  const problems: GeneratedProblem[] = [];
+  const kind = kindOf(spec.answer.compare);
+  // the answer, the mistakes and the options are defined
+  let task: Task;
+  try {
+    task = generate(spec, seed, data);
+  } catch (error) {
+    if (error instanceof StarvedError)
+      return [
+        {
+          path: ["where"],
+          message: `for seed ${seed} ${message(error).replace(/ for seed \d+$/, "")}; loosen it or narrow given`,
+        },
+      ];
+    const code = compiled(spec);
+    for (let attempt = 0; attempt < ATTEMPTS; attempt++) {
+      const env = values(spec, seed, attempt, data);
+      try {
+        if (!holds(spec, env)) continue;
+      } catch (inner) {
+        return [{ path: ["where"], message: `for seed ${seed} where is not defined: ${message(inner)}` }];
+      }
+      try {
+        evaluate(code.answer, env);
+      } catch (inner) {
+        return [
+          { path: ["answer", "value"], message: `for seed ${seed} the answer is not defined: ${message(inner)}` },
+        ];
+      }
+      for (const [index, mistake] of code.mistakes.entries()) {
+        try {
+          evaluate(mistake, env);
+        } catch (inner) {
+          return [
+            {
+              path: ["mistakes", index, "value"],
+              message: `for seed ${seed} the mistake is not defined: ${message(inner)}`,
+            },
+          ];
+        }
+      }
+      return [
+        { path: ["answer", "compare"], message: `for seed ${seed} the options are not defined: ${message(error)}` },
+      ];
+    }
+    return [{ path: ["where"], message: `for seed ${seed} ${message(error)}` }];
+  }
+  const lang = languages[0] ?? "de";
+  // the answer is in its format and passes its own comparator
+  let format: string | undefined;
+  try {
+    format = formatProblem(spec, task, lang);
+  } catch (error) {
+    format = message(error);
+  }
+  if (format !== undefined) return [{ path: ["answer", "value"], message: `for seed ${seed} ${format}` }];
+  let question: Question;
+  try {
+    question = questionFor(spec, task, lang, templates);
+  } catch (error) {
+    return [{ path: ["answer", "value"], message: `for seed ${seed} ${message(error)}` }];
+  }
+  if (!isCorrect(question, rightAnswer(question))) {
+    return [{ path: ["answer", "compare"], message: `for seed ${seed} the answer does not pass its own comparator` }];
+  }
+  // no mistake equals the answer, and the learner can give each
+  task.mistakes.forEach((mistake, index) => {
+    const path = ["mistakes", index, "value"];
+    let given: Answer;
+    try {
+      given = answerFor(spec, task, question, index);
+    } catch (error) {
+      problems.push({ path, message: `for seed ${seed} ${message(error)}; the learner could not give this mistake` });
+      return;
+    }
+    const equal =
+      kind.kind === "number" && mistake instanceof Fraction && task.answer instanceof Fraction
+        ? mistake.sub(task.answer).abs().compare(kind.tolerance) <= 0
+        : isCorrect(question, given);
+    if (equal) {
+      problems.push({
+        path,
+        message: `for seed ${seed} the mistake equals the answer (${answerLabel(spec, task, lang)}); a mistake is a wrong answer, so change its value or remove it`,
+      });
+    }
+  });
+  if (problems.length > 0) return problems;
+  // the task is the same for the same seed, and no placeholder is left
+  const again = generate(spec, seed, data);
+  for (const each of languages) {
+    if (
+      taskText(spec, again, each, templates) !== taskText(spec, task, each, templates) ||
+      !same(again.answer, task.answer)
+    ) {
+      return [{ path: ["given"], message: `for seed ${seed} the task is not the same each time` }];
+    }
+    const texts: [(string | number)[], string, string][] = [
+      [["task", "text"], "the task", taskText(spec, task, each, templates)],
+      ...stepsOf(spec, task, each, templates).map((step, index): [(string | number)[], string, string] => [
+        ["steps", index],
+        `step ${index + 1}`,
+        step,
+      ]),
+      ...(spec.hint === undefined
+        ? []
+        : [[["hint"], "the hint", hintOf(spec, task, each, templates) ?? ""] as [(string | number)[], string, string]]),
+      ...(spec.mistakes ?? []).map((_mistake, index): [(string | number)[], string, string] => [
+        ["mistakes", index, "message"],
+        "the message",
+        fill(templatesIn(templates, each).messages[index] ?? "", spec, task, each, "text"),
+      ]),
+    ];
+    for (const [path, what, text] of texts) {
+      const left = /\{\{?([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\}\}?/.exec(text);
+      if (left === null) continue;
+      const source = path.reduce<unknown>((at, part) => (at as Record<string, unknown> | undefined)?.[part], spec);
+      const keyed = typeof source === "object" && source !== null;
+      const root = left[1]?.split(".")[0] ?? "";
+      problems.push({
+        path: keyed ? [...path, each] : path,
+        ...(keyed ? { lang: each } : {}),
+        message:
+          root in spec.given || root === "answer"
+            ? `for seed ${seed} ${what} still has the placeholder ${left[0]}: it has no value there; name a field the records have`
+            : `for seed ${seed} ${what} still has the placeholder ${left[0]}; declare ${root} in given, or remove the braces`,
+      });
+    }
+  }
+  if (problems.length > 0) return problems;
+  return problems;
+}
+
+/**
+ * The property test of an exercise over the seeds 1 to `seeds`: `where` holds in at least one draw of ten; for each
+ * seed the answer is defined, in its format and passes its comparator, no mistake equals it, the task is the same each
+ * time and has no placeholder left The problems of the first seed
+ * that has any; none when the exercise is sound.
+ */
+export function soundness(
+  spec: Generated,
+  data: Data,
+  templates: Templates,
+  languages: readonly string[],
+  seeds = 200,
+): GeneratedProblem[] {
+  if ((spec.where ?? []).length > 0) {
+    let passes = 0;
+    for (let seed = 1; seed <= seeds; seed++) {
+      try {
+        if (holds(spec, values(spec, seed, 0, data))) passes++;
+      } catch (error) {
+        return [{ path: ["where"], message: `for seed ${seed} where is not defined: ${message(error)}` }];
+      }
+    }
+    if (passes * 10 < seeds) {
+      return [
+        {
+          path: ["where"],
+          message: `where holds in ${passes} of ${seeds} draws; it must hold in at least 1 of 10, so loosen it or narrow given`,
+        },
+      ];
+    }
+  }
+  for (let seed = 1; seed <= seeds; seed++) {
+    const problems = seedProblems(spec, seed, data, templates, languages);
+    if (problems.length > 0) return problems;
+  }
+  return [];
+}
+
+/**
+ * Whether the check of every seed's task, 1 to `seeds`, judges each example as it is listed: an accepted one passes,
+ * a rejected one fails. An example is an expression of given and of answer, the task's right answer (unless given
+ * declares its own answer), evaluated per seed. One problem per example, at the first seed that judges it otherwise.
+ */
+export function exampleProblems(
+  spec: Generated,
+  data: Data,
+  templates: Templates,
+  languages: readonly string[],
+  seeds = 200,
+): GeneratedProblem[] {
+  const examples = (["accepted", "rejected"] as const).flatMap((kind) =>
+    (spec.examples?.[kind] ?? []).map((source, index) => ({ kind, index, expression: parse(source) })),
+  );
+  if (examples.length === 0) return [];
+  const lang = languages[0] ?? "de";
+  const problems = new Map<string, GeneratedProblem>();
+  for (let seed = 1; seed <= seeds && problems.size < examples.length; seed++) {
+    const task = generate(spec, seed, data);
+    const question = questionFor(spec, task, lang, templates);
+    const env = { answer: task.answer, ...task.values };
+    for (const { kind, index, expression } of examples) {
+      const key = `${kind}#${index}`;
+      if (problems.has(key)) continue;
+      const path = ["examples", kind, index];
+      let given: Answer;
+      let shown: string;
+      try {
+        const value = evaluate(expression, env);
+        shown = label(value, lang);
+        given = exampleGiving(spec, task, value);
+      } catch (error) {
+        problems.set(key, {
+          path,
+          message: `for seed ${seed} the learner could not give this answer: ${message(error)}`,
+        });
+        continue;
+      }
+      const right = isCorrect(question, given);
+      if (right === (kind === "accepted")) continue;
+      problems.set(key, {
+        path,
+        message: `for seed ${seed} (answer ${answerLabel(spec, task, lang)}) ${verdict(`"${shown}"`, right, "the check", "change the exercise (e.g. another comparator, a tolerance) so that its check")}`,
+      });
+    }
+  }
+  return [...problems.values()];
+}