@learnpack/learnpack 5.0.363 → 5.0.364

package/src/utils/interactivity/structuredComponents.ts

@@ -0,0 +1,1004 @@
+// Turns the structured output of the interactivity template (v2) into the
+// lesson markdown that the LearnPack IDE renders.
+//
+// The model returns the lesson text with <!-- component:ID --> placeholders and
+// one list of data per component. This module validates each component, writes
+// its markdown and puts it where its placeholder is. The model never writes
+// component syntax, so a component whose data is invalid is dropped instead of
+// reaching the README broken.
+//
+// Every syntax rule below comes from how the IDE parses the markdown
+// (learnpack/ide: Markdowner.tsx, QuizRenderer.tsx, markdownConfig.ts).
+
+import { sanitizeReadmeNewlines } from "../readmeSanitizer"
+
+export const COMPONENT_NAMES = [
+  "multiple_choice_question",
+  "select_the_blank",
+  "ordering",
+  "open_text_question",
+  "code_challenge_proposal",
+  "ask_rigo_button",
+  "mermaid_diagram",
+  "technical_diagram",
+  "conceptual_illustration",
+  "realistic_photography",
+] as const
+
+export type ComponentName = typeof COMPONENT_NAMES[number];
+export type DecidedBy = "syllabus" | "interactivity";
+
+const IMAGE_COMPONENTS = new Set<ComponentName>([
+  "technical_diagram",
+  "conceptual_illustration",
+  "realistic_photography",
+])
+
+const ASSESSMENT_COMPONENTS = new Set<ComponentName>([
+  "multiple_choice_question",
+  "select_the_blank",
+  "ordering",
+  "open_text_question",
+  "code_challenge_proposal",
+])
+
+// Used when the components guide is not at hand. It matches
+// docs/components_v2.yml in learnpack/ide.
+export const DEFAULT_DECIDED_BY: Record<ComponentName, DecidedBy> = {
+  multiple_choice_question: "syllabus",
+  select_the_blank: "syllabus",
+  ordering: "syllabus",
+  open_text_question: "syllabus",
+  code_challenge_proposal: "syllabus",
+  ask_rigo_button: "syllabus",
+  mermaid_diagram: "interactivity",
+  technical_diagram: "interactivity",
+  conceptual_illustration: "interactivity",
+  realistic_photography: "interactivity",
+}
+
+const KNOWN_MERMAID_TYPES = [
+  "graph",
+  "flowchart",
+  "sequenceDiagram",
+  "classDiagram",
+  "stateDiagram",
+  "erDiagram",
+  "journey",
+  "gantt",
+  "pie",
+  "mindmap",
+  "timeline",
+  "%%{init",
+]
+
+// Code block languages of LearnPack components. A block with one of these
+// languages in the lesson text was copied by the model instead of returned as
+// data (see Markdowner.tsx CustomCodeBlock).
+const COMPONENT_FENCE_LANGUAGES = new Set([
+  "select_the_blank",
+  "select",
+  "ordering",
+  "order",
+  "question",
+  "fill_in_the_blank",
+  "fill",
+  "code_challenge_proposal",
+])
+
+export interface Issue {
+  component?: ComponentName;
+  id?: string;
+  message: string;
+}
+
+export interface ProcessContext {
+  // Type of the step: READ, QUIZ or CODE.
+  stepType?: string;
+  // First or last step of the course. Nothing is appended at their end.
+  isFirstOrLast?: boolean;
+  // Step id, such as "01.2". It prefixes image file names, which are
+  // shared by the whole course.
+  lessonId?: string;
+  // Which components the syllabus plans. Planned components keep their place
+  // at the end of the step when their placeholder is missing.
+  decidedBy?: Partial<Record<string, DecidedBy>>;
+}
+
+export interface ProcessResult {
+  markdown: string;
+  usedComponents: ComponentName[];
+  // Number of assessment components written into the lesson.
+  assessmentCount: number;
+  // Components or text that were removed. The course author should know.
+  dropped: Issue[];
+  // Questionable but rendered content. Only for the server log.
+  warnings: Issue[];
+}
+
+interface RenderedComponent {
+  name: ComponentName;
+  id: string;
+  markdown: string;
+}
+
+interface BuildState {
+  lessonId: string;
+  usedImageNames: Set<string>;
+  usedQuestions: Set<string>;
+}
+
+type Builder = (
+  data: any,
+  state: BuildState,
+  warn: (message: string) => void
+) => { markdown?: string; errors: string[]; removed?: string[] };
+
+// ---------------------------------------------------------------------------
+// Text utilities
+// ---------------------------------------------------------------------------
+
+const asText = (value: unknown): string =>
+  typeof value === "string" ?
+    value :
+    (value === null || value === undefined ?
+    "" :
+    String(value))
+
+// One line of text: line breaks and repeated spaces become a single space.
+export const collapse = (value: unknown): string =>
+  asText(value).replace(/\s+/g, " ").trim()
+
+// Removes markdown block markers at the start of a text: headings, quotes,
+// list markers, numbers and task list boxes. Used where a text must stay a
+// plain paragraph, such as a question or an option.
+export const stripBlockPrefix = (value: unknown): string => {
+  let text = collapse(value)
+  let previous = ""
+  while (text !== previous) {
+    previous = text
+    text = text
+      .replace(/^#{1,6}\s+/, "")
+      .replace(/^>\s*/, "")
+      .replace(/^[*+-]\s+/, "")
+      .replace(/^\d+[).]\s+/, "")
+      .replace(/^\[[ Xx]]\s*/, "")
+      .trim()
+  }
+
+  return text
+}
+
+// Encodes a value for the info string of a code block, such as
+// select_the_blank 1="..." or question eval="...". CommonMark decodes
+// backslash escapes and entities in the info string before the IDE reads the
+// value with /(\w+)="([^"]*)"/, a backtick ends the code block, and the value
+// cannot span lines.
+export const encodeMetaValue = (value: unknown): string =>
+  collapse(value)
+    .replace(/`/g, "")
+    .replace(/\\/g, "\\\\")
+    .replace(/&/g, "&amp;")
+    .replace(/"/g, "'")
+    .trim()
+
+// A code block whose fence is longer than any run of backticks in its body,
+// so that no line of the body can close it.
+export const fence = (language: string, meta: string, body: string): string => {
+  let longest = 0
+  for (const run of body.match(/`+/g) || [])
+    longest = Math.max(longest, run.length)
+  const marks = "`".repeat(Math.max(3, longest + 1))
+  const info = meta ? `${language} ${meta}` : language
+  return `${marks}${info}\n${body}\n${marks}`
+}
+
+// Text used to compare options and questions: no inline formatting, no case.
+const comparable = (value: string): string =>
+  value
+    .replace(/[*_`~]/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .toLowerCase()
+
+const wordCount = (value: string): number =>
+  value.split(/\s+/).filter(Boolean).length
+
+const headingLine = (value: unknown): string => {
+  const heading = stripBlockPrefix(value)
+  return heading ? `### ${heading}\n\n` : ""
+}
+
+// Combining marks left by NFD normalization (U+0300 to U+036F).
+const DIACRITICS = new RegExp(
+  `[${String.fromCharCode(0x3_00)}-${String.fromCharCode(0x3_6F)}]`,
+  "g"
+)
+
+// Lowercase ASCII slug with hyphens.
+export const toSlug = (value: string): string =>
+  value
+    .normalize("NFD")
+    .replace(DIACRITICS, "")
+    .toLowerCase()
+    .replace(/[^\da-z]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+
+// FNV-1a hash, used as a seed and to tell image descriptions apart.
+const hash = (value: string): number => {
+  let h = 0x81_1C_9D_C5
+  for (let i = 0; i < value.length; i++) {
+    h ^= value.charCodeAt(i)
+    h = Math.imul(h, 0x01_00_01_93)
+  }
+
+  return h >>> 0
+}
+
+// Deterministic shuffle: the same question always gets the same order, and
+// different questions get different orders. The IDE does not shuffle multiple
+// choice options, so without this the correct option would always sit where
+// the model wrote it.
+export const seededShuffle = <T>(items: T[], seed: string): T[] => {
+  let state = hash(seed) || 1
+  const random = () => {
+    // mulberry32. `| 0` wraps the sum to 32 bits, which Math.trunc does not.
+    // eslint-disable-next-line unicorn/prefer-math-trunc
+    state = (state + 0x6D_2B_79_F5) | 0
+    let t = state
+    t = Math.imul(t ^ (t >>> 15), t | 1)
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
+    return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296
+  }
+
+  const result = [...items]
+  for (let i = result.length - 1; i > 0; i--) {
+    const j = Math.floor(random() * (i + 1))
+    const tmp = result[i]
+    result[i] = result[j]
+    result[j] = tmp
+  }
+
+  return result
+}
+
+// Separator that ends any list around a quiz. It is an HTML block, which the
+// IDE does not render (react-markdown skipHtml), and without it a quiz next to
+// a numbered list in the text would merge with that list and never show its
+// submit button (QuizRenderer counts every item of the list as a question).
+const QUIZ_SEPARATOR = "<!-- -->"
+
+// ---------------------------------------------------------------------------
+// Builders, one per component
+// ---------------------------------------------------------------------------
+
+const buildMultipleChoice: Builder = (data, state, warn) => {
+  const errors: string[] = []
+  // Problems of single questions: the question is removed, the quiz stays.
+  const questionErrors: string[] = []
+  const questions = Array.isArray(data?.questions) ? data.questions : []
+  const blocks: string[] = []
+
+  for (const [index, raw] of (questions as any[]).entries()) {
+    const where = `question ${index + 1}`
+    const question = stripBlockPrefix(raw?.question)
+    if (!question) {
+      questionErrors.push(`${where} is empty`)
+      continue
+    }
+
+    const key = comparable(question)
+    if (state.usedQuestions.has(key)) {
+      questionErrors.push(`${where} repeats another question of the step`)
+      continue
+    }
+
+    const options: Array<{ text: string; correct: boolean }> = (
+      Array.isArray(raw?.options) ? raw.options : []
+    ).map((option: any) => ({
+      text: stripBlockPrefix(option?.text),
+      correct: option?.correct === true,
+    }))
+    if (options.some(option => !option.text)) {
+      questionErrors.push(`${where} has an empty option`)
+      continue
+    }
+
+    if (options.length < 2) {
+      questionErrors.push(`${where} has fewer than 2 options`)
+      continue
+    }
+
+    const correctCount = options.filter(option => option.correct).length
+    if (correctCount !== 1) {
+      questionErrors.push(
+        `${where} has ${correctCount} correct options instead of 1`
+      )
+      continue
+    }
+
+    // The IDE identifies an option by its text, so repeated options collide.
+    const optionKeys = options.map(option => comparable(option.text))
+    if (new Set(optionKeys).size !== optionKeys.length) {
+      questionErrors.push(`${where} has repeated options`)
+      continue
+    }
+
+    if (options.length < 3 || options.length > 5) {
+      warn(`${where} has ${options.length} options (3 to 5 expected)`)
+    }
+
+    const correct = options.find(
+      (option: { correct: boolean }) => option.correct
+    )
+    if (
+      correct &&
+      comparable(correct.text).length > 3 &&
+      key.includes(comparable(correct.text))
+    ) {
+      warn(`${where} contains its correct option`)
+    }
+
+    state.usedQuestions.add(key)
+    const lines = seededShuffle(options, question).map(
+      (option: { text: string; correct: boolean }) =>
+        `   - [${option.correct ? "x" : " "}] ${option.text}`
+    )
+    blocks.push(`1. ${question}\n\n${lines.join("\n")}`)
+  }
+
+  if (blocks.length === 0) {
+    errors.push(
+      ...questionErrors,
+      questions.length === 0 ?
+        "the quiz has no questions" :
+        "no question is valid"
+    )
+    return { errors }
+  }
+
+  // Questions are separated by a blank line so that the list is loose: in a
+  // tight list the question is not a paragraph and the IDE renders nothing.
+  const markdown = `${QUIZ_SEPARATOR}\n\n${headingLine(
+    data?.heading
+  )}${blocks.join("\n\n")}\n\n${QUIZ_SEPARATOR}`
+  return {
+    markdown,
+    errors,
+    removed: questionErrors.map(
+      error => `${error}; the question was removed`
+    ),
+  }
+}
+
+const buildSelectTheBlank: Builder = (data, _state, warn) => {
+  const errors: string[] = []
+  const text = asText(data?.text)
+    .split(/\r?\n/)
+    .map(line => line.replace(/\s+$/, ""))
+    .join("\n")
+    .replace(/^\n+|\n+$/g, "")
+  if (!text.trim()) return { errors: ["the text is empty"] }
+
+  const blanks = Array.isArray(data?.blanks) ? data.blanks : []
+  const markers = (text.match(/_(\d+)_/g) || []).map(marker =>
+    Number(marker.slice(1, -1))
+  )
+  const expected = blanks.map((_: unknown, index: number) => index + 1)
+  const sortedMarkers = [...markers].sort((a, b) => a - b)
+  if (blanks.length === 0) {
+    errors.push("there are no blanks")
+  } else if (sortedMarkers.join(",") !== expected.join(",")) {
+    errors.push(
+      `the text has blanks [${markers.join(", ")}] but ${
+        blanks.length
+      } blank options were given`
+    )
+  }
+
+  const metadata: string[] = []
+  for (const [index, blank] of (blanks as any[]).entries()) {
+    const where = `blank ${index + 1}`
+    const correct = encodeMetaValue(blank?.correct)
+    if (!correct) {
+      errors.push(`${where} has no correct option`)
+      continue
+    }
+
+    const seen = new Set<string>([comparable(correct)])
+    const distractors: string[] = []
+    for (const raw of Array.isArray(blank?.distractors) ?
+      blank.distractors :
+      []) {
+      const distractor = encodeMetaValue(raw)
+      if (!distractor || seen.has(comparable(distractor))) continue
+      seen.add(comparable(distractor))
+      distractors.push(distractor)
+    }
+
+    const options = [correct, ...distractors]
+    if (options.some(option => option.includes("|"))) {
+      errors.push(`${where} has an option with the character |`)
+      continue
+    }
+
+    if (distractors.length < 2) {
+      errors.push(
+        `${where} has ${distractors.length} distractors (at least 2 needed)`
+      )
+      continue
+    }
+
+    if (options.some(option => wordCount(option) > 4)) {
+      warn(`${where} has an option longer than 4 words`)
+    }
+
+    // The IDE takes the first option as the correct one and shuffles them.
+    metadata.push(`${index + 1}="${options.join("|")}"`)
+  }
+
+  if (errors.length > 0) return { errors }
+  if (blanks.length < 3)
+    warn(`only ${blanks.length} blanks (at least 3 expected)`)
+
+  const markdown =
+    headingLine(data?.heading) +
+    fence("select_the_blank", metadata.join(" "), text)
+  return { markdown, errors }
+}
+
+const buildOrdering: Builder = (data, _state, warn) => {
+  const errors: string[] = []
+  const items = (Array.isArray(data?.items) ? data.items : []).map(
+    stripBlockPrefix
+  )
+  if (items.some((item: string) => !item)) errors.push("an item is empty")
+  if (items.length < 3)
+    errors.push(`${items.length} items (at least 3 needed)`)
+  // The IDE uses each item as its key, so repeated items break the component.
+  const seen = new Set<string>()
+  for (const item of items) {
+    if (seen.has(comparable(item))) {
+      errors.push("there are repeated items")
+      break
+    }
+
+    seen.add(comparable(item))
+  }
+
+  if (errors.length > 0) return { errors }
+  if (items.length > 6) warn(`${items.length} items (at most 6 expected)`)
+
+  let title = encodeMetaValue(data?.instruction)
+  // The IDE turns "true" and "false" into booleans and ignores them.
+  if (/^(true|false)$/i.test(title)) title = ""
+  const meta = title ? `title="${title}"` : ""
+  const markdown =
+    headingLine(data?.heading) + fence("ordering", meta, items.join("\n"))
+  return { markdown, errors }
+}
+
+const buildOpenText: Builder = (data, _state, warn) => {
+  const errors: string[] = []
+  const question = asText(data?.question).trim()
+  const evaluation = encodeMetaValue(data?.eval)
+  if (!question) errors.push("the question is empty")
+  if (!evaluation) errors.push("the eval is empty")
+
+  const examples: Array<{ text: string; correct: boolean }> = (
+    Array.isArray(data?.examples) ? data.examples : []
+  )
+    .map((example: any) => ({
+      text: collapse(example?.text).replace(/^(?:in)?correct\s*:\s*/i, ""),
+      correct: example?.correct === true,
+    }))
+    .filter((example: { text: string }) => example.text)
+  if (errors.length > 0) return { errors }
+
+  // Examples help the evaluator but are not required: the IDE evaluates with
+  // the eval, and when a question has fewer than 3 examples it asks the
+  // course author to add them (OpenQuestion.tsx). Dropping the question would
+  // lose it, so missing examples are only reported.
+  if (examples.length < 3 || examples.length > 5) {
+    warn(`${examples.length} examples (3 to 5 expected)`)
+  }
+
+  if (examples.length > 0 && !examples.some(example => example.correct)) {
+    warn("no correct example")
+  }
+
+  if (examples.length > 0 && !examples.some(example => !example.correct)) {
+    warn("no incorrect example")
+  }
+
+  // The learner never sees the code block, so the question goes before it.
+  // The IDE reads one example per line.
+  const body = examples
+    .map(
+      (example: { text: string; correct: boolean }) =>
+        `${example.correct ? "CORRECT" : "INCORRECT"}: ${example.text}`
+    )
+    .join("\n")
+  const markdown =
+    headingLine(data?.heading) +
+    `${question}\n\n` +
+    fence("question", `eval="${evaluation}"`, body)
+  return { markdown, errors }
+}
+
+const buildCodeChallenge: Builder = data => {
+  // The IDE removes these blocks with a regex that expects a fence of exactly
+  // three backticks, so runs of backticks are removed from the text instead of
+  // lengthening the fence.
+  const description = asText(data?.description).replace(/`{3,}/g, "").trim()
+  if (!description) return { errors: ["the description is empty"] }
+  return {
+    markdown: fence("code_challenge_proposal", "", description),
+    errors: [],
+  }
+}
+
+const buildAskRigo: Builder = data => {
+  const errors: string[] = []
+  const label = collapse(data?.label).replace(/[[\]]/g, "")
+  const question = collapse(data?.question)
+  if (!label) errors.push("the label is empty")
+  if (wordCount(question) < 3)
+    errors.push("the question has fewer than 3 words")
+  if (errors.length > 0) return { errors }
+
+  // The IDE reads the query with URLSearchParams and turns hyphens into
+  // spaces, so hyphens of the question become spaces first. Encoding keeps
+  // accents and non-Latin scripts, and parentheses are encoded too because
+  // they would end the link.
+  const query = question
+    .toLowerCase()
+    .replace(/-/g, " ")
+    .trim()
+    .split(/\s+/)
+    .map(word =>
+      encodeURIComponent(word).replace(/\(/g, "%28").replace(/\)/g, "%29")
+    )
+    .join("-")
+  return {
+    markdown: `[${label}](https://4geeks.com/ask?query=${query})`,
+    errors,
+  }
+}
+
+const buildImage: Builder = (data, state, warn) => {
+  const alt = collapse(data?.alt).replace(/[[\]]/g, "")
+  if (!alt) return { errors: ["the alt text is empty"] }
+  if (wordCount(alt) < 30)
+    warn(`the alt text has ${wordCount(alt)} words (at least 30 expected)`)
+
+  // Image files are shared by the whole course and the image webhook skips
+  // files that already exist, so the name carries the step id and a hash of
+  // the description: the same description keeps its file, a new one gets a
+  // new file.
+  const words =
+    toSlug(alt.split(/\s+/).slice(0, 6).join(" ")) ||
+    toSlug(asText(data?.id)) ||
+    "image"
+  const base = `${toSlug(state.lessonId) || "step"}-${words}`
+    .slice(0, 72)
+    .replace(/-+$/, "")
+  let name = `${base}-${hash(alt).toString(16).padStart(8, "0").slice(0, 6)}`
+  let suffix = 2
+  while (state.usedImageNames.has(name)) {
+    name = `${base}-${hash(alt)
+      .toString(16)
+      .padStart(8, "0")
+      .slice(0, 6)}-${suffix++}`
+  }
+
+  state.usedImageNames.add(name)
+  return { markdown: `![${alt}](./.learn/assets/${name}.png)`, errors: [] }
+}
+
+const buildMermaid: Builder = (data, _state, warn) => {
+  const code = asText(data?.code)
+    .trim()
+    .replace(/^(`{3,}|~{3,})\s*mermaid\s*\n/i, "")
+    .replace(/\n?(`{3,}|~{3,})\s*$/, "")
+    .trim()
+  if (!code) return { errors: ["the code is empty"] }
+  if (!KNOWN_MERMAID_TYPES.some(type => code.startsWith(type))) {
+    warn("the diagram type is not one of the usual Mermaid types")
+  }
+
+  return { markdown: fence("mermaid", "", code), errors: [] }
+}
+
+const BUILDERS: Record<ComponentName, Builder> = {
+  multiple_choice_question: buildMultipleChoice,
+  select_the_blank: buildSelectTheBlank,
+  ordering: buildOrdering,
+  open_text_question: buildOpenText,
+  code_challenge_proposal: buildCodeChallenge,
+  ask_rigo_button: buildAskRigo,
+  mermaid_diagram: buildMermaid,
+  technical_diagram: buildImage,
+  conceptual_illustration: buildImage,
+  realistic_photography: buildImage,
+}
+
+// ---------------------------------------------------------------------------
+// Lesson text
+// ---------------------------------------------------------------------------
+
+const PLACEHOLDER = /<!--\s*component\s*:\s*(.*?)\s*-->/gi
+const FENCE_OPEN = /^\s*(`{3,}|~{3,})\s*([^\s`]*)/
+const NESTED_TASK_ITEM = /^(?: {2,}|\t)(?:[*+-]|\d+[).])\s+\[[ Xx]]\s/
+const ASK_RIGO_LINK =
+  /\[([^\]]*)]\(\s*https?:\/\/(?:www\.)?4geeks\.com\/ask\/?\?[^)]*\)/g
+
+const normalizeId = (value: unknown): string => collapse(value).toLowerCase()
+
+const isFenceClose = (line: string, open: string): boolean => {
+  const match = line.match(/^\s*(`{3,}|~{3,})\s*$/)
+  return Boolean(
+    match && match[1][0] === open[0] && match[1].length >= open.length
+  )
+}
+
+// Removes component markup that the model wrote in the lesson text instead of
+// returning it as data. It usually copies it from the previous lesson, which
+// reaches the model already rendered. Removing it keeps a broken or duplicated
+// component out of the README.
+export const removeCopiedSyntax = (
+  text: string
+): { text: string; removed: string[] } => {
+  const removed: string[] = []
+  const output: string[] = []
+  const lines = text.split(/\r?\n/)
+  let openFence: string | null = null
+  let dropping = false
+  let removedTaskItems = 0
+  // Only task items nested under a numbered item have the shape of a quiz;
+  // other checklists in the text are left alone.
+  let underNumberedItem = false
+
+  for (const line of lines) {
+    if (openFence) {
+      if (!dropping) output.push(line)
+      if (isFenceClose(line, openFence)) {
+        openFence = null
+        dropping = false
+      }
+
+      continue
+    }
+
+    const fenceMatch = line.match(FENCE_OPEN)
+    if (fenceMatch) {
+      openFence = fenceMatch[1]
+      const language = fenceMatch[2].split("-")[0].toLowerCase()
+      dropping =
+        COMPONENT_FENCE_LANGUAGES.has(language) ||
+        COMPONENT_FENCE_LANGUAGES.has(fenceMatch[2].toLowerCase())
+      if (dropping) {
+        removed.push(`a ${fenceMatch[2]} block written in the lesson text`)
+      } else {
+        output.push(line)
+      }
+
+      continue
+    }
+
+    // Only lines that start at column 0 change the parent; indented lines
+    // belong to the current item.
+    if (/^\d+[).]\s+/.test(line)) {
+      underNumberedItem = true
+    } else if (line.trim() && !/^\s/.test(line)) {
+      underNumberedItem = false
+    }
+
+    if (underNumberedItem && NESTED_TASK_ITEM.test(line)) {
+      removedTaskItems++
+      continue
+    }
+
+    output.push(
+      line.replace(ASK_RIGO_LINK, (_match, label: string) => {
+        removed.push("an Ask Rigo link written in the lesson text")
+        return label
+      })
+    )
+  }
+
+  if (removedTaskItems > 0) {
+    removed.push(
+      `${removedTaskItems} quiz option line(s) written in the lesson text`
+    )
+  }
+
+  return { text: output.join("\n"), removed }
+}
+
+// Splits the text into lines and placeholders. Placeholders inside code blocks
+// are kept as text.
+type Piece =
+  | { kind: "text"; line: string }
+  | { kind: "placeholder"; id: string };
+
+const splitPlaceholders = (text: string): Piece[] => {
+  const pieces: Piece[] = []
+  let openFence: string | null = null
+
+  for (const line of text.split(/\r?\n/)) {
+    if (openFence) {
+      pieces.push({ kind: "text", line })
+      if (isFenceClose(line, openFence)) openFence = null
+      continue
+    }
+
+    const fenceMatch = line.match(FENCE_OPEN)
+    if (fenceMatch) {
+      openFence = fenceMatch[1]
+      pieces.push({ kind: "text", line })
+      continue
+    }
+
+    PLACEHOLDER.lastIndex = 0
+    if (!PLACEHOLDER.test(line)) {
+      pieces.push({ kind: "text", line })
+      continue
+    }
+
+    PLACEHOLDER.lastIndex = 0
+    let last = 0
+    let match: RegExpExecArray | null
+    let first = true
+    while ((match = PLACEHOLDER.exec(line)) !== null) {
+      const before = line.slice(last, match.index)
+      // A placeholder alone in a list item or a quote leaves its container, so
+      // the component is a block of its own. When the line has other text,
+      // that text keeps its list or quote marker.
+      const withoutContainer = first ?
+        before.replace(/^\s*(?:>\s*)*(?:(?:[*+-]|\d+[).])\s+)?/, "") :
+        before
+      if (withoutContainer.trim())
+        pieces.push({ kind: "text", line: before.replace(/\s+$/, "") })
+      pieces.push({ kind: "placeholder", id: normalizeId(match[1]) })
+      last = match.index + match[0].length
+      first = false
+    }
+
+    const after = line.slice(last)
+    if (after.trim()) pieces.push({ kind: "text", line: after.trim() })
+  }
+
+  return pieces
+}
+
+// Joins text lines and component blocks, with a blank line around each block
+// and never more than one blank line in a row.
+const joinPieces = (
+  parts: Array<
+    { kind: "text"; line: string } | { kind: "block"; markdown: string }
+  >
+): string => {
+  const out: string[] = []
+  const pushBlank = () => {
+    if (out.length > 0 && out[out.length - 1] !== "") out.push("")
+  }
+
+  for (const part of parts) {
+    if (part.kind === "block") {
+      pushBlank()
+      out.push(...part.markdown.split("\n"), "")
+    } else {
+      out.push(part.line)
+    }
+  }
+
+  return out.join("\n")
+}
+
+const normalizeBlankLines = (text: string): string => {
+  const cleaned = text
+    .split("\n")
+    .map(line => (line.trim() === "" ? "" : line.replace(/\s+$/, "")))
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .replace(/^\n+/, "")
+    .replace(/\n*$/, "\n")
+  // The IDE applies this when it reads a README; the result must not change.
+  return sanitizeReadmeNewlines(cleaned)
+}
+
+const removePlaceholders = (text: string): string =>
+  splitPlaceholders(text)
+    .filter(
+      (piece): piece is { kind: "text"; line: string } => piece.kind === "text"
+    )
+    .map(piece => piece.line)
+    .join("\n")
+
+// ---------------------------------------------------------------------------
+// Entry point
+// ---------------------------------------------------------------------------
+
+const processUnsafe = (parsed: any, ctx: ProcessContext): ProcessResult => {
+  const dropped: Issue[] = []
+  const warnings: Issue[] = []
+  const decidedBy = { ...DEFAULT_DECIDED_BY, ...ctx.decidedBy } as Record<
+    string,
+    DecidedBy
+  >
+  const state: BuildState = {
+    lessonId: asText(ctx.lessonId),
+    usedImageNames: new Set(),
+    usedQuestions: new Set(),
+  }
+
+  // Build every component. Order: as the lists come, which is the order the
+  // model wrote them.
+  const rendered = new Map<string, RenderedComponent>()
+  const unidentified: RenderedComponent[] = []
+  for (const name of COMPONENT_NAMES) {
+    const list = Array.isArray(parsed?.[name]) ? parsed[name] : []
+    for (const [index, data] of (list as any[]).entries()) {
+      const id = normalizeId(data?.id)
+      const label = id || `${name} #${index + 1}`
+      if (id && rendered.has(id)) {
+        dropped.push({
+          component: name,
+          id,
+          message: `${name} "${label}" repeats an id`,
+        })
+        continue
+      }
+
+      const result = BUILDERS[name](data, state, message =>
+        warnings.push({
+          component: name,
+          id: label,
+          message: `${name} "${label}": ${message}`,
+        })
+      )
+      if (!result.markdown || result.errors.length > 0) {
+        dropped.push({
+          component: name,
+          id: label,
+          message: `${name} "${label}" was removed: ${result.errors.join(
+            "; "
+          )}`,
+        })
+        continue
+      }
+
+      for (const message of result.removed || []) {
+        dropped.push({
+          component: name,
+          id: label,
+          message: `${name} "${label}": ${message}`,
+        })
+      }
+
+      const component = { name, id: label, markdown: result.markdown }
+      if (id) rendered.set(id, component)
+      else unidentified.push(component)
+    }
+  }
+
+  let content = asText(parsed?.final_content)
+  // Cache-busting suffix that the CLI adds to the draft (see serve.ts).
+  content = content.replace(/-\d{6}\s*$/, "")
+  const copied = removeCopiedSyntax(content)
+  for (const message of copied.removed)
+    dropped.push({ message: `Removed ${message}` })
+
+  const placed = new Set<string>()
+  const parts: Array<
+    { kind: "text"; line: string } | { kind: "block"; markdown: string }
+  > = []
+  const written: RenderedComponent[] = []
+  for (const piece of splitPlaceholders(copied.text)) {
+    if (piece.kind === "text") {
+      parts.push(piece)
+      continue
+    }
+
+    const component = rendered.get(piece.id)
+    if (!component) {
+      // Its data was invalid (already reported) or never returned.
+      if (!dropped.some(issue => issue.id === piece.id)) {
+        warnings.push({
+          id: piece.id,
+          message: `placeholder "${piece.id}" has no component`,
+        })
+      }
+
+      continue
+    }
+
+    if (placed.has(piece.id)) {
+      warnings.push({
+        id: piece.id,
+        message: `placeholder "${piece.id}" appears more than once`,
+      })
+      continue
+    }
+
+    placed.add(piece.id)
+    written.push(component)
+    parts.push({ kind: "block", markdown: component.markdown })
+  }
+
+  // Components with data but no placeholder.
+  const unplaced = [
+    ...[...rendered.values()].filter(c => !placed.has(c.id)),
+    ...unidentified,
+  ]
+  for (const component of unplaced) {
+    if (decidedBy[component.name] === "syllabus" && !ctx.isFirstOrLast) {
+      warnings.push({
+        component: component.name,
+        id: component.id,
+        message: `${component.name} "${component.id}" had no placeholder and was added at the end`,
+      })
+      written.push(component)
+      parts.push({ kind: "block", markdown: component.markdown })
+    } else {
+      dropped.push({
+        component: component.name,
+        id: component.id,
+        message: `${component.name} "${component.id}" was removed: it had no placeholder in the lesson text`,
+      })
+    }
+  }
+
+  const usedComponents = COMPONENT_NAMES.filter(name =>
+    written.some(c => c.name === name)
+  )
+  const assessmentCount = written.filter(c =>
+    ASSESSMENT_COMPONENTS.has(c.name)
+  ).length
+  return {
+    markdown: normalizeBlankLines(joinPieces(parts)),
+    usedComponents,
+    assessmentCount,
+    dropped,
+    warnings,
+  }
+}
+
+// Never throws: a failure here would mark the lesson as failed with no retry
+// (see the interactivity webhook in serve.ts), so an internal error returns the
+// lesson text without placeholders.
+export const processInteractivityResult = (
+  parsed: any,
+  ctx: ProcessContext = {}
+): ProcessResult => {
+  try {
+    return processUnsafe(parsed, ctx)
+  } catch (error) {
+    let markdown = ""
+    try {
+      markdown = normalizeBlankLines(
+        removePlaceholders(asText(parsed?.final_content))
+      )
+    } catch {
+      markdown = asText(parsed?.final_content)
+    }
+
+    return {
+      markdown,
+      usedComponents: [],
+      assessmentCount: 0,
+      dropped: [
+        {
+          message: `Components could not be processed: ${
+            (error as Error).message
+          }`,
+        },
+      ],
+      warnings: [],
+    }
+  }
+}
+
+export const isImageComponent = (name: string): boolean =>
+  IMAGE_COMPONENTS.has(name as ComponentName)