lens-content-processor 0.78.0 → 0.80.0

package/dist/index.js

@@ -1,16 +1,19 @@
-import { flattenModule, tagCourseOutcomes, suggestedLensPaths, flattenLens, flattenStandaloneArticle, expandPromptImports, toKebabCase, } from "./flattener/index.js";
-import { dedupeErrors } from "./dedupe-errors.js";
+import { declaredSections, flattenModule, tagLensOutcomes, suggestedLensPaths, flattenLens, flattenStandaloneArticle, expandPromptImports, toKebabCase, } from "./flattener/index.js";
+import { attachDroppedSegments, crossCheck, droppedSegmentsOf, failedModule, failedPage, pageEntries, placedAfter, } from "./dropped.js";
+import { fileNameToSlug } from "./utils/slug.js";
+import { errorKey, reportOnce } from "./dedupe-errors.js";
 import { populateCardModuleSlugs, resolveInlineLensModuleSlugs, } from "./flattener/resolve-text-links.js";
 import { parseModule } from "./parser/module.js";
 import { parseCourse } from "./parser/course.js";
 import { parseSurvey } from "./parser/survey.js";
 import { MEETING_DOC_SHARED_DIR, parseMeetingDoc, } from "./parser/meeting-doc.js";
 import { parseLearningOutcome, } from "./parser/learning-outcome.js";
-import { parseLens } from "./parser/lens.js";
+import { convertRawSegments, parseLens, } from "./parser/lens.js";
+import { isQuestionFilePath, parseQuestionFile, questionEmbedCacheKey, questionEmbedsOutsideLenses, } from "./parser/question-embeds.js";
 import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion, } from "./parser/wikilink.js";
 import { validateUuids, } from "./validator/uuid.js";
 import { detectDuplicateSlugs, } from "./validator/duplicates.js";
-import { validateOutputIntegrity } from "./validator/output-integrity.js";
+import { outputIntegrityCandidates } from "./validator/output-integrity.js";
 import { parseArticleBody } from "./bundler/article.js";
 import { parseFrontmatter } from "./parser/frontmatter.js";
 import { parseArticle } from "./parser/article.js";
@@ -22,19 +25,21 @@ import { applyGlossary, isGlossaryPath, parseGlossaryEntry, validateGlossaryAlia
 import { validateTimestamps } from "./validator/timestamps.js";
 import { levenshtein } from "./validator/field-typos.js";
 import { buildTagMap, buildTierMap, checkTierViolation, } from "./validator/tier.js";
-import { validateCardModuleMembership } from "./validator/card-module-membership.js";
+import { cardModuleGroups, checkCardModuleMembership, } from "./validator/card-module-membership.js";
 import { collectOptionalLensSets, } from "./validator/optional-lens-sets.js";
 import { detectPendingCriticMarkup, isCriticMarkupCheckedPath, } from "./validator/criticmarkup.js";
 import { validateMath } from "./validator/math.js";
 import { validateArticleStructure } from "./validator/article-structure.js";
 import { validateArticleReviewProvenance } from "./validator/article-review-provenance.js";
-import { stripAuthoringMarkup } from "./authoring-markup.js";
+import { stripAuthoringMarkup, stripAuthoringMarkupForValidation, } from "./authoring-markup.js";
 export { checkTierViolation } from "./validator/tier.js";
 import { buildSkillTree, } from "./skill-tree.js";
 import { DOMAIN_METADATA_FILE, parseDomainMetadata, parseDomainPath, parseTopicFile, } from "./parser/domain.js";
 import { RunInputs } from "./dependency-cache.js";
 export { FileResultCache } from "./cache.js";
 import { clearParsedMarkdown } from "./markdown-parse.js";
+import { ContentView, inPathOrder, } from "./engine/reader.js";
+import { pageKey } from "./engine/units.js";
 /** Authored field name of each flattened prompt-import-capable property. */
 const PROMPT_FIELD_NAMES = {
     assessmentInstructions: "assessment-instructions",
@@ -47,58 +52,59 @@ const PROMPT_FIELD_NAMES = {
  * their sourcePath, so their own id tells them apart. The backend finds a question by (id, module), so two
  * different copies in one module would both be graded, length-checked and
  * coached by the first. This also catches a lens and the article it embeds
- * using one id, which flatten into a single section.
+ * using one id, which flatten into a single section. A lens's own
+ * "appears twice" error is reported once over all pages (`once`), not once
+ * per module that copies the lens.
  */
-function detectSharedQuestionsInOneModule(modules) {
-    const errors = [];
-    const reported = new Set(); // one error per lens, not per module
-    for (const mod of modules) {
-        // lowercased id → where it first appeared in this module
-        const first = new Map();
-        for (const section of mod.sections) {
-            const lens = section.sourcePath ?? mod.sourcePath ?? mod.slug;
-            // Inline lenses share the module file as their path: name them.
-            const where = section.sourcePath === mod.sourcePath &&
-                lens.startsWith("modules/") &&
-                section.meta.title
-                ? `lens '${section.meta.title}' in ${lens}`
-                : lens;
-            for (const seg of section.segments) {
-                if (seg.type !== "question" || !seg.responseId)
+function sharedQuestionCandidates(mod) {
+    const candidates = [];
+    // lowercased id → where it first appeared in this module
+    const first = new Map();
+    for (const section of mod.sections) {
+        const lens = section.sourcePath ?? mod.sourcePath ?? mod.slug;
+        // Inline lenses share the module file as their path: name them.
+        const where = section.sourcePath === mod.sourcePath &&
+            lens.startsWith("modules/") &&
+            section.meta.title
+            ? `lens '${section.meta.title}' in ${lens}`
+            : lens;
+        for (const seg of section.segments) {
+            if (seg.type !== "question" || !seg.responseId)
+                continue;
+            const from = seg.partOfArticle ? `an article in ${where}` : where;
+            const id = seg.responseId.toLowerCase();
+            const seen = first.get(id);
+            if (seen === undefined) {
+                first.set(id, { section, from, inArticle: !!seg.partOfArticle });
+            }
+            else if (seen.section === section) {
+                // Two in the lens's own text are collectResponseSegmentIds's error.
+                if (!seen.inArticle && !seg.partOfArticle)
                     continue;
-                const from = seg.partOfArticle ? `an article in ${where}` : where;
-                const id = seg.responseId.toLowerCase();
-                const seen = first.get(id);
-                if (seen === undefined) {
-                    first.set(id, { section, from, inArticle: !!seg.partOfArticle });
-                }
-                else if (seen.section === section) {
-                    // Two in the lens's own text are collectResponseSegmentIds's error.
-                    if (!seen.inArticle && !seg.partOfArticle)
-                        continue;
-                    if (reported.has(`${lens}\n${id}`))
-                        continue;
-                    reported.add(`${lens}\n${id}`);
-                    errors.push({
+                candidates.push({
+                    error: {
                         file: lens,
                         message: `Question id '${seg.responseId}' appears twice in this lens: in ${seen.from} and in ${from}`,
                         suggestion: "Give one of the two questions its own UUID",
                         severity: "error",
-                    });
-                }
-                else if (seen.section.sourcePath !== section.sourcePath ||
-                    seen.section.contentId !== section.contentId) {
-                    errors.push({
+                    },
+                    once: `${lens}\n${id}`,
+                });
+            }
+            else if (seen.section.sourcePath !== section.sourcePath ||
+                seen.section.contentId !== section.contentId) {
+                candidates.push({
+                    error: {
                         file: mod.sourcePath ?? lens,
                         message: `Module '${mod.slug}' holds question id '${seg.responseId}' twice: in ${seen.from} and in ${from}`,
                         suggestion: "A question id may appear only once per module. Remove one of the copies from this module, or give one its own UUID",
                         severity: "error",
-                    });
-                }
+                    },
+                });
             }
         }
     }
-    return errors;
+    return candidates;
 }
 /**
  * Collect the stable id:: of every question segment (from lenses, inline
@@ -107,19 +113,14 @@ function detectSharedQuestionsInOneModule(modules) {
  * files share an id only for the same question (validator/uuid.ts).
  * validateUuids deliberately allows same-file, same-field duplicates
  * (contentId sharing across submodule groups), so duplicates within one file
- * are detected here instead, via the caller's per-file seen map.
+ * are detected here instead, via the file's own set of ids already seen.
  */
-function collectResponseSegmentIds(segments, context, path, uuidEntries, seenResponseIds, errors) {
+function collectResponseSegmentIds(segments, context, path, uuidEntries, seenInFile, errors) {
     for (const seg of segments) {
         if (seg.type !== "question" || !("responseId" in seg) || !seg.responseId) {
             continue;
         }
         const lower = seg.responseId.toLowerCase();
-        let seenInFile = seenResponseIds.get(path);
-        if (!seenInFile) {
-            seenInFile = new Set();
-            seenResponseIds.set(path, seenInFile);
-        }
         if (seenInFile.has(lower)) {
             errors.push({
                 file: path,
@@ -141,7 +142,7 @@ function collectResponseSegmentIds(segments, context, path, uuidEntries, seenRes
 /**
  * Recursively collect UUIDs from inline lens sections (including submodule children).
  */
-function collectInlineLensUUIDs(sections, path, uuidEntries, seenResponseIds, errors) {
+function collectInlineLensUUIDs(sections, path, uuidEntries, seenInFile, errors) {
     for (const section of sections) {
         if (section.type === "lens") {
             if (section.inlineLens) {
@@ -151,7 +152,7 @@ function collectInlineLensUUIDs(sections, path, uuidEntries, seenResponseIds, er
                     file: path,
                     field: "section id",
                 });
-                collectResponseSegmentIds(section.inlineLens.segments, "lens", path, uuidEntries, seenResponseIds, errors);
+                collectResponseSegmentIds(section.inlineLens.segments, "lens", path, uuidEntries, seenInFile, errors);
             }
             else if (!section.fields.source && !section.fields.id) {
                 // No section-level source:: and no id:: → missing required id
@@ -173,598 +174,727 @@ function collectInlineLensUUIDs(sections, path, uuidEntries, seenResponseIds, er
             }
         }
         if (section.children) {
-            collectInlineLensUUIDs(section.children, path, uuidEntries, seenResponseIds, errors);
+            collectInlineLensUUIDs(section.children, path, uuidEntries, seenInFile, errors);
         }
     }
 }
-function collectLearningOutcomeTestUUIDs(learningOutcome, path, uuidEntries, seenResponseIds, errors) {
+function collectLearningOutcomeTestUUIDs(learningOutcome, path, uuidEntries, seenInFile, errors) {
     if (learningOutcome.test) {
         uuidEntries.push({
             uuid: learningOutcome.test.id,
             file: path,
             field: "test section id",
         });
-        collectResponseSegmentIds(learningOutcome.test.segments, "test", path, uuidEntries, seenResponseIds, errors);
+        collectResponseSegmentIds(learningOutcome.test.segments, "test", path, uuidEntries, seenInFile, errors);
     }
 }
-export function processContent(files, cache) {
-    // Parsed markdown is memoized for the length of one run (markdown-parse.ts).
-    // Drop the trees afterwards so a long-lived caller does not keep them.
-    try {
-        return processContentRun(files, cache);
-    }
-    finally {
-        clearParsedMarkdown();
+/**
+ * The LOs reachable from a module's `sections`, each with the submodule it
+ * sits in. A "# Submodule:" heading opens a group that runs until the next one
+ * (its siblings belong to it, as in splitAtBoundaries), and any children nest
+ * under it too. Works off the raw module parse, so an LO counts as "taught by"
+ * the module even when it contributes no flattened sections (e.g. it has no
+ * test yet).
+ */
+function collectModuleLoRefs(sections, modulePath, files, refs = [], submodule = null) {
+    for (const section of sections) {
+        if (section.type === "submodule") {
+            // Same slug rule as splitAtBoundaries, so the link lands on the page.
+            submodule = {
+                slug: section.fields.slug ?? toKebabCase(section.title),
+                title: section.title,
+            };
+        }
+        if (section.type === "learning-outcome" && section.fields.source) {
+            const wikilink = parseWikilink(section.fields.source, modulePath);
+            if (wikilink && !wikilink.error) {
+                const resolved = resolveWikilinkPath(wikilink.path, modulePath);
+                const actualPath = findFileWithExtension(resolved, files);
+                if (actualPath)
+                    refs.push({ lo: actualPath, submodule });
+            }
+        }
+        if (section.children) {
+            collectModuleLoRefs(section.children, modulePath, files, refs, submodule);
+        }
     }
+    return refs;
 }
-function processContentRun(files, cache) {
-    // Iteration order of `files` decides the order of every output array below, so it must
-    // not depend on how a caller happened to enumerate the filesystem. Callers do differ:
-    // the Node vault reader and the Python backend both walk directories without sorting,
-    // and the --stdin path takes whatever order its JSON arrived in. Sorting here once
-    // makes the output a function of the content alone, for every caller.
-    files = new Map([...files].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
-    // Memoize a pure function of one file's content (see cache.ts for the key's
-    // correctness argument; its entry wrapper rejects any bytes put() did not
-    // write). Without a cache this is just fn(): the pipeline below runs
-    // identically either way — same passes, same order — a hit only skips
-    // recomputing what an identical input already produced.
-    function cachedPerFile(kind, path, content, fn) {
-        const hit = cache?.get(kind, path, content);
-        if (hit !== undefined)
-            return hit;
-        const value = fn();
-        cache?.put(kind, path, content, value);
-        return value;
+/** One file's pass: parse it, run its own checks, and flatten its pages. */
+export function runPathUnit(path, { files, tierMap, tagMap }, steps) {
+    const unit = {
+        path,
+        errors: [],
+        modules: [],
+        urls: [],
+        uuids: [],
+    };
+    const { errors, modules, uuids: uuidEntries } = unit;
+    // Response segment ids already seen in this file (within-file duplicate check)
+    const seenResponseIds = new Set();
+    const recordDropped = (entries) => {
+        if (entries.length > 0)
+            unit.dropped = entries;
+    };
+    // Skip ignored files entirely
+    if (tierMap.get(path) === "ignored") {
+        return unit;
     }
-    const modules = [];
-    const articleSegments = {};
-    const courses = [];
-    const surveys = [];
-    const surveyPathToId = new Map();
-    const meetingDocs = {};
-    const errors = [];
-    const urlsToValidate = [];
-    const uuidEntries = [];
-    // Response segment ids already seen, per file (within-file duplicate check)
-    const seenResponseIds = new Map();
-    const slugEntries = [];
-    const slugToPath = new Map();
-    const filePathToSlug = new Map(); // Reverse: file path → slug (survives duplicate slugs)
-    const courseSlugToFile = new Map();
-    const parentSlugToChildren = new Map();
-    // Pre-scan: frontmatter tags of every file, and the tier they imply
-    const tagMap = buildTagMap(files);
-    const tierMap = buildTierMap(tagMap);
-    // Steps that read other files are cached by what they read (see
-    // dependency-cache.ts); each is a pure function of the file and tier maps.
-    const inputs = new RunInputs(files, tierMap, cache);
-    // --- Skill-tree collections (assembled after the main loop) ---
-    // Domain folder path → folder; topic file path → topic (+ its folder).
-    const domainFolders = new Map();
-    const topicFiles = new Map();
-    const learningOutcomesByPath = new Map();
-    // LO path → module path → the submodules (local slug → title) that
-    // reference it; empty when the module is not split or the LO is module-level.
-    const moduleLoRefs = new Map();
-    const glossaryEntries = new Map();
-    const skillTreeModules = new Map();
-    // Record that `modulePath` references each LO reachable from `sections`,
-    // remembering which submodule it sits in. A "# Submodule:" heading opens a
-    // group that runs until the next one (its siblings belong to it, as in
-    // splitAtBoundaries), and any children nest under it too. Works off the raw
-    // module parse, so an LO counts as "taught by" the module even when it
-    // contributes no flattened sections (e.g. it has no test yet).
-    function collectModuleLoRefs(sections, modulePath, submodule = null) {
-        for (const section of sections) {
-            if (section.type === "submodule") {
-                // Same slug rule as splitAtBoundaries, so the link lands on the page.
-                submodule = {
-                    slug: section.fields.slug ?? toKebabCase(section.title),
-                    title: section.title,
-                };
-            }
-            if (section.type === "learning-outcome" && section.fields.source) {
-                const wikilink = parseWikilink(section.fields.source, modulePath);
-                if (wikilink && !wikilink.error) {
-                    const resolved = resolveWikilinkPath(wikilink.path, modulePath);
-                    const actualPath = findFileWithExtension(resolved, files);
-                    if (actualPath) {
-                        let byModule = moduleLoRefs.get(actualPath);
-                        if (!byModule) {
-                            byModule = new Map();
-                            moduleLoRefs.set(actualPath, byModule);
-                        }
-                        let submodules = byModule.get(modulePath);
-                        if (!submodules) {
-                            submodules = new Map();
-                            byModule.set(modulePath, submodules);
-                        }
-                        if (submodule)
-                            submodules.set(submodule.slug, submodule.title);
-                    }
-                }
-            }
-            if (section.children) {
-                collectModuleLoRefs(section.children, modulePath, submodule);
-            }
-        }
+    const content = files.get(path);
+    // processContent passes `steps`, its result cache: a pure function of this
+    // file's content is cached by that content (cachedPerFile), and a step that
+    // reads other files by what it read (crossFile, dependency-cache.ts). The
+    // engine passes none: a hit returns a result without the reads that made
+    // it, and the engine's recording views must see every read. So a step
+    // reads only the views it is given. Without `steps` the pass runs the same
+    // functions; a hit only skips recomputing what the same input produced.
+    // `extraKey` adds what else a result depends on to its cache key. It runs
+    // only with `steps`, so the engine's views record no reads for it.
+    function cachedPerFile(kind, fn, extraKey) {
+        if (!steps)
+            return fn();
+        return steps.perFile(kind, path, content + (extraKey?.() ?? ""), fn);
     }
-    // Identify file types by path
-    for (const [path, content] of files.entries()) {
-        // Skip ignored files entirely
-        if (tierMap.get(path) === "ignored") {
-            continue;
-        }
-        if (path.endsWith(".md") || isCriticMarkupCheckedPath(path)) {
-            const prechecks = cachedPerFile("prechecks", path, content, () => {
-                const pre = [];
-                // A widget body is HTML in an iframe, never rendered as math. Only
-                // the root widgets/ folder: a nested widgets/ under Lenses/ or
-                // articles/ is parsed as a lens or article below, and renders math.
-                if (path.endsWith(".md") && !path.startsWith("widgets/")) {
-                    const mathErrors = validateMath(content, path);
-                    if (path.startsWith("articles/") || path.includes("/articles/")) {
-                        for (const error of mathErrors) {
-                            error.code ??= error.message.includes("unclosed $$")
-                                ? "article.math-unclosed"
-                                : "article.math-invalid";
-                        }
-                    }
-                    pre.push(...mathErrors);
-                }
-                // Warn about pending CriticMarkup suggestions on the raw content — the
-                // parsers strip them (reject-all behavior) before any other processing.
-                if (isCriticMarkupCheckedPath(path)) {
-                    pre.push(...detectPendingCriticMarkup(content, path));
-                }
-                return pre;
-            });
-            errors.push(...prechecks);
-        }
-        if (path.startsWith("modules/")) {
-            const result = inputs.cached("flatten-module", path, content, (f, t) => {
-                const { modules, errors } = flattenModule(path, f, new Set(), t);
-                return { modules, errors };
-            });
-            for (const mod of result.modules) {
-                modules.push(mod);
-                slugToPath.set(mod.slug, path);
-                // Track parent→children mapping for course expansion
-                if (mod.parentSlug) {
-                    if (!parentSlugToChildren.has(mod.parentSlug)) {
-                        parentSlugToChildren.set(mod.parentSlug, []);
+    function crossFile(kind, step, extraKey) {
+        if (!steps)
+            return step(files, tierMap);
+        return steps.cached(kind, path, content + (extraKey?.() ?? ""), step);
+    }
+    if (path.endsWith(".md") || isCriticMarkupCheckedPath(path)) {
+        const prechecks = cachedPerFile("prechecks", () => {
+            const pre = [];
+            // A widget body is HTML in an iframe, never rendered as math. Only
+            // the root widgets/ folder: a nested widgets/ under Lenses/ or
+            // articles/ is parsed as a lens or article below, and renders math.
+            if (path.endsWith(".md") && !path.startsWith("widgets/")) {
+                const mathErrors = validateMath(content, path);
+                if (path.startsWith("articles/") || path.includes("/articles/")) {
+                    for (const error of mathErrors) {
+                        error.code ??= error.message.includes("unclosed $$")
+                            ? "article.math-unclosed"
+                            : "article.math-invalid";
                     }
-                    parentSlugToChildren.get(mod.parentSlug).push(mod.slug);
-                }
-                else {
-                    filePathToSlug.set(path, mod.slug);
-                }
-                slugEntries.push({ slug: mod.slug, file: path });
-                if (mod.contentId) {
-                    uuidEntries.push({
-                        uuid: mod.contentId,
-                        file: path,
-                        field: "contentId",
-                    });
                 }
+                pre.push(...mathErrors);
             }
-            // If no submodules, set filePathToSlug for the primary module
-            if (result.modules.length === 1 && !result.modules[0].parentSlug) {
-                filePathToSlug.set(path, result.modules[0].slug);
+            // Warn about pending CriticMarkup suggestions on the raw content — the
+            // parsers strip them (reject-all behavior) before any other processing.
+            if (isCriticMarkupCheckedPath(path)) {
+                pre.push(...detectPendingCriticMarkup(content, path));
             }
-            else if (result.modules.length > 0 && result.modules[0].parentSlug) {
-                // For split modules, map the file path to the parent slug
-                filePathToSlug.set(path, result.modules[0].parentSlug);
+            // A lone ![[../Questions/...]] line is a question only in a lens file
+            const isLensFile = path.startsWith("Lenses/") || path.includes("/Lenses/");
+            if (path.endsWith(".md") && !isLensFile && !isQuestionFilePath(path)) {
+                pre.push(...questionEmbedsOutsideLenses(content, path));
             }
-            // Collect section-level id:: fields from Lens sections (inline lenses).
-            // Referenced lenses (with source::) get their id from the lens file itself.
-            if (result.modules.length > 0) {
-                const rawParse = parseModule(content, path);
-                if (rawParse.module) {
-                    collectInlineLensUUIDs(rawParse.module.sections, path, uuidEntries, seenResponseIds, errors);
-                    collectModuleLoRefs(rawParse.module.sections, path);
-                }
-                // Skill tree points at the user-visible module page: the primary
-                // (parent) module for split modules.
-                const primary = result.modules[0];
-                skillTreeModules.set(path, {
-                    slug: primary.parentSlug ?? primary.slug,
-                    title: primary.parentTitle ?? primary.title,
-                    tags: tagMap.get(path) ?? [],
+            return pre;
+        });
+        errors.push(...prechecks);
+    }
+    if (path.startsWith("modules/")) {
+        const result = crossFile("flatten-module", (f, t) => {
+            const { modules, errors, dropped } = flattenModule(path, f, new Set(), t);
+            return { modules, errors, dropped };
+        });
+        const pageItems = result.dropped;
+        for (const mod of result.modules) {
+            modules.push(mod);
+            if (!mod.parentSlug)
+                unit.moduleSlug = mod.slug;
+            if (mod.contentId) {
+                uuidEntries.push({
+                    uuid: mod.contentId,
+                    file: path,
+                    field: "contentId",
                 });
             }
-            errors.push(...result.errors);
         }
-        else if (path.startsWith("courses/")) {
-            const result = parseCourse(content, path);
-            if (result.course) {
-                courses.push(result.course);
-                courseSlugToFile.set(result.course.slug, path);
-            }
-            errors.push(...result.errors);
+        // If no submodules, the file's slug is the primary module's; a split
+        // module's file maps to the parent slug.
+        if (result.modules.length === 1 && !result.modules[0].parentSlug) {
+            unit.moduleSlug = result.modules[0].slug;
         }
-        else if (path.startsWith("meetings/") &&
-            path.endsWith(".md") &&
-            !path.startsWith(MEETING_DOC_SHARED_DIR)) {
-            const result = inputs.cached("meeting-doc", path, content, (f) => parseMeetingDoc(content, path, f));
-            errors.push(...result.errors);
-            if (result.markdown !== null)
-                meetingDocs[path] = result.markdown;
+        else if (result.modules.length > 0 && result.modules[0].parentSlug) {
+            unit.moduleSlug = result.modules[0].parentSlug;
         }
-        else if (path.startsWith("surveys/") || path.includes("/surveys/")) {
-            const result = parseSurvey(content, path);
-            errors.push(...result.errors);
-            if (result.survey) {
-                // Response segments and interviews carry prompt-capable instruction
-                // fields. Surveys have no flatten pass, so their prompt imports are
-                // expanded here; a failed import drops the survey, like a failed
-                // import drops a lens segment in the flattener.
-                let expansionFailed = false;
-                for (const seg of result.survey.segments) {
-                    const promptProps = seg.type === "interview"
-                        ? ["assessmentInstructions"]
-                        : seg.type === "question" && "questionType" in seg
-                            ? ["assessmentInstructions", "feedbackInstructions"]
-                            : [];
-                    for (const prop of promptProps) {
-                        const target = seg;
-                        const value = target[prop];
-                        if (!value)
-                            continue;
-                        const resolved = expandPromptImports(value, PROMPT_FIELD_NAMES[prop], path, files, tierMap);
-                        errors.push(...resolved.errors);
-                        if (resolved.value === null) {
-                            expansionFailed = true;
-                        }
-                        else {
-                            target[prop] = resolved.value;
-                        }
-                    }
-                }
-                if (!expansionFailed) {
-                    surveys.push({ ...result.survey, sourcePath: path });
-                    surveyPathToId.set(path, result.survey.id);
-                    uuidEntries.push({
-                        uuid: result.survey.id,
-                        file: path,
-                        field: "id",
-                    });
-                    collectResponseSegmentIds(result.survey.segments, "survey", path, uuidEntries, seenResponseIds, errors);
-                }
+        // Collect section-level id:: fields from Lens sections (inline lenses).
+        // Referenced lenses (with source::) get their id from the lens file itself.
+        if (result.modules.length > 0) {
+            const rawParse = parseModule(content, path);
+            if (rawParse.module) {
+                collectInlineLensUUIDs(rawParse.module.sections, path, uuidEntries, seenResponseIds, errors);
+                const loRefs = collectModuleLoRefs(rawParse.module.sections, path, files);
+                if (loRefs.length > 0)
+                    unit.loRefs = loRefs;
+                crossCheck(path, result.modules, pageItems, declaredSections(rawParse.module, path, files), rawParse.refused);
             }
+            // Skill tree points at the user-visible module page: the primary
+            // (parent) module for split modules.
+            const primary = result.modules[0];
+            unit.skillTreeModule = {
+                slug: primary.parentSlug ?? primary.slug,
+                title: primary.parentTitle ?? primary.title,
+                tags: tagMap.get(path) ?? [],
+            };
         }
-        else if (path.startsWith("Learning Outcomes/") ||
-            path.includes("/Learning Outcomes/")) {
-            // Fully validate Learning Outcome (structure, fields, wikilink syntax)
-            const result = parseLearningOutcome(content, path);
-            errors.push(...result.errors);
-            // Check that suggested lens files exist
-            if (result.learningOutcome) {
-                collectLearningOutcomeTestUUIDs(result.learningOutcome, path, uuidEntries, seenResponseIds, errors);
-                for (const lensRef of result.learningOutcome.suggestedLenses) {
-                    const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
-                    if (!lensPath) {
-                        // Find similar files to suggest
-                        const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
-                        const suggestion = formatSuggestion(similarFiles, path) ??
-                            "Check the file path in the wiki-link";
-                        errors.push({
-                            file: path,
-                            message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
-                            suggestion,
-                            severity: "error",
-                        });
-                        continue;
-                    }
-                    // Check tier violation (LO → Lens)
-                    const parentTier = tierMap.get(path) ?? "production";
-                    const childTier = tierMap.get(lensPath) ?? "production";
-                    const violation = checkTierViolation(path, parentTier, lensPath, childTier, "lens");
-                    if (violation) {
-                        errors.push(violation);
+        errors.push(...result.errors);
+        recordDropped(result.modules.length
+            ? pageEntries(path, result.modules, pageItems)
+            : failedModule(content, path, result.errors));
+    }
+    else if (path.startsWith("courses/")) {
+        const result = parseCourse(content, path);
+        if (result.course)
+            unit.course = result.course;
+        errors.push(...result.errors);
+    }
+    else if (path.startsWith("meetings/") &&
+        path.endsWith(".md") &&
+        !path.startsWith(MEETING_DOC_SHARED_DIR)) {
+        const result = crossFile("meeting-doc", (f) => parseMeetingDoc(content, path, f));
+        errors.push(...result.errors);
+        if (result.markdown !== null)
+            unit.meetingDoc = result.markdown;
+    }
+    else if (path.startsWith("surveys/") || path.includes("/surveys/")) {
+        const result = parseSurvey(content, path);
+        errors.push(...result.errors);
+        if (result.survey) {
+            // Response segments and interviews carry prompt-capable instruction
+            // fields. Surveys have no flatten pass, so their prompt imports are
+            // expanded here; a failed import drops the survey, like a failed
+            // import drops a lens segment in the flattener.
+            let expansionFailed = false;
+            for (const seg of result.survey.segments) {
+                const promptProps = seg.type === "interview"
+                    ? ["assessmentInstructions"]
+                    : seg.type === "question" && "questionType" in seg
+                        ? ["assessmentInstructions", "feedbackInstructions"]
+                        : [];
+                for (const prop of promptProps) {
+                    const target = seg;
+                    const value = target[prop];
+                    if (!value)
                         continue;
+                    const resolved = expandPromptImports(value, PROMPT_FIELD_NAMES[prop], path, files, tierMap);
+                    errors.push(...resolved.errors);
+                    if (resolved.value === null) {
+                        expansionFailed = true;
                     }
-                    if (childTier === "ignored") {
-                        continue;
+                    else {
+                        target[prop] = resolved.value;
                     }
                 }
             }
-            // Collect id for UUID validation
-            if (result.learningOutcome?.id) {
+            if (!expansionFailed) {
+                unit.survey = { ...result.survey, sourcePath: path };
                 uuidEntries.push({
-                    uuid: result.learningOutcome.id,
+                    uuid: result.survey.id,
                     file: path,
                     field: "id",
                 });
-            }
-            if (result.learningOutcome) {
-                learningOutcomesByPath.set(path, result.learningOutcome);
+                collectResponseSegmentIds(result.survey.segments, "survey", path, uuidEntries, seenResponseIds, errors);
             }
         }
-        else if (path.startsWith("Domains and Topics/") ||
-            path.includes("/Domains and Topics/")) {
-            // Skill-tree domain folders: `Domains and Topics/<N> <Title>/` holds the
-            // domain's topic files and an optional _domain_metadata.md. The folder
-            // exists as soon as any file inside it does.
-            if (!path.endsWith(".md"))
-                continue;
-            const located = parseDomainPath(path);
-            if (!located.location) {
-                errors.push(located.error);
-                continue;
-            }
-            const { dir, number, title, fileName } = located.location;
-            let folder = domainFolders.get(dir);
-            if (!folder) {
-                folder = { number, title, declaredBy: path };
-                domainFolders.set(dir, folder);
-            }
-            if (fileName === DOMAIN_METADATA_FILE) {
-                const result = parseDomainMetadata(content, path);
-                errors.push(...result.errors);
-                if (result.metadata)
-                    folder.metadata = result.metadata;
-            }
-            else {
-                const result = parseTopicFile(content, path);
-                errors.push(...result.errors);
-                if (result.topic) {
-                    topicFiles.set(path, { ...result.topic, domainDir: dir });
-                    uuidEntries.push({ uuid: result.topic.id, file: path, field: "id" });
+    }
+    else if (path.startsWith("Learning Outcomes/") ||
+        path.includes("/Learning Outcomes/")) {
+        // Fully validate Learning Outcome (structure, fields, wikilink syntax)
+        const result = parseLearningOutcome(content, path);
+        errors.push(...result.errors);
+        // Check that suggested lens files exist
+        if (result.learningOutcome) {
+            collectLearningOutcomeTestUUIDs(result.learningOutcome, path, uuidEntries, seenResponseIds, errors);
+            for (const lensRef of result.learningOutcome.suggestedLenses) {
+                const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
+                if (!lensPath) {
+                    // Find similar files to suggest
+                    const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
+                    const suggestion = formatSuggestion(similarFiles, path) ??
+                        "Check the file path in the wiki-link";
+                    errors.push({
+                        file: path,
+                        message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
+                        suggestion,
+                        severity: "error",
+                    });
+                    continue;
+                }
+                // Check tier violation (LO → Lens)
+                const parentTier = tierMap.get(path) ?? "production";
+                const childTier = tierMap.get(lensPath) ?? "production";
+                const violation = checkTierViolation(path, parentTier, lensPath, childTier, "lens");
+                if (violation) {
+                    errors.push(violation);
+                    continue;
+                }
+                if (childTier === "ignored") {
+                    continue;
                 }
             }
         }
-        else if (path.startsWith("Lenses/") || path.includes("/Lenses/")) {
-            // Fully validate Lens (structure, segments, fields)
-            const result = cachedPerFile("lens", path, content, () => parseLens(content, path));
+        // Collect id for UUID validation
+        if (result.learningOutcome?.id) {
+            uuidEntries.push({
+                uuid: result.learningOutcome.id,
+                file: path,
+                field: "id",
+            });
+        }
+        if (result.learningOutcome) {
+            unit.learningOutcome = result.learningOutcome;
+        }
+    }
+    else if (path.startsWith("Domains and Topics/") ||
+        path.includes("/Domains and Topics/")) {
+        // Skill-tree domain folders: `Domains and Topics/<N> <Title>/` holds the
+        // domain's topic files and an optional _domain_metadata.md. The folder
+        // exists as soon as any file inside it does.
+        if (!path.endsWith(".md"))
+            return unit;
+        const located = parseDomainPath(path);
+        if (!located.location) {
+            errors.push(located.error);
+            return unit;
+        }
+        const { dir, number, title, fileName } = located.location;
+        unit.domainFolder = { dir, number, title };
+        if (fileName === DOMAIN_METADATA_FILE) {
+            const result = parseDomainMetadata(content, path);
             errors.push(...result.errors);
-            // Collect id for UUID validation
-            if (result.lens?.id) {
-                uuidEntries.push({
-                    uuid: result.lens.id,
-                    file: path,
-                    field: "id",
-                });
-            }
-            if (result.lens) {
-                collectResponseSegmentIds(result.lens.segments, "lens", path, uuidEntries, seenResponseIds, errors);
-            }
-            // Flatten lens as standalone module (pass pre-parsed lens to avoid re-parsing)
-            if (result.lens) {
-                const lens = result.lens;
-                const lensModuleResult = inputs.cached("flatten-lens", path, content, (f, t) => {
-                    const { module, errors } = flattenLens(path, f, t, lens);
-                    return { module, errors };
-                });
-                if (lensModuleResult.module) {
-                    modules.push(lensModuleResult.module);
-                    slugEntries.push({ slug: lensModuleResult.module.slug, file: path });
-                    slugToPath.set(lensModuleResult.module.slug, path);
-                }
-                errors.push(...lensModuleResult.errors);
+            if (result.metadata)
+                unit.domainFolder.metadata = result.metadata;
+        }
+        else {
+            const result = parseTopicFile(content, path);
+            errors.push(...result.errors);
+            if (result.topic) {
+                unit.topic = { ...result.topic, domainDir: dir };
+                uuidEntries.push({ uuid: result.topic.id, file: path, field: "id" });
             }
         }
-        else if (path.endsWith(".timestamps.json")) {
-            const tsErrors = validateTimestamps(content, path);
-            errors.push(...tsErrors);
+    }
+    else if (isQuestionFilePath(path)) {
+        // A question file: one question, embedded by lenses with ![[...]]
+        const fileResult = cachedPerFile("question", () => {
+            const parsed = parseQuestionFile(stripAuthoringMarkupForValidation(content), path);
+            const fileErrors = [...parsed.errors];
+            // Checked as the question it becomes in a lens
+            const converted = parsed.segment
+                ? convertRawSegments([parsed.segment], path)
+                : { segments: [], errors: [] };
+            fileErrors.push(...converted.errors);
+            return { errors: fileErrors, segments: converted.segments };
+        });
+        errors.push(...fileResult.errors);
+        // Its id is taken: anything else using it must be this same question
+        if (fileResult.segments.length > 0) {
+            collectResponseSegmentIds(fileResult.segments, "lens", path, uuidEntries, seenResponseIds, errors);
         }
-        else if (path.startsWith("articles/") || path.includes("/articles/")) {
-            // The article's whole per-file pass (parse + structure + provenance) is
-            // pure in (content, path); its two effects — errors and URL entries —
-            // are the cached value.
-            const fileResult = cachedPerFile("article", path, content, () => {
-                const result = parseArticle(content, path);
-                const parsedFrontmatter = parseFrontmatter(stripAuthoringMarkup(content), path);
-                const body = parsedFrontmatter.error
-                    ? undefined
-                    : parsedFrontmatter.body;
-                // Standalone-page segments are emitted only when the page differs
-                // from the raw body. Without inline imports they depend on this file
-                // alone, so they are built (and cached) here; with imports they need
-                // the file map and are built below.
-                const parsedBody = body === undefined ? undefined : parseArticleBody(content, path);
-                // Elements resolve against other files (transcripts, widgets, prompt
-                // imports), so an article that has any is flattened with the file map
-                // below.
-                const hasImports = parsedBody?.markers.some((m) => m.kind === "element") ?? false;
-                let standaloneSegments;
-                if (parsedBody && !hasImports) {
-                    const { segments } = flattenStandaloneArticle(path, new Map([[path, content]]));
-                    // Linked author names (author_url) only exist on processor
-                    // segments; the API's plain fallback carries the name alone.
-                    const plain = segments.length === 1 &&
-                        segments[0].type === "article" &&
-                        segments[0].content === parsedBody.body &&
-                        !segments[0].authorLinks;
-                    if (!plain)
-                        standaloneSegments = segments;
-                }
-                const fileErrors = [
-                    ...result.errors,
-                    ...validateArticleStructure(content, path),
-                    ...validateArticleReviewProvenance(content, path),
-                ];
-                const urls = [];
-                if (result.article) {
-                    // Only carry the flag when set true, so URL entries (and thus the
-                    // serialized output / Python parity) are unchanged for normal articles.
-                    const overrideFlag = result.article.allowUnreachableUrl
-                        ? { allowUnreachable: true }
-                        : {};
+    }
+    else if (path.startsWith("Lenses/") || path.includes("/Lenses/")) {
+        // Fully validate Lens (structure, segments, fields). The segments the
+        // parser refused ride on the lens as a Symbol (src/dropped.ts), which a
+        // cache entry drops, so the entry keeps them as a field and a hit puts
+        // them back. The parse also reads the question files the lens embeds,
+        // and the flatten step below reuses it, so both cache keys cover them.
+        const embeds = () => questionEmbedCacheKey(content, path, files);
+        const result = cachedPerFile("lens", () => {
+            const parsed = parseLens(content, path, files);
+            const refused = parsed.lens ? droppedSegmentsOf(parsed.lens) : [];
+            return { ...parsed, refused };
+        }, embeds);
+        if (result.lens && droppedSegmentsOf(result.lens).length === 0) {
+            attachDroppedSegments(result.lens, result.refused);
+        }
+        errors.push(...result.errors);
+        // Collect id for UUID validation
+        if (result.lens?.id) {
+            uuidEntries.push({
+                uuid: result.lens.id,
+                file: path,
+                field: "id",
+            });
+        }
+        if (result.lens) {
+            collectResponseSegmentIds(result.lens.segments, "lens", path, uuidEntries, seenResponseIds, errors);
+        }
+        // Flatten lens as standalone module (pass pre-parsed lens to avoid re-parsing)
+        if (result.lens) {
+            const lens = result.lens;
+            const lensModuleResult = crossFile("flatten-lens", (f, t) => {
+                const { modules, errors, dropped } = flattenLens(path, f, t, lens);
+                return { modules, errors, dropped };
+            }, embeds);
+            modules.push(...lensModuleResult.modules);
+            errors.push(...lensModuleResult.errors);
+            recordDropped(pageEntries(path, lensModuleResult.modules, lensModuleResult.dropped));
+        }
+        else {
+            const page = "lens/" + fileNameToSlug(path);
+            recordDropped([failedPage(page, path, result.errors)]);
+        }
+    }
+    else if (path.endsWith(".timestamps.json")) {
+        const tsErrors = validateTimestamps(content, path);
+        errors.push(...tsErrors);
+    }
+    else if (path.startsWith("articles/") || path.includes("/articles/")) {
+        // The article's whole per-file pass (parse + structure + provenance) is
+        // pure in (content, path); its two effects — errors and URL entries —
+        // are the cached value.
+        const fileResult = cachedPerFile("article", () => {
+            const result = parseArticle(content, path);
+            const parsedFrontmatter = parseFrontmatter(stripAuthoringMarkup(content), path);
+            const body = parsedFrontmatter.error ? undefined : parsedFrontmatter.body;
+            // Standalone-page segments are emitted only when the page differs
+            // from the raw body. Without inline imports they depend on this file
+            // alone, so they are built (and cached) here; with imports they need
+            // the file map and are built below.
+            const parsedBody = body === undefined ? undefined : parseArticleBody(content, path);
+            // Elements resolve against other files (transcripts, widgets, prompt
+            // imports), so an article that has any is flattened with the file map
+            // below.
+            const hasImports = parsedBody?.markers.some((m) => m.kind === "element") ?? false;
+            let standaloneSegments;
+            if (parsedBody && !hasImports) {
+                const { segments } = flattenStandaloneArticle(path, new ContentView(new Map([[path, content]])));
+                // Linked author names (author_url) only exist on processor
+                // segments; the API's plain fallback carries the name alone.
+                const plain = segments.length === 1 &&
+                    segments[0].type === "article" &&
+                    segments[0].content === parsedBody.body &&
+                    !segments[0].authorLinks;
+                if (!plain)
+                    standaloneSegments = segments;
+            }
+            const fileErrors = [
+                ...result.errors,
+                ...validateArticleStructure(content, path),
+                ...validateArticleReviewProvenance(content, path),
+            ];
+            const urls = [];
+            if (result.article) {
+                // Only carry the flag when set true, so URL entries (and thus the
+                // serialized output / Python parity) are unchanged for normal articles.
+                const overrideFlag = result.article.allowUnreachableUrl
+                    ? { allowUnreachable: true }
+                    : {};
+                urls.push({
+                    url: result.article.sourceUrl,
+                    file: path,
+                    line: 2,
+                    label: "source_url",
+                    ...overrideFlag,
+                });
+                for (const url of result.article.authorUrls) {
                     urls.push({
-                        url: result.article.sourceUrl,
+                        url,
                         file: path,
                         line: 2,
-                        label: "source_url",
+                        label: "author_url",
+                        ...overrideFlag,
+                    });
+                }
+                for (const img of result.article.imageUrls) {
+                    urls.push({
+                        url: img.url,
+                        file: path,
+                        line: img.line,
+                        label: "Image URL",
                         ...overrideFlag,
                     });
-                    for (const url of result.article.authorUrls) {
-                        urls.push({
-                            url,
-                            file: path,
-                            line: 2,
-                            label: "author_url",
-                            ...overrideFlag,
-                        });
-                    }
-                    for (const img of result.article.imageUrls) {
-                        urls.push({
-                            url: img.url,
-                            file: path,
-                            line: img.line,
-                            label: "Image URL",
-                            ...overrideFlag,
-                        });
-                    }
                 }
-                return {
-                    errors: fileErrors,
-                    urls,
-                    hasImports,
-                    standaloneSegments,
-                    elements: result.article?.elements ?? [],
-                };
-            });
-            errors.push(...fileResult.errors);
-            urlsToValidate.push(...fileResult.urls);
-            // Questions declared inline in the article body carry ids like any
-            // response segment
-            collectResponseSegmentIds(fileResult.elements, "lens", path, uuidEntries, seenResponseIds, errors);
-            if (fileResult.hasImports) {
-                const standalone = inputs.cached("standalone-article", path, content, (f, t) => {
-                    const { segments, errors } = flattenStandaloneArticle(path, f, t);
-                    return { segments, errors };
-                });
-                errors.push(...standalone.errors);
-                articleSegments[path] = standalone.segments;
-            }
-            else if (fileResult.standaloneSegments) {
-                articleSegments[path] = fileResult.standaloneSegments;
             }
+            return {
+                errors: fileErrors,
+                urls,
+                hasImports,
+                standaloneSegments,
+                elements: result.article?.elements ?? [],
+            };
+        });
+        errors.push(...fileResult.errors);
+        unit.urls.push(...fileResult.urls);
+        // Questions declared inline in the article body carry ids like any
+        // response segment
+        collectResponseSegmentIds(fileResult.elements, "lens", path, uuidEntries, seenResponseIds, errors);
+        if (fileResult.hasImports) {
+            const standalone = crossFile("standalone-article", (f, t) => flattenStandaloneArticle(path, f, t));
+            errors.push(...standalone.errors);
+            unit.articleSegments = standalone.segments;
+            // The platform shows the article as one section, with no contentId,
+            // on page article/<slug>.
+            const page = { slug: "article/" + fileNameToSlug(path) };
+            const items = standalone.dropped.map((drop) => placedAfter(0, [], drop));
+            recordDropped(pageEntries(path, [page], [items]));
         }
-        else if (path.startsWith("video_transcripts/") ||
-            path.includes("/video_transcripts/")) {
-            const result = parseVideoTranscript(content, path);
-            errors.push(...result.errors);
-            if (result.transcript) {
-                urlsToValidate.push({
-                    url: result.transcript.url,
-                    file: path,
-                    line: 2,
-                    label: "url",
-                    ...(result.transcript.allowUnreachableUrl
-                        ? { allowUnreachable: true }
-                        : {}),
-                });
-            }
+        else if (fileResult.standaloneSegments) {
+            unit.articleSegments = fileResult.standaloneSegments;
         }
-        else if (isGlossaryPath(path) && path.endsWith(".md")) {
-            const result = parseGlossaryEntry(content, path);
-            errors.push(...result.errors);
-            if (result.entry)
-                glossaryEntries.set(path, result.entry);
+    }
+    else if (path.startsWith("video_transcripts/") ||
+        path.includes("/video_transcripts/")) {
+        const result = parseVideoTranscript(content, path);
+        errors.push(...result.errors);
+        if (result.transcript) {
+            unit.urls.push({
+                url: result.transcript.url,
+                file: path,
+                line: 2,
+                label: "url",
+                ...(result.transcript.allowUnreachableUrl
+                    ? { allowUnreachable: true }
+                    : {}),
+            });
         }
-        else if (isPromptPath(path)) {
-            // Fully validate prompt file (frontmatter + non-empty prompt text)
-            const result = parsePrompt(content, path);
-            errors.push(...result.errors);
+    }
+    else if (isGlossaryPath(path) && path.endsWith(".md")) {
+        const result = parseGlossaryEntry(content, path);
+        errors.push(...result.errors);
+        if (result.entry)
+            unit.glossaryEntry = result.entry;
+    }
+    else if (isPromptPath(path)) {
+        // Fully validate prompt file (frontmatter + non-empty prompt text)
+        const result = parsePrompt(content, path);
+        errors.push(...result.errors);
+    }
+    else if (isWidgetFile(path)) {
+        // Widget file: frontmatter + HTML body (validity, size, visible text)
+        if (path.endsWith(".html") && files.has(path.replace(/\.html$/, ".md"))) {
+            errors.push({
+                file: path,
+                message: `Two files for one widget: ${path} and its .md twin`,
+                suggestion: "A link to the widget cannot tell them apart. Delete the .md file once the .html one is complete",
+                severity: "error",
+            });
         }
-        else if (isWidgetFile(path)) {
-            // Widget file: frontmatter + HTML body (validity, size, visible text)
-            if (path.endsWith(".html") && files.has(path.replace(/\.html$/, ".md"))) {
-                errors.push({
-                    file: path,
-                    message: `Two files for one widget: ${path} and its .md twin`,
-                    suggestion: "A link to the widget cannot tell them apart. Delete the .md file once the .html one is complete",
-                    severity: "error",
-                });
+        const widgetResult = cachedPerFile("widget", () => {
+            const result = parseWidget(content, path);
+            return { errors: result.errors, id: result.widget?.id };
+        });
+        errors.push(...widgetResult.errors);
+        if (widgetResult.id) {
+            uuidEntries.push({ uuid: widgetResult.id, file: path, field: "id" });
+        }
+    }
+    else {
+        // File didn't match any known directory pattern — check for near-misses via Levenshtein distance
+        const dir = path.split("/")[0];
+        const VALID_DIRS = [
+            "modules",
+            "courses",
+            "surveys",
+            "articles",
+            "Lenses",
+            "Questions",
+            "video_transcripts",
+            "Learning Outcomes",
+            "Domains and Topics",
+            "prompts",
+            "widgets",
+            "glossary",
+        ];
+        let closest = "";
+        let minDist = Infinity;
+        for (const valid of VALID_DIRS) {
+            const dist = levenshtein(dir.toLowerCase(), valid.toLowerCase());
+            if (dist < minDist) {
+                minDist = dist;
+                closest = valid;
             }
-            const widgetResult = cachedPerFile("widget", path, content, () => {
-                const result = parseWidget(content, path);
-                return { errors: result.errors, id: result.widget?.id };
+        }
+        // Threshold: distance <= 3 or <= 40% of the directory name length (whichever is smaller)
+        const threshold = Math.min(3, Math.ceil(dir.length * 0.4));
+        if (minDist > 0 && minDist <= threshold) {
+            errors.push({
+                file: path,
+                message: `File in directory '${dir}/' not recognized as content`,
+                suggestion: `Did you mean '${closest}/'?`,
+                severity: "warning",
             });
-            errors.push(...widgetResult.errors);
-            if (widgetResult.id) {
-                uuidEntries.push({ uuid: widgetResult.id, file: path, field: "id" });
-            }
         }
-        else {
-            // File didn't match any known directory pattern — check for near-misses via Levenshtein distance
-            const dir = path.split("/")[0];
-            const VALID_DIRS = [
-                "modules",
-                "courses",
-                "surveys",
-                "articles",
-                "Lenses",
-                "video_transcripts",
-                "Learning Outcomes",
-                "Domains and Topics",
-                "prompts",
-                "widgets",
-                "glossary",
-            ];
-            let closest = "";
-            let minDist = Infinity;
-            for (const valid of VALID_DIRS) {
-                const dist = levenshtein(dir.toLowerCase(), valid.toLowerCase());
-                if (dist < minDist) {
-                    minDist = dist;
-                    closest = valid;
+    }
+    return unit;
+}
+export function processContent(files, cache) {
+    // Parsed markdown is memoized for the length of one run (markdown-parse.ts).
+    // Drop the trees afterwards so a long-lived caller does not keep them.
+    try {
+        return processContentRun(files, cache);
+    }
+    finally {
+        clearParsedMarkdown();
+    }
+}
+function processContentRun(input, cache) {
+    // Iteration order of `files` decides the order of every output array below, so it must
+    // not depend on how a caller happened to enumerate the filesystem. Callers do differ:
+    // the Node vault reader and the Python backend both walk directories without sorting,
+    // and the --stdin path takes whatever order its JSON arrived in. Sorting here once
+    // makes the output a function of the content alone, for every caller.
+    const files = inPathOrder(input);
+    // Pre-scan: frontmatter tags of every file, and the tier they imply
+    const tags = buildTagMap(files);
+    const tiers = buildTierMap(tags);
+    const views = {
+        files,
+        tierMap: new ContentView(tiers),
+        tagMap: new ContentView(tags),
+    };
+    // With a cache, each file's steps are cached (runPathUnit, dependency-cache.ts).
+    const steps = cache && new RunInputs(new Map(files), tiers, cache);
+    const units = [...files.keys()].map((path) => runPathUnit(path, views, steps));
+    return assembleContent(units, views);
+}
+/**
+ * Everything after the file units: the whole-content step, every page
+ * finished, and the errors in today's order. `units` holds one entry per
+ * content file, in path order; their pages are finished in place. The engine
+ * runs the same three steps on stored results (engine/build.ts), so a rule
+ * changes in one place for both.
+ */
+export function assembleContent(units, views) {
+    const whole = wholeContentStep(units.map(summarizeUnit), views);
+    const pages = units.flatMap((unit) => unit.modules.map((page, index) => ({
+        page,
+        key: pageKey(unit.path, index),
+    })));
+    const finished = pages.map(({ page, key }) => ({
+        source: key,
+        finish: finishPage(page, key, whole.tables, views),
+    }));
+    const { errors } = assembleErrors(units.map((unit) => ({ source: unit.path, errors: unit.errors })), whole.errors, finished, views.tierMap);
+    return processResult(pages.map(({ page }) => page), whole, errors);
+}
+export function summarizeUnit({ modules, ...unit }) {
+    return {
+        ...unit,
+        pages: modules.map((page) => ({
+            slug: page.slug,
+            parentSlug: page.parentSlug,
+            contentIds: page.sections.flatMap((section) => section.contentId ? [section.contentId] : []),
+            tests: page.sections.flatMap((section) => section.type === "test" && section.sourcePath
+                ? [
+                    {
+                        sourcePath: section.sourcePath,
+                        outcomes: section.learningOutcomes,
+                    },
+                ]
+                : []),
+        })),
+    };
+}
+/**
+ * The whole-content step: merge every file's summary into the tables that
+ * span files and run the checks across files. It reads facts and page
+ * summaries, never a page body. `units` holds one entry per content file, in
+ * path order; their courses and question entries are changed in place.
+ */
+export function wholeContentStep(units, { files, tierMap, tagMap }) {
+    const articleSegments = {};
+    const courses = [];
+    const surveys = [];
+    const surveyPathToId = new Map();
+    const meetingDocs = {};
+    const urlsToValidate = [];
+    const uuidEntries = [];
+    const slugEntries = [];
+    const slugToPath = new Map();
+    const filePathToSlug = new Map(); // Reverse: file path → slug (survives duplicate slugs)
+    const courseSlugToFile = new Map();
+    const parentSlugToChildren = new Map();
+    const pages = [];
+    // --- Skill-tree collections (assembled after the merge) ---
+    // Domain folder path → folder; topic file path → topic (+ its folder).
+    const domainFolders = new Map();
+    const topicFiles = new Map();
+    const learningOutcomesByPath = new Map();
+    // LO path → module path → the submodules (local slug → title) that
+    // reference it; empty when the module is not split or the LO is module-level.
+    const moduleLoRefs = new Map();
+    const glossaryEntries = new Map();
+    const skillTreeModules = new Map();
+    for (const unit of units) {
+        const { path } = unit;
+        unit.pages.forEach((page, index) => {
+            pages.push({ ...page, key: pageKey(path, index) });
+            slugToPath.set(page.slug, path);
+            // Track parent→children mapping for course expansion
+            if (page.parentSlug) {
+                if (!parentSlugToChildren.has(page.parentSlug)) {
+                    parentSlugToChildren.set(page.parentSlug, []);
                 }
+                parentSlugToChildren.get(page.parentSlug).push(page.slug);
             }
-            // Threshold: distance <= 3 or <= 40% of the directory name length (whichever is smaller)
-            const threshold = Math.min(3, Math.ceil(dir.length * 0.4));
-            if (minDist > 0 && minDist <= threshold) {
-                errors.push({
-                    file: path,
-                    message: `File in directory '${dir}/' not recognized as content`,
-                    suggestion: `Did you mean '${closest}/'?`,
-                    severity: "warning",
-                });
+            slugEntries.push({ slug: page.slug, file: path });
+        });
+        if (unit.moduleSlug !== undefined) {
+            filePathToSlug.set(path, unit.moduleSlug);
+        }
+        urlsToValidate.push(...unit.urls);
+        uuidEntries.push(...unit.uuids);
+        if (unit.course) {
+            courses.push(unit.course);
+            courseSlugToFile.set(unit.course.slug, path);
+        }
+        if (unit.survey) {
+            surveys.push(unit.survey);
+            surveyPathToId.set(path, unit.survey.id);
+        }
+        if (unit.meetingDoc !== undefined)
+            meetingDocs[path] = unit.meetingDoc;
+        if (unit.articleSegments)
+            articleSegments[path] = unit.articleSegments;
+        if (unit.skillTreeModule)
+            skillTreeModules.set(path, unit.skillTreeModule);
+        if (unit.learningOutcome) {
+            learningOutcomesByPath.set(path, unit.learningOutcome);
+        }
+        for (const { lo, submodule } of unit.loRefs ?? []) {
+            let byModule = moduleLoRefs.get(lo);
+            if (!byModule) {
+                byModule = new Map();
+                moduleLoRefs.set(lo, byModule);
+            }
+            let submodules = byModule.get(path);
+            if (!submodules) {
+                submodules = new Map();
+                byModule.set(path, submodules);
             }
+            if (submodule)
+                submodules.set(submodule.slug, submodule.title);
         }
-    }
-    errors.push(...validateGlossaryAliases(glossaryEntries));
-    // Build contentId → moduleSlug mapping and populate moduleSlug in cross-module card links.
-    // Prefer real modules over standalone lenses (lens/ prefix) — a lens that appears in
-    // both a module and as a standalone should map to the module.
-    const contentIdToModuleSlug = new Map();
-    for (const mod of modules) {
-        if (mod.slug.startsWith("lens/"))
-            continue; // Skip standalone lenses
-        for (const section of mod.sections) {
-            if (section.contentId) {
-                contentIdToModuleSlug.set(section.contentId, mod.slug);
+        if (unit.domainFolder) {
+            // The first file in a folder (in path order) declares it.
+            const { dir, number, title, metadata } = unit.domainFolder;
+            let folder = domainFolders.get(dir);
+            if (!folder) {
+                folder = { number, title, declaredBy: path };
+                domainFolders.set(dir, folder);
             }
+            if (metadata)
+                folder.metadata = metadata;
         }
+        if (unit.topic)
+            topicFiles.set(path, unit.topic);
+        if (unit.glossaryEntry)
+            glossaryEntries.set(path, unit.glossaryEntry);
     }
-    for (const mod of modules) {
-        populateCardModuleSlugs(mod.sections, contentIdToModuleSlug);
-        resolveInlineLensModuleSlugs(mod.sections, contentIdToModuleSlug);
-    }
-    // Glossary marks are offsets into segment text, so they are placed after
-    // every pass that rewrites it (card links and lens links above, card
-    // metadata in the flattener).
-    for (const mod of modules) {
-        for (const section of mod.sections) {
-            if (section.glossaryTerms === undefined)
-                continue;
-            const result = applyGlossary(section.glossaryTerms, section.segments, section.sourcePath ?? mod.sourcePath ?? mod.slug, files, glossaryEntries, tierMap);
-            errors.push(...result.errors);
-            section.segments = result.segments;
-            if (result.glossary.length)
-                section.glossary = result.glossary;
-            delete section.glossaryTerms;
+    const glossaryAliases = validateGlossaryAliases(glossaryEntries);
+    // Which module a card or a lens link names: the module page that holds the
+    // lens. Real modules only: a lens that appears in both a module and as a
+    // standalone page maps to the module.
+    const contentIdToModule = new Map();
+    for (const page of pages) {
+        if (page.slug.startsWith("lens/"))
+            continue; // Skip standalone lenses
+        for (const contentId of page.contentIds) {
+            contentIdToModule.set(contentId, page.slug);
         }
     }
-    // Card-to-module membership check. Runs after card link resolution above so
-    // every card has its data-lens-card JSON populated, including allowExternal.
-    errors.push(...validateCardModuleMembership(modules));
-    // After card resolution: each require_x_optional_lenses list becomes its
-    // section's optionalLensSets (the lens gate reads those).
-    errors.push(...collectOptionalLensSets(modules));
+    const courseErrors = [];
     /**
      * Resolve one survey wikilink (from a meeting's survey:: /
      * facilitator-survey::, or the course's application-survey::) to the
@@ -780,11 +910,11 @@ function processContentRun(files, cache) {
             const childTier = tierMap.get(actualFile) ?? "production";
             const violation = checkTierViolation(courseFile, parentTier, actualFile, childTier, label);
             if (violation) {
-                errors.push(violation);
+                courseErrors.push(violation);
             }
             return surveyPathToId.get(actualFile);
         }
-        errors.push({
+        courseErrors.push({
             file: courseFile,
             message: `Survey reference could not be resolved: "${surveyPath}"`,
             suggestion: `Check that ${label}:: points to an existing, error-free file in surveys/`,
@@ -818,7 +948,7 @@ function processContentRun(files, cache) {
                         }
                     }
                     if (!matched) {
-                        errors.push({
+                        courseErrors.push({
                             file: courseFile,
                             message: `Module reference could not be resolved: "${item.path}"`,
                             suggestion: "Check that the wikilink path points to an existing module file",
@@ -853,7 +983,7 @@ function processContentRun(files, cache) {
                         item.docPath = actualFile;
                     }
                     else {
-                        errors.push({
+                        courseErrors.push({
                             file: courseFile,
                             message: `Meeting doc reference could not be resolved: "${item.docPath}"`,
                             suggestion: "Check that meeting-doc:: points to an existing, error-free file in meetings/",
@@ -923,7 +1053,7 @@ function processContentRun(files, cache) {
                 const childTier = tierMap.get(modulePath) ?? "production";
                 const violation = checkTierViolation(coursePath, parentTier, modulePath, childTier, "module");
                 if (violation) {
-                    errors.push(violation);
+                    courseErrors.push(violation);
                 }
             }
         }
@@ -944,7 +1074,7 @@ function processContentRun(files, cache) {
             for (const alias of course.slugAliases ?? []) {
                 const existing = allCourseSlugs.get(alias);
                 if (existing) {
-                    errors.push({
+                    courseErrors.push({
                         file,
                         message: `Course slug alias '${alias}' collides with ${existing === file ? "its own primary slug" : `slug in ${existing}`}`,
                         suggestion: `Choose a different alias or remove the conflicting slug`,
@@ -968,12 +1098,9 @@ function processContentRun(files, cache) {
         }
     }
     // Validate all collected UUIDs
-    const uuidValidation = validateUuids(uuidEntries);
-    errors.push(...uuidValidation.errors);
-    errors.push(...detectSharedQuestionsInOneModule(modules));
+    const uuidErrors = validateUuids(uuidEntries).errors;
     // Validate for duplicate slugs
-    const duplicateSlugErrors = detectDuplicateSlugs(slugEntries);
-    errors.push(...duplicateSlugErrors);
+    const slugsAndTranscripts = detectDuplicateSlugs(slugEntries);
     // Validate video transcript / timestamps.json pairing
     const transcriptPaths = [...files.keys()].filter((p) => (p.startsWith("video_transcripts/") ||
         p.includes("/video_transcripts/")) &&
@@ -983,7 +1110,7 @@ function processContentRun(files, cache) {
     for (const mdPath of transcriptPaths) {
         const expectedTs = mdPath.replace(/\.md$/, ".timestamps.json");
         if (!timestampPaths.has(expectedTs)) {
-            errors.push({
+            slugsAndTranscripts.push({
                 file: mdPath,
                 message: `Missing timestamps.json: expected ${expectedTs}`,
                 severity: "error",
@@ -995,20 +1122,42 @@ function processContentRun(files, cache) {
         if (tierMap.get(expectedMd) === "ignored")
             continue;
         if (!files.has(expectedMd)) {
-            errors.push({
+            slugsAndTranscripts.push({
                 file: tsPath,
                 message: `Orphaned timestamps file: no matching .md transcript found`,
                 severity: "warning",
             });
         }
     }
-    // Safety-net: catch empty sections/segments in final output
-    const integrityErrors = validateOutputIntegrity(modules, slugToPath);
-    errors.push(...integrityErrors);
-    tagCourseOutcomes(modules, courses, (loPath) => {
+    // A course teaches an outcome across units but tests it once, so a lens
+    // also carries every outcome that suggests it and is tested in any module
+    // of a course the lens's module is in (marked testedElsewhere, after the
+    // tags from the lens's own submodule). Courses link to module slugs (split
+    // modules expanded above); the last page with a slug is the one a slug
+    // names. Each page gets one outcome list per course it is in, in course
+    // order, and finishPage tags its lens sections with them.
+    const lensPathsOf = (loPath) => {
         const lo = learningOutcomesByPath.get(loPath);
-        return lo ? suggestedLensPaths(lo, files) : new Set();
-    });
+        return lo ? suggestedLensPaths(lo, files) : [];
+    };
+    const pageBySlug = new Map(pages.map((page) => [page.slug, page]));
+    const courseOutcomes = new Map();
+    for (const course of courses) {
+        const coursePages = course.progression.flatMap((item) => {
+            const page = item.slug ? pageBySlug.get(item.slug) : undefined;
+            return item.type === "module" && page ? [page] : [];
+        });
+        const outcomes = coursePages.flatMap((page) => page.tests.flatMap((test) => test.outcomes.map((ref) => ({
+            ref: { ...ref, testedElsewhere: true },
+            lensPaths: lensPathsOf(test.sourcePath),
+        }))));
+        for (const page of new Set(coursePages)) {
+            let lists = courseOutcomes.get(page.key);
+            if (!lists)
+                courseOutcomes.set(page.key, (lists = []));
+            lists.push(outcomes);
+        }
+    }
     // --- Skill tree assembly ---
     // Course progression was expanded to submodule slugs above; map back to the
     // parent module slug, which is what taughtBy points at.
@@ -1042,44 +1191,167 @@ function processContentRun(files, cache) {
         courses: skillTreeCourses,
         files,
     });
-    errors.push(...skillTreeResult.errors);
-    // Sections learned their outcomes before topics were resolved; name each
-    // outcome's primary topic now that the tree knows it.
-    const topicByOutcomeId = new Map();
+    // Sections learned their outcomes before topics were resolved; finishPage
+    // names each outcome's primary topic from the tree.
+    const topics = new Map();
     const topicBySlug = new Map(skillTreeResult.skillTree.topics.map((t) => [t.slug, t]));
     for (const card of skillTreeResult.skillTree.cards) {
         const topic = topicBySlug.get(card.topic);
         if (topic) {
-            topicByOutcomeId.set(card.id, { slug: topic.slug, title: topic.title });
+            topics.set(card.id, { slug: topic.slug, title: topic.title });
         }
     }
-    for (const module of modules) {
-        for (const section of module.sections) {
-            for (const ref of section.learningOutcomes) {
-                const topic = topicByOutcomeId.get(ref.id);
-                if (topic)
-                    ref.topic = topic;
-            }
+    return {
+        errors: {
+            glossaryAliases,
+            courses: courseErrors,
+            uuids: uuidErrors,
+            slugsAndTranscripts,
+            skillTree: skillTreeResult.errors,
+        },
+        tables: {
+            contentIdToModule,
+            glossary: glossaryEntries,
+            moduleContentIds: cardModuleGroups(pages),
+            slugToPath,
+            courseOutcomes,
+            topics,
+        },
+        courses,
+        surveys,
+        meetingDocs,
+        urlsToValidate,
+        skillTree: skillTreeResult.skillTree,
+        articleSegments,
+    };
+}
+/**
+ * Finish one flattened page with what other pages contribute: a card's and a
+ * lens link's module, glossary marks, optional-lens lists, outcomes tested
+ * elsewhere in a course, and outcome topics; and run the checks on the
+ * finished page. Changes `page` in place, so it starts from a page fresh
+ * from flattening: a card's module, once set, is never cleared. `key` is
+ * pageKey of the page.
+ *
+ * Anything a page takes from other pages belongs here, never in flattening.
+ * Flattening runs in one file's unit before the other units have run, so a
+ * value it took from another page would go stale when that page changes,
+ * with no test failing at once.
+ */
+export function finishPage(page, key, tables, { files, tierMap }) {
+    populateCardModuleSlugs(page.sections, tables.contentIdToModule);
+    resolveInlineLensModuleSlugs(page.sections, tables.contentIdToModule);
+    // Glossary marks are offsets into segment text, so they are placed after
+    // every pass that rewrites it (card links and lens links above, card
+    // metadata in the flattener).
+    const glossary = [];
+    for (const section of page.sections) {
+        if (section.glossaryTerms === undefined)
+            continue;
+        const result = applyGlossary(section.glossaryTerms, section.segments, section.sourcePath ?? page.sourcePath ?? page.slug, files, tables.glossary, tierMap);
+        glossary.push(...result.errors);
+        section.segments = result.segments;
+        if (result.glossary.length)
+            section.glossary = result.glossary;
+        delete section.glossaryTerms;
+    }
+    // Card-to-module membership check. Runs after card link resolution above so
+    // every card has its data-lens-card JSON populated, including allowExternal.
+    const cardMembership = checkCardModuleMembership(page, tables.moduleContentIds);
+    // After card resolution: each require_x_optional_lenses list becomes its
+    // section's optionalLensSets (the lens gate reads those).
+    const optionalLensSets = collectOptionalLensSets([page]);
+    const sharedQuestions = sharedQuestionCandidates(page);
+    // Safety-net: catch empty sections/segments in final output
+    const integrity = outputIntegrityCandidates(page, tables.slugToPath.get(page.slug) ?? page.slug);
+    for (const outcomes of tables.courseOutcomes.get(key) ?? []) {
+        for (const section of page.sections)
+            tagLensOutcomes(section, outcomes);
+    }
+    for (const section of page.sections) {
+        for (const ref of section.learningOutcomes) {
+            const topic = tables.topics.get(ref.id);
+            if (topic)
+                ref.topic = topic;
         }
     }
-    // Post-process: assign category to errors that don't already have one
-    for (const error of errors) {
+    return {
+        glossary,
+        cardMembership,
+        optionalLensSets,
+        sharedQuestions,
+        integrity,
+    };
+}
+/**
+ * The final error list in today's order: every unit's errors in path order,
+ * then the checks after the file units in the order they ran, the page
+ * checks in page order within their step. Each error gets its category (the
+ * tier of its file), and dedupeErrors' rule keeps the first copy. `sources`
+ * names, for each final error, the sources (as given) that reported it.
+ */
+export function assembleErrors(units, whole, pages, tierMap) {
+    const reported = [];
+    const add = (source, errors) => {
+        for (const error of errors)
+            reported.push({ error, source });
+    };
+    const fromPages = (step) => {
+        for (const { source, finish } of pages)
+            add(source, finish[step]);
+    };
+    // A cross-page check: the first page (in page order) with a key reports it.
+    const onceFromPages = (step) => reported.push(...reportOnce(pages.flatMap(({ source, finish }) => finish[step].map((candidate) => ({ ...candidate, source })))));
+    for (const { source, errors } of units)
+        add(source, errors);
+    add(WHOLE_CONTENT, whole.glossaryAliases);
+    fromPages("glossary");
+    fromPages("cardMembership");
+    fromPages("optionalLensSets");
+    add(WHOLE_CONTENT, whole.courses);
+    add(WHOLE_CONTENT, whole.uuids);
+    onceFromPages("sharedQuestions");
+    add(WHOLE_CONTENT, whole.slugsAndTranscripts);
+    onceFromPages("integrity");
+    add(WHOLE_CONTENT, whole.skillTree);
+    // The same file is validated once per pass (standalone + once per module
+    // that flattens it), so identical errors are emitted multiple times. Collapse
+    // them so each problem is reported once.
+    const errors = [];
+    const sources = [];
+    const at = new Map();
+    for (const { error, source } of reported) {
         if (!error.category) {
             const tier = tierMap.get(error.file);
             error.category = tier === "wip" ? "wip" : "production";
         }
+        const key = errorKey(error);
+        const index = at.get(key);
+        if (index === undefined) {
+            at.set(key, errors.length);
+            errors.push(error);
+            sources.push([source]);
+        }
+        else if (!sources[index].includes(source)) {
+            sources[index].push(source);
+        }
     }
-    // The same file is validated once per pass (standalone + once per module
-    // that flattens it), so identical errors are emitted multiple times. Collapse
-    // them so each problem is reported once.
+    return { errors, sources };
+}
+/** The source assembleErrors gives errors of the whole-content step. */
+const WHOLE_CONTENT = "whole";
+/** The ProcessResult of finished pages, the whole-content step and the final errors. */
+export function processResult(modules, whole, errors) {
+    const { courses, surveys, meetingDocs, urlsToValidate, skillTree } = whole;
+    const { articleSegments } = whole;
     return {
         modules,
         courses,
         surveys,
         ...(Object.keys(meetingDocs).length > 0 ? { meetingDocs } : {}),
-        errors: dedupeErrors(errors),
+        errors,
         urlsToValidate,
-        skillTree: skillTreeResult.skillTree,
+        skillTree,
         ...(Object.keys(articleSegments).length > 0 ? { articleSegments } : {}),
     };
 }